% =====================================================================
%  VulX Technical Whitepaper
%  "Proving Authorization Bugs in AI-Generated Code"
%  arXiv-ready single-column article. Compiles with pdflatex.
%  Required packages are all in a standard TeX Live install.
% =====================================================================
\documentclass[11pt]{article}

\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{lmodern}
\usepackage[margin=1in]{geometry}
\usepackage{microtype}
\usepackage{amsmath,amssymb}
\usepackage{booktabs}
\usepackage{array}
\usepackage{longtable}
\usepackage{tabularx}
\newcolumntype{L}{>{\raggedright\arraybackslash}X}
% breakable underscore for long identifiers inside tables
\newcommand{\bu}{\_\allowbreak}
\usepackage{xcolor}
\usepackage{listings}
\usepackage{enumitem}
\usepackage{tikz}
\usetikzlibrary{arrows.meta,positioning,fit,backgrounds}
\usepackage[hidelinks]{hyperref}
\usepackage{caption}

% ---- brand-ish colours -------------------------------------------------
\definecolor{vulxred}{HTML}{C8102E}
\definecolor{vulxgreen}{HTML}{1B7F4B}
\definecolor{vulxink}{HTML}{111111}
\definecolor{vulxgrey}{HTML}{6B6B6B}
\definecolor{codebg}{HTML}{F6F6F4}

% ---- listings ----------------------------------------------------------
\lstdefinestyle{vulx}{
  basicstyle=\ttfamily\small,
  backgroundcolor=\color{codebg},
  frame=single,
  rulecolor=\color{vulxgrey!40},
  breaklines=true,
  showstringspaces=false,
  keywordstyle=\color{vulxred},
  commentstyle=\color{vulxgrey}\itshape,
  numberstyle=\tiny\color{vulxgrey},
  xleftmargin=1.2em,
  framexleftmargin=1.0em,
  language=Python,
}
\lstset{style=vulx}

% ---- a small "claim status" tag macro ---------------------------------
\newcommand{\shipped}{\textcolor{vulxgreen}{\textsf{\footnotesize[MEASURED]}}}
\newcommand{\roadmap}{\textcolor{vulxred}{\textsf{\footnotesize[ROADMAP]}}}
\newcommand{\stale}{\textcolor{vulxgrey}{\textsf{\footnotesize[ANECDOTAL]}}}

\newcommand{\code}[1]{\texttt{#1}}

\title{\textbf{Proving LLM-Found Authorization Bugs\\ in AI-Generated Code}\\[4pt]
\large A Deterministic Oracle that Certifies Exploits with \emph{No} LLM Verdict---and Refuses What It Cannot Prove}

\author{Minh Danh Ngo\\
\small Founder, VULX LTD\\
\small 71--75 Shelton Street, London WC2H 9JQ, United Kingdom\\
\small \href{mailto:contact@vulx.ai}{contact@vulx.ai} \quad\textbar\quad \href{https://vulx.ai}{vulx.ai}}

\date{VulX Technical Whitepaper~\textbf{v1.0} --- \today \\[2pt]
\small\textcolor{vulxgrey}{For design partners and reviewers. Not peer-reviewed. Third-party targets anonymized (\S\ref{sec:ethics}).}}

\begin{document}
\maketitle

% =====================================================================
\begin{abstract}
\noindent
Code is now written faster than it can be reviewed: a large and growing fraction
of new software is generated by large language models (LLMs). The vulnerability
class that benefits least from this speed-up, and suffers most from it, is
\emph{broken authorization}---IDOR / Broken Object-Level Authorization
(BOLA)---because authorization is a property of an application's intended access
\emph{policy}, not a syntactic pattern a scanner can match. We argue that for
AI-generated code the binding constraint is not detection but \emph{discovery}
(localizing where an access-control decision is missing relative to the declared
policy) and \emph{proof} (showing the gap is actually exploitable, without
guessing). Concretely, this paper asks the question a security tool must answer
before it can act on an LLM: \emph{when a model claims an authorization bug
exists, how do we know it is real---without trusting the model's word?}

We present \textbf{VulX}, whose central contribution is a \emph{deterministic
exploit oracle}. An LLM proposes; the oracle disposes. The ship-gate is a pure
rule---\code{decide()} $=$ \code{before.present} $\wedge\ \neg\,$\code{after.present}
$\wedge$ \code{control\_ok\_after}---and \emph{never} an LLM verdict: a fix ships
only when a real exploit is observed before the patch, is gone after it, and a
benign request still succeeds (a mandatory negative control). The same rule makes
the oracle a \emph{precision gate}: when a proposed exploit does not actually fire
at runtime, the oracle returns \textsc{not\_reproducible} and \emph{refuses} the
finding rather than reporting it. Around this oracle, discovery and
attack-authoring are autonomous (no hand-fed finding); the test-bed is
harness-provisioned; and the fix is a reference fix used as a proof vehicle, not
an autonomous fixer.

We report what is measured and label what is not. Across eleven real deployed
applications, the oracle \emph{confirmed} six broken-authorization exploits (each
measured before-leak / after-fix / passing-control) and \emph{declined} five more
in three distinct ways---unproven impact, no measurable control, and an
un-buildable target. Only one HIT was found \emph{end-to-end} by autonomous
discovery; the rest were proven on hand-fed shapes. Separately, a deterministic,
LLM-free discovery floor recalls \textbf{5/6} of the proven shapes from source at
high confidence ($\geq 0.7$; recall-first, with precision left to the oracle) and
\textbf{2/2} on synthetic fixtures---advancing proof, breadth, and a discovery
recall floor, though not yet end-to-end autonomous discovery at scale. The
refusals---a tool declining real-looking findings---are the result we most want to
show. As a baseline, two
industry-standard static analyzers (Semgrep, CodeQL) flagged \emph{none} of these
authorization bugs---$0/5$ on the proven HITs; two controlled fixtures additionally
exercise the full loop. We
are explicit about scope: the evidence is small-$n$, early-stage, and---apart from
one app---hand-fed; the reasoning model is hosted Claude Opus 4.8 (not a
VulX-trained model); and autonomous discovery of these shapes, cross-codebase
variant analysis, and a trained attacker are roadmap. The thesis is not that an LLM
can hack---others have shown that---but that its output can be made
\emph{trustworthy}: capability comes from the model; trust comes from the rule.
\end{abstract}

\noindent\textcolor{vulxgrey}{\footnotesize\textbf{Honesty note.} Tags
\shipped\ and \roadmap\ appear throughout. All figures are
\emph{internal and self-reported} unless a primary external source is cited.}

\vspace{3pt}\noindent\textcolor{vulxgrey}{\footnotesize\textbf{Keywords:} broken
access control; BOLA / IDOR; LLM security; deterministic exploit oracle; exploit
verification; AI-generated code; Supabase / Postgres Row-Level Security.}

\tableofcontents

% =====================================================================
\section{Introduction}

The economics of software creation changed when LLMs began writing production
code. Speed went up; review did not. The result is a widening gap between code
that \emph{works} and code that has been \emph{reasoned about for security}.
Some of that gap is well served by existing tooling: hardcoded secrets,
known-vulnerable dependencies, and unsafe primitives (\code{eval}, string-built
SQL) are syntactic and a good scanner finds them.

Authorization is not like that. Whether a route is allowed to return an object
depends on \emph{who is asking} and on the application's \emph{intended policy}
(``a user may read only their own documents''). That policy lives across several
artifacts---database Row-Level Security (RLS) rules, ownership columns and
foreign keys, framework route guards---and the bug is the \emph{absence} of a
check relative to that policy. No pattern match expresses ``this query should
have been scoped to the caller and is not.'' This is precisely the class that
AI-generated code gets wrong most often, because the model emits a plausible,
working query and silently omits the ownership predicate.

The research question is narrow on purpose---not ``can an LLM hack a site''
(others have shown that it can), but \emph{when an LLM claims a
broken-access-control bug, how do we prove it is real---and decline it when it is
not---without ever trusting the model's verdict?} This paper makes four claims, in
order of importance:

\begin{enumerate}[leftmargin=1.4em,itemsep=2pt]
  \item \textbf{Correctness must come from a deterministic oracle, not a model}
    (\S\ref{sec:oracle}, \S\ref{sec:eval})---\emph{this is the contribution.} A
    pure before/after/negative-control rule decides whether anything ships; no LLM
    renders the verdict, so model hallucination cannot produce a false ``proven.''
    The same rule makes the oracle a \emph{precision gate}: a real-looking finding
    whose exploit does not fire is returned \textsc{not\_reproducible} and
    \emph{refused}---declining a false finding, not merely confirming a true one.
  \item \textbf{Authorization is the dominant blind spot of AI-generated code}
    (\S\ref{sec:gap}). We motivate this with a public benchmark of real CVEs in
    AI-generated code and a real-world authorization CVE class.
  \item \textbf{An LLM can autonomously discover and exploit these bugs}
    (\S\ref{sec:loop}, \S\ref{sec:anatomy})---supporting, early-stage evidence,
    not a benchmarked rate. Opus 4.8 autonomously reconstructs the policy gap and
    authors a working cross-user attack; the applied fix is a \emph{reference fix}
    used as a proof vehicle, not an autonomous fixer. We present this as
    capability evidence at small $n$, deliberately not as the headline.
  \item \textbf{Discovery is the hard input, not a solved one} (\S\ref{sec:discover}).
    A position, not a result we measure here, well supported by prior work:
    access-control intent is \emph{implicit} in code rather than stated as a
    spec~\cite{sun2011}, and detecting \emph{missing} checks is specification-free
    and false-positive-prone even for strong tools~\cite{near2016}. We treat
    discovery as our candidate source and make no generalization claim for it.
\end{enumerate}

We describe the pipeline (discover $\to$ author attack $\to$ oracle $\to$ apply
reference fix $\to$ oracle re-check), then report results and an unusually explicit
limitations section, because a security vendor that overclaims has already lost.

% =====================================================================
\section{The authorization gap in AI-generated code}
\label{sec:gap}

\paragraph{Real CVEs in AI-generated code.}
The Georgia Tech SSLab \emph{Vibe Security Radar}~\cite{viberadar} collects real
CVEs introduced by AI coding tools. In separate testing of our \emph{production
static scanner}---a different system from the oracle described in this paper, used
here only to characterize the problem---the pattern of \emph{misses} is the
telling signal: syntactic classes (e.g.\ JavaScript injection-style bugs) are
caught reliably, while the residual failures cluster on authorization logic---an
empty allow-list that returns ``allowed,'' a trusted-proxy header that skips a
role check---and the authorization CWEs (CWE-863 ``incorrect authorization,''
CWE-306 ``missing authentication'') are exactly where pattern-based detection is
weakest. Pattern matching plateaus where authorization logic begins. That is the
motivation for a discovery-and-proof approach rather than more patterns; it is
\emph{not} a result of the system this paper evaluates.

\paragraph{An authorization CVE class, at scale.}
The risk is not hypothetical. CVE-2025-48757~\cite{lovable} describes RLS
misconfiguration and anonymous-key exposure in applications built with the
AI app-builder Lovable: an analysis found $170$ of $1{,}645$ projects
($10.3\%$) affected, with $303$ vulnerable endpoints, rated CVSS $9.3$
(Critical). The pattern---an AI-generated app fronting a Supabase/Postgres
database where the intended RLS policy is absent, bypassed via the service-role
key, or never enforced in the route---is the exact target this paper addresses.

\paragraph{Why incumbents stop at the static half.}
Recent AI-assisted static tools have improved IDOR detection markedly. Semgrep's
multimodal detector~\cite{semgrep} reports $61\%$ IDOR precision (versus $22\%$
for a raw LLM baseline), $1.9\times$ recall, and at least one IDOR found in
$80\%$ of repositories (in private beta). This is real progress on
\emph{candidate generation}---and it sharpens, rather than refutes, our thesis:
once you have candidates at $\sim\!61\%$ precision, the remaining value is
\emph{proof}. A static claim that an IDOR ``exists'' is still a probability; a
deploy decision needs certainty. VulX's contribution is the runtime oracle that
converts a candidate into a proven, fixed, re-attack-survived result---or into
nothing at all.

% =====================================================================
\section{Threat model and scope}
\label{sec:threat}

\paragraph{Target.}
Web applications with a clear data-ownership model, exemplified by the
Next.js / Supabase / Postgres stack: framework routes or Supabase Edge Functions
that read and write rows in tables whose ownership is declared by RLS policies
and ownership columns (\code{owner\_id = auth.uid()}).

\paragraph{Adversary.}
An authenticated but unprivileged user (``attacker'') who can address objects
they do not own---by manipulating an object identifier in a path or query
parameter---and attempts to read or modify another user's (``victim'') data.
This is Broken Object-Level Authorization (BOLA / IDOR; CWE-639, and the related
CWE-285/862/863)---the top-ranked risk in the OWASP API Security Top
10~\cite{owaspapi}.

\paragraph{What we prove vs.\ what we assume.}
The system \emph{assumes} a discovered candidate names the suspect route, the
owned resource, and the object-identifier source. The system \emph{proves}, at
runtime against a live instance, whether a cross-user read or write actually
occurs, whether a proposed patch eliminates it, and whether the patch leaves
legitimate same-user access working. The oracle's deterministic guarantees
(\S\ref{sec:oracle}) cover time-based and boolean-blind SQLi, reflected XSS,
command injection, LFI/path-traversal, and read/write BOLA; \emph{all end-to-end
evidence reported here is BOLA} (\S\ref{sec:eval}). Stored/DOM XSS, second-order
SQLi, and SSRF are explicitly out of the oracle's scope.

% =====================================================================
\section{System overview}
\label{sec:system}

VulX is a pipeline of five stages with a strict separation of duties: LLMs
propose, a deterministic oracle disposes.

\begin{figure}[h]
\centering
\begin{tikzpicture}[
  node distance=6mm and 15mm,
  box/.style={draw, rounded corners, align=center, minimum height=10mm,
    minimum width=20mm, font=\footnotesize, thick},
  llm/.style={box, draw=vulxred, fill=vulxred!5},
  det/.style={box, draw=vulxgreen, fill=vulxgreen!6},
  arr/.style={-{Stealth[length=2mm]}, thick, vulxink}
]
\node[llm] (disc) {Discover\\\scriptsize static + LLM re-rank};
\node[llm, right=of disc] (hack) {Author attack\\\scriptsize Opus 4.8};
\node[det, right=of hack, draw=vulxink, fill=vulxink!4, line width=0.6mm] (orac) {\textbf{Oracle}\\\scriptsize pure rule};
\node[det, right=of orac] (fix) {Reference fix\\\scriptsize proof vehicle};
\node[det, right=of fix] (fly) {Flywheel\\\scriptsize de-id, $\to$ store};

\draw[arr] (disc) -- node[above,font=\tiny]{candidate} (hack);
\draw[arr] (hack) -- node[above,font=\tiny]{attack} (orac);
\draw[arr] (orac) -- node[above,font=\tiny]{before-proof} (fix);
\draw[arr] (fix) to[bend left=28] node[below,font=\tiny]{patched} (orac);
\draw[arr] (fix) -- node[above,font=\tiny]{certified} (fly);
\end{tikzpicture}
\caption{The VulX pipeline. Red = LLM (proposes, may hallucinate); green =
deterministic. The \textbf{oracle} (black, the contribution) is invoked twice:
once on the authored attack (must fire), once after the reference fix is applied
(must not fire, control must pass). It alone issues a verdict; the LLM never does.}
\label{fig:pipeline}
\end{figure}

\begin{enumerate}[leftmargin=1.4em,itemsep=3pt]
  \item \textbf{Discover} (\S\ref{sec:discover}) reconstructs the declared
    authorization model from SQL migrations and routes and emits
    \emph{candidates} with a confidence score. Deterministic; an optional LLM
    layer only re-ranks.
  \item \textbf{Hacker} (\S\ref{sec:loop}) authors a concrete two-principal
    attack (attacker tries to read/write the victim's object). The model emits a
    \emph{plan}; the leak/control signals are filled from ground truth, so the
    signal is never invented by the model.
  \item \textbf{Oracle} (\S\ref{sec:oracle}) measures exploitability before and
    after a patch plus a negative control, and returns one of five verdicts via
    a pure rule.
  \item \textbf{Reference fix} (\S\ref{sec:loop}) is applied as the proof
    vehicle---the change whose effect the oracle measures. On controlled fixtures
    an LLM authors the candidate patch (passing a backdoor-safety gate); on real
    apps the fix is a known-good reference. An autonomous production fixer is
    roadmap (\S\ref{sec:limits}), so we never claim the fix step is autonomous.
  \item \textbf{Flywheel} (\S\ref{sec:flywheel}) de-identifies an oracle-confirmed
    finding into a code-free signature for cross-codebase reuse. The
    de-identification gate contains no LLM.
\end{enumerate}

% =====================================================================
\section{Discovery: the hard problem}
\label{sec:discover}

Discovery localizes a candidate BOLA without executing anything. Its design
principle is recall-first: ``a missed BOLA is worse than a false positive the
oracle removes.'' It therefore emits \emph{candidates, never verdicts}.

\subsection{Reconstructing the declared policy}
Discovery unions three deterministic extractors:

\begin{itemize}[leftmargin=1.4em,itemsep=2pt]
  \item \textbf{Route / sink loci} via structural matching (ast-grep) over
    Next.js App-Router handlers, Server Actions, and Supabase Deno Edge
    Functions, plus a body-scoped scan for untrusted inputs, auth guards, and the
    auth subject.
  \item \textbf{Data sinks and client kind}: every \code{supabase.from(\dots)} /
    \code{.rpc(\dots)} chain, with resolution of whether the call runs as
    \code{anon}, \code{authenticated}, or---critically---\code{service-role}
    (which \emph{bypasses RLS}), by following import aliases and
    \code{Deno.env.get("\dots SERVICE\_ROLE\_KEY")} bindings.
  \item \textbf{RLS / ownership model}: a purpose-built Postgres parser that
    extracts RLS policies (including quoted policy names with spaces, which naive
    parsers drop), ownership columns, and predicates such as
    \code{owner\_id = auth.uid()}, mapping \code{auth.uid()} to the JWT
    \code{sub} claim.
\end{itemize}

A candidate is raised when a handler reaches an attacker-controlled object id,
queries an \emph{owned} table, is \emph{not} scoped by the ownership column, RLS
cannot save it (service-role bypass, or an unknown client treated fail-open), and
no post-fetch ownership comparison re-imposes scope. Confidence is a transparent
additive prior (e.g.\ $+0.25$ if ownership is RLS-declared, $+0.10$ if RLS is
bypassed, $-0.10$ if the client kind is unknown), clamped to $[0.05, 0.99]$.

\subsection{The LLM is a re-ranker, not an oracle}
An optional inference layer asks a frontier model whether a route's enforcement
matches the declared access model. By construction it (i) never drops a candidate
(recall-first), (ii) only refines confidence and reasoning, and (iii) fails soft
on any model error. The unit handed downstream is a ``context dossier'' (intended
access, missing checks, exploit scaffolding, evidence)---not a snippet and a line
number. The principle is stated in the code: ``an oracle proves verdicts later,
an LLM's say-so does not.''

\subsection{What is measured}
On the one \emph{real, unmodified} open-source target
(\code{sharespace-connect}, a Lovable-built document app on a Vite/React +
Supabase Edge-Function stack), discovery localizes the BOLA at confidence
$\mathbf{0.99}$ with \emph{zero} LLM calls \shipped: it walks the Deno edge
function, resolves the \code{SUPABASE\_SERVICE\_ROLE\_KEY} binding (RLS bypass),
parses the quoted RLS policy ``\emph{Users can view own documents}'' keyed to the
JWT \code{sub}, and identifies that the route checks only that an auth header is
\emph{present}---never that the bearer \emph{owns} the document---\emph{deterministically,
with no model}. The optional LLM inference layer (the re-ranker, \S\ref{sec:discover})
has been exercised against a real model only in the autonomous runs---the crafted
\code{bola-notes} fixture and the \code{sharespace} autonomous run---at roughly
\$0.06--0.07 each; it refines confidence but cannot fabricate a candidate. We separate the two questions
discovery raises---can it \emph{recall} a known bug shape, and can it find one
end-to-end---and report them apart.

\paragraph{A measured recall floor (deterministic, LLM-free) \shipped.}
We measured the \emph{recall} of the deterministic discovery floor---the LLM-free
structural pass---by running it over each app's source and asking whether it flags
the oracle-proven vulnerable shape. On the six oracle-proven real-app HITs it
recalls \textbf{5/6 at high confidence ($\geq 0.7$)} and \textbf{6/6 at a loose
threshold} (the sixth, App~E, only as a weak $0.45$ incidental hit); on synthetic
fixtures, \textbf{2/2}. We report the real and synthetic figures separately and
never blend them. An independent overfit audit confirms the floor
\emph{generalizes}: it flagged never-seen tables and skipped a benign public-data
lookup purely from structural signals (RLS state, anonymous grants, PII/ownership),
with no LLM in the loop.

\paragraph{What this is---and is not.}
This is \emph{recall}, by design (discovery is recall-first); we make \emph{no}
precision claim. The floor is deliberately noisy on positives---App~D's true hit,
for instance, sits among $139$ candidates---and the oracle, not discovery, is what
removes that noise (the $\leq 5$ noise cap applies only to negatives). It also
measures recall of the proven shapes \emph{from source}: the external HITs were
still oracle-proven on \emph{hand-fed} shapes, and \emph{end-to-end} autonomous
discovery beyond \code{sharespace-connect} (the one app auto-found end-to-end)
remains the open follow-up (\S\ref{sec:future}). Discovery is the \emph{input} to
this paper's contribution; the contribution we defend is the deterministic proof of
a fix (\S\ref{sec:oracle}, \S\ref{sec:eval}), which holds regardless of where a
candidate originates. Comparative discovery against baselines
(Sun~et~al.~\cite{sun2011}, Near and Jackson~\cite{near2016}) is future work.

% =====================================================================
\section{The deterministic exploit oracle}
\label{sec:oracle}

The oracle is the heart of the system and the reason it is safe to run against
real code. It never asks a model whether a fix worked; it \emph{measures}.

\subsection{Why ``verified'' usually lies}
A common shortcut---one VulX's own production scanner used and which we now label
``heuristically verified''---is to re-test a patched endpoint and grep the
response for an error string. This passes blind/time-based injections (which leak
nothing into the body), passes routes that ``got fixed'' by crashing (a 500 has
no error string), and, in the static case, can mark a finding fixed with
\emph{zero} execution. A substring scan is not a proof.

\subsection{Three measurements, one pure rule}
The oracle runs a probe three times and decides with a deterministic rule:

\begin{itemize}[leftmargin=1.4em,itemsep=1pt]
  \item \textbf{Before}: fire the exploit at the unpatched app. The signal must
    be \emph{present} (else the finding was a false positive).
  \item \textbf{After}: apply the patch, fire the \emph{identical} probe. The
    signal must be \emph{gone}.
  \item \textbf{Control} (negative control): a benign, legitimate request must
    \emph{still succeed}. This catches ``fixed it by breaking the app.''
\end{itemize}

\begin{equation}
\textsf{BLOCKED} \;\Longleftrightarrow\;
\underbrace{\text{exploitable\_before}}_{\text{real}}
\;\wedge\;
\underbrace{\neg\,\text{exploitable\_after}}_{\text{killed}}
\;\wedge\;
\underbrace{\text{control\_ok\_after}}_{\text{still works}}
\end{equation}

The decision function is pure and total over five verdicts; only
\textsf{BLOCKED} ships.

\begin{lstlisting}[caption={The ship-gate. Pure, deterministic, never an LLM call.},captionpos=b]
def decide(before, after, control_ok_after) -> Verdict:
    if before.error is not None or after.error is not None:
        return Verdict.INCONCLUSIVE      # couldn't measure -> NO PR
    if not before.present:
        return Verdict.NOT_REPRODUCIBLE  # false positive    -> NO PR
    if after.present:
        return Verdict.STILL_EXPLOITABLE # fix failed         -> NO PR
    if not control_ok_after:
        return Verdict.FIX_BROKE_APP     # broke the app      -> NO PR
    return Verdict.BLOCKED               # proven & non-breaking -> SHIP
\end{lstlisting}

\noindent A hardened wrapper can only ever \emph{downgrade} a would-be
\textsf{BLOCKED} (e.g.\ to \textsf{FIX\_BROKE\_APP} if the post-patch app is
unhealthy, or \textsf{INCONCLUSIVE} if the absence is unstable across repeated
measurement); it never upgrades. \textbf{Four of five verdicts open no pull
request.} ``No PR is safer than a false proven PR.''

\begin{figure}[h]
\centering
\begin{tikzpicture}[
  node distance=4mm and 12mm,
  m/.style={draw, rounded corners, align=left, font=\scriptsize, minimum width=33mm, minimum height=7mm},
  dec/.style={draw, vulxink, line width=0.6mm, rounded corners, align=center, font=\footnotesize\bfseries, minimum width=20mm, minimum height=13mm, fill=vulxink!4},
  vbox/.style={draw, rounded corners, align=left, font=\scriptsize, minimum width=40mm},
  ar/.style={-{Stealth[length=2mm]}, thick}
]
\node[m] (b) {\textbf{Before:} exploit \emph{must} fire};
\node[m, below=of b] (a) {\textbf{After:} \emph{must} vanish};
\node[m, below=of a] (c) {\textbf{Control:} benign \emph{must} pass};
\node[dec, right=of a] (d) {decide()\\\scriptsize pure rule};
\node[vbox, right=18mm of d, fill=vulxgreen!10, draw=vulxgreen] (ship) {\textcolor{vulxgreen}{\textbf{BLOCKED} $\to$ SHIP}};
\node[vbox, below=5mm of ship, fill=vulxred!6, draw=vulxred] (nopr)
  {\textcolor{vulxred}{\textbf{NO PR}} $=$ \textsc{not\_reproducible} $\cdot$\\
   \textsc{still\_exploitable} $\cdot$ \textsc{fix\_broke\_app} $\cdot$ \textsc{inconclusive}};
\draw[ar] (b) -- (d); \draw[ar] (a) -- (d); \draw[ar] (c) -- (d);
\draw[ar] (d) -- (ship);
\draw[ar] (d) -- (nopr);
\end{tikzpicture}
\caption{The ship-gate as a precision gate. Three measurements feed one pure
rule. Exactly one outcome opens a pull request; the other four---including
\textsc{not\_reproducible}, which fires when a real-looking exploit does not
actually occur---open none.}
\label{fig:oracle}
\end{figure}

\subsection{The precision gate: refusing what cannot be proven}
The most important verdict is not \textsf{BLOCKED} but \textsf{NOT\_REPRODUCIBLE}.
When discovery and the LLM produce a confident, plausible candidate but the
exploit \emph{does not fire} against the live app, \code{before.present} is false
and the oracle returns \textsf{NOT\_REPRODUCIBLE}---\textbf{the finding is refused,
not reported.} This is where an LLM-only pipeline fails: it surfaces the
plausible-but-false bug as real, whereas the oracle cannot, because the verdict is
a measurement, not an opinion. We show this on a real application in
\S\ref{sec:anatomy} (App~F). A tool that refuses an unexploitable finding is
the entire value proposition.

\subsection{The BOLA signal: a leaked secret, not an HTTP 200}
For read-BOLA, the signal is whether the \emph{victim's} secret marker (a value
seeded into the victim's row, never present in the attacker's request) appears in
the \emph{attacker's} response---regardless of HTTP status. This is deliberate:
gating on \code{2xx} is both unnecessary and unsafe, because a ``fix'' that
returns 403 \emph{but still leaks the body} must not read as fixed. Because the
marker originates in the victim's data, there is no input-reflection false
positive.

For write-BOLA, the signal is a ground-truth \emph{read-back} as the owner: did
the attacker's tampering land in the victim's row (\code{readback\_contains}), or
did the attacker's delete remove it (\code{readback\_absent})? The write's own
HTTP status is evidence only.

\subsection{The negative control is what makes it sound}
The control for read-BOLA is \emph{owner-reads-own}: after the patch, the owner
must still get \code{2xx} plus their own marker. This is the single
distinguishing test between a real fix and a deny-all ``fix'' that locks everyone
out---the latter fails the control and is correctly ruled
\textsf{FIX\_BROKE\_APP}. The control for write-BOLA is \emph{owner-writes-own}
(write then read-back), catching both deny-all-writes and silent no-op-write
``fixes.'' BOLA is treated as an explicitly two-principal attack, so it is routed
through dedicated factories rather than the single-finding probe registry.

\subsection{Resistant signals and a clean reset}
The injection probes are engineered against being fooled. The timing-SQLi probe
requires the induced delay to \emph{scale} with the injected sleep
(\code{SLEEP(2)} vs \code{SLEEP(6)}); a flat delay (a WAF tarpit or rate limiter)
reads as \emph{absent}, not as a false positive. The differential probe
normalizes nonces (CSRF tokens, timestamps) and returns \emph{inconclusive}
rather than ``absent'' on a nondeterministic endpoint. The canary-reflection
probe checks token-level encoding so a fully entity-encoded reflection is
correctly judged safe.

Soundness of the before/after comparison depends on a clean state reset between
measurements. The Docker-Compose backend snapshots the seeded database as a
golden template and resets via Postgres
\code{CREATE DATABASE \dots TEMPLATE} (after terminating connections and
dropping the live database), then warms the connection pool. The reset path is
fail-closed and is exercised in every committed run (each records its
\code{db\_resets} count).

% =====================================================================
\section{From candidate to certified fix: the loop}
\label{sec:loop}

Around the oracle sits a thin loop---\emph{author attack $\to$ prove $\to$ apply
fix $\to$ re-prove}---whose brains and sandbox are injected behind Python
\code{Protocol}s, so the identical loop runs offline with fakes (tests) or with a
live model and sandbox. The reasoning model is hosted \textbf{Claude Opus 4.8}; a
VulX-trained model is roadmap behind the same seam (\S\ref{sec:future}).

\paragraph{The LLM's job: reason about the policy, author the attack.}
Given a candidate dossier (suspect route, owned resource, object-id source, and
the reconstructed intended rule), Opus 4.8 answers one question: \emph{does the
route's enforcement match the intended access policy, and if not, how is the gap
exploited?} Its output is concrete and checkable---a victim object, an actor, and
a rationale---e.g.\ verbatim: ``\emph{the handler fetches the note with the
service-role (BYPASSRLS) client by id only and never compares note.user\_id to
claims.sub, so any authenticated user can read another user's note.}'' Crucially
the model emits a \emph{plan}; the leak and control signals are filled from ground
truth, so the model can never fabricate the proof (\S\ref{sec:oracle}). This is
supporting capability evidence (\S\ref{sec:anatomy}), not a benchmarked rate.

\paragraph{The fix is a proof vehicle, not an autonomous fixer.}
What the oracle measures is the \emph{effect} of a fix. On real applications the
fix is a known-good \emph{reference} patch---a vehicle to exercise the
before/after/control proof---\emph{not} the output of an autonomous fixer. On
controlled fixtures we additionally let the LLM author the candidate patch
(passing a backdoor-safety gate: no removed auth checks, no new
\code{eval}/\code{exec}/shell primitives) and run the adversarial battle below. A
productized autonomous fixer is roadmap (\S\ref{sec:limits}); we never claim the
fix step is autonomous on real apps.

\paragraph{The adversarial battle (fixtures).}
On fixtures, once a fix holds against the first exploit, the LLM is shown the
\emph{applied patch} and asked for a bypass, up to $5$ rounds, and is required to
\emph{give up honestly}---``a fix that truly holds should make you give up''---rather
than invent a bogus payload. A landing bypass forces another fix, after which the
oracle re-proves the new bypass \emph{and} every prior probe. A round counts as
``survived'' only when a \emph{confirmed} bypass was attempted and blocked, so
``survived $N$ rounds'' is always literally true. A run from a simulated sandbox is
permanently barred from emitting a ``proven'' result---the real sandbox does not
carry the ``simulated'' marker, and its absence is load-bearing.

% =====================================================================
\section{The oracle in action: confirm, then refuse}
\label{sec:anatomy}

We show the oracle doing the two things that matter: \emph{confirming} real
exploits, and \emph{refusing} real-looking ones. Every figure is from a committed
run-log (\code{simulated=false}; model \code{claude-opus-4-8}); apps were stood up
in isolated, harness-provisioned test-beds (\S\ref{sec:ethics}).

\subsection{Confirm --- a corpus of proven authorization bugs on real apps}
We walk through one HIT in full, then summarize five more. \textbf{Honest scope:}
of these six real-app HITs, only \code{sharespace-connect} was found by autonomous
discovery; the other five were proven on \emph{hand-fed} exploit shapes. They
therefore strengthen the \emph{proof} (and the range of flaw shapes the oracle
handles), \emph{not} a discovery rate (\S\ref{sec:discover}).

\code{sharespace-connect} is a real Lovable-built document app on a
Supabase/Deno-edge stack. Discovery auto-found the candidate \emph{from source} at
confidence $0.97$ in the autonomous run (no hand-feeding; the zero-LLM static
localizer alone scores $0.99$, \S\ref{sec:discover}); Opus 4.8 then reasoned,
verbatim, that the edge function ``\emph{selects the document solely by id with
.single() and never compares the row's owner\_id to the authenticated caller
(auth.uid()) nor checks is\_public \dots\ so any authenticated user can read
another user's private document}.'' The oracle fired the authored attack: the
attacker's response leaked the victim canary \code{CANARY-SHARESPACE-3df9a1c7e2} at
HTTP \code{200} (\emph{before}: present). A reference fix was applied; the identical
probe then returned the canary \emph{absent} at \code{404} (\emph{after}: gone),
and the negative control---the owner reading their own document---still succeeded.
Verdict: \textcolor{vulxgreen}{\textbf{BLOCKED}}; $3$ LLM calls (cost in
\S\ref{sec:eval}). Discovery and attack-authoring were autonomous; the test-bed
was harness-provisioned; the fix was a reference patch.

\paragraph{Five more, across distinct flaw shapes.}
Batch-2 added five more oracle-proven HITs on real deployed apps---each measured
before/after/control on a warm-E2B test-bed, byte-identical to its as-built
source---spanning five different ways authorization breaks:

\begin{table}[h]
\centering\scriptsize
\setlength{\tabcolsep}{4pt}\renewcommand{\arraystretch}{1.3}
\begin{tabularx}{\textwidth}{@{}>{\raggedright\arraybackslash}p{1.5cm} >{\raggedright\arraybackslash}p{3.2cm} L@{}}
\toprule
\textbf{App} & \textbf{Flaw shape} & \textbf{Proof (before $\to$ after; control)} \\
\midrule
App~A & unauthenticated broadcast-read: an admin endpoint returns all users' rows with no auth & all-users data leaked $\to$ \code{401} after an auth check; control holds \\
App~B & share-token enumeration exposing private records & anon enumerates share tokens and reads others' records $\to$ token secrecy restored; legitimate-user control holds \\
App~C & anon table-read via a permissive RLS policy (\code{USING(true)}) & anon reads all private rows $\to$ policy dropped, read returns \code{[]}; owner control holds \\
App~D & anon read via a view over \code{auth.users} exposing user PII & the view exposes PII to anon $\to$ \code{REVOKE} $\to$ \code{401}; owner control holds \\
App~E & edge-read-BOLA: a service-role edge function reads by a caller-supplied identifier with no owner check (\code{verify\_jwt} satisfied by the anon key) & any record's PII read by identifier (CWE-639) $\to$ \code{401} after an owner check; owner control \code{200} \\
\bottomrule
\end{tabularx}
\caption{Five further real-app HITs (anonymized; \S\ref{sec:ethics}), all
\textcolor{vulxgreen}{BLOCKED}: a real leak before, gone after a reference fix, with
a passing legitimate-user control. Hand-fed shapes (not autonomous discovery); they
broaden the proof across flaw types, not a discovery rate.}
\label{tab:confirm}
\end{table}

App~E additionally produced an \emph{identical} verdict on local Docker and on
E2B---a backend-independent proof point: the verdict tracks the measured exploit,
not the sandbox it ran in.

\subsection{Refuse --- the honesty contract, three ways}
The system declines to confirm in three distinct ways, each a different reason a
finding cannot be stood behind. (Third-party apps are anonymized; \S\ref{sec:ethics}.)

\paragraph{Mode 1 --- \textsc{not\_reproducible}: a real flaw whose impact does not
reproduce (App~F).}
This is the result we most want to show, and it is fully \emph{deterministic}: the
run made \textbf{zero LLM calls}. App~F was a \emph{hand-fed oracle test}---a focused
precision-gate demonstration, not an autonomous discovery run---and it contains a
\emph{genuine} authorization flaw: a sharing-grant table whose \textsc{insert}
policy is \code{WITH CHECK (auth.uid() = <granter>)} and never checks that the caller
owns the shared record, so any user can self-grant a share on another user's record.
The exploit is a two-step chain---a self-grant write, then a cross-actor read of the
victim's record.

The write layer behaves exactly like a real bug: the attacker's self-grant returns
\code{201} before the fix and \code{403} after, while a legitimate owner-initiated
share still returns \code{201} (flaw real; fix rejects the attacker; control holds).
\emph{Yet the oracle returns} \textcolor{vulxred}{\textbf{NOT\_REPRODUCIBLE}}. After
the \code{201} self-grant, the attacker's cross-actor read returns \code{200} but an
empty body \code{[]}---the planted canary never appears. The cause is a latent
platform footgun: the share-view \textsc{select} policy resolves the caller's email
through a subquery on \code{auth.users}, but that table is RLS-enabled with zero
policies, so the subquery yields \code{NULL}, the \code{EXISTS} never matches, and
the read comes back empty (the idiomatic fix would key on \code{auth.jwt()->>'email'}).

\textbf{The authorization flaw is real; the confidentiality impact is not.} A
scanner that flagged the missing ownership check would report a false positive. The
oracle---which requires a \emph{measured} leak, not a plausible one---declines to
confirm, with no model in the loop. It is not saying ``no bug''; it is saying ``this
real flaw does not yield the exploitable confidentiality breach we require to
confirm.'' That distinction is the precision gate (\S\ref{sec:oracle}) at its most
valuable.

\paragraph{Mode 2 --- \textsc{inconclusive}: no baseline to form a control (App~G).}
App~G leaks real data---but it is a \emph{missing-authentication} flaw (CWE-306), not
object-level authorization (CWE-639). The app exposes everything to anonymous
callers, so there is no authenticated owner-read baseline against which a negative
control could be defined. Rather than fabricate one, the oracle returns
\textcolor{vulxred}{\textbf{INCONCLUSIVE}} and refuses to issue \textsc{blocked}---%
even though the leak is real. Declining to invent a control it cannot measure is the
same discipline as declining to confirm an unmeasured leak, and it cleanly separates
CWE-306 (missing auth) from the CWE-639 (BOLA) the oracle is built to prove.

\paragraph{Mode 3 --- standup-blocked: the target cannot be built from source.}
Three targets (App~H--App~J) keep core tables in the Supabase/Lovable dashboard, not
the repository; their RLS policies reference tables the repo never \code{CREATE}s.
The harness cannot stand up a faithful instance without fabricating schema, so a
cheap schema-completeness gate detects the gap and \emph{defers} rather than guess.
This is not an oracle verdict but a provisioning limit---the off-repo-configuration
ceiling, which turns out to be the dominant real-world wall (\S\ref{sec:limits}).

Together these are honest refusals in three modes---unproven impact (App~F), no
measurable control (App~G), and an un-buildable target (App~H--App~J)---each chosen
over reporting a finding the system cannot prove.

\subsection{The full loop, on a controlled fixture (bola-notes)}
On controlled fixtures we exercise the entire loop, including the LLM-authored fix
and the adversarial battle. On \code{bola-notes} (route
\code{web/app/api/notes/[id]/route.ts}, a service-role read by id), Opus 4.8
authored the attack ((\emph{before}) leaked \code{CANARY-BOB-SECRET-7f3a9c} at
\code{200}), then proposed a minimal, framework-idiomatic patch that passed the
safety gate:
\begin{lstlisting}[language={},caption={LLM-authored patch (verbatim, fixture). The single added line re-imposes the ownership check RLS would have enforced.},captionpos=b]
   const { data: note, error } = await supabaseService
     .from("notes")
     .select("*")
     .eq("id", params.id)
 +   .eq("user_id", claims.sub)
     .single();
\end{lstlisting}
The oracle re-fired the probe (\emph{after}: canary absent at \code{404}), the
owner-reads-own control still passed, and across $8$ database resets the verdict
was \textcolor{vulxgreen}{\textbf{BLOCKED}}. Shown its own patch, Opus 4.8 then
tried to bypass it and \emph{conceded honestly}---``\emph{\dots\ the symmetric
bob$\to$alice access \dots\ is equally blocked \dots\ so the broken authorization
is genuinely closed}''---rather than invent a payload, making the round a genuine
``survived.'' Total LLM cost \$0.084. (This is where the LLM-authored fix and
battle are exercised; on the real apps above, the fix is a reference patch.)

\section{The data flywheel}
\label{sec:flywheel}

Each oracle-confirmed finding is distilled into a \emph{code-free} signature so
that one proven bug can drive variant analysis across other codebases. The
privacy posture is the trust gate: \emph{metadata, never code}.

The de-identification gate is a \textbf{type-total positive allowlist}---it walks
the entire record and \emph{raises} on anything not provably safe, rather than
returning a partially-clean record. Keys must be known structural names; string
values must be controlled categorical tokens, allowlisted CWEs, or fixed
structural shapes; integers are digit-capped and floats are range/precision
capped (closing a float-mantissa side channel); subclasses, bytes, and sets are
refused. Defence-in-depth detectors reject emails, UUIDs, JWTs, PEM/API-key
shapes, and long digit/hex runs over normalized text. \textbf{No LLM is in the
gate}---a model there would itself be an exfiltration surface.

Concretely, a route like \code{/api/tickets/[id]} becomes the pattern
\code{/SEG/OWNED\_RESOURCE/[dyn]}; the resource becomes \code{OWNED\_RESOURCE};
the canary value is recorded only as the category \code{marker}; and the
cross-repo join key is a role-abstracted \emph{authorization-pattern signature}
carrying no customer noun, e.g.
\begin{quote}\small\ttfamily
bola|object|missing|src:route\_param|sink:select:OWNED\_RESOURCE\_READ|\\
check:OWNERSHIP\_PREDICATE(OWNER(RESOURCE)==ACTOR\_ID)|rls\_bypass|\\
fw:next-app-router|db:supabase-postgres
\end{quote}
A build-time self-check enumerates every raw identifier from the source run as a
``declared secret'' and asserts none survive into the serialized payload; a ship
script independently re-scans and refuses to ship on any leak. The gate was
adversarially red-teamed across four passes ($24$ leaks found $\to$ fixed $\to$
converged to $0$). The gate has now had its \emph{first real multi-finding fill}:
the four batch-2 HITs were de-identified into four distinct, idempotent,
zero-raw-identifier records---the first time the flywheel was loaded with measured
findings rather than modeled ones, exercising the additive, closed gate end-to-end.
The remaining step is the cross-machine live transmission to the central store,
which we have not yet executed; today's round-trip runs against an in-process
replica of the server (\S\ref{sec:limits}).

% =====================================================================
\section{Evaluation}
\label{sec:eval}

This paper's contribution is the oracle, so the evaluation asks one question:
\emph{does the oracle issue correct, mechanically-grounded verdicts---confirming
real exploits and, just as importantly, refusing non-reproducible ones?} We report
(A) the committed end-to-end oracle runs and (B) cost.

We deliberately do \emph{not} report a detection-rate benchmark. At this stage $n$
is far too small to support a rate, and a headline ``we find $X\%$'' would invert
the paper's thesis, which is about \emph{proof}, not volume. (A separate
production SAST/DAST scanner has its own benchmark numbers; we exclude them here
on purpose---it is a different system, and presenting its detection rates beside
the oracle would conflate two tools and overstate what \emph{this} system has
shown.)

\subsection{The committed oracle runs \shipped}
\label{sec:runs}
Every row below is a \emph{verdict}, not a flag---a committed run-log with
\code{"simulated": false} and (where a model is used) \code{model:
claude-opus-4-8}. The wider real-application corpus---\textbf{six confirmed HITs and
five declined (two refused by the oracle, three deferred as un-buildable) across
eleven real apps}---is detailed in \S\ref{sec:anatomy}
(Table~\ref{tab:confirm}); this section's table zooms in on the proving-ground
detail: the two controlled fixtures (\code{bola-notes}, \code{bola-helpdesk}) plus
the autonomously-discovered \code{sharespace-connect} HIT and the App~F refusal.
The corpus is small on purpose: a verdict that
survives independent cross-checking---and a gate that \emph{refuses} a real-looking
finding---is worth more than a large table of unverified candidates, and inflating
the count would contradict the discipline the oracle exists to enforce.

\begin{table}[h]
\centering
\scriptsize
\setlength{\tabcolsep}{4pt}
\renewcommand{\arraystretch}{1.35}
\begin{tabularx}{\textwidth}{@{}>{\raggedright\arraybackslash}p{2.0cm} >{\raggedright\arraybackslash}p{2.0cm} >{\raggedright\arraybackslash}p{1.35cm} >{\raggedright\arraybackslash}p{2.2cm} L@{}}
\toprule
\textbf{Run} & \textbf{Target} & \textbf{Class} & \textbf{Verdict} & \textbf{Signal (before $\to$ after / control)} \\
\midrule
\texttt{m3\bu read} & bola-notes & read-BOLA & \textcolor{vulxgreen}{BLOCKED} & victim canary @200 $\to$ absent @404; control ok \\
\texttt{m3\bu write} & bola-notes & write-BOLA & \textcolor{vulxgreen}{BLOCKED} & attacker marker in owner read-back $\to$ absent; control ok \\
\texttt{m3\bu denyall} \emph{(counter-factual)} & bola-notes & read-BOLA & \textcolor{vulxred}{FIX\bu BROKE\bu APP} & leak gone but \emph{control fails} (deny-all) \\
\texttt{m4\bu loop\bu again} & bola-notes & read+write & \textcolor{vulxgreen}{BLOCKED}, then STILL\bu EXPLOITABLE (sibling), then \textcolor{vulxgreen}{BLOCKED} & incomplete fix; loop converges over 2 rounds \\
\texttt{m5\bu autonomous} & bola-notes & read-BOLA & \textcolor{vulxgreen}{BLOCKED} & Discover auto-finds (conf 0.98); 4 LLM calls \\
\texttt{track1\bu helpdesk} & bola-helpdesk & read-BOLA & \textcolor{vulxgreen}{BLOCKED} & tenant canary @200 $\to$ absent @404; 3 LLM calls \\
\texttt{track1\bu helpdesk\bu run2} & bola-helpdesk & read-BOLA & \textcolor{vulxgreen}{BLOCKED} & reproduction; identical result \\
\texttt{sharespace\bu handfed} & sharespace-connect \emph{(our app)} & read-BOLA & \textcolor{vulxgreen}{BLOCKED} & doc canary @200 $\to$ absent @404; control ok \\
\texttt{sharespace\bu autonomous} & sharespace-connect \emph{(our app)} & read-BOLA & \textcolor{vulxgreen}{BLOCKED} & Discover auto-finds (conf 0.97); 3 LLM calls \\
App~F \emph{(hand-fed)} & third-party \emph{(anon.)} & write-BOLA & \textcolor{vulxred}{NOT\bu REPRODUCIBLE} & self-grant write 201 (flaw real), fix 403, owner-share 201; \emph{but} read-back empty---canary absent $\to$ \textbf{refused}. 0 LLM calls \\
\bottomrule
\end{tabularx}
\caption{Committed end-to-end oracle runs (all \code{simulated=false}). The two
rows in red are the point: \textsc{fix\_broke\_app} (a leak-removing fix that
breaks legitimate access, correctly rejected) and \textsc{not\_reproducible}
(a \emph{genuine} write-authz flaw whose unauthorized write succeeds but whose
required data leak never materializes, so the oracle refuses to confirm a
confidentiality breach). A tool that declines an unexploitable finding is the pitch.}
\label{tab:runs}
\end{table}

Three results carry disproportionate weight, and two of them are \emph{refusals}.
First, the \textbf{precision gate on a real app} (App~F,
\S\ref{sec:anatomy}): a \emph{genuine} write-authz flaw the oracle declined as
\textsf{NOT\_REPRODUCIBLE}---the unauthorized write succeeded but the data leak did
not---refused deterministically with zero LLM calls, where a scanner would report a
false positive. Second, the \textbf{deny-all counterfactual}
(\code{m3\_denyall}): a fix that stops the leak by locking everyone out is ruled
\textsf{FIX\_BROKE\_APP} because the negative control fails---soundness, not just
the happy path. Third, the \textbf{real-app confirmation} (\code{sharespace\_*}):
the same bug was proven both hand-fed and via the fully autonomous discovery-plus-%
attack pipeline, and cross-checked from four independent ``lenses,'' including a
live deny-all disproof and a \code{diff}
confirming the application code is unchanged by the run (we change only platform
configuration to provision a test-bed; the app is left as-built and still
vulnerable afterward).

\subsection{Baseline: do standard static tools catch these? \shipped}
\label{sec:baseline}
We ran two industry-standard static analyzers, with their standard security
suites, over five real-app targets (the two oracle-adjudicated apps above plus
three further candidates pending the runner): \textbf{Semgrep} (project rules plus
\code{p/typescript}, \code{p/javascript}, \code{p/security-audit}; $311$ rules) and
\textbf{CodeQL} $2.25.6$ \code{security-extended} ($104$ queries). The vulnerable
files were confirmed in analysis scope, so a miss is a query/capability gap, not a
coverage gap.

\begin{table}[h]
\centering\small
\setlength{\tabcolsep}{6pt}
\renewcommand{\arraystretch}{1.15}
\begin{tabular}{@{}lccl@{}}
\toprule
\textbf{Target (real app)} & \textbf{Semgrep} & \textbf{CodeQL} & \textbf{VulX oracle} \\
 & \scriptsize 311 rules & \scriptsize 104 queries & \\
\midrule
sharespace-connect & 0 & 0 & \textcolor{vulxgreen}{BLOCKED (proven)} \\
App~F & 0 & 0 & \textcolor{vulxred}{\textsc{not\_reproducible}} \\
App~K & 0 & 1\textsuperscript{$\dagger$} & candidate (pending) \\
2 further apps & 0 & 0 & candidate (pending) \\
\midrule
\textbf{authz bugs flagged} & \textbf{0} & \textbf{0} & --- \\
\bottomrule
\end{tabular}
\caption{Standard static analyzers vs.\ the VulX oracle on five real apps.
\textsuperscript{$\dagger$}CodeQL's single finding is \code{js/insecure-randomness}
in a test-data helper---not an authorization issue. Neither tool flagged any
authorization bug; on \code{sharespace-connect} (the one oracle-proven BOLA) both
returned zero, independently re-verified.}
\label{tab:baseline}
\end{table}

Neither tool flagged a single authorization bug. On \code{sharespace-connect}---the
one target with an oracle-\emph{proven} BOLA---both returned zero, which we
independently re-verified: standard SAST missed the exact bug VulX proved. Two
honest qualifications. First, App~F is invisible to both tools
\emph{by construction}: the flaw is a declarative Supabase RLS policy in a
\code{.sql} migration, CodeQL's JavaScript extractor does not analyze SQL, and the
Semgrep suites carry no Supabase-policy rules---so this is not a head-to-head
detection win (and recall the oracle itself \emph{refused} App~F,
\S\ref{sec:anatomy}). Second, the remaining three targets are \emph{candidates
pending} oracle adjudication by our batch runner, so their zero baseline is
recorded, not yet a VulX result. \textbf{Fairness caveats:} free/standard suites
only (no Supabase-specific or paid rules), a single run each, and CodeQL's JS
extractor does not reach SQL. The takeaway is narrow but real: on the one proven bug in this first set---and, in
Table~\ref{tab:baseline2} below, on all five proven HITs---two mature static tools
find nothing; runtime proof is \emph{complementary} to pattern/dataflow SAST, not
redundant with it. \paragraph{The same baseline on the five proven HITs.}
We then ran both tools over the five batch-2 HIT targets---each an
oracle-\emph{proven} BOLA (\S\ref{sec:anatomy}). The result is again \textbf{0/5}:
neither tool flags any authorization bug.

\begin{table}[h]
\centering\small
\setlength{\tabcolsep}{6pt}\renewcommand{\arraystretch}{1.15}
\begin{tabular}{@{}lccl@{}}
\toprule
\textbf{Target (proven HIT)} & \textbf{Semgrep} & \textbf{CodeQL} & \textbf{VulX oracle} \\
 & \scriptsize 90 rules & \scriptsize 102 queries & \\
\midrule
App~A & 0 & 0 & \textcolor{vulxgreen}{BLOCKED (proven)} \\
App~B & 0 & 2\textsuperscript{$\dagger$} & \textcolor{vulxgreen}{BLOCKED (proven)} \\
App~C & 0 & ---\textsuperscript{$\ddagger$} & \textcolor{vulxgreen}{BLOCKED (proven)} \\
App~D & 0 & 0 & \textcolor{vulxgreen}{BLOCKED (proven)} \\
App~E & 0 & 0 & \textcolor{vulxgreen}{BLOCKED (proven)} \\
\midrule
\textbf{authz bugs flagged} & \textbf{0} & \textbf{0} & --- \\
\bottomrule
\end{tabular}
\caption{Standard static analyzers vs.\ the VulX oracle on the five batch-2 HIT
targets (anonymized, App~A--App~E; \S\ref{sec:ethics}), each oracle-proven. Both
tools flag zero authorization bugs. \textsuperscript{$\dagger$}CodeQL's only
findings are 2$\times$ \code{js/stack-trace-exposure} (CWE-209) on App~B---%
error-handling, not authorization. \textsuperscript{$\ddagger$}App~C is a pure-SQL
RLS bug; CodeQL's JavaScript extractor does not analyze it.}
\label{tab:baseline2}
\end{table}

Extraction was real (e.g.\ App~D $5{,}988$ LoC, App~A $433$, App~B $195$, App~E
$78$) and all files were in scope, so these are capability gaps, not coverage gaps;
re-running Semgrep with \code{p/owasp-top-ten} added ($677$ rules loaded) still
yielded $0/5$. The reason is structural: four of the five bugs live in the
Postgres/RLS layer neither tool models (CodeQL has no SQL analysis; Semgrep ships no
Supabase-policy rules), and the fifth (App~E) is a pure-TypeScript
authorization-logic flaw outside both tools' modeled vulnerability classes. As
before, this is not a head-to-head score---it shows the class sits structurally
outside pattern/dataflow SAST.

\emph{Methodology note.} This table counts the named community packs only
(\code{p/security-audit}, \code{p/typescript}, \code{p/javascript}; $90$ applicable
rules per TypeScript target, $3$ on the pure-SQL App~C); we exclude VulX's
own project rules from a standard-tool baseline (Table~\ref{tab:baseline} included
them in its Semgrep count---removing them, as here, changes nothing: still $0$).
CodeQL \code{security-extended} resolved to $102$ queries here versus $104$ in
Table~\ref{tab:baseline} (same $2.25.6$, a trivial suite-resolution difference).
Full method and reproduce commands are in \code{baseline\_sast\_eval.md}
(\S\ref{sec:repro}).

\subsection{Cost \shipped\ (measured) / \roadmap\ (modeled)}
Measured cost is \textbf{\$0.065--0.11 per solve} across runs (dominated by model
tokens; sandbox compute is comparatively negligible). In the batch-2 corpus, cost
was \textbf{$\sim$\$0.09--0.10 per target}---e.g.\ the autonomous
\code{sharespace-connect} solve at \textbf{\$0.0996} on a warm-E2B template. The
batch runner amortizes the sandbox: a warmed template plus parallel fan-out makes
$N$ targets cost roughly one boot (\S\ref{sec:limits}).
For context only, MAPTA~\cite{mapta} reports $83\%$ on the XBOW broken-authorization
set at an average \$3.67 per \emph{repository} assessment; our per-\emph{solve}
figure is a different unit and scope, so we offer it as an order-of-magnitude data
point, not a head-to-head comparison. Broader per-scan and pricing economics are
\emph{modeled, not measured}: token cost dominates ($\sim\!85$--$96\%$), and the
deterministic oracle avoids paying a model to render a verdict---a structural cost
edge, since the verdict is computation, not generation.

% =====================================================================
\section{Limitations and honesty}
\label{sec:limits}

We state these plainly because the product's entire premise is not overclaiming.

\begin{itemize}[leftmargin=1.4em,itemsep=3pt]
  \item \textbf{Small $n$, early-stage, and hand-fed.} End-to-end evidence covers
    eleven real apps (six \emph{confirmed} BLOCKED, two refused by the oracle, three
    deferred as un-buildable) plus two controlled fixtures. Crucially, only
    \code{sharespace-connect} was found by autonomous discovery; the other real-app
    HITs were proven on \emph{hand-fed} exploit shapes. We measure a deterministic
    discovery \emph{recall} floor (5/6 high-confidence on the real HITs, 2/2
    synthetic; \S\ref{sec:discover}), but make \emph{no} precision claim
    (recall-first; the oracle removes the noise) and report \emph{no} end-to-end
    autonomous discovery beyond the one app---both await a labeled corpus
    (\S\ref{sec:future}).
  \item \textbf{The fix is a reference patch on real apps.} The
    before/after/control proof measures the effect of a known-good reference fix;
    an autonomous production fixer is \emph{not} claimed and is roadmap. The
    LLM-authored fix and the adversarial battle are exercised on controlled
    fixtures only.
  \item \textbf{``Real model'' $\neq$ trained model.} Every model-driven result
    uses \emph{hosted Claude Opus 4.8}. No VulX-trained or distilled model has
    been exercised anywhere; the training-data pipeline exists but the model is
    roadmap (\S\ref{sec:future}).
  \item \textbf{Model-guided, not unscaffolded.} The LLM emits an attack
    \emph{plan} (target object, actor, rationale); the concrete leak and control
    \emph{signals} are filled from candidate ground truth, and the test-bed
    (accounts, canary, credentials) is harness-provisioned. This is a deliberate
    soundness choice---the model cannot fabricate the proof---but it means we do
    \emph{not} claim the LLM autonomously compromises a host with no scaffolding.
  \item \textbf{Autonomy boundary.} On the real OSS app the \emph{finding} is
    autonomous, but the runtime test-bed (victim/attacker accounts, canary,
    credentials) is harness-provisioned, not autonomously discovered. The
    autonomous discovery path's own LLM re-ranker has been run against a real
    model only on a crafted fixture.
  \item \textbf{Off-repo configuration ceiling (the dominant real-world wall).}
    Many Supabase/Lovable apps keep schema and RLS in the dashboard, never in git;
    their RLS references tables the repo never creates, so we cannot stand up a
    faithful instance without fabricating schema. Three of our targets
    (App~H--App~J) were confirmed-but-unprovable for this reason---caught cheaply by a
    schema-completeness gate that \emph{defers} rather than guess. In practice this
    is the most common provisioning blocker, and it is the empirical case for a
    ``connect-your-project'' step over ``paste-a-URL'' auto-provisioning.
  \item \textbf{Non-authz coverage is unproven end-to-end.} The SQLi/XSS/cmdi/LFI
    probes are implemented and unit-tested offline, but \emph{no} end-to-end
    run-log proves them against a live app. Every committed proof is BOLA.
  \item \textbf{Flywheel not live.} The de-identification gate is implemented and
    red-teamed, but the live transmission of signatures to the central store has
    never been executed; the round-trip uses an in-process replica.
  \item \textbf{Production auto-fix is heuristic.} The shipped scanner's
    auto-fix ``verified'' label is a heuristic check (substring/status, no
    negative control; the static branch executes nothing). The deterministic
    oracle described here is \emph{not yet wired into the production scanner}.
  \item \textbf{Tests not re-run here.} Test counts cited from the codebase are
    static counts and documented claims; the suites were not executed while
    preparing this document.
  \item \textbf{Self-reported.} All results are internal and not independently
    audited; the run-logs are available for inspection (\S\ref{sec:repro}).
\end{itemize}

% =====================================================================
\section{Ethics and responsible disclosure}
\label{sec:ethics}

This work studies \emph{publicly available, open-source} applications. All
exploitation was performed \emph{solely against local clones we stood up in
isolated sandboxes}---never against any third party's production system,
infrastructure, or real user data. Each target ran with synthetic accounts and
canary data only, and the application code was left byte-identical to its public
source. One target (\code{sharespace-connect}, \url{github.com/NgoTomek}) is our
own application; the others are third-party open-source projects, which we identify
in this paper only in \emph{anonymized} form.

The third-party targets are \emph{production-grade, open-source} repositories.
Several of the confirmed findings are real, not-yet-remediated authorization flaws,
so \textbf{we do not disclose their identities}: every third-party app appears only
in anonymized form (App~A--App~K), and we are in the process of privately informing
the affected maintainers of the vulnerabilities we found. We report flaw
\emph{shapes}, CWE classes, and oracle results---the contribution---and deliberately
omit anything that could fingerprint an app (names, verbatim policy or file names,
distinctive endpoints, identifying descriptions). Going forward, external scanning
operates under an explicit, scoped responsible-disclosure protocol
(\S\ref{sec:future}).

% =====================================================================
\section{Related work}
\label{sec:related}

\paragraph{Static IDOR detection.}
Semgrep's multimodal detector~\cite{semgrep} represents the current AI-assisted
static frontier for IDOR ($61\%$ precision; $1.9\times$ recall). It produces
strong \emph{candidates}; VulX is complementary in producing \emph{proof}.
CodeQL~\cite{codeql} pioneered multi-repository variant analysis, the model our
cross-codebase flywheel adopts (\S\ref{sec:flywheel}). In our own baseline
(\S\ref{sec:baseline}), both Semgrep and CodeQL, run with standard security suites,
flagged zero authorization bugs on our targets---consistent with the view that
this class needs runtime proof, not more patterns.

\paragraph{Runtime proof and authorization agents.}
MAPTA~\cite{mapta}, an autonomous LLM pen-tester, reports $83\%$ on broken
authorization (XBOW benchmark); we cite it for cost context (\S\ref{sec:eval}).
BACFuzz~\cite{bacfuzz} argues for instrumenting the data layer rather than trusting
HTTP status---the principle behind our read-back BOLA signal. Autonomous offensive
agents such as XBOW demonstrate canary/oracle assemblies; our differentiator is the
\emph{deterministic} ship-gate (never an LLM verdict), the stack-grounded policy
inference that scopes the attack, and the requirement that a fix survive re-attack
before anything ships.

\paragraph{AI-native risk.}
Beyond authorization, AI-generated software introduces new classes: package
hallucination / ``slopsquatting'' (one study reports $19.7\%$ of LLM-generated
dependencies are hallucinated)~\cite{slopsquatting} and retrieval poisoning
(PoisonedRAG: $90\%$ attack success with five poisoned
texts)~\cite{poisonedrag}. These motivate, but are out of scope for, the
authorization oracle.

% =====================================================================
\section{Future work}
\label{sec:future}

\paragraph{From hosted model to a trained attacker.}
Every oracle-confirmed run banks one labeled trajectory
(schema \code{vulx.trajectory.v1}) whose label is the \emph{oracle's} verdict,
never a model's or a human's. Positive examples are attacker payloads that
defeated a patch; negatives are bypasses the oracle blocked. This corpus is the
intended training set for a distilled attacker model (SFT then RL-from-oracle),
gated behind an explicit bar: a meaningful number of connected repositories
\emph{and} beating hosted Claude on held-out, oracle-confirmed exploits. Until
both hold, the ``trained model'' remains roadmap, not product.

\paragraph{Cross-codebase immunity.}
The flywheel's end state is variant analysis: a single proven-and-fixed
authorization bug, abstracted to a code-free signature, drives prove-before-surface
queries across every connected codebase of the same class---isolate one
vulnerability, immunize the rest. This is the most valuable idea in the system
and the least built; we present it as direction, not result.

\paragraph{Hardening the evidence.}
The batch runner (warm-E2B template plus parallel fan-out), the first real
multi-finding flywheel fill, a deterministic discovery \emph{recall} floor
(\S\ref{sec:discover}), and baselines on the proven HITs are now built and measured.
The near-term priorities that remain follow from \S\ref{sec:limits}: discovery
\emph{precision} and \emph{end-to-end} autonomous discovery of the new flaw shapes
on a labeled corpus (the batch-2 HITs were hand-fed); end-to-end proofs for the
non-authz probe families; and re-attack survival demonstrated on a real app (today
it is fixture-only). An autonomous
production fixer remains genuinely roadmap---real-app fixes are reference patches.
The cross-machine live flywheel transmission is the last integration step.

% =====================================================================
\section{Conclusion}

We asked: when an LLM claims a broken-access-control bug, how do we know it is
real---without trusting the model? Our answer is a deterministic oracle: a pure
before/after/negative-control rule that issues every verdict, so model
hallucination can never produce a false ``proven.'' The same rule is a precision
gate---on a real application it \emph{refused} a genuine flaw whose unauthorized
write succeeded but whose data leak did not (\textsc{not\_reproducible}), exactly
where an LLM-only pipeline would have cried wolf. The supporting capability is real too: Claude Opus 4.8
autonomously reasoned about a real app's access policy and authored a working
cross-user exploit, which the oracle confirmed and, with a reference fix, certified
closed under a passing control; on controlled fixtures the model even read its own
fix and conceded honestly when it held. AI writes more code every month, and
authorization is the class review was already worst at and that models are worst at
emitting. The instinct is a bigger model or more patterns; we argue the answer is
\emph{discipline}---let the model propose, and let a rule, never the model,
dispose. Four of five verdicts open no pull request, and that is the point: a
security tool earns trust by refusing to cry wolf. The evidence here is
deliberately small and labeled, but it is \emph{proof}, not a confidence score.
That is the foundation we intend to scale.

% =====================================================================
\appendix
\section{Reproducibility}
\label{sec:repro}
The artifacts behind every claim in \S\ref{sec:eval} are committed:
\begin{itemize}[leftmargin=1.4em,itemsep=2pt]
  \item \textbf{Run-logs.} Each row of Table~\ref{tab:runs} is a committed JSON
    record carrying \code{simulated:false}, the model id, per-step token cost, the
    before/after exploit signals, the negative-control result, the database-reset
    count, and the model's verbatim rationale.
  \item \textbf{The ship-gate.} The verdict is the pure function \code{decide()}
    reproduced in \S\ref{sec:oracle}; it consumes only the three measured booleans
    and is independent of any model, so a third party can re-derive every verdict
    from the logged measurements.
  \item \textbf{The de-identification gate.} The flywheel signature format and its
    fail-closed allowlist (\S\ref{sec:flywheel}) are unit-tested, including a
    red-team suite that drove residual leakage to zero.
  \item \textbf{The baseline.} The Semgrep/CodeQL methodology, exact packs, query
    counts, and reproduce commands for Tables~\ref{tab:baseline}
    and~\ref{tab:baseline2} are in \code{baseline\_sast\_eval.md}.
\end{itemize}
Exact run-log identifiers and the harness are available to design partners under
NDA; a public artifact release will accompany the empirical version of this paper.

% =====================================================================
\begin{thebibliography}{99}
\small
\bibitem{viberadar} Georgia Tech SSLab.
  \emph{Vibe Security Radar: Real CVEs in AI-Generated Code.}
  \url{https://github.com/HQ1995/vibe-security-radar}.

\bibitem{lovable} M.~Palmer et al.
  \emph{CVE-2025-48757: Row-Level Security misconfiguration in
  AI-generated (Lovable) Supabase applications.} 2025.
  Reported affected: $170/1{,}645$ projects ($10.3\%$), $303$ endpoints,
  CVSS $9.3$. \url{https://www.superblocks.com/blog/lovable-vulnerabilities}.

\bibitem{semgrep} Semgrep.
  \emph{AI-Powered Detection with Semgrep (Multimodal IDOR detection).} 2026.
  \url{https://semgrep.dev/blog/2025/ai-powered-detection-with-semgrep/}.

\bibitem{codeql} GitHub.
  \emph{CodeQL and Multi-Repository Variant Analysis (MRVA).}
  \url{https://codeql.github.com/}.

\bibitem{mapta} I.~David et al.
  \emph{Multi-Agent Penetration Testing AI for the Web.} arXiv:2508.20816, 2025.
  Reports $83\%$ on broken authorization (XBOW benchmark), at an average \$3.67 per
  repository assessment.

\bibitem{bacfuzz} \emph{BACFuzz: Exposing the Silence on Broken Access Control
  Vulnerabilities in Web Applications.} arXiv:2507.15984, 2025.

\bibitem{slopsquatting} J.~Spracklen et al.
  \emph{We Have a Package for You! A Comprehensive Analysis of Package
  Hallucinations by Code-Generating LLMs.} In \emph{Proc.\ USENIX Security
  Symposium}, 2025. Reported: $19.7\%$ of generated dependencies hallucinated
  across $576$K samples.

\bibitem{poisonedrag} W.~Zou, R.~Geng, B.~Wang, and J.~Jia.
  \emph{PoisonedRAG: Knowledge Corruption Attacks to Retrieval-Augmented
  Generation of Large Language Models.} In \emph{Proc.\ 34th USENIX Security
  Symposium}, 2025. Reported: $90\%$ attack success with five poisoned texts.

\bibitem{owaspapi} OWASP. \emph{API Security Top 10 (2023): API1:2023
  Broken Object Level Authorization.}
  \url{https://owasp.org/API-Security/editions/2023/en/0xa1-broken-object-level-authorization/}.

\bibitem{sun2011} F.~Sun, L.~Xu, and Z.~Su.
  \emph{Static Detection of Access Control Vulnerabilities in Web Applications.}
  In \emph{Proc.\ 20th USENIX Security Symposium}, 2011.

\bibitem{near2016} J.~P.~Near and D.~Jackson.
  \emph{Finding Security Bugs in Web Applications using a Catalog of Access
  Control Patterns.} In \emph{Proc.\ 38th International Conference on Software
  Engineering (ICSE)}, 2016.
\end{thebibliography}

\end{document}
