From 2bbfecc22834d0a60b9a6940fe4c0063296bfbe7 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 12 Nov 2026 12:28:49 -0700 Subject: [PATCH 01/17] test(oracle-gaps): law 9 over an edit that lands during a check (WIP) A new edit, midrun: a test of file a, whose context loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it. That check is not compared; the next check through the daemon must equal a cold run of the state after the edit. Constructed history "an edit during a check through the daemon" enters "an edit landed during a check through the daemon" and "edit midrun changed the cold result". GREEN on main and the RED mutant are still to come. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 49 ++++++++++- .../gap-report-oracle.property.test.mjs | 85 ++++++++++++++++++- packages/oracles/scripts/e2e-shards.mjs | 4 + 3 files changed, 134 insertions(+), 4 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index dc96d11..79d9560 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -429,6 +429,10 @@ const edits = [ `cli`, // The module that only file b's mutated runs import (`lazyGate`). `lazy`, + // File b's helper edited while a check through the daemon runs (law 9's + // mid-run edit): a test of file a, which has loaded the helper, rewrites + // it the first time it runs, before file b's runs read it. + `midrun`, `restart`, `revert`, // Events that end a daemon, not edits: a crash (killed, its socket and @@ -2630,6 +2634,9 @@ function resolveHistory(history) { stability: history.stability, steps: history.steps ?? [], lazyGate: history.lazyGate === true, + // Whether a step edits file b's helper during a check (`midrun`): then + // file a loads the helper and holds the test that edits it. + midrun: (history.steps ?? []).includes(`midrun`), ordinary: history.ordinary === true, // An `owns` key in an assertion's arguments, in the ordinary file: its // assertion form and whether its list is a literal, or none. @@ -3849,7 +3856,10 @@ function blockingOf(plan) { function baselineOf(plan) { const failing = blockingOf(plan) return [...oracleFilesOf(plan), `c`].map((file) => { - const own = plan.tests.filter((t) => t.file === file).length + // File a's mid-run edit test (`midrun`) runs in every baseline too. + const own = + plan.tests.filter((t) => t.file === file).length + + (plan.midrun && file === `a` ? 1 : 0) const broken = failing.find((f) => f.file === file) // The failing owner's suite holds two tests and a third of its name. const extra = broken?.test === undefined ? 0 : 3 @@ -4211,6 +4221,14 @@ function oracleTestFile(plan, file) { `import { join as pidPath } from 'node:path'`, ] : []), + // The mid-run edit (`midrun`): file a loads file b's helper, so its + // context holds the helper's old text when the edit lands. + ...(plan.midrun && file === `a` + ? [ + `import { readFileSync as readMarker, writeFileSync as writeEdit } from 'node:fs'`, + `import { strict as strictAtLoad } from './strict'`, + ] + : []), // A module's text through a query import, which Vite loads (see rawImport). ...(plan.tests.some((t) => t.file === file && t.rawImport) ? [ @@ -4392,8 +4410,27 @@ function oracleTestFile(plan, file) { `})`, `test('fixed seed 999', () => {})`, ) + // The mid-run edit: the first run of this test after the driver marks an + // edit pending (`tests/midrun.txt`, `pending:`) rewrites file b's + // helper with that value and marks the edit done, so every later run of + // it, in this check or another, changes nothing. It calls nothing of the + // source under test, so no mutant reaches it. + if (plan.midrun && file === `a`) + out.push( + `test('edits the helper during the check', () => {`, + ` void strictAtLoad`, + ` const marker = new URL('./midrun.txt', import.meta.url)`, + ` const text = readMarker(marker, 'utf8')`, + ` if (!text.startsWith('pending:')) return`, + ` writeEdit(new URL('./strict.ts', import.meta.url), \`export const strict = \${text.slice('pending:'.length).trim()}\\n\`)`, + ` writeEdit(marker, 'done\\n')`, + `})`, + ) // A file with no tests still loads, so the CLI can read what it owns. - if (!plan.tests.some((t) => t.file === file)) + if ( + !plan.tests.some((t) => t.file === file) && + !(plan.midrun && file === `a`) + ) out.push(`test.skip('no tests', () => {})`) return `${out.join(`\n`)}\n` } @@ -4694,6 +4731,10 @@ function fixtureFiles(plan, state = initialState(plan)) { ...(plan.dotGenSource ? { [DOTGEN]: plan.dotGenSource } : {}), ...plan.exportSources, 'tests/strict.ts': `export const strict = ${state.strict}\n`, + // The mid-run edit's marker: done in every state, and pending only in + // the check through the daemon during which the edit lands (the + // driver's `midrun` step). + ...(plan.midrun ? { 'tests/midrun.txt': `done\n` } : {}), ...(plan.lazyGate ? { 'tests/lazy.ts': `export const open = ${state.lazyOpen}\n` } : {}), @@ -4960,7 +5001,9 @@ function applyEdit(state, edit) { : { stmt: (h.authored?.stmt ?? -1) + 1, kind: `apply` }, }, } - if (edit === `helper`) return { ...state, strict: !state.strict } + // A mid-run edit leaves file b's helper as the helper edit does. + if (edit === `helper` || edit === `midrun`) + return { ...state, strict: !state.strict } if (edit === `lazy`) return { ...state, lazyOpen: !state.lazyOpen } if (edit === `seed`) return { ...state, seed: state.seed === 1 ? 2 : 1 } if (edit === `config`) return { ...state, altConfig: !state.altConfig } diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index b74f4fc..a8d553a 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -220,6 +220,12 @@ * requires an edit that changed the cold result after a hang replaced * the only context (which ran every file), and an edit of a module that * only mutated runs load, which no unmutated run records as an input. + * One edit lands during a check (`midrun`): a test of file a, whose + * context has loaded file b's helper, rewrites it before file b's runs + * read it. A run that the edit overlapped may report either text, so + * that check is not compared; the runner must not store such a run, so + * the next check, with nothing changed, equals a cold run of the state + * after the edit. * One daemon serves a package, across crashes, reboots, and concurrent * starts: two checks started at once, a check after the daemon was * killed (its socket and lock left behind), after a reboot (its socket @@ -1222,6 +1228,32 @@ async function incremental(dir, plan, cli, copy, observe, states) { // before it reads the request, which resets the connection // (`resetBeforeRead`). async function daemonChecks(step, files, how) { + // The mid-run edit: the check starts from the state before the edit, + // with the edit marked pending, and file a's test lands it while the + // check runs, after its context loaded the helper and before file b's + // runs read it. That check may report either state for a run the edit + // overlapped, so it is not compared; the runner must store no such run + // (settleIn), so the next check, with nothing changed, equals a cold + // run of the state after the edit. + if (step === `midrun`) { + const after = files[`tests/strict.ts`] + const value = after.includes(`true`) + runCli( + dir, + { + ...files, + 'tests/strict.ts': `export const strict = ${!value}\n`, + 'tests/midrun.txt': `pending:${value}\n`, + }, + { ...how, daemon: true }, + ) + if ( + readFileSync(join(dir, `tests/midrun.txt`), `utf8`) === `done\n` && + readFileSync(join(dir, `tests/strict.ts`), `utf8`) === after + ) + states.add(`an edit landed during a check through the daemon`) + return [runCli(dir, files, { ...how, daemon: true })] + } if (step !== `dieMidCheck` && step !== `resetBeforeRead`) return [runCli(dir, files, { ...how, daemon: true })] // A daemon that this check's CLI starts itself is the one that dies. @@ -2492,6 +2524,7 @@ const incrementalStates = [ `two checks at once after a code change`, `a check from another install of the CLI`, `a check after the OS cleaned the daemon's state`, + `an edit landed during a check through the daemon`, `two checks started the daemon at once`, `the daemon ran from a path with a space and #`, ] @@ -2731,7 +2764,7 @@ oracleProperty({ law: { statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10).`, - limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come only between checks, so a file saved during a check is not generated (the runner invalidates a known input that changed while a run ran and does not store that run, which only inspection fixes; a witness needs a step that edits a file between a unit's start and its end), and a module that a run first loads and that changes during that same run is stored under its later hash; law 9's edits leave out files read through fs, other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export.`, + limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; that check is not compared, because a run that the edit overlapped may report either text (the runner's rule: a run whose input changed while it ran is invalidated and not stored, settleIn), and the next check, with nothing changed, must equal a cold run of the state after the edit; an edit during a mutant run, or of a module that the run evaluating it reads, is not generated, and a module that a run first loads and that changes during that same run is stored under its later hash; law 9's edits leave out files read through fs, other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs @@ -3325,6 +3358,56 @@ oracleProperty({ steps: [`helper`], }), }, + { + // A check through the daemon during which file b's helper changes + // (law 9's mid-run edit): one context runs file a, which loads the + // helper and then rewrites it, before file b's runs read it. The + // edit turns b's verdict off, so the cold run then reports b's + // mutant as a survivor; the daemon's next check must not reuse a + // run that read the old helper. + name: `an edit during a check through the daemon`, + enters: [ + `an edit landed during a check through the daemon`, + `edit midrun changed the cold result`, + ], + build: (generated) => ({ + ...generated, + functions: [ + [{ kind: `push`, quote: `'` }], + [{ kind: `push`, quote: `'` }], + ], + containers: [], + overloads: [], + owners: [`a`, `b`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [ + { ...judging(`a`, `fixed seed`, 1), fn: 0 }, + { ...judging(`b`, `fixed seed`, 1), fn: 1 }, + ], + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + contexts: 1, + steps: [`midrun`], + }), + }, { // File b's verdict waits on a module that it imports only where the // output differs, so only mutated runs load it. The edit closes that diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index e445bd0..bdc80ee 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -67,6 +67,10 @@ export const shards = { `fixed seed 20261105: an edit that changed the cold result after a hang replaced the only context`, `random seed: an edit that changed the cold result after a hang replaced the only context`, `determinism: constructed history: an edit that changed the cold result after a hang replaced the only context`, + // The mid-run edit's history (law 9), not yet measured. + `fixed seed 20261105: an edit during a check through the daemon`, + `random seed: an edit during a check through the daemon`, + `determinism: constructed history: an edit during a check through the daemon`, // Added with the CLI's parent-config discovery (not yet measured). `fixed seed 20261105: a package whose parent directory has its config`, `random seed: test.projects in a parent's config`, From 15cc6dd3c7a33d60bd2b2a835d6a04c713fc8812 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 12 Nov 2026 14:32:55 -0700 Subject: [PATCH 02/17] test(oracle-gaps): each mid-run edit changes file a's text as well Through the daemon, file a's text had not changed since the previous check, so the daemon reused its baseline and its edit test never ran: the marker is read through fs, which no run records as an input. Each mid-run edit now also changes file a (a count of them), so the edit lands. That fs reads are no input is law 9's own gap; the next commits judge it. Co-authored-by: Isaac --- .../oracles/oracle-gaps/gap-report-model-oracle.mjs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 79d9560..1e8a577 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -4549,6 +4549,8 @@ const initialState = (plan) => ({ strict: true, // What `tests/lazy.ts` exports: whether file b's lazy gate is open. lazyOpen: true, + // How many mid-run edits have landed (`midrun`). + midruns: 0, seed: 1, // Whether the check's `--config` names the second config (`altConfig`). altConfig: false, @@ -4859,6 +4861,12 @@ function fixtureFiles(plan, state = initialState(plan)) { } for (const file of oracleFilesOf(plan)) files[`tests/${file}.test.ts`] = oracleTestFile(plan, file) + // Each mid-run edit changes file a's own text too (a count of them), so + // the daemon runs file a's tests again in the check the edit lands in, + // instead of reusing their evidence: the marker is read through fs, + // which no run records as an input. + if (plan.midrun) + files[`tests/a.test.ts`] += `// mid-run edits: ${state.midruns}\n` // The new oracle file d owns the first generated function and judges it. if (state.newOracle) { const target = plan.functions.find((fn) => fn.generated && !fn.library) @@ -5002,8 +5010,9 @@ function applyEdit(state, edit) { }, } // A mid-run edit leaves file b's helper as the helper edit does. - if (edit === `helper` || edit === `midrun`) - return { ...state, strict: !state.strict } + if (edit === `helper`) return { ...state, strict: !state.strict } + if (edit === `midrun`) + return { ...state, strict: !state.strict, midruns: state.midruns + 1 } if (edit === `lazy`) return { ...state, lazyOpen: !state.lazyOpen } if (edit === `seed`) return { ...state, seed: state.seed === 1 ? 2 : 1 } if (edit === `config`) return { ...state, altConfig: !state.altConfig } From 5772215af5c53725a44aecd1fc39efb20317a71d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 12 Nov 2026 14:56:08 -0700 Subject: [PATCH 03/17] test(oracle-gaps): law 9 over a data file that a test reads through fs A file that a test reads through fs is no input of its run, so a check through the daemon reused evidence after an edit of only that file, while a cold run sees the edit (found while the mid-run witness was debugged). A new dimension, fsRead, makes file b's verdicts also wait on tests/data.json, read by readFileSync, fs/promises readFile, or the callback readFile; the new edit fsData changes only that file. Three constructed histories, one per kind of read, enter "edit fsData changed the cold result" and "an edit to data read through changed the cold result". Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 45 +++++- .../gap-report-oracle.property.test.mjs | 137 ++++++++++++++++++ packages/oracles/scripts/e2e-shards.mjs | 10 ++ 3 files changed, 190 insertions(+), 2 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 1e8a577..f1d8440 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -429,6 +429,9 @@ const edits = [ `cli`, // The module that only file b's mutated runs import (`lazyGate`). `lazy`, + // The data file that file b's verdict reads through fs (`fsRead`): no + // test or source changes, only a file a test reads at run time. + `fsData`, // File b's helper edited while a check through the daemon runs (law 9's // mid-run edit): a test of file a, which has loaded the helper, rewrites // it the first time it runs, before file b's runs read it. @@ -614,6 +617,18 @@ const drawn = fc.record({ // where the output differs, so only a mutated run loads it // (`tests/lazy.ts`, which law 9's `lazy` edit changes). lazyGate: fc.constantFrom(false, false, false, true), + // File b's verdicts also wait on `tests/data.json`, which the test reads + // through fs at run time, by a sync, a promise, or a callback read (law + // 9: a file a test reads is an input of its run, as a module it imports + // is). The `fsData` edit changes only that file. + fsRead: fc.constantFrom( + `none`, + `none`, + `none`, + `sync`, + `promise`, + `callback`, + ), // A second authored mutant, on the next statement, so that two patched // sources are judged in the same run and neither may see the other's. secondAuthored: fc.boolean(), @@ -2634,6 +2649,7 @@ function resolveHistory(history) { stability: history.stability, steps: history.steps ?? [], lazyGate: history.lazyGate === true, + fsRead: history.fsRead ?? `none`, // Whether a step edits file b's helper during a check (`midrun`): then // file a loads the helper and holds the test that edits it. midrun: (history.steps ?? []).includes(`midrun`), @@ -4158,6 +4174,24 @@ const quoteEntry = (plan, entry) => // Whether a test's verdict waits on file b's lazy gate (`lazyGate`). const lazyGated = (plan, file, t) => plan.lazyGate && file === `b` && t.judges && t.judgeBy === `verdict` +// Whether a test's verdict waits on the data file it reads through fs. +const fsGated = (plan, file, t) => + plan.fsRead !== `none` && file === `b` && t.judges && t.judgeBy === `verdict` +// How file b reads `tests/data.json`, by each kind of fs read. +const dataReads = { + sync: [ + `import { readFileSync as readDataSync } from 'node:fs'`, + `const readData = async () => JSON.parse(readDataSync(new URL('./data.json', import.meta.url), 'utf8'))`, + ], + promise: [ + `import { readFile as readDataFile } from 'node:fs/promises'`, + `const readData = async () => JSON.parse(await readDataFile(new URL('./data.json', import.meta.url), 'utf8'))`, + ], + callback: [ + `import { readFile as readDataCallback } from 'node:fs'`, + `const readData = () => new Promise((resolve, reject) => readDataCallback(new URL('./data.json', import.meta.url), 'utf8', (error, text) => (error ? reject(error) : resolve(JSON.parse(text)))))`, + ], +} function oracleTestFile(plan, file) { const sites = sitesOf(plan) @@ -4241,6 +4275,7 @@ function oracleTestFile(plan, file) { : []), `import * as sut from '../src/sut'`, ...(file === `b` ? [`import { strict } from './strict'`] : []), + ...(file === `b` && plan.fsRead !== `none` ? dataReads[plan.fsRead] : []), ...stubs, // State that tests of this file share (the `w_shared` witness). `const shared = { value: 0 }`, @@ -4371,7 +4406,7 @@ function oracleTestFile(plan, file) { // A lazy gate is imported only where the output differs, so // the unmutated run never loads it. It is open before any // edit, so the model need not read it. - ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}${lazyGated(plan, file, t) ? ` && (await import('./lazy')).open` : ``}) {`, + ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}${lazyGated(plan, file, t) ? ` && (await import('./lazy')).open` : ``}${fsGated(plan, file, t) ? ` && (await readData()).open` : ``}) {`, ` g.__oracleVerdict?.({ property: 'fixture-${file}', law: '${t.law}', checkpoint: 'trace' })`, ` throw new Error('violation')`, ` }`, @@ -4389,7 +4424,7 @@ function oracleTestFile(plan, file) { : []), t.asProp ? ` fcTest.prop([fcv.integer({ min: 0, max: 10 })], { numRuns: 20, seed: 1 })('${testName(t, index)}', async (n) => {` - : ` test('${testName(t, index)}', ${(t.hangs && !t.flaky) || t.slow || t.ioWait || lazyGated(plan, file, t) ? `async ` : ``}() => {`, + : ` test('${testName(t, index)}', ${(t.hangs && !t.flaky) || t.slow || t.ioWait || lazyGated(plan, file, t) || fsGated(plan, file, t) ? `async ` : ``}() => {`, ...partsOf(t).flatMap(campaign), ` })`, `})`, @@ -4549,6 +4584,8 @@ const initialState = (plan) => ({ strict: true, // What `tests/lazy.ts` exports: whether file b's lazy gate is open. lazyOpen: true, + // What `tests/data.json` holds: whether file b's data gate is open. + dataOpen: true, // How many mid-run edits have landed (`midrun`). midruns: 0, seed: 1, @@ -4740,6 +4777,9 @@ function fixtureFiles(plan, state = initialState(plan)) { ...(plan.lazyGate ? { 'tests/lazy.ts': `export const open = ${state.lazyOpen}\n` } : {}), + ...(plan.fsRead !== `none` + ? { 'tests/data.json': `${JSON.stringify({ open: state.dataOpen })}\n` } + : {}), // A workspace dependency with module state, linked through node_modules. 'deps/state-dep/package.json': manifest({ name: `state-dep`, @@ -5014,6 +5054,7 @@ function applyEdit(state, edit) { if (edit === `midrun`) return { ...state, strict: !state.strict, midruns: state.midruns + 1 } if (edit === `lazy`) return { ...state, lazyOpen: !state.lazyOpen } + if (edit === `fsData`) return { ...state, dataOpen: !state.dataOpen } if (edit === `seed`) return { ...state, seed: state.seed === 1 ? 2 : 1 } if (edit === `config`) return { ...state, altConfig: !state.altConfig } // A seed no other step uses, so the check has fresh work to be killed in. diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index a8d553a..96fe46a 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1195,6 +1195,11 @@ async function incremental(dir, plan, cli, copy, observe, states) { // An edit that changes the cold result is one a stale cache could miss. if (step !== `start` && result(previous) !== result(coldRun)) { states.add(`edit ${step} changed the cold result`) + // An edit of only a file a test reads through fs, by each kind of read. + if (step === `fsData`) + states.add( + `an edit to data read through ${plan.fsRead} changed the cold result`, + ) // The daemon's previous check cut a run off, so the only context, which // ran every file, was replaced. Whether the replacement read the edited // file again is not observed; the constructed history's order makes it @@ -2525,6 +2530,9 @@ const incrementalStates = [ `a check from another install of the CLI`, `a check after the OS cleaned the daemon's state`, `an edit landed during a check through the daemon`, + `an edit to data read through sync changed the cold result`, + `an edit to data read through promise changed the cold result`, + `an edit to data read through callback changed the cold result`, `two checks started the daemon at once`, `the daemon ran from a path with a space and #`, ] @@ -3408,6 +3416,135 @@ oracleProperty({ steps: [`midrun`], }), }, + { + // File b's verdict also waits on a data file that the test reads + // through fs (sync read). The edit changes only that file, which + // closes the gate: the cold run then reports b's mutant as a + // survivor, and the daemon must not reuse a kill that read the old + // data (law 9: a file a test reads is an input of its run). + name: `an edit to data a test reads through fs (sync)`, + enters: [ + `edit fsData changed the cold result`, + `an edit to data read through sync changed the cold result`, + ], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`b`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [{ ...judging(`b`, `fixed seed`, 1), fn: 0 }], + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + cliPath: `plain`, + concurrentStart: false, + lazyGate: false, + fsRead: `sync`, + steps: [`fsData`], + }), + }, + { + // File b's verdict also waits on a data file that the test reads + // through fs (promise read). The edit changes only that file, which + // closes the gate: the cold run then reports b's mutant as a + // survivor, and the daemon must not reuse a kill that read the old + // data (law 9: a file a test reads is an input of its run). + name: `an edit to data a test reads through fs (promise)`, + enters: [ + `edit fsData changed the cold result`, + `an edit to data read through promise changed the cold result`, + ], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`b`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [{ ...judging(`b`, `fixed seed`, 1), fn: 0 }], + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + cliPath: `plain`, + concurrentStart: false, + lazyGate: false, + fsRead: `promise`, + steps: [`fsData`], + }), + }, + { + // File b's verdict also waits on a data file that the test reads + // through fs (callback read). The edit changes only that file, which + // closes the gate: the cold run then reports b's mutant as a + // survivor, and the daemon must not reuse a kill that read the old + // data (law 9: a file a test reads is an input of its run). + name: `an edit to data a test reads through fs (callback)`, + enters: [ + `edit fsData changed the cold result`, + `an edit to data read through callback changed the cold result`, + ], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`b`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [{ ...judging(`b`, `fixed seed`, 1), fn: 0 }], + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + cliPath: `plain`, + concurrentStart: false, + lazyGate: false, + fsRead: `callback`, + steps: [`fsData`], + }), + }, { // File b's verdict waits on a module that it imports only where the // output differs, so only mutated runs load it. The edit closes that diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index bdc80ee..1694217 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -71,6 +71,16 @@ export const shards = { `fixed seed 20261105: an edit during a check through the daemon`, `random seed: an edit during a check through the daemon`, `determinism: constructed history: an edit during a check through the daemon`, + // The fs-read data histories (law 9), not yet measured. + `fixed seed 20261105: an edit to data a test reads through fs (sync)`, + `random seed: an edit to data a test reads through fs (sync)`, + `determinism: constructed history: an edit to data a test reads through fs (sync)`, + `fixed seed 20261105: an edit to data a test reads through fs (promise)`, + `random seed: an edit to data a test reads through fs (promise)`, + `determinism: constructed history: an edit to data a test reads through fs (promise)`, + `fixed seed 20261105: an edit to data a test reads through fs (callback)`, + `random seed: an edit to data a test reads through fs (callback)`, + `determinism: constructed history: an edit to data a test reads through fs (callback)`, // Added with the CLI's parent-config discovery (not yet measured). `fixed seed 20261105: a package whose parent directory has its config`, `random seed: test.projects in a parent's config`, From d98af2e62c56541e4c2cbce4a72fca86a58f432d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 12 Nov 2026 18:23:49 -0700 Subject: [PATCH 04/17] fix(oracle-gaps): a file a test reads through fs is an input of its run Law 9: after an edit of only a data file that a test reads through fs, the daemon reused the run that read the old data, while a cold run sees the edit. The capture setup now records each copy's reads of files in the package (readFileSync, readFile, existsSync, and statSync, and the fs/promises readFile and stat; under node_modules and the runner's own files left out), and lists them with the modules the copy evaluated, so the run is stored under their content hashes. syncBuiltinESMExports makes an ES module's imports of node:fs and node:fs/promises read the recorded functions. RED on main (the three fs histories): "after fsData, the daemon's report differs from the cold run's at src/sut.ts#f0:delete-call". GREEN: the three fs histories and the mid-run history pass. Co-authored-by: Isaac --- .../oracles/oracle-gaps/capture-setup.mjs | 56 +++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/packages/oracles/oracle-gaps/capture-setup.mjs b/packages/oracles/oracle-gaps/capture-setup.mjs index 81a4388..a1de662 100644 --- a/packages/oracles/oracle-gaps/capture-setup.mjs +++ b/packages/oracles/oracle-gaps/capture-setup.mjs @@ -24,7 +24,10 @@ // In a mutant run, fast-check stops a property at its first failure instead // of shrinking it: a kill needs one failing history, not the smallest one. import { createHash } from 'node:crypto' -import { appendFileSync, readFileSync } from 'node:fs' +import fs, { appendFileSync, readFileSync } from 'node:fs' +import { syncBuiltinESMExports } from 'node:module' +import { resolve as resolvePath } from 'node:path' +import { fileURLToPath } from 'node:url' import { afterAll, beforeEach, chai, expect, inject, vi } from 'vitest' const run = @@ -151,6 +154,46 @@ if (run.units && !skippedCopy) { vi.resetModules() } const state = unit ? { ...run, ...unit } : run +// The files this copy reads through fs inside the package: by a sync, a +// promise, or a callback read, or a check of whether a file exists or what +// it is. The daemon stores the run under them as it does the modules the +// copy evaluated (the closing row's `loaded`), so an edit of only such a +// file runs it again, as a cold run sees the edit (law 9). Reads under +// node_modules and of the runner's own files are left out. +globalThis.__oracleGapsReads = new Set() +if (!globalThis.__oracleGapsReading) { + globalThis.__oracleGapsReading = true + const root = `${process.cwd()}/` + const record = (target) => { + let path + try { + if (target instanceof URL) path = fileURLToPath(target) + else if (typeof target === `string`) path = resolvePath(target) + else if (Buffer.isBuffer(target)) path = resolvePath(target.toString()) + } catch { + return + } + if ( + path?.startsWith(root) && + !path.includes(`/node_modules/`) && + !path.includes(`/.oracle-gaps`) + ) + globalThis.__oracleGapsReads.add(path) + } + const recorded = (owner, name) => { + const original = owner[name] + owner[name] = function (target, ...rest) { + record(target) + return original.call(this, target, ...rest) + } + } + for (const name of [`readFileSync`, `readFile`, `existsSync`, `statSync`]) + recorded(fs, name) + for (const name of [`readFile`, `stat`]) recorded(fs.promises, name) + // An ES module's `import { readFileSync } from 'node:fs'` (and from + // 'node:fs/promises') reads these from now on. + syncBuiltinESMExports() +} // A batch's watchdog reads this row to know which copy is running. A copy // whose test file fails to load runs no `afterAll` and so writes no closing // row; the next copy's start ends its turn. @@ -855,10 +898,13 @@ afterAll(() => { const evaluated = globalThis.__vitest_worker__?.evaluatedModules // The map keeps every module the worker evaluated, and the copy's module // reset only marks them unevaluated, so the copy's own are those marked. - const loaded = [...(evaluated?.idToModuleMap ?? [])] - .filter(([, node]) => node.evaluated) - .map(([id]) => id.split(`?`)[0]) - .filter((path) => path.startsWith(`/`)) + const loaded = [ + ...[...(evaluated?.idToModuleMap ?? [])] + .filter(([, node]) => node.evaluated) + .map(([id]) => id.split(`?`)[0]) + .filter((path) => path.startsWith(`/`)), + ...globalThis.__oracleGapsReads, + ] appendFileSync( state.capture, `${JSON.stringify({ file: thisFile, closed: true, loaded, ...(skippedCopy ? { skipped: true } : {}) })}\n`, From 4be4a1c1fa37bc65e6efd0dd4d29354cafb6f47d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 12 Nov 2026 19:27:25 -0700 Subject: [PATCH 05/17] docs(oracle-gaps): the daemon's fs inputs, their limits, and a changeset Co-authored-by: Isaac --- .changeset/gap-cli-daemon-fs-inputs.md | 5 +++++ README.md | 9 +++++++++ .../oracle-gaps/gap-report-oracle.property.test.mjs | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 .changeset/gap-cli-daemon-fs-inputs.md diff --git a/.changeset/gap-cli-daemon-fs-inputs.md b/.changeset/gap-cli-daemon-fs-inputs.md new file mode 100644 index 0000000..0765b29 --- /dev/null +++ b/.changeset/gap-cli-daemon-fs-inputs.md @@ -0,0 +1,5 @@ +--- +'@tanstack/oracles': patch +--- + +A check through `--daemon` now reruns a test after an edit to a file in the package that the test reads through `fs` (`readFileSync`, `readFile`, `existsSync`, `statSync`, or `fs/promises` `readFile` or `stat`), such as a JSON fixture or a snapshot. Before, the daemon reused the run that read the old file, while a cold run saw the edit. Reads outside the package, under `node_modules`, or through other APIs are no inputs. diff --git a/README.md b/README.md index 80ffd88..2257803 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,15 @@ every commit with [pkg.pr.new](https://pkg.pr.new). it with `oracle-gaps --stop-daemon --package `. On Windows, which has no process groups, the run is not supervised. + A check through the daemon reuses a run while its inputs are unchanged: + the modules the run loaded, and the files in the package that its tests + read through `fs` (`readFileSync`, `readFile`, `existsSync`, `statSync`, + and `fs/promises` `readFile` and `stat`), each by its content. A file that + a test reads outside the package or under `node_modules`, or through + another API (`readdir`, `open`, a stream, or a watcher), is no input, so an + edit to it alone does not rerun the tests that read it; run the check + cold (without `--daemon`) after such an edit. + - `packages/eslint-plugin-oracles` (`@tanstack/eslint-plugin-oracles`): the `oracle/*` lint rules. The rules recognize the framework by its package specifier and by a relative module with its file name. A project names its diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 96fe46a..fd93da8 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -2772,7 +2772,7 @@ oracleProperty({ law: { statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10).`, - limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; that check is not compared, because a run that the edit overlapped may report either text (the runner's rule: a run whose input changed while it ran is invalidated and not stored, settleIn), and the next check, with nothing changed, must equal a cold run of the state after the edit; an edit during a mutant run, or of a module that the run evaluating it reads, is not generated, and a module that a run first loads and that changes during that same run is stored under its later hash; law 9's edits leave out files read through fs, other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export.`, + limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; that check is not compared, because a run that the edit overlapped may report either text (the runner's rule: a run whose input changed while it ran is invalidated and not stored, settleIn), and the next check, with nothing changed, must equal a cold run of the state after the edit; an edit during a mutant run, or of a module that the run evaluating it reads, is not generated, and a module that a run first loads and that changes during that same run is stored under its later hash; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat; a read outside the package or under node_modules, or through another API, such as readdir, open, a stream, or a watcher, is no input, and none is generated); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs From 6a789a219d01c3aba1e31214c32b562fbfa83cf8 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 07:26:36 -0700 Subject: [PATCH 06/17] test(oracle-gaps): a test that stats its directory, fs functions keep their properties, owned law 9 states From the review of #55: - A `statDir` read stats the test's own directory and tests that it exists before it reads data.json, through a constructed history. The recorder records the directory, and law 9 owes no crash. - Each generated data read first checks that its fs function keeps its own name and length (law 3: a test that passes alone passes under the CLI). - The mid-run and fs-read states are reached by their own constructed histories, not by the edit sequence, which enters every state in `incrementalStates` (CI: "an edit sequence through the daemon" did not enter them). - Law 9's limits name reads from a test's worker threads or child processes, and a file that changes while a run first reads it. RED on 2dcb803: "an edit to data a test reads through fs (statDir)" fails its baseline (the wrapper loses statSync's name and length). With only the property copy applied, the sync history passes and statDir ends in "oracle-gaps internal error: EISDIR: illegal operation on a directory". Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 19 ++++++ .../gap-report-oracle.property.test.mjs | 63 +++++++++++++++++-- packages/oracles/scripts/e2e-shards.mjs | 3 + 3 files changed, 81 insertions(+), 4 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 3383118..93e9500 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -625,6 +625,9 @@ const drawn = fc.record({ // through fs at run time, by a sync, a promise, or a callback read (law // 9: a file a test reads is an input of its run, as a module it imports // is). The `fsData` edit changes only that file. + // A `statDir` read stats the test's own directory first, as a test that + // looks for a fixtures folder does (review of #55: a directory that a + // test stats is no file to hash). fsRead: fc.constantFrom( `none`, `none`, @@ -632,6 +635,7 @@ const drawn = fc.record({ `sync`, `promise`, `callback`, + `statDir`, ), // A second authored mutant, on the next statement, so that two patched // sources are judged in the same run and neither may see the other's. @@ -4255,17 +4259,32 @@ const lazyGated = (plan, file, t) => const fsGated = (plan, file, t) => plan.fsRead !== `none` && file === `b` && t.judges && t.judgeBy === `verdict` // How file b reads `tests/data.json`, by each kind of fs read. +// Each read first checks that the fs function it uses keeps its own name +// and length, as code that dispatches on a function's arity does: a test +// that passes alone passes under the CLI (law 3), whatever the CLI does to +// see the read (review of #55). +const keptFs = (local, name, length) => + `if (${local}.name !== '${name}' || ${local}.length !== ${length}) throw new Error('fs.${name} lost its own properties')` const dataReads = { sync: [ `import { readFileSync as readDataSync } from 'node:fs'`, + keptFs(`readDataSync`, `readFileSync`, 2), `const readData = async () => JSON.parse(readDataSync(new URL('./data.json', import.meta.url), 'utf8'))`, ], promise: [ `import { readFile as readDataFile } from 'node:fs/promises'`, + keptFs(`readDataFile`, `readFile`, 2), `const readData = async () => JSON.parse(await readDataFile(new URL('./data.json', import.meta.url), 'utf8'))`, ], + statDir: [ + `import { existsSync as dataDirExists, readFileSync as readDataSync, statSync as statDataDir } from 'node:fs'`, + keptFs(`statDataDir`, `statSync`, 1), + keptFs(`dataDirExists`, `existsSync`, 1), + `const readData = async () => (statDataDir(new URL('./', import.meta.url)).isDirectory() && dataDirExists(new URL('./', import.meta.url)) ? JSON.parse(readDataSync(new URL('./data.json', import.meta.url), 'utf8')) : { open: false })`, + ], callback: [ `import { readFile as readDataCallback } from 'node:fs'`, + keptFs(`readDataCallback`, `readFile`, 3), `const readData = () => new Promise((resolve, reject) => readDataCallback(new URL('./data.json', import.meta.url), 'utf8', (error, text) => (error ? reject(error) : resolve(JSON.parse(text)))))`, ], } diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 5349437..17e9366 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -3012,7 +3012,8 @@ const incrementalStates = [ ...edits .filter( (edit) => - !daemonEvents.has(edit) && edit !== `fixSession` && edit !== `lazy`, + !daemonEvents.has(edit) && + ![`fixSession`, `lazy`, `fsData`, `midrun`].includes(edit), ) .map((edit) => `edit ${edit} changed the cold result`), `a check whose daemon died during it`, @@ -3026,12 +3027,20 @@ const incrementalStates = [ `two checks at once after a code change`, `a check from another install of the CLI`, `a check after the OS cleaned the daemon's state`, + `two checks started the daemon at once`, + `the daemon ran from a path with a space and #`, +] +// Law 9's states that a constructed history of their own reaches, not the +// edit sequence: the mid-run edit, and an edit to data a test reads through +// fs, by each read. +const ownedIncrementalStates = [ + `edit midrun changed the cold result`, `an edit landed during a check through the daemon`, + `edit fsData changed the cold result`, `an edit to data read through sync changed the cold result`, `an edit to data read through promise changed the cold result`, `an edit to data read through callback changed the cold result`, - `two checks started the daemon at once`, - `the daemon ran from a path with a space and #`, + `an edit to data read through statDir changed the cold result`, ] // The static states (probe structure and ownership through the reference // tree) are gap-report-static-oracle's: it reaches each in every campaign, @@ -3177,6 +3186,7 @@ const required = [ `a session failure larger than a socket buffer`, `a config fault in a run that owns nothing`, ...incrementalStates, + ...ownedIncrementalStates, `a timed-out run reused after an edit`, `an edit that changed the cold result after a hang replaced the only context`, `edit lazy changed the cold result`, @@ -3259,6 +3269,7 @@ const separate = [ `a run with eleven contexts`, `a failing owner replayed under a config that --config names`, ...incrementalStates, + ...ownedIncrementalStates, `a timed-out run reused after an edit`, `an edit that changed the cold result after a hang replaced the only context`, `edit lazy changed the cold result`, @@ -3289,7 +3300,7 @@ oracleProperty({ law: { statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, - limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; that check is not compared, because a run that the edit overlapped may report either text (the runner's rule: a run whose input changed while it ran is invalidated and not stored, settleIn), and the next check, with nothing changed, must equal a cold run of the state after the edit; an edit during a mutant run, or of a module that the run evaluating it reads, is not generated, and a module that a run first loads and that changes during that same run is stored under its later hash; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat; a read outside the package or under node_modules, or through another API, such as readdir, open, a stream, or a watcher, is no input, and none is generated); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export.`, + limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; that check is not compared, because a run that the edit overlapped may report either text (the runner's rule: a run whose input changed while it ran is invalidated and not stored, settleIn), and the next check, with nothing changed, must equal a cold run of the state after the edit; an edit during a mutant run, or of a module that the run evaluating it reads, is not generated, and a module or a file that a run first reads and that changes during that same run is stored under its later hash (not generated); a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat; a read outside the package or under node_modules, through another API, such as readdir, open, a stream, or a watcher, or from a worker thread or a child process that a test starts, is no input, and none is generated); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs @@ -4078,6 +4089,50 @@ oracleProperty({ steps: [`fsData`], }), }, + { + // File b's verdict also waits on a data file that the test reads + // after it stats its own directory (review of #55: the directory is + // recorded too, and it is no file to hash, so no crash is owed). The edit changes only that file, which + // closes the gate: the cold run then reports b's mutant as a + // survivor, and the daemon must not reuse a kill that read the old + // data (law 9: a file a test reads is an input of its run). + name: `an edit to data a test reads through fs (statDir)`, + enters: [ + `edit fsData changed the cold result`, + `an edit to data read through statDir changed the cold result`, + ], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`b`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [{ ...judging(`b`, `fixed seed`, 1), fn: 0 }], + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + cliPath: `plain`, + concurrentStart: false, + lazyGate: false, + fsRead: `statDir`, + steps: [`fsData`], + }), + }, { // File b's verdict waits on a module that it imports only where the // output differs, so only mutated runs load it. The edit closes that diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 5ae7fbb..455b07e 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -81,6 +81,9 @@ export const shards = { `fixed seed 20261105: an edit to data a test reads through fs (callback)`, `random seed: an edit to data a test reads through fs (callback)`, `determinism: constructed history: an edit to data a test reads through fs (callback)`, + `fixed seed 20261105: an edit to data a test reads through fs (statDir)`, + `random seed: an edit to data a test reads through fs (statDir)`, + `determinism: constructed history: an edit to data a test reads through fs (statDir)`, // Added with the CLI's parent-config discovery (not yet measured). `fixed seed 20261105: a package whose parent directory has its config`, `random seed: test.projects in a parent's config`, From 0d57c3574f90b7ebd8498d648f81aa7eb7badf9d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 07:26:36 -0700 Subject: [PATCH 07/17] fix(oracle-gaps): hash a directory a test stats as a marker; keep the fs functions' own properties `fileHashes()` read every recorded input with readFileSync, which throws EISDIR for a directory that a test stats or tests for (a fixtures folder), and the error escaped as an internal error that left the session directory. A path that is not a regular file now hashes as `not a file`: its input is that it is there. The recorder's fs wrappers copy the original's own properties (name, length), so code that reads them sees the same functions as without the CLI. Co-authored-by: Isaac --- .changeset/gap-cli-daemon-fs-inputs.md | 2 +- README.md | 14 +++++++++----- packages/oracles/oracle-gaps/capture-setup.mjs | 13 ++++++++++++- packages/oracles/oracle-gaps/check.mjs | 3 +++ 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/.changeset/gap-cli-daemon-fs-inputs.md b/.changeset/gap-cli-daemon-fs-inputs.md index 0765b29..53380eb 100644 --- a/.changeset/gap-cli-daemon-fs-inputs.md +++ b/.changeset/gap-cli-daemon-fs-inputs.md @@ -2,4 +2,4 @@ '@tanstack/oracles': patch --- -A check through `--daemon` now reruns a test after an edit to a file in the package that the test reads through `fs` (`readFileSync`, `readFile`, `existsSync`, `statSync`, or `fs/promises` `readFile` or `stat`), such as a JSON fixture or a snapshot. Before, the daemon reused the run that read the old file, while a cold run saw the edit. Reads outside the package, under `node_modules`, or through other APIs are no inputs. +A check through `--daemon` now reruns a test after an edit to a file in the package that the test reads through `fs` (`readFileSync`, `readFile`, `existsSync`, `statSync`, or `fs/promises` `readFile` or `stat`), such as a JSON fixture or a snapshot. Before, the daemon reused the run that read the old file, while a cold run saw the edit. A directory that a test stats is an input as a directory. Reads outside the package, under `node_modules`, through other APIs, or from a test's own worker threads or child processes are no inputs. diff --git a/README.md b/README.md index fd4f096..f15c5a4 100644 --- a/README.md +++ b/README.md @@ -119,11 +119,15 @@ exited with ` and exits 3; after an out-of-memory crash A check through the daemon reuses a run while its inputs are unchanged: the modules the run loaded, and the files in the package that its tests read through `fs` (`readFileSync`, `readFile`, `existsSync`, `statSync`, - and `fs/promises` `readFile` and `stat`), each by its content. A file that - a test reads outside the package or under `node_modules`, or through - another API (`readdir`, `open`, a stream, or a watcher), is no input, so an - edit to it alone does not rerun the tests that read it; run the check - cold (without `--daemon`) after such an edit. + and `fs/promises` `readFile` and `stat`), each by its content. A + directory that a test stats or tests for is an input as a directory, not + by its content. A file that a test reads outside the package or under + `node_modules`, through another API (`readdir`, `open`, a stream, or a + watcher), or from a worker thread or child process that the test starts, + is no input, so an edit to it alone does not rerun the tests that read it; + run the check cold (without `--daemon`) after such an edit. A file that + changes while the test that reads it runs is stored under its content + after the change. - `packages/eslint-plugin-oracles` (`@tanstack/eslint-plugin-oracles`): the `oracle/*` lint rules. The rules recognize the framework by its package diff --git a/packages/oracles/oracle-gaps/capture-setup.mjs b/packages/oracles/oracle-gaps/capture-setup.mjs index 565a560..af04cfd 100644 --- a/packages/oracles/oracle-gaps/capture-setup.mjs +++ b/packages/oracles/oracle-gaps/capture-setup.mjs @@ -180,12 +180,23 @@ if (!globalThis.__oracleGapsReading) { ) globalThis.__oracleGapsReads.add(path) } + // A wrapper keeps the function's own properties (its name and length, + // which code that dispatches on arity reads), so a test that passes alone + // passes under the capture (law 3; review of #55). const recorded = (owner, name) => { const original = owner[name] - owner[name] = function (target, ...rest) { + const wrapper = function (target, ...rest) { record(target) return original.call(this, target, ...rest) } + for (const key of Reflect.ownKeys(original)) + if (key !== `prototype`) + Object.defineProperty( + wrapper, + key, + Object.getOwnPropertyDescriptor(original, key), + ) + owner[name] = wrapper } for (const name of [`readFileSync`, `readFile`, `existsSync`, `statSync`]) recorded(fs, name) diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index 3521a5f..eb7bc98 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -188,6 +188,9 @@ function fileHashes() { } catch { return `missing` } + // A test can stat or test for a directory (a fixtures folder): its input + // is that a directory is there, not its content (review of #55). + if (!stat.isFile()) return `not a file` const stamp = `${stat.size}:${stat.mtimeMs}` const known = memo.get(path) if (known?.stamp === stamp) return known.hash From 40527b4d63ebcc4bd10486c6c24aecb2d2569075 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 08:21:06 -0700 Subject: [PATCH 08/17] test(oracle-gaps): a run that edits the input it read must not be reused (law 9) Re-review of #55, F1: the runner stored a run under each input's hash at the end of the run, so a file or module that the run read and then edited was stored under its edited text with the old verdict, and the daemon's next check served it. The mid-run edit gains who lands it (`midrunBy`) and what it edits (`midrunTarget`, the helper module or the data file read through fs). File b's own test now reads what it edits, fails unless it reads the first state's text, and then lands the edit, so the run that lands it passes and every run that reads the edited text fails. Two constructed histories reach it: "a test edits the helper it read, during a check through the daemon" and "a test edits the data it read, ...". The new grammar fields come last, so the fields before them draw as before. Law 9's limits name the fs APIs that are no input (F2, a maintainer decision asked on 2026-11-13) and the hash memo's size-and-mtime key (F3, D10). RED on ef40cbb: both histories fail with "after midrun, the daemon's report differs from the cold run's": the daemon passes the editor test, and the cold run fails it ("the helper this run read is not strict", "the data this run read is closed"). Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 69 +++++++-- .../gap-report-oracle.property.test.mjs | 143 ++++++++++++++++-- packages/oracles/scripts/e2e-shards.mjs | 6 + 3 files changed, 194 insertions(+), 24 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 9fef83f..c3405a8 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -792,6 +792,13 @@ const drawn = fc.record({ }), // Who names the importer's function in an owns entry, if any. importer: fc.constantFrom(`none`, `a`, `b`), + // Last, so the fields above draw as before. + // Which file's test lands a mid-run edit (`midrun`), and what it edits: + // file b's helper, or the data file when file b reads one (`midrunTarget` + // is the helper otherwise). Only a check through the daemon takes the + // `midrun` step. + midrunBy: fc.constantFrom(`a`, `b`), + midrunTarget: fc.constantFrom(`helper`, `data`), }) // The edit sequence over the witness module, the one list of steps that a @@ -2844,6 +2851,14 @@ function resolveHistory(history) { // Whether a step edits file b's helper during a check (`midrun`): then // file a loads the helper and holds the test that edits it. midrun: (history.steps ?? []).includes(`midrun`), + // Which file's test lands the mid-run edit (`midrunBy`), and what it + // edits (`midrunTarget`): file a edits file b's helper after its + // context loaded it, or file b edits its own helper or the data file + // that its verdict reads through fs, after its judged tests read them. + // Then the run that lands the edit read the old text, and its verdict + // follows it (review of #55, F1). + midrunBy: history.midrunBy ?? `a`, + midrunTarget: midrunTargetOf(history), consoleOutput: history.consoleOutput === true, // Whether a run aborts its process (a stop run `abort`): then file a has // the test that aborts when the driver asks. @@ -4181,7 +4196,7 @@ function baselineOf(plan) { // that aborts when the driver asks. const own = plan.tests.filter((t) => t.file === file).length + - (plan.midrun && file === `a` ? 1 : 0) + + (plan.midrun && file === plan.midrunBy ? 1 : 0) + (file === `a` && plan.consoleOutput ? 1 : 0) + (file === `a` && plan.aborts ? 1 : 0) const broken = failing.find((f) => f.file === file) @@ -4641,10 +4656,12 @@ function oracleTestFile(plan, file) { : []), // The mid-run edit (`midrun`): file a loads file b's helper, so its // context holds the helper's old text when the edit lands. - ...(plan.midrun && file === `a` + ...(plan.midrun && file === plan.midrunBy ? [ `import { readFileSync as readMarker, writeFileSync as writeEdit } from 'node:fs'`, - `import { strict as strictAtLoad } from './strict'`, + ...(file === `a` + ? [`import { strict as strictAtLoad } from './strict'`] + : []), ] : []), // A module's text through a query import, which Vite loads (see rawImport). @@ -4859,17 +4876,33 @@ function oracleTestFile(plan, file) { ) // The mid-run edit: the first run of this test after the driver marks an // edit pending (`tests/midrun.txt`, `pending:`) rewrites file b's - // helper with that value and marks the edit done, so every later run of - // it, in this check or another, changes nothing. It calls nothing of the - // source under test, so no mutant reaches it. - if (plan.midrun && file === `a`) + // helper, or the data file, with that value and marks the edit done, so + // every later run of it, in this check or another, changes nothing. It + // calls nothing of the source under test, so no mutant reaches it. In + // file b it first reads what it edits and fails unless the helper is + // strict or the data open, as in the history's first state: the run that + // lands the edit passes, and every run that reads the edited text fails, + // so a run stored under the edited text with the old verdict shows in + // the baseline (review of #55, F1). + if (plan.midrun && file === plan.midrunBy) out.push( - `test('edits the helper during the check', () => {`, - ` void strictAtLoad`, + `test('edits ${plan.midrunTarget === `data` ? `the data` : `the helper`} during the check', async () => {`, + ...(file === `a` + ? [` void strictAtLoad`] + : plan.midrunTarget === `data` + ? [ + ` if (!(await readData()).open) throw new Error('the data this run read is closed')`, + ] + : [ + ` if (!strict) throw new Error('the helper this run read is not strict')`, + ]), ` const marker = new URL('./midrun.txt', import.meta.url)`, ` const text = readMarker(marker, 'utf8')`, ` if (!text.startsWith('pending:')) return`, - ` writeEdit(new URL('./strict.ts', import.meta.url), \`export const strict = \${text.slice('pending:'.length).trim()}\\n\`)`, + ` const value = text.slice('pending:'.length).trim()`, + plan.midrunTarget === `data` + ? ` writeEdit(new URL('./data.json', import.meta.url), \`{"open":\${value}}\\n\`)` + : ` writeEdit(new URL('./strict.ts', import.meta.url), \`export const strict = \${value}\\n\`)`, ` writeEdit(marker, 'done\\n')`, `})`, ) @@ -4886,7 +4919,8 @@ function oracleTestFile(plan, file) { // A file with no tests still loads, so the CLI can read what it owns. if ( !plan.tests.some((t) => t.file === file) && - !(file === `a` && (plan.midrun || plan.consoleOutput || plan.aborts)) + !(plan.midrun && file === plan.midrunBy) && + !(file === `a` && (plan.consoleOutput || plan.aborts)) ) out.push(`test.skip('no tests', () => {})`) // A test that writes to the process's own streams, as a test's debugging @@ -5009,6 +5043,12 @@ function edited(plan) { const AUTHORED = `mutants/authored-mutant.json` +/** What a mid-run edit changes: the data file only when file b reads one. */ +const midrunTargetOf = (history) => + history.midrunTarget === `data` && (history.fsRead ?? `none`) !== `none` + ? `data` + : `helper` + // Law 9's fixture state: the history, and what the edits outside it set. const initialState = (plan) => ({ history: plan.history, @@ -5349,7 +5389,8 @@ function fixtureFiles(plan, state = initialState(plan)) { // instead of reusing their evidence: the marker is read through fs, // which no run records as an input. if (plan.midrun) - files[`tests/a.test.ts`] += `// mid-run edits: ${state.midruns}\n` + files[`tests/${plan.midrunBy}.test.ts`] += + `// mid-run edits: ${state.midruns}\n` // The new oracle file d owns the first generated function and judges it. if (state.newOracle) { const target = plan.functions.find((fn) => fn.generated && !fn.library) @@ -5495,7 +5536,9 @@ function applyEdit(state, edit) { // A mid-run edit leaves file b's helper as the helper edit does. if (edit === `helper`) return { ...state, strict: !state.strict } if (edit === `midrun`) - return { ...state, strict: !state.strict, midruns: state.midruns + 1 } + return midrunTargetOf(state.history) === `data` + ? { ...state, dataOpen: !state.dataOpen, midruns: state.midruns + 1 } + : { ...state, strict: !state.strict, midruns: state.midruns + 1 } if (edit === `lazy`) return { ...state, lazyOpen: !state.lazyOpen } if (edit === `fsData`) return { ...state, dataOpen: !state.dataOpen } if (edit === `seed`) return { ...state, seed: state.seed === 1 ? 2 : 1 } diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 915a8a6..39ebc35 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1461,29 +1461,39 @@ async function incremental(dir, plan, cli, copy, observe, states) { // (`resetBeforeRead`). async function daemonChecks(step, files, how) { // The mid-run edit: the check starts from the state before the edit, - // with the edit marked pending, and file a's test lands it while the - // check runs, after its context loaded the helper and before file b's - // runs read it. That check may report either state for a run the edit - // overlapped, so it is not compared; the runner must store no such run - // (settleIn), so the next check, with nothing changed, equals a cold - // run of the state after the edit. + // with the edit marked pending, and a test lands it while the check + // runs. File a's test edits file b's helper after its context loaded + // it and before file b's runs read it; file b's own test edits its + // helper, or the data file that it reads through fs, after b's judged + // tests read them, so the run that lands the edit judged the old text. + // That check may report either state for a run the edit overlapped, so + // it is not compared; the runner must store no run under a text it did + // not read (settleIn), so the next check, with nothing changed, equals + // a cold run of the state after the edit. if (step === `midrun`) { - const after = files[`tests/strict.ts`] + const target = + plan.midrunTarget === `data` ? `tests/data.json` : `tests/strict.ts` + const after = files[target] const value = after.includes(`true`) runCli( dir, { ...files, - 'tests/strict.ts': `export const strict = ${!value}\n`, + [target]: after.replace(String(value), String(!value)), 'tests/midrun.txt': `pending:${value}\n`, }, { ...how, daemon: true }, ) if ( readFileSync(join(dir, `tests/midrun.txt`), `utf8`) === `done\n` && - readFileSync(join(dir, `tests/strict.ts`), `utf8`) === after - ) + readFileSync(join(dir, target), `utf8`) === after + ) { states.add(`an edit landed during a check through the daemon`) + if (plan.midrunBy === `b`) + states.add( + `a run edited the ${plan.midrunTarget} it had read, during a check through the daemon`, + ) + } return [runCli(dir, files, { ...how, daemon: true })] } if (step !== `dieMidCheck` && step !== `resetBeforeRead`) @@ -3117,6 +3127,8 @@ const incrementalStates = [ const ownedIncrementalStates = [ `edit midrun changed the cold result`, `an edit landed during a check through the daemon`, + `a run edited the helper it had read, during a check through the daemon`, + `a run edited the data it had read, during a check through the daemon`, `edit fsData changed the cold result`, `an edit to data read through sync changed the cold result`, `an edit to data read through promise changed the cold result`, @@ -3387,7 +3399,7 @@ oracleProperty({ law: { statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, - limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; that check is not compared, because a run that the edit overlapped may report either text (the runner's rule: a run whose input changed while it ran is invalidated and not stored, settleIn), and the next check, with nothing changed, must equal a cold run of the state after the edit; an edit during a mutant run, or of a module that the run evaluating it reads, is not generated, and a module or a file that a run first reads and that changes during that same run is stored under its later hash (not generated); a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat; a read outside the package or under node_modules, through another API, such as readdir, open, a stream, or a watcher, or from a worker thread or a child process that a test starts, is no input, and none is generated); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated.`, + limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check is not compared, because a run that the edit overlapped may report either text (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn), and the next check, with nothing changed, must equal a cold run of the state after the edit; an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, createReadStream, realpath, lstat, access, open, watch, readlink, the callback stat, exists, and the fs/promises access and open, is no input, and none is generated: whether to record these APIs is the maintainer's decision, asked 2026-11-13, and this oracle owns the gap); the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs @@ -4057,6 +4069,115 @@ oracleProperty({ steps: [`midrun`], }), }, + { + // File b's own test edits its helper after b's judged test read it, + // in the run that judges with the old text: its verdict follows the + // helper, so a run stored under the helper's text after the edit + // makes the daemon's next check report the old verdict, which the + // cold run of the edited state does not (review of #55, F1: a + // module input changed during the run that first read it). + name: `a test edits the helper it read, during a check through the daemon`, + enters: [ + `an edit landed during a check through the daemon`, + `a run edited the helper it had read, during a check through the daemon`, + `edit midrun changed the cold result`, + ], + build: (generated) => ({ + ...generated, + functions: [ + [{ kind: `push`, quote: `'` }], + [{ kind: `push`, quote: `'` }], + ], + containers: [], + overloads: [], + owners: [`a`, `b`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [ + { ...judging(`a`, `fixed seed`, 1), fn: 0 }, + { ...judging(`b`, `fixed seed`, 1), fn: 1 }, + ], + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + contexts: 1, + steps: [`midrun`], + midrunBy: `b`, + midrunTarget: `helper`, + fsRead: `none`, + lazyGate: false, + }), + }, + { + // File b's own test edits the data file that its judged test read + // through fs (a sync read), in the run that judges with the old + // data: a run stored under the data's hash after the edit makes the + // daemon's next check report the old verdict (review of #55, F1: a + // file that a test reads, then writes in the same run). + name: `a test edits the data it read, during a check through the daemon`, + enters: [ + `an edit landed during a check through the daemon`, + `a run edited the data it had read, during a check through the daemon`, + `edit midrun changed the cold result`, + ], + build: (generated) => ({ + ...generated, + functions: [ + [{ kind: `push`, quote: `'` }], + [{ kind: `push`, quote: `'` }], + ], + containers: [], + overloads: [], + owners: [`a`, `b`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [ + { ...judging(`a`, `fixed seed`, 1), fn: 0 }, + { ...judging(`b`, `fixed seed`, 1), fn: 1 }, + ], + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + contexts: 1, + steps: [`midrun`], + midrunBy: `b`, + midrunTarget: `data`, + fsRead: `sync`, + lazyGate: false, + }), + }, { // File b's verdict also waits on a data file that the test reads // through fs (sync read). The edit changes only that file, which diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 455b07e..2b25599 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -71,6 +71,12 @@ export const shards = { `fixed seed 20261105: an edit during a check through the daemon`, `random seed: an edit during a check through the daemon`, `determinism: constructed history: an edit during a check through the daemon`, + `fixed seed 20261105: a test edits the helper it read, during a check through the daemon`, + `random seed: a test edits the helper it read, during a check through the daemon`, + `determinism: constructed history: a test edits the helper it read, during a check through the daemon`, + `fixed seed 20261105: a test edits the data it read, during a check through the daemon`, + `random seed: a test edits the data it read, during a check through the daemon`, + `determinism: constructed history: a test edits the data it read, during a check through the daemon`, // The fs-read data histories (law 9), not yet measured. `fixed seed 20261105: an edit to data a test reads through fs (sync)`, `random seed: an edit to data a test reads through fs (sync)`, From b5578ee4f02b9d122362f2003c4b599e615ffb2f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 08:40:35 -0700 Subject: [PATCH 09/17] test(oracle-gaps): the mid-run marker sits in the daemon's session directory, and is never an input The test that lands a mid-run edit read its marker (tests/midrun.txt) through readFileSync and rewrote it in the same run. Once a run is stored under the text it read, that marker read alone kept the run from being stored, so a runner without its module transform hash still passed the helper history: the marker masked the module case. The marker now sits in the running daemon's session directory, beside the runner's harness, which is no input of a run by design (ownHarness). The test finds it through the worker's first setup file; a run with no harness or no marker edits nothing. After each mid-run step the driver reads the daemon's evidence store, and law 9 fails if any stored run has the marker among its inputs, so a runner that starts recording it fails loudly. Law 9's limits record the maintainer's F2 decision of 2026-11-13 (every fs read except watch is an input), which a follow-up implements. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 23 +++++--- .../gap-report-oracle.property.test.mjs | 58 +++++++++++++++---- 2 files changed, 62 insertions(+), 19 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index c3405a8..05cb6c5 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -4659,6 +4659,7 @@ function oracleTestFile(plan, file) { ...(plan.midrun && file === plan.midrunBy ? [ `import { readFileSync as readMarker, writeFileSync as writeEdit } from 'node:fs'`, + `import { dirname as markerDir, join as markerPath } from 'node:path'`, ...(file === `a` ? [`import { strict as strictAtLoad } from './strict'`] : []), @@ -4875,7 +4876,7 @@ function oracleTestFile(plan, file) { `test('fixed seed 999', () => {})`, ) // The mid-run edit: the first run of this test after the driver marks an - // edit pending (`tests/midrun.txt`, `pending:`) rewrites file b's + // edit pending (`midrun.txt`, `pending:`) rewrites file b's // helper, or the data file, with that value and marks the edit done, so // every later run of it, in this check or another, changes nothing. It // calls nothing of the source under test, so no mutant reaches it. In @@ -4896,8 +4897,16 @@ function oracleTestFile(plan, file) { : [ ` if (!strict) throw new Error('the helper this run read is not strict')`, ]), - ` const marker = new URL('./midrun.txt', import.meta.url)`, - ` const text = readMarker(marker, 'utf8')`, + // The marker sits in the run's own session directory, beside the + // runner's harness (the worker's first setup file), which is no input + // of a run by design (check.mjs, ownHarness). So the marker's own edit + // does not decide whether the run is stored: only the helper's or the + // data's text does. A run with no harness, or no marker, edits nothing. + ` const harness = (globalThis as { __vitest_worker__?: { config?: { setupFiles?: Array } } }).__vitest_worker__?.config?.setupFiles?.[0]`, + ` if (!harness) return`, + ` const marker = markerPath(markerDir(harness), 'midrun.txt')`, + ` let text = ''`, + ` try { text = readMarker(marker, 'utf8') } catch { return }`, ` if (!text.startsWith('pending:')) return`, ` const value = text.slice('pending:'.length).trim()`, plan.midrunTarget === `data` @@ -5253,10 +5262,6 @@ function fixtureFiles(plan, state = initialState(plan)) { ...(plan.dotGenSource ? { [DOTGEN]: plan.dotGenSource } : {}), ...plan.exportSources, 'tests/strict.ts': `export const strict = ${state.strict}\n`, - // The mid-run edit's marker: done in every state, and pending only in - // the check through the daemon during which the edit lands (the - // driver's `midrun` step). - ...(plan.midrun ? { 'tests/midrun.txt': `done\n` } : {}), ...(plan.lazyGate ? { 'tests/lazy.ts': `export const open = ${state.lazyOpen}\n` } : {}), @@ -5386,8 +5391,8 @@ function fixtureFiles(plan, state = initialState(plan)) { files[`tests/${file}.test.ts`] = oracleTestFile(plan, file) // Each mid-run edit changes file a's own text too (a count of them), so // the daemon runs file a's tests again in the check the edit lands in, - // instead of reusing their evidence: the marker is read through fs, - // which no run records as an input. + // instead of reusing their evidence: the marker sits in the daemon's + // session directory, which is no input of a run. if (plan.midrun) files[`tests/${plan.midrunBy}.test.ts`] += `// mid-run edits: ${state.midruns}\n` diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 39ebc35..21b74b5 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1209,6 +1209,18 @@ function copyCli(plan) { * The daemons that serve a package, by process id: the processes running * daemon.mjs for it, as the operating system lists them. */ +/** + * The paths that the package's daemon stored any run under, from its + * evidence store (evidence.mjs), or none when it has stored nothing. + */ +function storedInputs(dir) { + const file = join(stateDirOf(realpathSync(dir)), `evidence.json`) + if (!existsSync(file)) return [] + return JSON.parse(readFileSync(file, `utf8`)).flatMap((row) => + row.variants.flatMap((variant) => variant.inputs.map(([path]) => path)), + ) +} + function daemonsOf(dir) { const listed = spawnSync(`ps`, [`-A`, `-o`, `pid=,command=`], { encoding: `utf8`, @@ -1260,6 +1272,8 @@ async function incremental(dir, plan, cli, copy, observe, states) { let replacedOnly = false // Whether the daemon's last check cut a batch off. let cutBefore = false + // Whether the daemon stored a run under the mid-run marker. + let markerStored = false // A second install of the CLI, at another path, whose summary reads // otherwise, so its code differs from the first install's (`otherInstall`). let other @@ -1319,6 +1333,7 @@ async function incremental(dir, plan, cli, copy, observe, states) { for (const [i, run] of [daemon, ...others].entries()) observe(`incremental`, { step: i === 0 ? step : `${step} (a second check at once)`, + markerStored, cold: { exit: coldRun.exit, refusal: coldRun.refusal, @@ -1475,26 +1490,40 @@ async function incremental(dir, plan, cli, copy, observe, states) { plan.midrunTarget === `data` ? `tests/data.json` : `tests/strict.ts` const after = files[target] const value = after.includes(`true`) + // The marker goes in the running daemon's session directory, where + // the test finds it beside the runner's harness. With no daemon + // running (a step before killed it), no edit lands. + const [daemonPid] = daemonsOf(dir) + const marker = + daemonPid && join(dir, `.oracle-gaps-${daemonPid}`, `midrun.txt`) + if (marker && existsSync(dirname(marker))) + writeFileSync(marker, `pending:${value}\n`) runCli( dir, - { - ...files, - [target]: after.replace(String(value), String(!value)), - 'tests/midrun.txt': `pending:${value}\n`, - }, + { ...files, [target]: after.replace(String(value), String(!value)) }, { ...how, daemon: true }, ) - if ( - readFileSync(join(dir, `tests/midrun.txt`), `utf8`) === `done\n` && + const landed = + marker && + existsSync(marker) && + readFileSync(marker, `utf8`) === `done\n` && readFileSync(join(dir, target), `utf8`) === after - ) { + if (marker) rmSync(marker, { force: true }) + if (landed) { states.add(`an edit landed during a check through the daemon`) if (plan.midrunBy === `b`) states.add( `a run edited the ${plan.midrunTarget} it had read, during a check through the daemon`, ) } - return [runCli(dir, files, { ...how, daemon: true })] + const next = runCli(dir, files, { ...how, daemon: true }) + // The marker is out of a run's inputs by design; a runner that stored + // a run under it would let the marker's edit, and not the helper's or + // the data's, decide whether a run is stored. + markerStored ||= storedInputs(dir).some((path) => + path.endsWith(`/midrun.txt`), + ) + return [next] } if (step !== `dieMidCheck` && step !== `resetBeforeRead`) return [runCli(dir, files, { ...how, daemon: true })] @@ -2130,6 +2159,15 @@ function incrementalCheck(observations) { (o) => o.checkpoint === `incremental`, )) { const { step, cold, daemon } = value + // The mid-run marker sits in the daemon's own session directory, which + // is no input of a run by design (ownHarness); a run stored under it + // would let the marker hide whether the run read an edited input. + if (value.markerStored) + return violation( + `incremental`, + `after ${step}, the daemon stored a run under the mid-run marker, a file in its own session directory`, + `incremental equals cold`, + ) if ( daemon.exit !== cold.exit || daemon.refusal !== cold.refusal || @@ -3399,7 +3437,7 @@ oracleProperty({ law: { statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, - limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check is not compared, because a run that the edit overlapped may report either text (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn), and the next check, with nothing changed, must equal a cold run of the state after the edit; an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, createReadStream, realpath, lstat, access, open, watch, readlink, the callback stat, exists, and the fs/promises access and open, is no input, and none is generated: whether to record these APIs is the maintainer's decision, asked 2026-11-13, and this oracle owns the gap); the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated.`, + limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check is not compared, because a run that the edit overlapped may report either text (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn), and the next check, with nothing changed, must equal a cold run of the state after the edit; an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, createReadStream, realpath, lstat, access, open, watch, readlink, the callback stat, exists, and the fs/promises access and open, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs From 82b5c173ee667524a42d0f16cc2b258c25ac8859 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 08:40:35 -0700 Subject: [PATCH 10/17] fix(oracle-gaps): store a run under the text it read Re-review of #55: - F1: settleIn stored each input under its hash at the end of the run, so a file or module that the run read and that then changed was stored under the changed text with the old verdict. Now a file read through fs is stored under its hash at the copy's first read (the closing row's `readAt`), a module under the hash of its text when the run's context transformed it (a transform hook in the wrapper config, cleared when the module is invalidated), and any other input under its hash at the start of the run, or now. A path that changed after the run read it is invalidated and the run is not stored. main has the module case too; this fixes both with one mechanism. - F4: a file removed between the hash's stat and its read hashes as `missing`, not an internal error. - F8: the fs wrappers are one loop over an owner list (9 bytes smaller minified, the same gzip). - The README names each fs API that is no input yet. Evidence: RED on ef40cbb for both self-edit histories; GREEN for the mid-run, self-edit, and fs histories; the mutants "end-of-run hash", "no readAt", "no module transform hash", and "records the marker" each fail the oracle. Co-authored-by: Isaac --- README.md | 22 +++-- .../oracles/oracle-gaps/capture-setup.mjs | 51 +++++++---- packages/oracles/oracle-gaps/check.mjs | 89 ++++++++++++++----- packages/oracles/scripts/e2e-shards.mjs | 4 +- 4 files changed, 115 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index be67d29..964a306 100644 --- a/README.md +++ b/README.md @@ -157,15 +157,19 @@ exited with ` and exits 3; after an out-of-memory crash A check through the daemon reuses a run while its inputs are unchanged: the modules the run loaded, and the files in the package that its tests read through `fs` (`readFileSync`, `readFile`, `existsSync`, `statSync`, - and `fs/promises` `readFile` and `stat`), each by its content. A - directory that a test stats or tests for is an input as a directory, not - by its content. A file that a test reads outside the package or under - `node_modules`, through another API (`readdir`, `open`, a stream, or a - watcher), or from a worker thread or child process that the test starts, - is no input, so an edit to it alone does not rerun the tests that read it; - run the check cold (without `--daemon`) after such an edit. A file that - changes while the test that reads it runs is stored under its content - after the change. + and `fs/promises` `readFile` and `stat`), each by the content that the + run read. A run that an edit overlapped, because a file changed after the + run read it, is not reused. A directory that a test stats or tests for is + an input as a directory, not by its content. + + These reads are no input, so an edit to the file alone does not rerun the + tests that read it: a read outside the package or under `node_modules`; a + read from a worker thread or child process that the test starts; and a + read through any other `fs` API, which includes `readdir`, `opendir`, + `createReadStream`, `realpath`, `lstat`, `access`, `open`, `watch`, + `readlink`, the callback `stat`, `exists`, and the `fs/promises` + `access` and `open`. Run the check cold (without `--daemon`) after such + an edit. - `packages/eslint-plugin-oracles` (`@tanstack/eslint-plugin-oracles`): the `oracle/*` lint rules. The rules recognize the framework by its package diff --git a/packages/oracles/oracle-gaps/capture-setup.mjs b/packages/oracles/oracle-gaps/capture-setup.mjs index b0e45fc..1174bb6 100644 --- a/packages/oracles/oracle-gaps/capture-setup.mjs +++ b/packages/oracles/oracle-gaps/capture-setup.mjs @@ -243,11 +243,25 @@ const state = unit ? { ...run, ...unit } : run // it is. The daemon stores the run under them as it does the modules the // copy evaluated (the closing row's `loaded`), so an edit of only such a // file runs it again, as a cold run sees the edit (law 9). Reads under -// node_modules and of the runner's own files are left out. -globalThis.__oracleGapsReads = new Set() +// node_modules and of the runner's own files are left out. Each path keeps +// its hash from the copy's first read of it, taken as the runner hashes an +// input (check.mjs, fileHashes), so the run is stored under the text it +// read and not under an edit that lands later in the run (review of #55, +// F1). +globalThis.__oracleGapsReads = new Map() if (!globalThis.__oracleGapsReading) { globalThis.__oracleGapsReading = true const root = `${process.cwd()}/` + const { readFileSync: readNow, statSync: statNow } = fs + const hashNow = (path) => { + try { + return statNow(path).isFile() + ? createHash(`sha1`).update(readNow(path)).digest(`hex`) + : `not a file` + } catch { + return `missing` + } + } const record = (target) => { let path try { @@ -257,26 +271,29 @@ if (!globalThis.__oracleGapsReading) { } catch { return } + const reads = globalThis.__oracleGapsReads if ( path?.startsWith(root) && !path.includes(`/node_modules/`) && - !path.includes(`/.oracle-gaps`) + !path.includes(`/.oracle-gaps`) && + !reads.has(path) ) - globalThis.__oracleGapsReads.add(path) + reads.set(path, hashNow(path)) } // A wrapper keeps the function's own properties (its name and length, // which code that dispatches on arity reads), so a test that passes alone // passes under the capture (law 3; review of #55). - const recorded = (owner, name) => { - const original = owner[name] - owner[name] = keepProperties(original, function (target, ...rest) { - record(target) - return original.call(this, target, ...rest) - }) - } - for (const name of [`readFileSync`, `readFile`, `existsSync`, `statSync`]) - recorded(fs, name) - for (const name of [`readFile`, `stat`]) recorded(fs.promises, name) + for (const [owner, names] of [ + [fs, [`readFileSync`, `readFile`, `existsSync`, `statSync`]], + [fs.promises, [`readFile`, `stat`]], + ]) + for (const name of names) { + const original = owner[name] + owner[name] = keepProperties(original, function (target, ...rest) { + record(target) + return original.call(this, target, ...rest) + }) + } // An ES module's `import { readFileSync } from 'node:fs'` (and from // 'node:fs/promises') reads these from now on. syncBuiltinESMExports() @@ -1069,10 +1086,12 @@ afterAll(() => { .filter(([, node]) => node.evaluated) .map(([id]) => id.split(`?`)[0]) .filter((path) => path.startsWith(`/`)), - ...globalThis.__oracleGapsReads, + ...globalThis.__oracleGapsReads.keys(), ] + // The hash of each file the copy read, from its first read. + const readAt = Object.fromEntries(globalThis.__oracleGapsReads) appendFileSync( state.capture, - `${JSON.stringify({ file: thisFile, closed: true, loaded, ...(skippedCopy ? { skipped: true } : {}) })}\n`, + `${JSON.stringify({ file: thisFile, closed: true, loaded, readAt, ...(skippedCopy ? { skipped: true } : {}) })}\n`, ) }) diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index d242ce4..81e30c8 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -194,7 +194,13 @@ function fileHashes() { const stamp = `${stat.size}:${stat.mtimeMs}` const known = memo.get(path) if (known?.stamp === stamp) return known.hash - const hash = sha1(readFileSync(path)) + let hash + try { + hash = sha1(readFileSync(path)) + } catch { + // Removed between the stat and the read. + return `missing` + } memo.set(path, { stamp, hash }) return hash } @@ -455,7 +461,8 @@ export async function openSession(pkg) { const wrapper = wrapperFor(index, config) writeFileSync( wrapper, - `import { realpathSync } from 'node:fs' + `import { createHash } from 'node:crypto' +import { realpathSync } from 'node:fs' import { mergeConfig } from 'vitest/config' ${ config @@ -488,6 +495,13 @@ export default async (env) => { if (copies?.has(path)) return path return globalThis.__oracleGapsBundles?.get(source) }, + // The hash of each module's text as this context transforms it, so a + // run is stored under the text its context served, not under an edit + // that lands later in the run (review of #55, F1). + transform(code, id) { + const loads = (globalThis.__oracleGapsLoads[${index}] ??= new Map()) + loads.set(id.split('?')[0], createHash('sha1').update(code).digest('hex')) + }, load(id) { // Vite's own asset and worker queries ('?raw', '?url', '?worker&url', ...) // are loaded as written, so their module is not instrumented (a stated @@ -516,6 +530,8 @@ export default async (env) => { ) } globalThis.__oracleGapsServe = [] + // Per context: the hash of each module's text when it last transformed it. + globalThis.__oracleGapsLoads = [] // The text of each copy of a test file that a batch runs, by its path. globalThis.__oracleGapsCopies = new Map() serveCopies() @@ -766,6 +782,8 @@ export default async (env) => { const hash = servedHash(path, index) if (served.get(path) === hash) continue context.invalidateFile(path) + // Its next transform records the text it then serves. + globalThis.__oracleGapsLoads[index]?.delete(path) served.set(path, hash) } } @@ -918,7 +936,7 @@ export default async (env) => { }) } const rows = [] - let loaded = [] + let closing = {} if (existsSync(capture)) { for (const line of readFileSync(capture, `utf8`) .split(`\n`) @@ -926,7 +944,7 @@ export default async (env) => { const row = JSON.parse(line) // A closing row marks the end of a file for a batch's watchdog. if (!row.closed && !row.idle) rows.push([row.key, row]) - if (row.closed && row.file === file) loaded = row.loaded ?? [] + if (row.closed && row.file === file) closing = row } rmSync(capture) } @@ -971,7 +989,8 @@ export default async (env) => { String(error?.message ?? error), ), rows, - inputs: importsOf(file, modules, loaded), + inputs: importsOf(file, modules, closing.loaded ?? []), + readAt: closing.readAt, } } @@ -1125,11 +1144,12 @@ export default async (env) => { // Each copy's rows, and the copies that skipped their tests because an // earlier copy in the batch killed their mutant. const skipped = new Set() - // The files of the modules each copy evaluated (its closing row). - const loaded = new Map() + // Each copy's closing row: the files of the modules it evaluated, and the + // files it read with their hashes. + const closings = new Map() const rowsOf = () => { const rows = new Map(copies.map((copy) => [copy, []])) - loaded.clear() + closings.clear() if (existsSync(capture)) for (const line of readFileSync(capture, `utf8`) .split(`\n`) @@ -1139,7 +1159,7 @@ export default async (env) => { (c) => row.key?.startsWith(`${c} > `) || row.file === c, ) if (copy && row.skipped) skipped.add(copy) - if (copy && row.closed) loaded.set(copy, row.loaded ?? []) + if (copy && row.closed) closings.set(copy, row) if (copy && row.spin) spun.add(copy) if (copy && !row.closed && !row.idle && !row.spin && !row.started) rows.get(copy).push(row) @@ -1200,7 +1220,12 @@ export default async (env) => { })), unhandled: [], rows: own.map((row) => [row.key, row]), - inputs: importsOf(item.file, [], loaded.get(copies[i]) ?? []), + inputs: importsOf( + item.file, + [], + closings.get(copies[i])?.loaded ?? [], + ), + readAt: closings.get(copies[i])?.readAt, } }), timedOutAt: current, @@ -1253,7 +1278,12 @@ export default async (env) => { rows: rows .get(copy) .map((row) => [map(row.key), { ...row, key: map(row.key) }]), - inputs: importsOf(item.file, modules, loaded.get(copy) ?? []), + inputs: importsOf( + item.file, + modules, + closings.get(copy)?.loaded ?? [], + ), + readAt: closings.get(copy)?.readAt, } }), } @@ -1428,20 +1458,26 @@ export default async (env) => { } // Records what a settled run read in its context's ledger, so the next // check invalidates any of it that changes, and returns the inputs to - // store the run under: a path known at the start with its hash from - // then, any other with its hash now. A path that changed while the run - // ran is invalidated, and the run, which may have read either text, is + // store the run under: each path with the hash of the text the run read, + // a file's from the copy's first read of it (`readAt`), a module's from + // its context's transform of it, or else its hash at the start, or now. + // A path that changed while the run ran is invalidated, and the run is // not stored (undefined); nor is a run with no known input. - function settleIn(context, started, paths) { + function settleIn(context, started, paths, readAt = {}) { const index = indexOf.get(context) const served = servedBy(index) + const loads = globalThis.__oracleGapsLoads[index] let stable = paths.length > 0 const inputs = paths.map((path) => { const now = servedHash(path, index) - const hash = started.get(path) ?? now + const hash = + readAt[path] ?? loads?.get(path) ?? started.get(path) ?? now const read = served.get(path) ?? hash // The context may hold other text than the path's text now. - if (read !== now || hash !== now) context.invalidateFile(path) + if (read !== now || hash !== now) { + context.invalidateFile(path) + loads?.delete(path) + } if (hash !== now) stable = false served.set(path, now) return [path, hash] @@ -1467,12 +1503,17 @@ export default async (env) => { options.seed ?? null, ) const inputs = items.map(({ file }, k) => - settleIn(context, started, [ - ...new Set([ - ...(fileInputs.get(file) ?? []), - ...(batch.units[k]?.inputs ?? []), - ]), - ]), + settleIn( + context, + started, + [ + ...new Set([ + ...(fileInputs.get(file) ?? []), + ...(batch.units[k]?.inputs ?? []), + ]), + ], + batch.units[k]?.readAt, + ), ) return { ...batch, inputs } } @@ -1505,7 +1546,7 @@ export default async (env) => { if (!result.timedOut && spec.startsWith(`unmutated:`)) fileInputs.set(file, result.inputs) if (paths) { - const inputs = settleIn(context, started, paths) + const inputs = settleIn(context, started, paths, result.readAt) if (keep) remember(id, result, inputs) } return result diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 2b25599..8d68a48 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -67,7 +67,7 @@ export const shards = { `fixed seed 20261105: an edit that changed the cold result after a hang replaced the only context`, `random seed: an edit that changed the cold result after a hang replaced the only context`, `determinism: constructed history: an edit that changed the cold result after a hang replaced the only context`, - // The mid-run edit's history (law 9), not yet measured. + // The mid-run edit's histories (law 9): 3.5 to 5 s each here, fixed seed. `fixed seed 20261105: an edit during a check through the daemon`, `random seed: an edit during a check through the daemon`, `determinism: constructed history: an edit during a check through the daemon`, @@ -77,7 +77,7 @@ export const shards = { `fixed seed 20261105: a test edits the data it read, during a check through the daemon`, `random seed: a test edits the data it read, during a check through the daemon`, `determinism: constructed history: a test edits the data it read, during a check through the daemon`, - // The fs-read data histories (law 9), not yet measured. + // The fs-read data histories (law 9): 2.7 to 5.4 s each here, fixed seed. `fixed seed 20261105: an edit to data a test reads through fs (sync)`, `random seed: an edit to data a test reads through fs (sync)`, `determinism: constructed history: an edit to data a test reads through fs (sync)`, From 4b765a4c5782135f8ac2d462bd881237aa69ff5c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 09:42:13 -0700 Subject: [PATCH 11/17] test(oracle-gaps): unchanged reuse under a pre plugin, and the mid-run check compared Third review of #55: - F-A: a generated config dimension, `prePlugin`, adds a config plugin of the package's own with `enforce: 'pre'` that appends a comment to each module it transforms. A new law, `unchanged reuse`: a check through the daemon of a state that nothing changed computes nothing, where the cold run cut nothing off and no test reads the clock (the README's reuse contract). The constructed history "an unchanged check through the daemon under a config's pre plugin" reaches it. - F-C: the check that a mid-run edit overlaps is compared, not thrown away. It must exit as a cold run of the state before or after the edit does, report the same mutants, stream as any check does, and report each mutant as one of the two cold runs does. When file b's own test lands the edit, b's mutants may instead come out as a run of the edited text can: killed, survived, blocked, or crashed by the editing test's own assertion (coordinator decision, 2026-11-13). One extra cold run of the state before the edit. - F-B: the marker assertion is a witness guard, a sensitivity precondition for law 9's helper history. - Limits: the comparison's exclusion and the stronger follow-up, a timed-out run's own inputs (F-D), and the exact list of fs APIs that are no input (F-E). RED on 82b5c17: the unchanged check through the daemon under the pre plugin "computed 6 runs, owed none". Before the narrowing, both self-edit histories failed only on file b's own mutant, crashed by the editing test ("the helper this run read is not strict", "the data this run read is closed"); the a-editor history passed. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 41 +++- .../gap-report-oracle.property.test.mjs | 211 ++++++++++++++++-- packages/oracles/scripts/e2e-shards.mjs | 3 + 3 files changed, 232 insertions(+), 23 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 05cb6c5..1b3f6be 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -799,6 +799,12 @@ const drawn = fc.record({ // `midrun` step. midrunBy: fc.constantFrom(`a`, `b`), midrunTarget: fc.constantFrom(`helper`, `data`), + // The package's config also has a plugin of its own, `enforce: 'pre'`, + // whose transform appends a comment to every module in the package, as a + // code-rewriting plugin does: the runner must store a run under what it + // served, not under the text after another pre plugin (third review of + // #55, F-A). + prePlugin: fc.constantFrom(false, false, false, true), }) // The edit sequence over the witness module, the one list of steps that a @@ -2868,6 +2874,7 @@ function resolveHistory(history) { // assertion form and whether its list is a literal, or none. matcherOwns: history.matcherOwns ?? `none`, config: history.config ?? `object`, + prePlugin: history.prePlugin === true, ...configOf(history), configEnv: history.configEnv === true, native: history.native ?? `none`, @@ -4887,16 +4894,12 @@ function oracleTestFile(plan, file) { // the baseline (review of #55, F1). if (plan.midrun && file === plan.midrunBy) out.push( - `test('edits ${plan.midrunTarget === `data` ? `the data` : `the helper`} during the check', async () => {`, + `test('${MIDRUN_EDITORS[plan.midrunTarget].test}', async () => {`, ...(file === `a` ? [` void strictAtLoad`] - : plan.midrunTarget === `data` - ? [ - ` if (!(await readData()).open) throw new Error('the data this run read is closed')`, - ] - : [ - ` if (!strict) throw new Error('the helper this run read is not strict')`, - ]), + : [ + ` if (!(${plan.midrunTarget === `data` ? `(await readData()).open` : `strict`})) throw new Error('${MIDRUN_EDITORS[plan.midrunTarget].message}')`, + ]), // The marker sits in the run's own session directory, beside the // runner's harness (the worker's first setup file), which is no input // of a run by design (check.mjs, ownHarness). So the marker's own edit @@ -5052,6 +5055,21 @@ function edited(plan) { const AUTHORED = `mutants/authored-mutant.json` +/** + * The test that lands a mid-run edit, by what it edits: its name, and the + * error it throws in file b when it reads the edited text. + */ +const MIDRUN_EDITORS = { + helper: { + test: `edits the helper during the check`, + message: `the helper this run read is not strict`, + }, + data: { + test: `edits the data during the check`, + message: `the data this run read is closed`, + }, +} + /** What a mid-run edit changes: the data file only when file b reads one. */ const midrunTargetOf = (history) => history.midrunTarget === `data` && (history.fsRead ?? `none`) !== `none` @@ -5092,6 +5110,10 @@ const altConfigPath = (plan) => * fault, the top-level await, or the `require` calls the plan holds. `test` * is the config's `test` object. */ +// The config's own pre plugin (`prePlugin`): it rewrites each module of the +// package that it transforms, keeping what the module does. +const PRE_PLUGIN = `{ name: 'gap-pre-rewrite', enforce: 'pre', transform(code, id) { const path = id.split('?')[0]; return /\\.(ts|js|mjs)$/.test(path) && !path.includes('/node_modules/') ? code + '\\n// rewritten by a pre plugin\\n' : undefined } }` + function configText(plan, test) { const commonjs = plan.configSyntax === `cjs` const config = `defineConfig(${ @@ -5102,7 +5124,7 @@ function configText(plan, test) { plan.fault === `projects` ? `{ projects: [{ test: { name: 'unit', ...${test} } }] }` : test - } }${plan.config === `object` || plan.config === `promise` ? `` : `)`})` + }${plan.prePlugin ? `, plugins: [${PRE_PLUGIN}]` : ``} }${plan.config === `object` || plan.config === `promise` ? `` : `)`})` // A promise of the config, a null default, or a config cut off mid-object. const body = plan.fault === `configNull` @@ -5751,6 +5773,7 @@ function readSummary(summary) { } export { + MIDRUN_EDITORS, captureWitness, outsideRepository, endingOf, diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 21b74b5..cd1e3dc 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -538,6 +538,7 @@ import { MUTANT_LIMIT_MS, HEARTBEAT_MS, FULL_LIMIT_MS, + MIDRUN_EDITORS, } from './gap-report-model-oracle.mjs' const here = dirname(fileURLToPath(import.meta.url)) @@ -1274,6 +1275,12 @@ async function incremental(dir, plan, cli, copy, observe, states) { let cutBefore = false // Whether the daemon stored a run under the mid-run marker. let markerStored = false + // How many runs the daemon computed in a check of a state that nothing + // changed, where it owes none (undefined elsewhere). + let unchangedComputed + // The check through the daemon during which a mid-run edit landed, and a + // cold run of the state before the edit (the `midrun` step). + let midrunChecks // A second install of the CLI, at another path, whose summary reads // otherwise, so its code differs from the first install's (`otherInstall`). let other @@ -1330,10 +1337,45 @@ async function incremental(dir, plan, cli, copy, observe, states) { : await daemonChecks(step, files, how) if (!cold.has(key)) cold.set(key, runCli(dir, files, how)) const coldRun = cold.get(key) + // A check through the daemon of a state that nothing changed reuses + // every run that the store keeps (the README's reuse contract: a run is + // reused while its inputs are unchanged), so where the cold run cut + // nothing off and no test reads the clock, it computes nothing (third + // review of #55, F-A). The incremental check judges the count. + unchangedComputed = + step === `recheck` && + (coldRun.evidence?.cut ?? 0) === 0 && + plan.tests.every((t) => !t.flaky) + ? (daemon.evidence?.computed ?? 0) + : undefined for (const [i, run] of [daemon, ...others].entries()) observe(`incremental`, { step: i === 0 ? step : `${step} (a second check at once)`, markerStored, + unchangedComputed: i === 0 ? unchangedComputed : undefined, + midrun: + i === 0 && step === `midrun` && midrunChecks + ? { + during: { + exit: midrunChecks.during.exit, + report: midrunChecks.during.report, + stream: midrunChecks.during.stream, + stdoutStream: midrunChecks.during.stdoutStream, + }, + before: { + exit: midrunChecks.before.exit, + report: midrunChecks.before.report, + }, + // File b's own test, when it lands the edit (see below). + editing: + plan.midrunBy === `b` + ? { + file: `tests/b.test.ts`, + ...MIDRUN_EDITORS[plan.midrunTarget], + } + : undefined, + } + : undefined, cold: { exit: coldRun.exit, refusal: coldRun.refusal, @@ -1402,6 +1444,13 @@ async function incremental(dir, plan, cli, copy, observe, states) { states.add(`incremental check reused some evidence and recomputed some`) if (step === `restart` && reused > 0) states.add(`evidence reused after a daemon restart`) + if (unchangedComputed === 0 && reused > 0) { + states.add(`an unchanged check through the daemon computed nothing`) + if (plan.prePlugin) + states.add( + `an unchanged check through the daemon computed nothing under a config's pre plugin`, + ) + } // The witness module's hangs are cut off in every cold run. A daemon // check after an edit that cut off fewer runs than the cold run of the // same state reused a timed-out run, so law 9 judged a timeout that came @@ -1481,10 +1530,11 @@ async function incremental(dir, plan, cli, copy, observe, states) { // it and before file b's runs read it; file b's own test edits its // helper, or the data file that it reads through fs, after b's judged // tests read them, so the run that lands the edit judged the old text. - // That check may report either state for a run the edit overlapped, so - // it is not compared; the runner must store no run under a text it did - // not read (settleIn), so the next check, with nothing changed, equals - // a cold run of the state after the edit. + // That check may report, for each mutant, what a cold run of either + // state reports (third review of #55, F-C: one extra cold run, of the + // state before the edit); the runner must store no run under a text it + // did not read (settleIn), so the next check, with nothing changed, + // equals a cold run of the state after the edit. if (step === `midrun`) { const target = plan.midrunTarget === `data` ? `tests/data.json` : `tests/strict.ts` @@ -1498,11 +1548,11 @@ async function incremental(dir, plan, cli, copy, observe, states) { daemonPid && join(dir, `.oracle-gaps-${daemonPid}`, `midrun.txt`) if (marker && existsSync(dirname(marker))) writeFileSync(marker, `pending:${value}\n`) - runCli( - dir, - { ...files, [target]: after.replace(String(value), String(!value)) }, - { ...how, daemon: true }, - ) + const before = { + ...files, + [target]: after.replace(String(value), String(!value)), + } + const during = runCli(dir, before, { ...how, daemon: true }) const landed = marker && existsSync(marker) && @@ -1523,6 +1573,7 @@ async function incremental(dir, plan, cli, copy, observe, states) { markerStored ||= storedInputs(dir).some((path) => path.endsWith(`/midrun.txt`), ) + midrunChecks = { during, before: runCli(dir, before, how) } return [next] } if (step !== `dieMidCheck` && step !== `resetBeforeRead`) @@ -2159,14 +2210,82 @@ function incrementalCheck(observations) { (o) => o.checkpoint === `incremental`, )) { const { step, cold, daemon } = value - // The mid-run marker sits in the daemon's own session directory, which - // is no input of a run by design (ownHarness); a run stored under it - // would let the marker hide whether the run read an edited input. + // The check that a mid-run edit overlapped reports, for each mutant, + // what a cold run of the state before the edit or of the state after + // it reports, and exits as one of them does (F-C). + if (value.midrun) { + const { during, before } = value.midrun + if (during.exit !== before.exit && during.exit !== cold.exit) + return violation( + `incremental`, + `the check during the mid-run edit exited ${during.exit}, and the cold runs before and after the edit exited ${before.exit} and ${cold.exit}`, + `incremental equals cold`, + ) + const resultOf = (run, key) => + JSON.stringify(run.report?.results.find((r) => r.key === key)) + const keys = (run) => + JSON.stringify((run.report?.results ?? []).map((r) => r.key)) + if (keys(during) !== keys(before) && keys(during) !== keys(cold)) + return violation( + `incremental`, + `the check during the mid-run edit reported other mutants than the cold runs before and after the edit`, + `incremental equals cold`, + ) + // When file b's own test lands the edit, b's baseline reads the old + // text and b's mutant runs read the new one, so b's mutants can come + // out as neither cold run reports them (coordinator decision, + // 2026-11-13). Each must still come out as a run of the edited text + // can: killed, survived, or blocked, or crashed by the editing test's + // own assertion on the edited text, never by another test or with + // another gap outcome. Every other mutant, and every mutant when file + // a lands the edit, keeps the full comparison. + const { editing } = value.midrun + const ownedByEditor = (result) => + editing !== undefined && + (result.oracleFiles ?? []).some((file) => file.endsWith(editing.file)) + const ranEdited = (result) => + [`killed`, `survived`, `blocked`].includes(result.outcome) || + (result.outcome === `crashed` && + `${result.detail}`.includes(editing.message) && + (result.by ?? []).length > 0 && + result.by.every((test) => `${test}`.includes(editing.test))) + for (const result of during.report?.results ?? []) + if ( + JSON.stringify(result) !== resultOf(before, result.key) && + JSON.stringify(result) !== resultOf(cold, result.key) && + !(ownedByEditor(result) && ranEdited(result)) + ) + return violation( + `incremental`, + `the check during the mid-run edit reported ${result.key} as neither cold run did, before or after the edit`, + `incremental equals cold`, + { + actual: result, + expected: [ + resultOf(before, result.key), + resultOf(cold, result.key), + ], + }, + ) + } + if (value.unchangedComputed > 0) + return violation( + `incremental`, + `the check through the daemon of a state that nothing changed computed ${value.unchangedComputed} runs, owed none`, + `unchanged reuse`, + ) + // A witness guard, not a law-9 requirement: the mid-run marker sits in + // the daemon's own session directory, which is no input of a run by + // design (ownHarness). The helper history is sensitive only while that + // holds: the run that lands the edit also rewrites the marker, so a run + // stored under the marker would not be stored at all, and the marker + // would hide whether the runner stores the run under the helper's text + // that it read. if (value.markerStored) return violation( `incremental`, `after ${step}, the daemon stored a run under the mid-run marker, a file in its own session directory`, - `incremental equals cold`, + `witness guard`, ) if ( daemon.exit !== cold.exit || @@ -2270,6 +2389,15 @@ function check(plan, observations, expected) { `greppable stream`, ends.extra, ) + // The check that a mid-run edit overlapped streams as any check does. + const duringEnds = value.midrun && endsBreach(value.midrun.during) + if (duringEnds) + return violation( + `incremental`, + `through the daemon during the mid-run edit: ${duringEnds.difference}`, + `greppable stream`, + duringEnds.extra, + ) // A refusal through the daemon prints the cold run's notes (review of // #46, F4): the notes of the session that failed reach the CLI. if ( @@ -3166,6 +3294,8 @@ const ownedIncrementalStates = [ `edit midrun changed the cold result`, `an edit landed during a check through the daemon`, `a run edited the helper it had read, during a check through the daemon`, + `an unchanged check through the daemon computed nothing`, + `an unchanged check through the daemon computed nothing under a config's pre plugin`, `a run edited the data it had read, during a check through the daemon`, `edit fsData changed the cold result`, `an edit to data read through sync changed the cold result`, @@ -3437,7 +3567,7 @@ oracleProperty({ law: { statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, - limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check is not compared, because a run that the edit overlapped may report either text (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn), and the next check, with nothing changed, must equal a cold run of the state after the edit; an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, createReadStream, realpath, lstat, access, open, watch, readlink, the callback stat, exists, and the fs/promises access and open, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated.`, + limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin), and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so the check mixes states for b's mutants, and those need only come out as a run of the edited text can (killed, survived, blocked, or crashed by the editing test's own assertion on the edited text, never another crash or gap outcome; coordinator decision, 2026-11-13); the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read, and makes the editing test take a law path instead of crashing; and the next check, with nothing changed, must equal a cold run of the state after the edit (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute nothing, judged only where the cold run cut nothing off and no test reads the clock (F-A); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access and open, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs @@ -4107,6 +4237,59 @@ oracleProperty({ steps: [`midrun`], }), }, + { + // A check through the daemon of a state that nothing changed, under + // a config whose own pre plugin rewrites every module: the runner + // must store each run under the text it served, which that plugin + // has not rewritten yet, so the second check reuses every run (third + // review of #55, F-A). + name: `an unchanged check through the daemon under a config's pre plugin`, + enters: [ + `an unchanged check through the daemon computed nothing`, + `an unchanged check through the daemon computed nothing under a config's pre plugin`, + ], + build: (generated) => ({ + ...generated, + functions: [ + [{ kind: `push`, quote: `'` }], + [{ kind: `push`, quote: `'` }], + ], + containers: [], + overloads: [], + owners: [`a`, `b`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [ + { ...judging(`a`, `fixed seed`, 1), fn: 0 }, + { ...judging(`b`, `fixed seed`, 1), fn: 1 }, + ], + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + contexts: 1, + steps: [`recheck`], + prePlugin: true, + configPlace: `discovered`, + configFile: `vite.config.mjs`, + config: `object`, + }), + }, { // File b's own test edits its helper after b's judged test read it, // in the run that judges with the old text: its verdict follows the diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 8d68a48..d05ed27 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -77,6 +77,9 @@ export const shards = { `fixed seed 20261105: a test edits the data it read, during a check through the daemon`, `random seed: a test edits the data it read, during a check through the daemon`, `determinism: constructed history: a test edits the data it read, during a check through the daemon`, + `fixed seed 20261105: an unchanged check through the daemon under a config's pre plugin`, + `random seed: an unchanged check through the daemon under a config's pre plugin`, + `determinism: constructed history: an unchanged check through the daemon under a config's pre plugin`, // The fs-read data histories (law 9): 2.7 to 5.4 s each here, fixed seed. `fixed seed 20261105: an edit to data a test reads through fs (sync)`, `random seed: an edit to data a test reads through fs (sync)`, From 1cfab2b7e27a9b2e07cb9871aa9bce40c60c7b80 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 09:58:25 -0700 Subject: [PATCH 12/17] fix(oracle-gaps): hash a transformed module as every input is hashed; a spun run keeps its reads Third review of #55: - F-A: the wrapper's transform hook hashed the code it received, which a config's own pre plugin had already rewritten, while servedHash hashes the served text, so the two never matched and an unchanged check through the daemon recomputed runs (6 of them in the witness). The hook now asks the runner to record servedHash for the module, one definition for both sides. - F-D: a run that spun ends, so it now returns the inputs its closing row names, and a timed-out unit is stored under the union of its file's known inputs and its own. A run cut off by its clock writes no closing row (a limit, by inspection). - F-E: the README names each fs API that is no input yet, and that a later release records all of them except watch and watchFile. The changeset also names the fix for a stale verdict after a module or file changed during the run that read it, which main has too. Evidence: RED on 82b5c17 ("computed 6 runs, owed none"); GREEN for the targeted units (10 of 10) and the fast suites (33,891); the mutants "hash the transformed code", "settleIn throws on a changed input", and "the runner records the marker" each fail the oracle. Co-authored-by: Isaac --- .changeset/gap-cli-daemon-fs-inputs.md | 2 ++ README.md | 13 +++++--- packages/oracles/oracle-gaps/check.mjs | 45 +++++++++++++++++++------- 3 files changed, 43 insertions(+), 17 deletions(-) diff --git a/.changeset/gap-cli-daemon-fs-inputs.md b/.changeset/gap-cli-daemon-fs-inputs.md index 53380eb..c7ab54e 100644 --- a/.changeset/gap-cli-daemon-fs-inputs.md +++ b/.changeset/gap-cli-daemon-fs-inputs.md @@ -3,3 +3,5 @@ --- A check through `--daemon` now reruns a test after an edit to a file in the package that the test reads through `fs` (`readFileSync`, `readFile`, `existsSync`, `statSync`, or `fs/promises` `readFile` or `stat`), such as a JSON fixture or a snapshot. Before, the daemon reused the run that read the old file, while a cold run saw the edit. A directory that a test stats is an input as a directory. Reads outside the package, under `node_modules`, through other APIs, or from a test's own worker threads or child processes are no inputs. + +The daemon also stores each run under the text that the run read. Before, it stored a run under each input's text at the end of the run, so a module or file that changed during the run that read it (a test that rewrites a fixture it read, or a helper edited mid-check) kept the old verdict for the new text, and the next check through the daemon reported that stale verdict. A run whose input changed after it read it is now run again. diff --git a/README.md b/README.md index 964a306..23a15f8 100644 --- a/README.md +++ b/README.md @@ -165,11 +165,14 @@ exited with ` and exits 3; after an out-of-memory crash These reads are no input, so an edit to the file alone does not rerun the tests that read it: a read outside the package or under `node_modules`; a read from a worker thread or child process that the test starts; and a - read through any other `fs` API, which includes `readdir`, `opendir`, - `createReadStream`, `realpath`, `lstat`, `access`, `open`, `watch`, - `readlink`, the callback `stat`, `exists`, and the `fs/promises` - `access` and `open`. Run the check cold (without `--daemon`) after such - an edit. + read through any other `fs` API. These include `readdir`, `opendir`, + `createReadStream`, `realpath`, `lstat`, `access`, `open`, a read through + a file descriptor (`read`, `readSync`, `readv`), `copyFile` and + `copyFileSync`, `cp` and `cpSync`, `openAsBlob`, `statfs`, `readlink`, the + callback `stat`, `exists`, `watch`, `watchFile`, and the `fs/promises` + `access` and `open`. Run the check cold (without `--daemon`) after such an + edit. A later release records all of them except `watch` and `watchFile` + (maintainer decision, 2026-11-13). - `packages/eslint-plugin-oracles` (`@tanstack/eslint-plugin-oracles`): the `oracle/*` lint rules. The rules recognize the framework by its package diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index 81e30c8..56f6a65 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -461,8 +461,7 @@ export async function openSession(pkg) { const wrapper = wrapperFor(index, config) writeFileSync( wrapper, - `import { createHash } from 'node:crypto' -import { realpathSync } from 'node:fs' + `import { realpathSync } from 'node:fs' import { mergeConfig } from 'vitest/config' ${ config @@ -495,12 +494,12 @@ export default async (env) => { if (copies?.has(path)) return path return globalThis.__oracleGapsBundles?.get(source) }, - // The hash of each module's text as this context transforms it, so a - // run is stored under the text its context served, not under an edit - // that lands later in the run (review of #55, F1). + // Each module this context transforms, so a run is stored under the + // text its context served, not under an edit that lands later in the + // run (review of #55, F1). The runner hashes it as it hashes every + // input (servedHash), not as another pre plugin has rewritten it (F-A). transform(code, id) { - const loads = (globalThis.__oracleGapsLoads[${index}] ??= new Map()) - loads.set(id.split('?')[0], createHash('sha1').update(code).digest('hex')) + globalThis.__oracleGapsTransformed?.(${index}, id.split('?')[0]) }, load(id) { // Vite's own asset and worker queries ('?raw', '?url', '?worker&url', ...) @@ -549,6 +548,13 @@ export default async (env) => { globalThis.__oracleGapsSchemata.get(path) return text === undefined ? hashOf(path) : sha1(text) } + // A context transformed a module (its wrapper's transform hook): the hash + // of what it then serves. + globalThis.__oracleGapsTransformed = (index, path) => + (globalThis.__oracleGapsLoads[index] ??= new Map()).set( + path, + servedHash(path, index), + ) // Per context: the hash of what it last served for each path. const lastServed = new Map() const servedBy = (index) => { @@ -912,10 +918,6 @@ export default async (env) => { rmSync(capture, { force: true }) return { file, timedOut: true, seconds, ...(spun ? { spin: true } : {}) } } - if (spun) { - rmSync(capture, { force: true }) - return { file, timedOut: true, spin: true, seconds } - } // The result also lists test modules from earlier runs, with their old // results. Only this run's file counts. const modules = result.testModules.filter((m) => m.moduleId === file) @@ -948,6 +950,16 @@ export default async (env) => { } rmSync(capture) } + // A run that spun ended, so its closing row names what it read (F-D). + if (spun) + return { + file, + timedOut: true, + spin: true, + seconds, + inputs: importsOf(file, modules, closing.loaded ?? []), + readAt: closing.readAt, + } const tests = modules.flatMap((module) => [...module.children.allTests()].map((testCase) => { const testResult = testCase.result() @@ -1542,7 +1554,16 @@ export default async (env) => { }) evidence.computed += 1 if (result.timedOut) evidence.cut += 1 - const paths = result.timedOut ? fileInputs.get(file) : result.inputs + // A run cut off keeps its file's known inputs and any it named itself + // (third review of #55, F-D). + const paths = result.timedOut + ? [ + ...new Set([ + ...(fileInputs.get(file) ?? []), + ...(result.inputs ?? []), + ]), + ] + : result.inputs if (!result.timedOut && spec.startsWith(`unmutated:`)) fileInputs.set(file, result.inputs) if (paths) { From 5dd0ab46b835d799e8ed3b4d2bb84b139af53796 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 10:50:31 -0700 Subject: [PATCH 13/17] test(oracle-gaps): an unchanged check reuses runs whose tests read a source through fs Fourth review of #55: - F-A2: `unchanged reuse` no longer skips a history whose cold run cut a run off. It counts the computed runs that were not cut off. A new constructed history, "an unchanged check through the daemon while a test reads a source through fs", has a test of file a read the source under test through readFileSync and as a `?raw` import, under the config's pre plugin. This also gives the pre-plugin state a second history. - F-C1: the editing file's exemption in the check that a mid-run edit overlaps holds only when the edit landed. The observation carries `landed`, and a calibration ("the check a mid-run edit did not reach flips a mutant of the editing file") binds the gate. - F-E and F-A1: the oracle's header prose says where law 9's edits come; the limits name glob, globSync, promises.glob, fstat, fstatSync, and fd reads, and the plugin and environment inputs that the maintainer decided on 2026-11-13 the daemon will track. The configText JSDoc is back above configText. RED on 1cfab2b (locally, -t): the new history fails `unchanged reuse`, "computed 2 runs that were not cut off, owed none". Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 8 +- .../gap-report-oracle.property.test.mjs | 137 +++++++++++++++--- packages/oracles/scripts/e2e-shards.mjs | 3 + 3 files changed, 126 insertions(+), 22 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 1b3f6be..b1416d4 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -5105,15 +5105,15 @@ const initialState = (plan) => ({ const altConfigPath = (plan) => `config/gap-vitest-alt.${plan.configFile.split(`.`).at(-1)}` +// The config's own pre plugin (`prePlugin`): it rewrites each module of the +// package that it transforms, keeping what the module does. +const PRE_PLUGIN = `{ name: 'gap-pre-rewrite', enforce: 'pre', transform(code, id) { const path = id.split('?')[0]; return /\\.(ts|js|mjs)$/.test(path) && !path.includes('/node_modules/') ? code + '\\n// rewritten by a pre plugin\\n' : undefined } }` + /** * The text of the package's config, in its syntax (`configOf`), with the * fault, the top-level await, or the `require` calls the plan holds. `test` * is the config's `test` object. */ -// The config's own pre plugin (`prePlugin`): it rewrites each module of the -// package that it transforms, keeping what the module does. -const PRE_PLUGIN = `{ name: 'gap-pre-rewrite', enforce: 'pre', transform(code, id) { const path = id.split('?')[0]; return /\\.(ts|js|mjs)$/.test(path) && !path.includes('/node_modules/') ? code + '\\n// rewritten by a pre plugin\\n' : undefined } }` - function configText(plan, test) { const commonjs = plan.configSyntax === `cjs` const config = `defineConfig(${ diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index cd1e3dc..c3811aa 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -417,13 +417,13 @@ * over those classes are not drawn. Each such step costs a cold run of the * witness module, and one campaign in six drew a history that ran past the * test's limit (maintainer decision, 2026-11-11; this oracle owns the gap). - * Law 9's edits come only between checks: a file saved during a check is - * not generated, so the runner's handling of an input that changes while a - * run runs (it invalidates the input and does not store the run) is fixed - * by inspection; a witness needs a step that edits a file between a unit's - * start and its end. - * Law 9's edits do not include a file that a test reads through `fs`, an - * environment variable other than the seed, a new file that changes how an + * Law 9's edits come between checks, except the mid-run edit (`midrun`), + * which a test of the package lands while a check through the daemon runs: + * the check it overlaps must report, for each mutant, what a cold run of + * the state before or after the edit reports, and the runner must store no + * run under a text it did not read (see the limits). An edit can change a + * file that a test reads through `fs` (`fsData`). Law 9's edits do not + * include an environment variable other than the seed, a new file that changes how an * unchanged import resolves, a change to the Vite config or the installed * packages, or checks at once other than the two that start the daemon. * The harness keeps every global a copy defines configurable, by wrapping @@ -1339,14 +1339,12 @@ async function incremental(dir, plan, cli, copy, observe, states) { const coldRun = cold.get(key) // A check through the daemon of a state that nothing changed reuses // every run that the store keeps (the README's reuse contract: a run is - // reused while its inputs are unchanged), so where the cold run cut - // nothing off and no test reads the clock, it computes nothing (third - // review of #55, F-A). The incremental check judges the count. + // reused while its inputs are unchanged), so where no test reads the + // clock it computes no run that was not cut off (third and fourth + // reviews of #55, F-A and F-A2). The incremental check judges the count. unchangedComputed = - step === `recheck` && - (coldRun.evidence?.cut ?? 0) === 0 && - plan.tests.every((t) => !t.flaky) - ? (daemon.evidence?.computed ?? 0) + step === `recheck` && plan.tests.every((t) => !t.flaky) + ? (daemon.evidence?.computed ?? 0) - (daemon.evidence?.cut ?? 0) : undefined for (const [i, run] of [daemon, ...others].entries()) observe(`incremental`, { @@ -1366,6 +1364,8 @@ async function incremental(dir, plan, cli, copy, observe, states) { exit: midrunChecks.before.exit, report: midrunChecks.before.report, }, + // Whether the edit landed in that check. + landed: Boolean(midrunChecks.landed), // File b's own test, when it lands the edit (see below). editing: plan.midrunBy === `b` @@ -1450,6 +1450,13 @@ async function incremental(dir, plan, cli, copy, observe, states) { states.add( `an unchanged check through the daemon computed nothing under a config's pre plugin`, ) + // A test that reads a source under test through fs and as a `?raw` + // import reads the file's text, while the context serves the + // instrumented text: two inputs (fourth review of #55, F-A2). + if (plan.tests.some((t) => t.rawImport)) + states.add( + `an unchanged check through the daemon computed nothing while a test read a source through fs`, + ) } // The witness module's hangs are cut off in every cold run. A daemon // check after an edit that cut off fewer runs than the cold run of the @@ -1573,7 +1580,7 @@ async function incremental(dir, plan, cli, copy, observe, states) { markerStored ||= storedInputs(dir).some((path) => path.endsWith(`/midrun.txt`), ) - midrunChecks = { during, before: runCli(dir, before, how) } + midrunChecks = { during, before: runCli(dir, before, how), landed } return [next] } if (step !== `dieMidCheck` && step !== `resetBeforeRead`) @@ -2239,8 +2246,11 @@ function incrementalCheck(observations) { // own assertion on the edited text, never by another test or with // another gap outcome. Every other mutant, and every mutant when file // a lands the edit, keeps the full comparison. - const { editing } = value.midrun + // The exemption holds only for a check that the edit reached: with + // no edit, the check reads one state (fourth review, F-C1). + const { editing, landed } = value.midrun const ownedByEditor = (result) => + landed && editing !== undefined && (result.oracleFiles ?? []).some((file) => file.endsWith(editing.file)) const ranEdited = (result) => @@ -2271,7 +2281,7 @@ function incrementalCheck(observations) { if (value.unchangedComputed > 0) return violation( `incremental`, - `the check through the daemon of a state that nothing changed computed ${value.unchangedComputed} runs, owed none`, + `the check through the daemon of a state that nothing changed computed ${value.unchangedComputed} runs that were not cut off, owed none`, `unchanged reuse`, ) // A witness guard, not a law-9 requirement: the mid-run marker sits in @@ -3106,6 +3116,36 @@ calibrations.push({ }, }) +// The editing file's exemption in the check that a mid-run edit overlaps +// holds only when the edit landed (fourth review of #55, F-C1): with no +// edit, that check reads one state, so a mutant of file b that it reports +// as neither cold run does, killed for survived, is a violation. +calibrations.push({ + name: `the check a mid-run edit did not reach flips a mutant of the editing file`, + law: `incremental equals cold`, + history: { + constructed: `a test edits the helper it read, during a check through the daemon`, + }, + corrupt: (trace) => { + const index = trace.findIndex( + (o) => o.checkpoint === `incremental` && o.value.midrun?.editing, + ) + if (index < 0) return undefined + const value = structuredClone(trace[index].value) + const owned = value.midrun.during.report?.results.find((r) => + (r.oracleFiles ?? []).some((file) => file.endsWith(`tests/b.test.ts`)), + ) + if (!owned) return undefined + // The edit did not land, and the check reports file b's mutant + // survived where it ran killed, or killed where it survived. + value.midrun.landed = false + owned.outcome = owned.outcome === `survived` ? `killed` : `survived` + delete owned.by + delete owned.detail + return trace.map((o, i) => (i === index ? { ...o, value } : o)) + }, +}) + calibrations.push({ name: `a daemon is left after --stop-daemon`, law: `one daemon`, @@ -3296,6 +3336,7 @@ const ownedIncrementalStates = [ `a run edited the helper it had read, during a check through the daemon`, `an unchanged check through the daemon computed nothing`, `an unchanged check through the daemon computed nothing under a config's pre plugin`, + `an unchanged check through the daemon computed nothing while a test read a source through fs`, `a run edited the data it had read, during a check through the daemon`, `edit fsData changed the cold result`, `an edit to data read through sync changed the cold result`, @@ -3567,7 +3608,7 @@ oracleProperty({ law: { statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, - limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin), and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so the check mixes states for b's mutants, and those need only come out as a run of the edited text can (killed, survived, blocked, or crashed by the editing test's own assertion on the edited text, never another crash or gap outcome; coordinator decision, 2026-11-13); the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read, and makes the editing test take a law path instead of crashing; and the next check, with nothing changed, must equal a cold run of the state after the edit (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute nothing, judged only where the cold run cut nothing off and no test reads the clock (F-A); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access and open, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated.`, + limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin); a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so the check mixes states for b's mutants, and those need only come out as a run of the edited text can (killed, survived, blocked, or crashed by the editing test's own assertion on the edited text, never another crash or gap outcome; coordinator decision, 2026-11-13); the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read, and makes the editing test take a law path instead of crashing; and the next check, with nothing changed, must equal a cold run of the state after the edit (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D); no history makes a cut run read a file that its file's unmutated run does not read, so this is fixed by inspection; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan (which no generated function's mutant reached in the 36 fixed-seed units), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated.`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs @@ -4290,6 +4331,66 @@ oracleProperty({ config: `object`, }), }, + { + // The same unchanged check, where a test of file a also reads the + // source under test through fs and as a `?raw` import: the file's + // text is that test's input, and the instrumented text that the + // context serves is another (fourth review of #55, F-A2). + name: `an unchanged check through the daemon while a test reads a source through fs`, + enters: [ + `an unchanged check through the daemon computed nothing`, + `an unchanged check through the daemon computed nothing under a config's pre plugin`, + `an unchanged check through the daemon computed nothing while a test read a source through fs`, + ], + build: (generated) => ({ + ...generated, + functions: [ + [{ kind: `push`, quote: `'` }], + [{ kind: `push`, quote: `'` }], + ], + containers: [], + overloads: [], + owners: [`a`, `b`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [ + { ...judging(`a`, `fixed seed`, 1), fn: 0 }, + { ...judging(`b`, `fixed seed`, 1), fn: 1 }, + { + ...judging(`a`, `fixed seed`, 1, { + judges: false, + rawImport: true, + }), + fn: 0, + }, + ], + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + contexts: 1, + steps: [`recheck`], + prePlugin: true, + configPlace: `discovered`, + configFile: `vite.config.mjs`, + config: `object`, + }), + }, { // File b's own test edits its helper after b's judged test read it, // in the run that judges with the old text: its verdict follows the diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index d05ed27..9cb1e1d 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -80,6 +80,9 @@ export const shards = { `fixed seed 20261105: an unchanged check through the daemon under a config's pre plugin`, `random seed: an unchanged check through the daemon under a config's pre plugin`, `determinism: constructed history: an unchanged check through the daemon under a config's pre plugin`, + `fixed seed 20261105: an unchanged check through the daemon while a test reads a source through fs`, + `random seed: an unchanged check through the daemon while a test reads a source through fs`, + `determinism: constructed history: an unchanged check through the daemon while a test reads a source through fs`, // The fs-read data histories (law 9): 2.7 to 5.4 s each here, fixed seed. `fixed seed 20261105: an edit to data a test reads through fs (sync)`, `random seed: an edit to data a test reads through fs (sync)`, From b687d5b8fcc9c6341232bcbbcddfe229c9a8da02 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 11:05:03 -0700 Subject: [PATCH 14/17] fix(oracle-gaps): a test's fs read of a file and the module served from it are two inputs Fourth review of #55, F-A2: the closing row's `readAt` held a file's disk hash, and servedHash held the instrumented module's hash, under one path. A test that read a source under test through fs (or as a `?raw` import) made settleIn see them disagree, so the daemon never stored its runs and an unchanged check recomputed them (191 of 237 at seed 2). A test's read is now its own input, `fs:`, hashed by the file's text; the module keeps its path and the served text's hash. The README names the remaining fs APIs that are no input (glob, globSync, promises.glob, fstat, fstatSync, and fd reads), and documents the plugin and environment inputs as a known limit with the maintainer's decision of 2026-11-13 to track them. The changeset names the fix. Evidence: RED (locally, -t, on 1cfab2b) "computed 2 runs that were not cut off, owed none"; GREEN 12 of 12 targeted units; the mutant "a test's fs read shares the served module's key" fails `unchanged reuse`, and the check without the landed gate fails the F-C1 calibration. Co-authored-by: Isaac --- .changeset/gap-cli-daemon-fs-inputs.md | 2 + README.md | 22 +++++++-- .../oracles/oracle-gaps/capture-setup.mjs | 13 +++-- packages/oracles/oracle-gaps/check.mjs | 47 +++++++++---------- 4 files changed, 52 insertions(+), 32 deletions(-) diff --git a/.changeset/gap-cli-daemon-fs-inputs.md b/.changeset/gap-cli-daemon-fs-inputs.md index c7ab54e..f05d027 100644 --- a/.changeset/gap-cli-daemon-fs-inputs.md +++ b/.changeset/gap-cli-daemon-fs-inputs.md @@ -5,3 +5,5 @@ A check through `--daemon` now reruns a test after an edit to a file in the package that the test reads through `fs` (`readFileSync`, `readFile`, `existsSync`, `statSync`, or `fs/promises` `readFile` or `stat`), such as a JSON fixture or a snapshot. Before, the daemon reused the run that read the old file, while a cold run saw the edit. A directory that a test stats is an input as a directory. Reads outside the package, under `node_modules`, through other APIs, or from a test's own worker threads or child processes are no inputs. The daemon also stores each run under the text that the run read. Before, it stored a run under each input's text at the end of the run, so a module or file that changed during the run that read it (a test that rewrites a fixture it read, or a helper edited mid-check) kept the old verdict for the new text, and the next check through the daemon reported that stale verdict. A run whose input changed after it read it is now run again. + +A test that reads a source under test through `fs` (or as a `?raw` import) no longer makes each check through the daemon run its tests again. The test's read of the file and the instrumented module that the runner serves from it are now two inputs, each hashed by its own text. diff --git a/README.md b/README.md index 23a15f8..c108d03 100644 --- a/README.md +++ b/README.md @@ -162,15 +162,27 @@ exited with ` and exits 3; after an out-of-memory crash run read it, is not reused. A directory that a test stats or tests for is an input as a directory, not by its content. + Two kinds of input are not tracked yet. A Vite plugin whose output depends + on anything but a module's text, such as a sidecar file that it reads, can + make the daemon serve a run that read the plugin's old output. And the + daemon's environment is fixed when it starts, so an environment variable + that a later check sets does not reach the daemon's runs. After such a + change, run the check cold (without `--daemon`), or stop the daemon with + `oracle-gaps --stop-daemon --package `. The maintainer decided on + 2026-11-13 that the daemon tracks both and restarts when they change; a + later release does that. + These reads are no input, so an edit to the file alone does not rerun the tests that read it: a read outside the package or under `node_modules`; a read from a worker thread or child process that the test starts; and a read through any other `fs` API. These include `readdir`, `opendir`, - `createReadStream`, `realpath`, `lstat`, `access`, `open`, a read through - a file descriptor (`read`, `readSync`, `readv`), `copyFile` and - `copyFileSync`, `cp` and `cpSync`, `openAsBlob`, `statfs`, `readlink`, the - callback `stat`, `exists`, `watch`, `watchFile`, and the `fs/promises` - `access` and `open`. Run the check cold (without `--daemon`) after such an + `glob`, `globSync`, `createReadStream`, `realpath`, `lstat`, `access`, + `open`, a read through a file descriptor (`read`, `readSync`, `readv`, + `fstat`, `fstatSync`, and `readFileSync` or `readFile` given a file + descriptor), `copyFile` and `copyFileSync`, `cp` and `cpSync`, + `openAsBlob`, `statfs`, `readlink`, the callback `stat`, `exists`, + `watch`, `watchFile`, and the `fs/promises` `access`, `open`, and + `glob`. Run the check cold (without `--daemon`) after such an edit. A later release records all of them except `watch` and `watchFile` (maintainer decision, 2026-11-13). diff --git a/packages/oracles/oracle-gaps/capture-setup.mjs b/packages/oracles/oracle-gaps/capture-setup.mjs index 1174bb6..b7adf22 100644 --- a/packages/oracles/oracle-gaps/capture-setup.mjs +++ b/packages/oracles/oracle-gaps/capture-setup.mjs @@ -1086,10 +1086,17 @@ afterAll(() => { .filter(([, node]) => node.evaluated) .map(([id]) => id.split(`?`)[0]) .filter((path) => path.startsWith(`/`)), - ...globalThis.__oracleGapsReads.keys(), ] - // The hash of each file the copy read, from its first read. - const readAt = Object.fromEntries(globalThis.__oracleGapsReads) + // The hash of each file the copy read, from its first read, under its own + // key (`fs:`): a test's read of a file is another input than the + // module that its context serves from that file, which can differ from + // the file's text (fourth review of #55, F-A2). + const readAt = Object.fromEntries( + [...globalThis.__oracleGapsReads].map(([path, hash]) => [ + `fs:${path}`, + hash, + ]), + ) appendFileSync( state.capture, `${JSON.stringify({ file: thisFile, closed: true, loaded, readAt, ...(skippedCopy ? { skipped: true } : {}) })}\n`, diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index 56f6a65..be7cf66 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -542,7 +542,10 @@ export default async (env) => { // context last served, so a changed file is invalidated in the warm // contexts before it is read again. globalThis.__oracleGapsSchemata = new Map() + // A test's read of a file (`fs:`) is hashed by the file's text, a + // module by the text that its context serves. const servedHash = (path, index) => { + if (path.startsWith(`fs:`)) return hashOf(path.slice(3)) const text = globalThis.__oracleGapsServe[index]?.get(path) ?? globalThis.__oracleGapsSchemata.get(path) @@ -957,7 +960,7 @@ export default async (env) => { timedOut: true, spin: true, seconds, - inputs: importsOf(file, modules, closing.loaded ?? []), + inputs: importsOf(file, modules, closing), readAt: closing.readAt, } const tests = modules.flatMap((module) => @@ -1001,7 +1004,7 @@ export default async (env) => { String(error?.message ?? error), ), rows, - inputs: importsOf(file, modules, closing.loaded ?? []), + inputs: importsOf(file, modules, closing), readAt: closing.readAt, } } @@ -1011,19 +1014,23 @@ export default async (env) => { * and those its copy evaluated (`loaded`, from the capture's closing row), * which include a module that only a mutated branch imports at run time. */ - function importsOf(file, modules, loaded) { + function importsOf(file, modules, closing = {}) { return [ - ...new Set([ - file, - ...modules.flatMap((m) => - Object.keys(m.diagnostic?.().importDurations ?? {}), - ), - ...loaded, - ]), - ].filter( - (path) => - path.startsWith(`/`) && !ownHarness(path) && !isRunnerCopy(path), - ) + ...[ + ...new Set([ + file, + ...modules.flatMap((m) => + Object.keys(m.diagnostic?.().importDurations ?? {}), + ), + ...(closing.loaded ?? []), + ]), + ].filter( + (path) => + path.startsWith(`/`) && !ownHarness(path) && !isRunnerCopy(path), + ), + // The files its tests read through fs, each its own input. + ...Object.keys(closing.readAt ?? {}), + ] } /** @@ -1232,11 +1239,7 @@ export default async (env) => { })), unhandled: [], rows: own.map((row) => [row.key, row]), - inputs: importsOf( - item.file, - [], - closings.get(copies[i])?.loaded ?? [], - ), + inputs: importsOf(item.file, [], closings.get(copies[i])), readAt: closings.get(copies[i])?.readAt, } }), @@ -1290,11 +1293,7 @@ export default async (env) => { rows: rows .get(copy) .map((row) => [map(row.key), { ...row, key: map(row.key) }]), - inputs: importsOf( - item.file, - modules, - closings.get(copy)?.loaded ?? [], - ), + inputs: importsOf(item.file, modules, closings.get(copy)), readAt: closings.get(copy)?.readAt, } }), From af4a0089d207c75696dcf5e3a7ddad00a1b35bb3 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 14:17:11 -0700 Subject: [PATCH 15/17] ci(oracle-gaps): put the F-C1 calibration in an e2e shard The shard check failed on 5dd0ab4: the new calibration was in no shard. Co-authored-by: Isaac --- packages/oracles/scripts/e2e-shards.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 9cb1e1d..a381525 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -77,6 +77,7 @@ export const shards = { `fixed seed 20261105: a test edits the data it read, during a check through the daemon`, `random seed: a test edits the data it read, during a check through the daemon`, `determinism: constructed history: a test edits the data it read, during a check through the daemon`, + `calibration: the check a mid-run edit did not reach flips a mutant of the editing file`, `fixed seed 20261105: an unchanged check through the daemon under a config's pre plugin`, `random seed: an unchanged check through the daemon under a config's pre plugin`, `determinism: constructed history: an unchanged check through the daemon under a config's pre plugin`, From 90d75206625473d1b614fcda1e7677a08e09c3ff Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 14:20:59 -0700 Subject: [PATCH 16/17] test(oracle-gaps): the incremental violation names a cold run's error log When the cold run exits 3, its stack is in its error.log. The incremental violation named only the exit code, so a failure printed no stack. It now appends the error log the driver already reads for a run that exits 3. Co-authored-by: Isaac --- .../oracles/oracle-gaps/gap-report-oracle.property.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index c3811aa..4f6127b 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -2304,7 +2304,7 @@ function incrementalCheck(observations) { ) return violation( `incremental`, - `after ${step}, the daemon's check exited ${daemon.exit} (${daemon.refusal ?? daemon.stderr ?? `no refusal`}) and the cold run exited ${cold.exit} (${cold.refusal ?? `no refusal`})`, + `after ${step}, the daemon's check exited ${daemon.exit} (${daemon.refusal ?? daemon.stderr ?? `no refusal`}) and the cold run exited ${cold.exit} (${cold.refusal ?? `no refusal`})${cold.exit === 3 && cold.stderr ? `; its error log: ${cold.stderr}` : ``}`, `incremental equals cold`, { actual: daemon, expected: cold }, ) From b959d4c9262bbf2dabfa3dadc879ea2804ad6212 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 14:34:34 -0700 Subject: [PATCH 17/17] test(oracle-gaps): the incremental observation carries a cold run's error log ce46f4b appends cold.stderr to the incremental violation when the cold run exits 3, but the incremental observation's cold run had no stderr, so the message stayed empty. It now carries the run's stderr, with its error.log, when it exits 3, and only then: a completed run's stderr holds timings, which differ between two runs of one history. Co-authored-by: Isaac --- .../oracles/oracle-gaps/gap-report-oracle.property.test.mjs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 4f6127b..ca4dac6 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -1381,6 +1381,11 @@ async function incremental(dir, plan, cli, copy, observe, states) { refusal: coldRun.refusal, report: coldRun.report, notes: coldRun.stream.notes, + // An internal error's stack (its error.log, which readRun reads + // for a run that exits 3), for the violation's message. Only + // then: a completed run's stderr holds timings, which differ + // between two runs of one history. + stderr: coldRun.exit === 3 ? coldRun.stderr : undefined, }, daemon: { exit: run.exit,