From 919c9a80fb39c8b0727eb5b14baf25075c0d0cf7 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 12 Nov 2026 20:17:04 -0700 Subject: [PATCH 1/4] test(oracle-gaps): law 15 owes a stream log that the first line names The maintainer's decision of 2026-11-12: an agent that runs a check in the background learns where to read its stream from the run itself. A run that starts a check names `/stream.log` in its first line on both streams (`oracle-gaps: note log `), and that file holds the stderr's lines, in order, cold, through the daemon, and after a stop or an abort. The driver records each run's first lines and whether its log equals its stderr; two calibrations give a log that misses a line and a naming line after another line. Co-authored-by: Isaac --- .../gap-report-oracle.property.test.mjs | 84 ++++++++++++++++++- packages/oracles/scripts/e2e-shards.mjs | 3 + 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index f3f3a63..1328130 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -330,7 +330,12 @@ * The run's last line is its done line: `completed, gaps[ * (provisional)], report ` on stdout, `refused[, report * ]` after a refusal's block, or `stopped ()` - * after a stop. A check through the daemon streams the same lines. + * after a stop. A check through the daemon streams the same lines. The + * run's first line on both streams names its stream log, `oracle-gaps: + * note log /stream.log`, and that file holds the stderr's lines, in + * order, cold and through the daemon (the maintainer's decision of + * 2026-11-12: an agent that runs a check in the background learns from + * the run itself where to read it). * * Authority: the maintainer's decisions in the gap-check design runs v11, v14, * and v19 (2026-11-04 and 2026-11-05; the design notes are kept by the project @@ -699,7 +704,7 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { writeFixture(dir, files) const before = new Set(filesUnder(dir)) const pids = mkdtempSync(join(realpathSync(tmpdir()), `gap-stop-`)) - const { args } = cliArgs(dir, files, { + const { args, out } = cliArgs(dir, files, { ...how, outName: `out-stop`, contexts: 1, @@ -809,6 +814,16 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { [dir, ``], ]), ), + outDir: namedScratch(out, [ + [how.cli ?? here, ``], + [dir, ``], + ]), + log: streamLogOf(out, stderr, stdout, (text) => + namedScratch(text, [ + [how.cli ?? here, ``], + [dir, ``], + ]), + ), // The session directories in the package once it has ended: a stopped // run leaves its own, and the next run removes it. sessions: readdirSync(dir).filter((name) => @@ -950,6 +965,31 @@ const streamedGap = * or a heartbeat that names a mutant came before the first verdict; * - `last`: the last line, which ends a run that did not complete. */ +/** + * What the run's stream log says (the maintainer's decision of 2026-11-12: + * the CLI writes its stream to `/stream.log`, and its first line on + * both streams names that log), as relations: the first line of each + * stream, with scratch paths named, whether the log exists, and whether its + * lines are the stderr's lines, in order. + */ +function streamLogOf(out, stderr, stdout, name) { + const lines = (text) => `${text ?? ``}`.split(`\n`).filter(Boolean) + let log + try { + log = readFileSync(join(out, `stream.log`), `utf8`) + } catch { + // No log: the run wrote none. + } + return { + stderrFirst: name(lines(stderr)[0] ?? ``), + stdoutFirst: name(lines(stdout)[0] ?? ``), + exists: log !== undefined, + equal: + log !== undefined && + JSON.stringify(lines(log)) === JSON.stringify(lines(stderr)), + } +} + function streamOf(lines, name) { const kinds = new RegExp(`^oracle-gaps: (?:${streamKinds.join(`|`)}) `) const strays = new Set() @@ -1076,6 +1116,10 @@ function readRun(dir, out, status, stderr, cli, stdout) { // The output directory the run was given, as the trace names it: the // final line names its report there. outDir: namedScratch(out, scratch), + // The run's stream log, and the first line that names it. + log: streamLogOf(out, stderr, stdout, (text) => + namedScratch(text, scratch), + ), // Whether the run captured its tests' own console output there. captured: (() => { try { @@ -1297,6 +1341,9 @@ async function incremental(dir, plan, cli, copy, observe, states) { // The stream through the daemon (law 15): both streams' lines. stream: run.stream, stdoutStream: run.stdoutStream, + // Its stream log, and the output directory it names. + log: run.log, + outDir: run.outDir, // The first line of a failure, for the message; readRun has named // the scratch package and CLI copy in it. stderr: run.stderr?.split(`\n`)[0], @@ -2490,6 +2537,19 @@ function endsBreach(run) { [`stderr`, run.stream], [`stdout`, run.stdoutStream], ] + // The run's first line on each stream names its stream log, and the log + // holds the stderr's lines, in order (the maintainer's decision of + // 2026-11-12), for every run that starts a check: each of these does. + const note = `oracle-gaps: note log ${run.outDir}/stream.log` + if (run.log.stderrFirst !== note || run.log.stdoutFirst !== note) + return { + difference: `the run's first lines are ${JSON.stringify([run.log.stderrFirst, run.log.stdoutFirst])}, not ${JSON.stringify(note)} on each stream`, + } + if (!run.log.exists) return { difference: `the run wrote no stream.log` } + if (!run.log.equal) + return { + difference: `the run's stream.log is not its stderr, line for line`, + } for (const [where, stream] of streams) if (stream.strays.length > 0) return { @@ -2954,6 +3014,22 @@ calibrations.push( }, ), }, + { + name: `the stream log misses a line`, + law: `greppable stream`, + corrupt: corruptCli((cli) => ({ + ...cli, + log: { ...cli.log, equal: false }, + })), + }, + { + name: `the line that names the stream log comes after another line`, + law: `greppable stream`, + corrupt: corruptCli((cli) => ({ + ...cli, + log: { ...cli.log, stderrFirst: `oracle-gaps: config vite.config.ts` }, + })), + }, { name: `a stopped run ends without its done line`, law: `greppable stream`, @@ -3246,8 +3322,8 @@ const withDeclaration = (constructed) => oracleProperty({ id: `oracle-gaps.report`, law: { - statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, - authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, + statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names each busy mutant in a heartbeat while it runs, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, a first line on both streams that names the stream log in the output directory, which holds the stderr's lines in order, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, + authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps); the maintainer's decision of 2026-11-12 that the CLI writes its stream to the output directory's stream.log and names it in its first line.`, limits: `Law 12 judges refusals only: no legal history reaches an internal error, so its shape (exit 3, the stack in error.log) is not judged; argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own and test.projects only as a refusal, and only under the package's own package.json; a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), and the budget witness's along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); law 9's edits come only between checks, so a file saved during a check is not generated (the runner invalidates a known input that changed while a run ran and does not store that run, which only inspection fixes; a witness needs a step that edits a file between a unit's start and its end), and a module that a run first loads and that changes during that same run is stored under its later hash; law 9's edits leave out files read through fs, other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged by the static oracle, and these runs name --out in the package; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; the stop law covers SIGTERM and SIGINT sent to the CLI, not a SIGKILL of the CLI (its group then ends at the next check of its parent, by inspection) or a signal sent to the daemon, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; a heartbeat is owed only where a slow test waits 3 s against a 2 s interval, twice the first budget, so that no copy cut at its first budget beats, and each beat is observed as its mutant and file, not how long or how often it beat; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, so it can be larger than the smallest that fails; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export.`, }, histories: { diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 986c404..9361d23 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -167,6 +167,9 @@ export const shards = { // 79 s and 70 s; the daemon-notes one, bound to its history, 2 s (it // took 433 s searching for one). `calibration: a streamed line has no prefix`, + // The stream log's calibrations, not yet measured. + `calibration: the stream log misses a line`, + `calibration: the line that names the stream log comes after another line`, `calibration: a refusal through the daemon drops the cold run's notes`, // Bound to its history (re-review N2), not yet measured. `calibration: a run names a test-output log it never captured`, From 17ea63427021a076d78fcb48d6f0423a750e1c70 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 07:24:39 -0700 Subject: [PATCH 2/4] test(oracle-gaps): a refusal after a check started names its stream log first The maintainer's decision of 2026-11-12 names the stream log first in every run that starts a check, refusals in it included. The refusal law (#46) owed stdout = the done line alone; it now owes [note log, done line] when the check started, and [done line] for a refusal of the run's arguments, which comes before any check (outsideSource, and the static oracle's argument refusals). Law 15's log checks apply to runs that start a check. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-model-oracle.mjs | 27 ++++++++---- .../gap-report-oracle.property.test.mjs | 41 ++++++++++++------- 2 files changed, 46 insertions(+), 22 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 1958c9f..063cd3b 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -3815,7 +3815,11 @@ function refusal(plan) { if (plan.fault === `badEntry`) return { cause: `${SOURCE}#nope` } if (plan.fault === `missingSource`) return { cause: `src/missing.ts` } if (plan.fault === `nonLiteral`) return { cause: `b.test.ts` } - if (plan.fault === `outsideSource`) return { cause: `outside the package` } + // A `--source` outside the package is an argument the CLI refuses before + // it starts a check (the argument rules that the static oracle judges), + // so its stdout has no line that names a stream log. + if (plan.fault === `outsideSource`) + return { cause: `outside the package`, arguments: true } // The analysis reads the test files that the package's Vitest config // includes (index.mjs, step 2), so a config that does not load is named // before anything the analysis would find. The cause names the config @@ -5174,8 +5178,13 @@ function endingOf(exit, stderr, out, name = (text) => text, stdout = ``) { * The first way a run's ending breaks the refusal law, or undefined. The * cause must name `cause`; with `outKnown`, the run knew its output * directory, so the block names it and the directory holds the refusal. + * `logNote` is the line that names the run's stream log, owed first on + * stdout when the run refused after it started a check, and undefined for + * a refusal of its arguments, before any check (the maintainer's decision + * of 2026-11-12 that every run that starts a check names its stream log + * first). */ -function refusalBreach(ending, { cause, outKnown }) { +function refusalBreach(ending, { cause, outKnown, logNote }) { if (ending.exit !== 2) return `the refusal exited ${ending.exit}, not 2` if (ending.stackFrames) return `the refusal printed a stack trace` if (ending.cause === undefined) @@ -5194,12 +5203,14 @@ function refusalBreach(ending, { cause, outKnown }) { !/^refused(?:, report .+)?$/.test(ending.done) ) return `the refusal's last line is not \`oracle-gaps: done refused\`` - // The done line ends stdout too, as its only line. - if ( - ending.stdout.length !== 1 || - ending.stdout[0] !== `oracle-gaps: done ${ending.done}` - ) - return `the refusal's stdout is ${JSON.stringify(ending.stdout)}, not its done line` + // The done line ends stdout too, as its only line after the line that + // names the stream log, if the run started a check. + const owedStdout = [ + ...(logNote === undefined ? [] : [logNote]), + `oracle-gaps: done ${ending.done}`, + ] + if (JSON.stringify(ending.stdout) !== JSON.stringify(owedStdout)) + return `the refusal's stdout is ${JSON.stringify(ending.stdout)}, not ${JSON.stringify(owedStdout)}` if (!outKnown) return undefined if (!ending.namesOutput) return `the refusal does not name its report in its output directory: done ${ending.done}` diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 1328130..85ed4d3 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -331,11 +331,14 @@ * (provisional)], report ` on stdout, `refused[, report * ]` after a refusal's block, or `stopped ()` * after a stop. A check through the daemon streams the same lines. The - * run's first line on both streams names its stream log, `oracle-gaps: - * note log /stream.log`, and that file holds the stderr's lines, in - * order, cold and through the daemon (the maintainer's decision of - * 2026-11-12: an agent that runs a check in the background learns from - * the run itself where to read it). + * first line, on both streams, of a run that starts a check names its + * stream log, `oracle-gaps: note log /stream.log`, and that file + * holds the stderr's lines, in order, cold and through the daemon (the + * maintainer's decision of 2026-11-12: an agent that runs a check in the + * background learns from the run itself where to read it). A refusal + * after the check started has that line and then its done line on + * stdout; a refusal of the run's arguments comes before any check, and + * its stdout is its done line alone. * * Authority: the maintainer's decisions in the gap-check design runs v11, v14, * and v19 (2026-11-04 and 2026-11-05; the design notes are kept by the project @@ -2061,7 +2064,10 @@ function incrementalCheck(observations) { // A refusal through the daemon ends as a cold refusal does (law 12): // the CLI that asked the daemon writes it to its own output directory. if (cold.refusal !== undefined) { - const breach = refusalBreach(daemon.ending, { outKnown: true }) + const breach = refusalBreach(daemon.ending, { + outKnown: true, + logNote: logNoteOf(daemon), + }) if (breach) return violation( `incremental`, @@ -2198,6 +2204,7 @@ function check(plan, observations, expected) { const breach = refusalBreach(cli.ending, { cause: expected.refused.cause, outKnown: true, + logNote: expected.refused.arguments ? undefined : logNoteOf(cli), }) if (breach) return violation(`cli`, breach, `report-shaped ending`) return incrementalCheck(observations) ?? daemonCheck(observations) ?? pass @@ -2483,7 +2490,7 @@ function check(plan, observations, expected) { function streamBreach(checkpoint, run, expected, plan) { const breach = (difference, extra) => violation(checkpoint, difference, `greppable stream`, extra) - const ends = endsBreach(run) + const ends = endsBreach(run, { checked: !expected.refused?.arguments }) if (ends) return breach(ends.difference, ends.extra) if (run.stream.mutantBeforeVerdict) return breach( @@ -2526,27 +2533,33 @@ function streamBreach(checkpoint, run, expected, plan) { return undefined } +/** The line that names a run's stream log. */ +const logNoteOf = (run) => `oracle-gaps: note log ${run.outDir}/stream.log` + /** * What law 15 judges of every run, on both streams, whatever its outcome: * no stray line, and exactly one done line, which ends each stream, the * same on both, of the kind its exit status says (0 completed, 2 refused, - * 3 error, and a signal's code stopped). Undefined when it holds. + * 3 error, and a signal's code stopped). With `checked`, the run started a + * check, so its first line names its stream log. Undefined when it holds. */ -function endsBreach(run) { +function endsBreach(run, { checked = true } = {}) { const streams = [ [`stderr`, run.stream], [`stdout`, run.stdoutStream], ] // The run's first line on each stream names its stream log, and the log // holds the stderr's lines, in order (the maintainer's decision of - // 2026-11-12), for every run that starts a check: each of these does. - const note = `oracle-gaps: note log ${run.outDir}/stream.log` - if (run.log.stderrFirst !== note || run.log.stdoutFirst !== note) + // 2026-11-12), for every run that starts a check. A refusal of the run's + // arguments comes before any check, and names no log. + const note = logNoteOf(run) + if (checked && (run.log.stderrFirst !== note || run.log.stdoutFirst !== note)) return { difference: `the run's first lines are ${JSON.stringify([run.log.stderrFirst, run.log.stdoutFirst])}, not ${JSON.stringify(note)} on each stream`, } - if (!run.log.exists) return { difference: `the run wrote no stream.log` } - if (!run.log.equal) + if (checked && !run.log.exists) + return { difference: `the run wrote no stream.log` } + if (checked && !run.log.equal) return { difference: `the run's stream.log is not its stderr, line for line`, } From ceeeaf6cbf468ef3840684400aee432d0bd776f4 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 07:24:39 -0700 Subject: [PATCH 3/4] feat(oracle-gaps): write the stream to /stream.log and name it first A run that starts a check creates `/stream.log`, writes `oracle-gaps: note log ` as its first line on both streams, and writes every stderr line to the log too: the cold check's lines, a daemon reply's stderr, an internal error's or refusal's ending, the completed done line, and the supervisor's own done line for a stopped or crashed run. Co-authored-by: Isaac --- .changeset/gap-cli-stream-log.md | 5 ++ README.md | 4 ++ packages/oracles/oracle-gaps/index.mjs | 64 ++++++++++++++++++-------- 3 files changed, 55 insertions(+), 18 deletions(-) create mode 100644 .changeset/gap-cli-stream-log.md diff --git a/.changeset/gap-cli-stream-log.md b/.changeset/gap-cli-stream-log.md new file mode 100644 index 0000000..cb23854 --- /dev/null +++ b/.changeset/gap-cli-stream-log.md @@ -0,0 +1,5 @@ +--- +'@tanstack/oracles': patch +--- + +The `oracle-gaps` CLI now writes the lines it streams on stderr to `stream.log` in the output directory too, in order, cold and through `--daemon`, and the first line of a check on both streams names that file: `oracle-gaps: note log /stream.log`. An agent that starts a check in the background can read the run's progress and its done line there. diff --git a/README.md b/README.md index 0a5001b..02664e7 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,10 @@ every commit with [pkg.pr.new](https://pkg.pr.new). to the process's stdout and stderr goes to `test-output.log` in the output directory, which a note names; `summary.md` holds the report for reading. A run through the daemon prints the same lines, at the end of its check. + The first line of a check, on both streams, is `oracle-gaps: note log +/stream.log`: the run writes its stderr lines to that file too, in + order, so a check started in the background can be read there while it + runs and after it ends. `--help` is not a check, so its usage text has no prefix. For contributors: two environment variables in the `ORACLE_GAPS_TEST_` diff --git a/packages/oracles/oracle-gaps/index.mjs b/packages/oracles/oracle-gaps/index.mjs index d2639fe..1e873aa 100755 --- a/packages/oracles/oracle-gaps/index.mjs +++ b/packages/oracles/oracle-gaps/index.mjs @@ -122,8 +122,9 @@ // ended 2 s later is killed, and so is any process left in the group when // the run ends. The run kills its group itself if the CLI dies first. import { spawn } from 'node:child_process' -import { realpathSync } from 'node:fs' +import { appendFileSync, mkdirSync, realpathSync, writeFileSync } from 'node:fs' import { constants } from 'node:os' +import { join } from 'node:path' import { pathToFileURL } from 'node:url' import { parseArgs, usage } from './args.mjs' import { codeIdentity, openSession } from './check.mjs' @@ -134,6 +135,19 @@ import { memoryStore } from './evidence.mjs' // How long the group has to end after a stop signal before it is killed. const STOP_GRACE_MS = 2000 +// The stream log (the maintainer's decision of 2026-11-12): a run that +// starts a check writes its stderr to `/stream.log` too, line for +// line, and names that file in its first line on both streams, so an agent +// that runs a check in the background learns from the run where to read +// it. Undefined until the run starts a check. +let streamLog +const logPathOf = (out) => join(out, `stream.log`) +/** Writes `text` to stderr and to the stream log. */ +const writeStream = (text) => { + process.stderr.write(text) + if (streamLog) appendFileSync(streamLog, text) +} + /** Signals every process of group `group`; false when none is left. */ function signalGroup(group, signal) { try { @@ -149,7 +163,9 @@ function signalGroup(group, signal) { * with it: with its exit code, or, once a stop signal has ended its group, * with 128 plus the signal's number. */ -function superviseRun() { +function superviseRun(out) { + // The supervisor's own done line goes to the run's stream log too. + streamLog = logPathOf(out) let child let stopped // How the run ended: its exit code, or the signal that ended it. @@ -192,11 +208,16 @@ function superviseRun() { const line = stopped ? `oracle-gaps: done stopped (${stopped})\n` : `oracle-gaps: done error, the run's process exited with ${how}${outOfMemory ? `: it ran out of memory; give Node more (NODE_OPTIONS=--max-old-space-size=) or run fewer --contexts` : ``}\n` - process.stdout.write(line, () => + process.stdout.write(line, () => { + try { + appendFileSync(streamLog, line) + } catch { + // A run stopped before it wrote its log has none to end. + } process.stderr.write(line, () => process.exit(stopped ? 128 + constants.signals[stopped] : 3), - ), - ) + ) + }) } if (child === undefined || closed) return decide() child.once(`close`, decide) @@ -271,9 +292,9 @@ function watchSupervisor(supervisor) { function asksForCheck(argv) { try { const options = parseArgs(argv) - return !options.help && !options.stop + return options.help || options.stop ? undefined : options } catch { - return false + return undefined } } @@ -281,7 +302,9 @@ function asksForCheck(argv) { async function checkCold(options) { const session = await openSession(options.pkg) try { - return await session.check(options, memoryStore()) + return await session.check(options, memoryStore(), (line) => + writeStream(`${line}\n`), + ) } finally { await session.close() } @@ -294,7 +317,7 @@ async function checkThroughDaemon(options) { process.argv.slice(2), codeIdentity(), ) - if (reply.stderr) process.stderr.write(reply.stderr) + if (reply.stderr) writeStream(reply.stderr) if (reply.failed) throw received(reply.failed) return reply.stdout } @@ -306,13 +329,11 @@ const main = // Windows has no process groups: there the run is the CLI's own process. const supervised = process.platform !== `win32` const supervisor = process.env.ORACLE_GAPS_GROUP -if ( - main && - supervised && - supervisor === undefined && - asksForCheck(process.argv.slice(2)) -) - superviseRun() +const checking = + main && supervised && supervisor === undefined + ? asksForCheck(process.argv.slice(2)) + : undefined +if (checking) superviseRun(checking.out) else if (main) { if (supervisor !== undefined) watchSupervisor(Number(supervisor)) // A test that runs this CLI starts a run of its own, in a group of its own. @@ -356,7 +377,7 @@ else if (main) { if (ended) return const ending = endRun(error, options?.out) holdStops() - process.stderr.write(ending.stderr) + writeStream(ending.stderr) process.stdout.write(`${ending.done}\n`) end(ending.exitCode) } @@ -369,6 +390,13 @@ else if (main) { if (options.help) console.log(usage) else if (options.stop) await stopDaemon(options.pkg) else { + // The first line on both streams names the stream log. + mkdirSync(options.out, { recursive: true }) + streamLog = logPathOf(options.out) + writeFileSync(streamLog, ``) + const note = `oracle-gaps: note log ${streamLog}\n` + process.stdout.write(note) + writeStream(note) const summary = await (options.daemon ? checkThroughDaemon : checkCold)( options, ) @@ -376,7 +404,7 @@ else if (main) { // The summary ends with the done line (report.mjs), which ends stderr // too. console.log(summary) - process.stderr.write(`${summary.split(`\n`).at(-1)}\n`) + writeStream(`${summary.split(`\n`).at(-1)}\n`) } end(0) } catch (error) { From 5a92a3e56effa82e98f0d57415d0a8c2cbefe960 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 13 Nov 2026 10:23:58 -0700 Subject: [PATCH 4/4] fix(oracle-gaps): own the stream log in the supervisor; refuse a second run on a live --out The supervisor opens /stream.log and writes the note before it spawns the child; it resolves --out once and passes it to the child and to the daemon request; it tees the child's stderr, V8 lines included, to the log. A run whose log cannot be opened is refused before any check (exit 2, one done line), and a second run on a live output directory is refused without writing into it (maintainer decision, 2026-11-13). Oracle: out-mode (explicit or default) and log-blocked and overlap histories, with the law that a completed run's results.json sits beside its stream log and that a refused run writes nothing into a live directory. Stop histories use the stop-only witness. Co-authored-by: Isaac --- .changeset/gap-cli-stream-log.md | 2 +- README.md | 10 +- packages/oracles/oracle-gaps/ending.mjs | 4 +- .../oracle-gaps/gap-report-model-oracle.mjs | 84 +++- .../gap-report-oracle.property.test.mjs | 370 ++++++++++++++++-- ...gap-report-static-oracle.property.test.mjs | 37 +- packages/oracles/oracle-gaps/index.mjs | 129 +++++- packages/oracles/scripts/e2e-shards.mjs | 14 + 8 files changed, 568 insertions(+), 82 deletions(-) diff --git a/.changeset/gap-cli-stream-log.md b/.changeset/gap-cli-stream-log.md index cb23854..1c8c225 100644 --- a/.changeset/gap-cli-stream-log.md +++ b/.changeset/gap-cli-stream-log.md @@ -2,4 +2,4 @@ '@tanstack/oracles': patch --- -The `oracle-gaps` CLI now writes the lines it streams on stderr to `stream.log` in the output directory too, in order, cold and through `--daemon`, and the first line of a check on both streams names that file: `oracle-gaps: note log /stream.log`. An agent that starts a check in the background can read the run's progress and its done line there. +The `oracle-gaps` CLI now writes the lines it streams on stderr to `stream.log` in the output directory too, in order, cold and through `--daemon`, and the first line of a check on both streams names that file: `oracle-gaps: note log /stream.log`. An agent that starts a cold check in the background can read the run's progress and its done line there while it runs. A check through `--daemon` prints its lines at the end of its check, so its progress is readable there only after it ends. diff --git a/README.md b/README.md index d89f87e..655a7d9 100644 --- a/README.md +++ b/README.md @@ -119,10 +119,12 @@ every commit with [pkg.pr.new](https://pkg.pr.new). to the process's stdout and stderr goes to `test-output.log` in the output directory, which a note names; `summary.md` holds the report for reading. A run through the daemon prints the same lines, at the end of its check. - The first line of a check, on both streams, is `oracle-gaps: note log -/stream.log`: the run writes its stderr lines to that file too, in - order, so a check started in the background can be read there while it - runs and after it ends. + The first line of a check, on both streams, is + `oracle-gaps: note log /stream.log`: the run writes its stderr lines + to that file too, in order, so a check started in the background can be + read there while it runs and after it ends. A cold run's lines reach the + file as they are printed; a run through `--daemon` prints its lines at the + end of its check, so the file fills then. `--help` is not a check, so its usage text has no prefix. For contributors: two environment variables in the `ORACLE_GAPS_TEST_` diff --git a/packages/oracles/oracle-gaps/ending.mjs b/packages/oracles/oracle-gaps/ending.mjs index f26a94c..e915354 100644 --- a/packages/oracles/oracle-gaps/ending.mjs +++ b/packages/oracles/oracle-gaps/ending.mjs @@ -37,8 +37,10 @@ export function endRun(error, out = error?.out) { // The last line says the run refused, and where its report is once the // run knows its output directory (the stream's done line, stream.mjs); // it ends stdout too (index.mjs). + // A reportless refusal (another run uses the directory) writes nothing + // there: the directory's files are the other run's. const done = write( - out, + error.reportless ? undefined : out, results, `Refused: ${message}\n\nFix: ${error.fix}\n`, ) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 08f0b89..bcbe550 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -541,9 +541,12 @@ const drawn = fc.record({ // The budget witness alone (`budget`) costs two 10 s confirmations per // mutant in every CLI run of a history, so generated histories rarely // draw it; its constructed history (the daemon recheck) always does. + // `stop` is the stop witness alone (`w_stop`), which a run that the driver + // stops needs: the capture witness's other outcome classes are not run. witness: fc.oneof( { weight: 30, arbitrary: fc.constantFrom(false, true, `capture`) }, { weight: 1, arbitrary: fc.constant(`budget`) }, + { weight: 1, arbitrary: fc.constant(`stop`) }, ), authored: fc.option( fc.record({ @@ -690,6 +693,39 @@ const drawn = fc.record({ // A second run with `--only`, of one owned function's mutants: its // baseline runs only their owners (law 14). It costs a run of its own. onlyRun: fc.constantFrom(false, false, false, false, false, true), + // Where the run writes: the output directory `--out` names (`explicit`), or + // the one the CLI picks when `--out` is left out (`default`, args.mjs), whose + // stream log, results, and done line must still agree (law 15). + outMode: fc.constantFrom(`explicit`, `explicit`, `explicit`, `default`), + // The run's `/stream.log` is a directory, so the log cannot be written + // (a problem the author can fix: the run refuses before it checks). + logBlocked: fc.constantFrom( + false, + false, + false, + false, + false, + false, + false, + false, + true, + ), + // A second run starts on the first run's output directory while the first + // is live (the coordinator's decision of 2026-11-13 on F4): the second + // refuses, and the first's stream log stays its own. + overlap: fc.constantFrom( + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + ), // Oracle file a names its own Vitest environment in a pragma // (`// @vitest-environment`), another than the config's, and fails to load // outside it: every copy of it must run in that environment too. @@ -2477,6 +2513,7 @@ function resolveHistory(history) { // so only histories of the `budget` mode carry it. (history.witness === true && w.id !== `w_slowpair`) || (history.witness === `capture` && captureWitness.has(w.id)) || + (history.witness === `stop` && w.id === `w_stop`) || (history.witness === `budget` && w.id === `w_slowpair`), ) .map((w) => ({ @@ -2837,6 +2874,12 @@ function resolveHistory(history) { depSubpath: history.depSubpath === true, unbuiltDep: history.unbuiltDep === true, budget: history.budget ?? `two`, + outMode: history.outMode ?? `explicit`, + // Both name the run's `--out` directory before it starts, so they need + // an explicit one (a default directory is chosen by the CLI). + logBlocked: + history.logBlocked === true && outModeOf(history) === `explicit`, + overlap: history.overlap === true && outModeOf(history) === `explicit`, stopRuns: history.stopRuns ?? [], onlyRun: history.onlyRun === true, pragma: history.pragma === true, @@ -4083,7 +4126,16 @@ function refusal(plan) { if (plan.fault === `brokenVitest`) return { cause: `vitest`, runtime: true } if (plan.fault === `projects`) return { cause: `test.projects`, runtime: true } + // A stream log that cannot be written is refused before the check starts + // (the CLI opens the log before its first line, index.mjs), so its stdout + // owes its done line alone. + if (plan.logBlocked === true) + return { cause: `the stream log`, arguments: true } if (plan.fault === `badEntry`) return { cause: `${SOURCE}#nope` } + // An owns entry that names a missing file (a test file's declaration) is + // found by the analysis, after the check has started, so this refusal + // owes the stream log's note. The argument-stage case, a `--source` that + // is not a file of the package, is the static oracle's `absentSource`. if (plan.fault === `missingSource`) return { cause: `src/missing.ts` } if (plan.fault === `nonLiteral`) return { cause: `b.test.ts` } // A `--source` outside the package is an argument the CLI refuses before @@ -4187,6 +4239,7 @@ const declarationFaults = [ `nonLiteral`, `outsideSource`, ] +const outModeOf = (history) => history.outMode ?? `explicit` const reachesRun = (history) => !declarationFaults.includes(history.fault ?? `none`) @@ -4239,21 +4292,22 @@ function model(plan) { // fault, a Vitest that does not load, a config that does not load, and a // config that defines test.projects, which a loaded context shows. Every // other run names it once, refused or not. - const configLines = [ - `outsideSource`, - `brokenVitest`, - `configThrows`, - `configNoDefault`, - `configNull`, - `configSyntax`, - `projects`, - ].includes(plan.fault) - ? [] - : [ - plan.configPath === undefined - ? `oracle-gaps: config none, Vitest's default discovery` - : `oracle-gaps: config ${plan.configPath}`, - ] + const configLines = + [ + `outsideSource`, + `brokenVitest`, + `configThrows`, + `configNoDefault`, + `configNull`, + `configSyntax`, + `projects`, + ].includes(plan.fault) || plan.logBlocked === true + ? [] + : [ + plan.configPath === undefined + ? `oracle-gaps: config none, Vitest's default discovery` + : `oracle-gaps: config ${plan.configPath}`, + ] if (refused) return { refused, configLines } const sites = sitesOf(plan) const owners = ownersOf(plan) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 380b12e..cddd89c 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -588,14 +588,17 @@ function cliArgs( sources = `${SOURCE},${LIB}`, budget = `two`, outName = daemon ? `out-daemon` : `out`, + // `default` leaves `--out` out, so the CLI picks its own output + // directory (args.mjs): the run's first stdout line names it. + outMode = `explicit`, config, contexts = 4, only, pool, } = {}, ) { - const out = join(dir, outName) - rmSync(out, { recursive: true, force: true }) + const out = outMode === `default` ? undefined : join(dir, outName) + if (out !== undefined) rmSync(out, { recursive: true, force: true }) // The CLI runs from the package directory. A daemon check names its paths // relative to there, as an author types them, and the daemon must read // them as the CLI's caller meant them. @@ -628,9 +631,8 @@ function cliArgs( // one copy: one slow owner's confirmation fits it, two do not (law 4). `--mutant-limit`, String(MUTANT_LIMIT_MS), - `--out`, - path(out), ] + if (out !== undefined) args.push(`--out`, path(out)) if (only !== undefined) args.push(`--only`, only.join(`,`)) if (pool !== undefined) args.push(`--pool`, pool) if (ordinary) args.push(`--ordinary`) @@ -655,6 +657,7 @@ function runCli(dir, files, how = {}) { writeFixture(dir, files) const before = new Set(filesUnder(dir)) const { args, out } = cliArgs(dir, files, how) + if (how.logBlocked) mkdirSync(join(out, `stream.log`), { recursive: true }) const child = spawnSync(process.execPath, args, { cwd: repo, encoding: `utf8`, @@ -727,7 +730,7 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { writeFixture(dir, files) const before = new Set(filesUnder(dir)) const pids = mkdtempSync(join(realpathSync(tmpdir()), `gap-stop-`)) - const { args, out } = cliArgs(dir, files, { + const { args, out: given } = cliArgs(dir, files, { ...how, outName: `out-stop`, contexts: 1, @@ -813,7 +816,18 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { } if (!ended) child.kill(`SIGKILL`) rmSync(pids, { recursive: true, force: true }) - return { + const out = + given ?? + `${stdout}` + .split(`\n`)[0] + .match(/^oracle-gaps: note log (.*)\/stream\.log$/)?.[1] ?? + join(dir, `no-output`) + const scratchNames = [ + [how.cli ?? here, ``], + [dir, ``], + ...(given === undefined ? [[out, ``]] : []), + ] + const result = { signal, pool, spun, @@ -826,26 +840,14 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { exit: exitCode, // Both streams' lines (law 15), with the scratch paths named. stream: streamOf(stderr.split(`\n`), (text) => - namedScratch(text, [ - [how.cli ?? here, ``], - [dir, ``], - ]), + namedScratch(text, scratchNames), ), stdoutStream: streamOf(stdout.split(`\n`), (text) => - namedScratch(text, [ - [how.cli ?? here, ``], - [dir, ``], - ]), + namedScratch(text, scratchNames), ), - outDir: namedScratch(out, [ - [how.cli ?? here, ``], - [dir, ``], - ]), + outDir: namedScratch(out, scratchNames), log: streamLogOf(out, stderr, stdout, (text) => - namedScratch(text, [ - [how.cli ?? here, ``], - [dir, ``], - ]), + namedScratch(text, scratchNames), ), // The session directories in the package once it has ended: a stopped // run leaves its own, and the next run removes it. @@ -857,6 +859,9 @@ async function stoppedRun(dir, files, how, ids, { signal, pool, when }) { (file) => !/^\.oracle-gaps-\d+\//.test(file), ), } + // A default output directory is the run's own scratch. + if (given === undefined) rmSync(out, { recursive: true, force: true }) + return result } /** @@ -905,6 +910,83 @@ function filesUnder(root, within = ``) { ) } +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) + +/** + * Runs the CLI twice on one output directory, the second while the first is + * live (the first has printed its note, so it has its stream log). The first + * run is the history's `cli`, and the second's ending is the `overlap` + * observation (the coordinator's decision of 2026-11-13 on F4: a second run + * refuses, and the first's stream log stays its own). + */ +async function overlappingRuns(dir, files, how) { + writeFixture(dir, files) + const before = new Set(filesUnder(dir)) + const { args, out } = cliArgs(dir, files, how) + const start = () => { + const child = spawn(process.execPath, args, { + cwd: repo, + env: cleanEnv(), + stdio: [`ignore`, `pipe`, `pipe`], + }) + const got = { stdout: ``, stderr: `` } + child.stdout.on(`data`, (chunk) => { + got.stdout += chunk + }) + child.stderr.on(`data`, (chunk) => { + got.stderr += chunk + }) + const status = new Promise((resolvePromise) => { + const timer = setTimeout(() => child.kill(), 300_000) + child.on(`close`, (code) => { + clearTimeout(timer) + resolvePromise(code) + }) + }) + return { got, status } + } + const first = start() + // The first run's note is its first stdout line: its stream log exists. + for ( + let waited = 0; + !first.got.stdout.includes(`oracle-gaps: note log`) && waited < 120_000; + waited += 20 + ) + await sleep(20) + const second = start() + const secondStatus = await second.status + const firstStatus = await first.status + const ended = readRun( + dir, + out, + firstStatus, + first.got.stderr, + how.cli ?? here, + first.got.stdout, + ) + const refused = readRun( + dir, + out, + secondStatus, + second.got.stderr, + how.cli ?? here, + second.got.stdout, + ) + const replay = first.got.stderr + .split(`\n`) + .find((line) => line.startsWith(` replay: `)) + ?.slice(` replay: `.length) + return { + cli: { ...ended, replay, strayWrites: strays(dir, before) }, + overlap: { + exit: refused.exit, + ending: refused.ending, + stream: refused.stream, + stdoutStream: refused.stdoutStream, + }, + } +} + // Where a run may write in the package: only the output directories the // driver names with `--out` (`out`, `out-daemon`, `out-stop`, and // `out-only`). The daemon's state, the workspace dependencies' one-file @@ -996,7 +1078,9 @@ const streamedGap = * lines are the stderr's lines, in order. */ function streamLogOf(out, stderr, stdout, name) { - const lines = (text) => `${text ?? ``}`.split(`\n`).filter(Boolean) + // The lines of a text, its final newline ending the last line. A blank line + // is a line: the log must equal the stderr's lines exactly (F10). + const lines = (text) => `${text ?? ``}`.replace(/\n$/, ``).split(`\n`) let log try { log = readFileSync(join(out, `stream.log`), `utf8`) @@ -1060,11 +1144,40 @@ function streamOf(lines, name) { } function readRun(dir, out, status, stderr, cli, stdout) { + // A run without `--out` writes to the directory its stream log is in, which + // its first stdout line names (law 15); that directory is removed after the + // run is read, as the run's own scratch. + const chosen = + out ?? + `${stdout ?? ``}` + .split(`\n`)[0] + .match(/^oracle-gaps: note log (.*)\/stream\.log$/)?.[1] + try { + return readRunIn( + dir, + chosen ?? join(dir, `no-output`), + status, + stderr, + cli, + stdout, + out === undefined ? chosen : undefined, + ) + } finally { + if (out === undefined && chosen !== undefined) + rmSync(chosen, { recursive: true, force: true }) + } +} + +// `derived` is the output directory the CLI chose (no `--out`), which is named +// `` so that two runs of one history, each with its own directory, record +// the same trace (law 9's determinism). +function readRunIn(dir, out, status, stderr, cli, stdout, derived) { // The output names the CLI directory that ran and the package, as scratch // directories do, before it is cut to its last characters. const scratch = [ [cli, ``], [dir, ``], + ...(derived === undefined ? [] : [[derived, ``]]), ] const named = status === 0 ? undefined : namedScratch(`${stderr}`, scratch) let results @@ -1332,6 +1445,7 @@ async function incremental(dir, plan, cli, copy, observe, states) { const key = fingerprint(files, state, step === `otherInstall`) const how = { seed: state.seed, + outMode: plan.outMode, cli: step === `otherInstall` ? otherCli() : copy, ordinary: plan.ordinary, sources: sourceArgument(plan, `${dir}.link`), @@ -1357,6 +1471,8 @@ async function incremental(dir, plan, cli, copy, observe, states) { : await daemonChecks(step, files, how) if (!cold.has(key)) cold.set(key, runCli(dir, files, how)) const coldRun = cold.get(key) + if (plan.outMode === `default`) + states.add(`a daemon check with the default output directory`) for (const [i, run] of [daemon, ...others].entries()) observe(`incremental`, { step: i === 0 ? step : `${step} (a second check at once)`, @@ -1377,6 +1493,7 @@ async function incremental(dir, plan, cli, copy, observe, states) { // Its stream log, and the output directory it names. log: run.log, outDir: run.outDir, + reported: run.reported, // The first line of a failure, for the message; readRun has named // the scratch package and CLI copy in it. stderr: run.stderr?.split(`\n`)[0], @@ -1647,10 +1764,18 @@ async function driver(plan, observe) { ordinary: plan.ordinary, sources: sourceArgument(plan, `${dir}.link`), budget: plan.budget, + outMode: plan.outMode, + logBlocked: plan.logBlocked, cli: plan.cliPath === `spaced` ? copy : here, ...configArgs(plan), } - const { replay, ...cli } = runCli(dir, files, how) + // Two runs on one output directory: the second's refusal is its own + // observation (overlappingRuns). + const overlapping = plan.overlap + ? await overlappingRuns(dir, files, how) + : undefined + const { replay, ...cli } = overlapping?.cli ?? runCli(dir, files, how) + if (overlapping) observe(`overlap`, overlapping.overlap) // How many units were computed or reused says how the result was // produced, not what it is, so it is not observed. observe(`cli`, { ...cli, evidence: undefined }) @@ -1659,6 +1784,12 @@ async function driver(plan, observe) { observe(`replay`, { command: replay, ran: replayed(dir, replay) }) // Reach comes from what the CLI reported, mapped onto the plan. const states = new Set() + if (plan.outMode === `default`) + states.add(`a run with the default output directory`) + if (plan.logBlocked && cli.exit === 2) + states.add(`a stream log that cannot be written`) + if (overlapping && overlapping.overlap.exit === 2) + states.add(`two runs on one output directory`) if (cli.exit === 0 && !plan.consoleOutput) states.add(`a completed run whose tests wrote nothing`) // Law 14: a partial run of one owned function's mutants, whose baseline @@ -1712,6 +1843,8 @@ async function driver(plan, observe) { }, } observe(`stopped`, stopped) + if (plan.outMode === `default`) + states.add(`a stop with the default output directory`) if (previousSessions.length > 0) states.add(`a run after a stopped one that left its session`) previousSessions = run.sessions @@ -2221,6 +2354,24 @@ function check(plan, observations, expected) { `config`, { actual: cli.configLines, expected: expected.configLines }, ) + // A second run on a live output directory refuses before it writes + // anything (F4, the coordinator's decision of 2026-11-13): the refusal + // names the directory, is one block with its fix, and its stdout is its + // done line alone. The first run's own laws are judged on `cli`. + const overlap = observations.find((o) => o.checkpoint === `overlap`)?.value + if (overlap !== undefined) { + if (overlap.exit !== 2) + return violation( + `overlap`, + `the second run on a live output directory exited ${overlap.exit}, not refusing it`, + `ownership`, + ) + const breach = refusalBreach(overlap.ending, { + cause: `another run is using`, + outKnown: false, + }) + if (breach) return violation(`overlap`, breach, `report-shaped ending`) + } const stream = streamBreach(`cli`, cli, expected, plan) if (stream) return stream // Through the daemon, each check streams the same kinds of line, and a @@ -2290,6 +2441,14 @@ function check(plan, observations, expected) { logNote: expected.refused.arguments ? undefined : logNoteOf(cli), }) if (breach) return violation(`cli`, breach, `report-shaped ending`) + // A refusal of the arguments, or before the check starts, writes no + // stream log (F8, the coordinator's review of #57). + if (expected.refused.arguments && cli.log.exists) + return violation( + `cli`, + `the refusal before the check started wrote a stream log`, + `greppable stream`, + ) return incrementalCheck(observations) ?? daemonCheck(observations) ?? pass } if (cli.exit !== 0 || !cli.reported) @@ -2644,6 +2803,13 @@ function endsBreach(run, { checked = true } = {}) { return { difference: `the run's stream.log is not its stderr, line for line`, } + // A completed run's results.json sits beside its stream log, in the + // directory the log names (law 15's stream log, the maintainer's decision + // of 2026-11-12). + if (checked && run.exit === 0 && run.reported === false) + return { + difference: `the run completed, but its results.json is not beside its stream log`, + } for (const [where, stream] of streams) if (stream.strays.length > 0) return { @@ -3109,20 +3275,31 @@ calibrations.push( ), }, { + // A completed run that starts a check, so its log is judged (law 15). name: `the stream log misses a line`, law: `greppable stream`, - corrupt: corruptCli((cli) => ({ - ...cli, - log: { ...cli.log, equal: false }, - })), + history: { constructed: `two authored mutants` }, + corrupt: corruptCli((cli) => + cli.exit !== 0 + ? undefined + : { ...cli, log: { ...cli.log, equal: false } }, + ), }, { name: `the line that names the stream log comes after another line`, law: `greppable stream`, - corrupt: corruptCli((cli) => ({ - ...cli, - log: { ...cli.log, stderrFirst: `oracle-gaps: config vite.config.ts` }, - })), + history: { constructed: `two authored mutants` }, + corrupt: corruptCli((cli) => + cli.exit !== 0 + ? undefined + : { + ...cli, + log: { + ...cli.log, + stderrFirst: `oracle-gaps: config vite.config.ts`, + }, + }, + ), }, { name: `a stopped run ends without its done line`, @@ -3188,6 +3365,11 @@ const staticStates = [ `owned through bind`, ] const required = [ + `a stream log that cannot be written`, + `two runs on one output directory`, + `a run with the default output directory`, + `a stop with the default output directory`, + `a daemon check with the default output directory`, `killed by a second owner`, `killed`, `random-only kill`, @@ -3488,6 +3670,130 @@ oracleProperty({ consoleOutput: true, }), }, + { + // Without `--out`, the CLI picks the run's output directory + // (args.mjs). A run stopped while its config warms must still leave + // its stream log beside its results, with the done line last on both + // streams (law 15). The stop witness runs alone (witness `stop`), and a history + // with the witness module takes no steps. + name: `a stop with the default output directory`, + enters: [ + `a run with the default output directory`, + `a stop with the default output directory`, + ], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`a`], + library: [], + libraryOwners: [], + runes: [], + wholeFile: `none`, + witness: `stop`, + tests: [{ ...judging(`a`, `fixed seed`, 1), fn: 0 }], + authored: null, + fault: `none`, + stability: `none`, + steps: null, + ordinary: false, + outMode: `default`, + budget: `two`, + stopRuns: [{ signal: `SIGINT`, pool: `threads`, when: `warm` }], + }), + }, + { + // Through the daemon, a check that the CLI parses without `--out` + // would write its results to a directory of its own, apart from the + // stream log the CLI names (law 15). + name: `a daemon check with the default output directory`, + enters: [ + `a run with the default output directory`, + `a daemon check with the default output directory`, + ], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`a`], + library: [], + libraryOwners: [], + runes: [], + wholeFile: `none`, + witness: false, + tests: [{ ...judging(`a`, `fixed seed`, 1), fn: 0 }], + authored: null, + fault: `none`, + stability: `none`, + steps: [`test`], + ordinary: false, + outMode: `default`, + budget: `two`, + }), + }, + { + // A stream log that is a directory cannot be written: the CLI refuses + // before its first line (index.mjs opens the log first), with its + // block, its fix, and its report in the output directory, and its + // stdout is its done line alone (law 12, law 15, and F3 of the + // coordinator's review of #57). No stack trace, and exit 2 or 3. + name: `a stream log that cannot be written`, + enters: [`a stream log that cannot be written`], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`a`], + library: [], + libraryOwners: [], + runes: [], + wholeFile: `none`, + witness: false, + tests: [{ ...judging(`a`, `fixed seed`, 1), fn: 0 }], + authored: null, + fault: `none`, + stability: `none`, + steps: null, + ordinary: false, + budget: `two`, + outMode: `explicit`, + logBlocked: true, + overlap: false, + }), + }, + { + // A second run on the first's output directory while the first is + // live refuses (F4, the coordinator's decision of 2026-11-13): the + // first run's stream log still equals its own stderr, line for line, + // and the refusal reports no directory it does not own. + name: `two runs on one output directory`, + enters: [`two runs on one output directory`], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`a`], + library: [], + libraryOwners: [], + runes: [], + wholeFile: `none`, + witness: false, + tests: [{ ...judging(`a`, `fixed seed`, 1), fn: 0 }], + authored: null, + fault: `none`, + stability: `none`, + steps: null, + ordinary: false, + budget: `two`, + outMode: `explicit`, + logBlocked: false, + overlap: true, + }), + }, { // The budget witness checked through the daemon twice with nothing // changed between: the second check reuses the confirmations the diff --git a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs index d2ef2d0..07849a8 100644 --- a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs @@ -89,6 +89,7 @@ * target. An authored mutant's `source` field is used as written. */ import { + existsSync, mkdirSync, mkdtempSync, readFileSync, @@ -371,6 +372,14 @@ const matcherOwns = fc.oneof( ) // Each history of the shared grammar, with its argument fault and its // matcher's `owns`. +// The end-to-end dimensions the static oracle does not draw: `logBlocked` +// (a stream log the run cannot write) and `overlap` (a second run on a live +// output directory). The static run creates no stream log and starts no +// second CLI process, so these are off for it by deliberate scope; the +// end-to-end oracle (gap-report-oracle) owns both (F3 and F4 of the review of +// #57). +const staticResolve = (history) => + resolveHistory({ ...history, logBlocked: false, overlap: false }) const argumentHistories = fc.tuple(histories, badArguments, matcherOwns).map( ([history, bad, matcher]) => ({ ...history, @@ -489,13 +498,19 @@ function argumentsRunOf(dir, plan) { encoding: `utf8`, timeout: 120_000, }) - return endingOf( - run.status, - run.stderr, - out, - (text) => text.replaceAll(dir, ``), - run.stdout, - ) + return { + ...endingOf( + run.status, + run.stderr, + out, + (text) => text.replaceAll(dir, ``), + run.stdout, + ), + // A refusal of the arguments, or before a check starts, writes no stream + // log (F8, the coordinator's review of #57). Judged where the run names + // its output directory; a default one lives in the OS temp directory. + streamLog: out !== undefined && existsSync(join(out, `stream.log`)), + } } // ## The derived class, run @@ -1082,6 +1097,12 @@ function check(plan, observations, expected) { `with ${bad.kind} arguments${bad.daemon ? ` and --daemon` : ``}, ${breach}`, `report-shaped ending`, ) + if (argumentsRun.streamLog) + return violation( + `arguments`, + `with ${bad.kind} arguments${bad.daemon ? ` and --daemon` : ``}, the refusal wrote a stream log`, + `greppable stream`, + ) // An owner that fails, and a config the runner cannot load or honor, are // found only when the runner starts, so the analysis must not refuse for // them: it refuses as it would for the same plan without the fault. @@ -1785,7 +1806,7 @@ oracleProperty({ }, ]), }, - resolve: resolveHistory, + resolve: staticResolve, model, driver, check, diff --git a/packages/oracles/oracle-gaps/index.mjs b/packages/oracles/oracle-gaps/index.mjs index 1e873aa..349b96e 100755 --- a/packages/oracles/oracle-gaps/index.mjs +++ b/packages/oracles/oracle-gaps/index.mjs @@ -122,14 +122,21 @@ // ended 2 s later is killed, and so is any process left in the group when // the run ends. The run kills its group itself if the CLI dies first. import { spawn } from 'node:child_process' -import { appendFileSync, mkdirSync, realpathSync, writeFileSync } from 'node:fs' +import { + appendFileSync, + mkdirSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from 'node:fs' import { constants } from 'node:os' import { join } from 'node:path' import { pathToFileURL } from 'node:url' import { parseArgs, usage } from './args.mjs' import { codeIdentity, openSession } from './check.mjs' import { received, requestDaemon, stopDaemon } from './daemon.mjs' -import { endRun } from './ending.mjs' +import { endRun, Refusal } from './ending.mjs' import { memoryStore } from './evidence.mjs' // How long the group has to end after a stop signal before it is killed. @@ -139,14 +146,86 @@ const STOP_GRACE_MS = 2000 // starts a check writes its stderr to `/stream.log` too, line for // line, and names that file in its first line on both streams, so an agent // that runs a check in the background learns from the run where to read -// it. Undefined until the run starts a check. +// it. The supervisor opens it before the child starts (startLog), and then +// is the only writer of the child's stderr to it (superviseRun). Undefined +// until the log is open. let streamLog const logPathOf = (out) => join(out, `stream.log`) -/** Writes `text` to stderr and to the stream log. */ +/** Writes `text` to stderr and, once the log is open, to the stream log. */ const writeStream = (text) => { process.stderr.write(text) if (streamLog) appendFileSync(streamLog, text) } +const cannotWrite = (path, error) => + new Refusal( + `the stream log ${path} cannot be written (${error.code ?? error.message})`, + `Remove ${path}, or pass --out a directory whose stream.log can be written.`, + ) +/** Whether process `pid` runs. */ +const alive = (pid) => { + try { + process.kill(pid, 0) + return true + } catch (error) { + return error.code === `EPERM` + } +} +/** + * Claims output directory `out` for this process: a run keeps it while it + * lives, and a second run on it refuses, reportless so that the live run's + * files stay its own (the maintainer's decision of 2026-11-13). A claim by a + * run that has ended is taken over. + */ +function claimOut(out) { + const claim = join(out, `run.pid`) + for (;;) { + try { + writeFileSync(claim, `${process.pid}\n`, { flag: `wx` }) + return + } catch (error) { + if (error.code !== `EEXIST`) throw cannotWrite(claim, error) + } + let holder + try { + holder = Number(readFileSync(claim, `utf8`)) + } catch (error) { + if (error.code !== `ENOENT`) throw cannotWrite(claim, error) + continue + } + if (holder !== process.pid && holder > 0 && alive(holder)) { + const refusal = new Refusal( + `another run is using ${out}`, + `Pass --out a directory no run is using, or wait for the run that uses it to end.`, + ) + refusal.reportless = true + throw refusal + } + rmSync(claim, { force: true }) + } +} +/** + * Opens the run's stream log in `out` and writes its first line, on both + * streams, which names the log. Throws a Refusal when the output directory + * is in use or the log cannot be written; nothing is written then. + */ +function startLog(out) { + const path = logPathOf(out) + try { + mkdirSync(out, { recursive: true }) + } catch (error) { + throw cannotWrite(path, error) + } + claimOut(out) + try { + writeFileSync(path, ``) + } catch (error) { + throw cannotWrite(path, error) + } + streamLog = path + const note = `oracle-gaps: note log ${path}\n` + process.stdout.write(note) + writeStream(note) +} /** Signals every process of group `group`; false when none is left. */ function signalGroup(group, signal) { @@ -164,8 +243,6 @@ function signalGroup(group, signal) { * with 128 plus the signal's number. */ function superviseRun(out) { - // The supervisor's own done line goes to the run's stream log too. - streamLog = logPathOf(out) let child let stopped // How the run ended: its exit code, or the signal that ended it. @@ -209,11 +286,8 @@ function superviseRun(out) { ? `oracle-gaps: done stopped (${stopped})\n` : `oracle-gaps: done error, the run's process exited with ${how}${outOfMemory ? `: it ran out of memory; give Node more (NODE_OPTIONS=--max-old-space-size=) or run fewer --contexts` : ``}\n` process.stdout.write(line, () => { - try { - appendFileSync(streamLog, line) - } catch { - // A run stopped before it wrote its log has none to end. - } + // The log was opened before the run started (startLog). + appendFileSync(streamLog, line) process.stderr.write(line, () => process.exit(stopped ? 128 + constants.signals[stopped] : 3), ) @@ -249,6 +323,10 @@ function superviseRun(out) { [ ...process.execArgv.filter((flag) => !/^--inspect/.test(flag)), ...process.argv.slice(1), + // The out directory, resolved once by this process (asksForCheck): the + // run and a daemon's check write there, not to a directory of their own. + `--out`, + out, ], { detached: true, @@ -259,10 +337,12 @@ function superviseRun(out) { child.on(`message`, (message) => { if (message === `ending`) ending = true }) - // The run's stderr, passed on as it comes, and read for V8's - // out-of-memory message. + // The run's stderr, passed on as it comes, written to the stream log as it + // is (V8's own lines included, which the run does not write itself), and + // read for V8's out-of-memory message. child.stderr.on(`data`, (chunk) => { process.stderr.write(chunk) + appendFileSync(streamLog, chunk) stderrTail = (stderrTail + chunk).slice(-200) if (/heap out of memory/i.test(stderrTail)) outOfMemory = true }) @@ -333,7 +413,18 @@ const checking = main && supervised && supervisor === undefined ? asksForCheck(process.argv.slice(2)) : undefined -if (checking) superviseRun(checking.out) +// A check the supervisor cannot open its log for is run in this process, which +// refuses it the same way (below). +const supervising = (() => { + if (checking === undefined) return false + try { + startLog(checking.out) + return true + } catch { + return false + } +})() +if (supervising) superviseRun(checking.out) else if (main) { if (supervisor !== undefined) watchSupervisor(Number(supervisor)) // A test that runs this CLI starts a run of its own, in a group of its own. @@ -390,13 +481,9 @@ else if (main) { if (options.help) console.log(usage) else if (options.stop) await stopDaemon(options.pkg) else { - // The first line on both streams names the stream log. - mkdirSync(options.out, { recursive: true }) - streamLog = logPathOf(options.out) - writeFileSync(streamLog, ``) - const note = `oracle-gaps: note log ${streamLog}\n` - process.stdout.write(note) - writeStream(note) + // The supervisor opened the log (or, on Windows, there is none and this + // process runs the check); its child writes stderr alone. + if (supervisor === undefined) startLog(options.out) const summary = await (options.daemon ? checkThroughDaemon : checkCold)( options, ) diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index 9361d23..c25528f 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -41,6 +41,10 @@ const suite = `oracle-gaps.report` export const shards = { // 522 s in the measured run, about 221 s idle. 'e2e-1': [ + `fixed seed 20261105: a stop with the default output directory`, + `fixed seed 20261105: a daemon check with the default output directory`, + `fixed seed 20261105: a stream log that cannot be written`, + `fixed seed 20261105: two runs on one output directory`, `fixed seed 20261105: a .cjs config that calls require`, `fixed seed 20261105: a declaration fault: missingSource`, `fixed seed 20261105: a declaration fault: noTypescript`, @@ -106,6 +110,10 @@ export const shards = { ], // 522 s in the measured run, about 221 s idle. 'e2e-3': [ + `random seed: a stop with the default output directory`, + `random seed: a daemon check with the default output directory`, + `random seed: a stream log that cannot be written`, + `random seed: two runs on one output directory`, `fixed seed 20261105: a config fault in a run that owns nothing`, `fixed seed 20261105: a declaration fault: brokenOwner`, `fixed seed 20261105: a declaration fault: configThrows`, @@ -181,6 +189,10 @@ export const shards = { ], // 522 s in the measured run, about 221 s idle. 'e2e-5': [ + `determinism: constructed history: a stop with the default output directory`, + `determinism: constructed history: a daemon check with the default output directory`, + `determinism: constructed history: a stream log that cannot be written`, + `determinism: constructed history: two runs on one output directory`, `fixed seed 20261105: an edit sequence through the daemon`, `fixed seed 20261105: ordinary tests left out by default`, `fixed seed 20261105: two authored mutants`, @@ -267,6 +279,8 @@ function listed() { const run = spawnSync( `pnpm`, [ + // --silent keeps pnpm's workspace banner off stdout, which is JSON. + `--silent`, `exec`, `vitest`, `list`,