From 9bcb4c54c812cd2dd91dbcfbc75f9b3f5dd9a452 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Wed, 4 Nov 2026 11:59:17 -0600 Subject: [PATCH 1/2] Stop fetching patch views during hosted scans A hosted scan fetched the full patch view (every patched file's blob content) for each confirmed redirect, only to judge installed copies and feed the same run's VEX. It now makes no view call at all: - In-run `--vex` fetches the records it attests from itself, through the same record path the standalone `vex` uses, so the scan pipeline never asks for them. - The gem stale-install probe judges an installed gem by the archive RubyGems installed it from (`/cache/.gem`) against the patched artifact's sha256. - The Python stale-install probe downloads the patched wheel the lock now pins, through its grant URL and verified against the pinned sha256, and compares the installed copy with the wheel's own RECORD when the installed WHEEL tags match. It downloads only when a copy is installed. - Both probes still treat an agent-mode manifest record of the same patch as proof of an in-place patch. - The vlt warm-tree heal judges a store entry by vlt's hidden lock alone: the pinned integrity there is healthy, another is stale, and without a readable hidden lock the copy is kept and reported as unchecked. - The in-memory hosted engine no longer calls the provider's `fetchPatch` or warns `record_fetch_failed`. The Node binding still accepts a `fetchPatch` provider method and ignores it, so existing providers keep working. Assisted-by: Claude Code:claude-opus-5-5 --- crates/socket-patch-cli/CLI_CONTRACT.md | 8 +- .../src/commands/scan/hosted.rs | 257 +++++++++------- .../src/commands/scan/hosted/python.rs | 235 +++++++++++++-- crates/socket-patch-cli/src/commands/vex.rs | 33 +-- .../src/commands/vex_sources.rs | 45 ++- .../socket-patch-cli/src/commands/vlt_heal.rs | 27 +- .../tests/common/patched_wheel.rs | 104 +++++++ .../tests/e2e_redirect_gem_stale_install.rs | 98 ++++--- .../tests/hosted_memory_engine.rs | 14 +- .../tests/in_process_redirect.rs | 31 +- .../tests/in_process_redirect/vlt.rs | 48 ++- .../tests/in_process_redirect_pdm.rs | 41 ++- .../tests/in_process_redirect_pipenv.rs | 42 ++- .../tests/in_process_redirect_poetry.rs | 44 ++- .../scan/scan_ordered_concurrency_e2e.rs | 40 +-- .../tests/vlt_e2e_common/fixture.rs | 4 +- .../tests/vlt_hosted_common/mod.rs | 4 +- crates/socket-patch-core/src/api/client.rs | 7 - crates/socket-patch-core/src/hosted/engine.rs | 13 - .../src/hosted/memory/discover.rs | 30 +- .../src/hosted/memory/mod.rs | 38 +-- crates/socket-patch-core/src/hosted/mod.rs | 1 + .../src/hosted/wheel_record.rs | 274 ++++++++++++++++++ .../src/patch/redirect/vlt_heal.rs | 237 +++------------ crates/socket-patch-node/npm/index.d.ts | 5 +- crates/socket-patch-node/npm/index.js | 1 - crates/socket-patch-node/npm/test/smoke.mjs | 5 +- crates/socket-patch-node/src/lib.rs | 3 - crates/socket-patch-node/src/provider.rs | 17 +- docs/testing/poetry-compatibility.md | 2 +- 30 files changed, 1063 insertions(+), 645 deletions(-) create mode 100644 crates/socket-patch-cli/tests/common/patched_wheel.rs create mode 100644 crates/socket-patch-core/src/hosted/wheel_record.rs diff --git a/crates/socket-patch-cli/CLI_CONTRACT.md b/crates/socket-patch-cli/CLI_CONTRACT.md index 13574a1da..2d2e22569 100644 --- a/crates/socket-patch-cli/CLI_CONTRACT.md +++ b/crates/socket-patch-cli/CLI_CONTRACT.md @@ -274,7 +274,7 @@ For a **9.0 root lock**, the CLI ensures `pnpm-workspace.yaml` carries `trustLoc `redirect_pnpm_no_lockfile` names pnpm when installer markers exist without a lock; `redirect_pnpm_entry_vendored` identifies a vendored entry instead of reporting it missing. Supported `shrinkwrap.yaml` files are writable lockfiles, not read-only markers. -**vlt hosted-mode contract**: `scan` / `get --mode hosted` rewrite, in `vlt-lock.json`, every default-registry node of a granted `name@version` (the `''` / `npm` segment or a URL segment equal to the lock's scalar `registry`, both DepID grammars, every peer and modifier variant): slot [2] becomes the granted sha512 and slot [3] the hosted URL (appended to a 3-tuple); the DepID, flags and trailing slots, the line ending and every other byte stay. `options` is never edited and `vlt.json` is only read. A lock with another `lockfileVersion` (decided on the raw JSON token), a BOM, a non-object body or a `nodes` section outside vlt's one-node-per-line layout refuses the whole lock (`redirect_vlt_lock_unsupported`). **Confirmation**: vlt drives when its install state (`node_modules/.vlt-lock.json` or `node_modules/.vlt/`) is present or no other npm-family lock is; then only `vlt-lock.json` confirms a uuid. Otherwise every lock is rewritten, `redirect_vlt_sibling_lockfiles` warns, and the other locks' rules confirm, including a dep `vlt-lock.json` merely does not wire (`redirect_vlt_entry_not_found`, `redirect_vlt_entry_vendored`, or `redirect_vlt_custom_registry_skipped` alone). Whichever lock drives, a dep the vlt rewriter refuses (`redirect_vlt_missing_sha512`, `redirect_vlt_unsupported_lock_key`) is never confirmed by any lock, although a sibling lock may already carry its rewritten URL. **Artifacts**: the scan never downloads the hosted artifacts; the patch server sends them unencoded (`Cache-Control: no-transform`), and vlt verifies slot [2] against the bytes it receives at install time. **Heal**: stale installed copies of Socket-owned nodes are removed so the next `vlt install` extracts the patched bytes, and `rollback` / `remove` do the same for the registry bytes (`--no-vlt-install-cleanup` keeps them; optional dependencies' copies are always kept); `redirect_vlt_reinstall_required` says what happened and what to run. The same-run `--vex` never attests a vlt package whose installed copy is stale or unchecked, whose lock a vlt release may ignore (`redirect_vlt_lockfile_version_missing`, `redirect_vlt_old_lockfile_ignored`, `redirect_vlt_scalar_registry_ignored`), or which also resolves from a non-default registry (`redirect_vlt_custom_registry_skipped`). `vlt.json` or vlt install state without `vlt-lock.json` warns `redirect_vlt_no_lockfile` instead of `redirect_npm_no_lockfile`. `rollback` / `remove` restore each hosted node's slots [2] and [3] from the version document of the registry the node resolves against (slot [3] is its `dist.tarball`, as vlt writes it; `upstream_registry_fallback` when that registry can't be read), following the lock's own slot-[3] convention (see "Hosted unwind coverage"). Tested releases: `docs/testing/vlt-compatibility.md`. +**vlt hosted-mode contract**: `scan` / `get --mode hosted` rewrite, in `vlt-lock.json`, every default-registry node of a granted `name@version` (the `''` / `npm` segment or a URL segment equal to the lock's scalar `registry`, both DepID grammars, every peer and modifier variant): slot [2] becomes the granted sha512 and slot [3] the hosted URL (appended to a 3-tuple); the DepID, flags and trailing slots, the line ending and every other byte stay. `options` is never edited and `vlt.json` is only read. A lock with another `lockfileVersion` (decided on the raw JSON token), a BOM, a non-object body or a `nodes` section outside vlt's one-node-per-line layout refuses the whole lock (`redirect_vlt_lock_unsupported`). **Confirmation**: vlt drives when its install state (`node_modules/.vlt-lock.json` or `node_modules/.vlt/`) is present or no other npm-family lock is; then only `vlt-lock.json` confirms a uuid. Otherwise every lock is rewritten, `redirect_vlt_sibling_lockfiles` warns, and the other locks' rules confirm, including a dep `vlt-lock.json` merely does not wire (`redirect_vlt_entry_not_found`, `redirect_vlt_entry_vendored`, or `redirect_vlt_custom_registry_skipped` alone). Whichever lock drives, a dep the vlt rewriter refuses (`redirect_vlt_missing_sha512`, `redirect_vlt_unsupported_lock_key`) is never confirmed by any lock, although a sibling lock may already carry its rewritten URL. **Artifacts**: the scan never downloads the hosted artifacts; the patch server sends them unencoded (`Cache-Control: no-transform`), and vlt verifies slot [2] against the bytes it receives at install time. **Heal**: an installed copy is stale when vlt's hidden lock (`node_modules/.vlt-lock.json`) does not record its DepID at the integrity the lock pins, and cannot be checked without a readable hidden lock; stale installed copies of Socket-owned nodes are removed so the next `vlt install` extracts the patched bytes, and `rollback` / `remove` do the same for the registry bytes (`--no-vlt-install-cleanup` keeps them; optional dependencies' copies are always kept); `redirect_vlt_reinstall_required` says what happened and what to run. The same-run `--vex` never attests a vlt package whose installed copy is stale or unchecked, whose lock a vlt release may ignore (`redirect_vlt_lockfile_version_missing`, `redirect_vlt_old_lockfile_ignored`, `redirect_vlt_scalar_registry_ignored`), or which also resolves from a non-default registry (`redirect_vlt_custom_registry_skipped`). `vlt.json` or vlt install state without `vlt-lock.json` warns `redirect_vlt_no_lockfile` instead of `redirect_npm_no_lockfile`. `rollback` / `remove` restore each hosted node's slots [2] and [3] from the version document of the registry the node resolves against (slot [3] is its `dist.tarball`, as vlt writes it; `upstream_registry_fallback` when that registry can't be read), following the lock's own slot-[3] convention (see "Hosted unwind coverage"). Tested releases: `docs/testing/vlt-compatibility.md`. **Takeover reconciliation (every hosted ecosystem, v5.0)**: plain `vendor` with at least one hosted pin in scope is not a per-purl takeover: it EJECTS exactly the lockfile's hosted pins as one planned transition (see "Vendor command contract") and ignores `.socket/manifest.json` even when it exists — a manifest-only patch is not vendored, and a manifest naming a different patch for a hosted purl does not displace the hosted one (vendoring the manifest's patches takes a full `scan --mode vendored`, whose edits then land in the lockfiles). The eject runs the same upstream restore and the same backend previews and refusals described below, but reports them as an eject (`eject_planned` / `eject_refused` / `eject_rolled_back`) rather than through the per-purl takeover advisories. Vendoring over a hosted pin with `scan --mode vendored` / `get --mode vendored` (and the legacy manifest-driven `vendor` path, reached only when no hosted pin is in scope) first RESTORES that purl's lock entries to their default upstream registry entry — the same restore `rollback` runs (core `patch::redirect::upstream::restore_upstream`; see "Hosted unwind coverage"), over the hosted pins lockfile discovery finds (v5 keeps no hosted ledger) — and then vendors, so the vendor ledger records the PRISTINE registry entry as its wiring `original` and `vendor --revert` lands back on upstream registry state, never on hosted. The run that takes over records a `vendor_takeover_reverted_redirect` advisory event (`skipped` action beside the purl's genuine outcome; detail ` was hosted; restored its upstream registry entry () before vendoring (mode takeover)`; the human path prints `Warning: …`), plus any advisory the restore raised (`npm_allow_remote_left`, …). **A takeover the vendored backend does not carry through keeps the hosted pin (#853, #944)**: the wet run holds the restore in its group commit, and when the backend then refuses the purl — whatever the code: a pnpm `catalog:` dependency (`vendor_lock_entry_unsupported`), a CRLF `pnpm-lock.yaml` (`vendor_lockfile_crlf_unsupported`), a conflicting `pnpm-workspace.yaml` override — a range, another version or a `>` selector (`vendor_override_conflict`), a uv inline `[tool.uv] sources` table, a prebuilt download that fails, … — or its apply fails with nothing recorded, the restore is rolled back before anything reaches disk: the purl is reported `failed ` with the backend's own code and detail, neither `vendor_takeover_reverted_redirect` nor the restore's advisories are recorded for it, and the hosted wiring stays byte-for-byte (exit 1 / `partialFailure`), so the package stays hosted-patched instead of being un-hosted and then refused. `vendor --dry-run` previews that same `failed ` by running the backend's dry run over the restored project staged in memory (nothing written). A restore that writes a file outside the group commit's captured set (`.socket/gradle/hosted-index.tsv`) is not rolled back. The `scan` / `get --mode vendored --dry-run` preview, which stages no restore (it never touches the registry), predicts the pnpm (lockfileVersion 9) part of this (#853): a hosted pin the pnpm vendored backend refuses on lock or manifest text (a `catalog:` dependency, a CRLF `pnpm-lock.yaml` / `pnpm-workspace.yaml`, a conflicting override, …) is listed `would_refuse` with the wet run's `errorCode` and the backend's own `error` detail, evaluated on the still-hosted project (the pnpm restore only rewrites the entry's `resolution:`, which none of those gates reads); a hosted pin of any other lock flavor or ecosystem — a legacy pnpm 7/8 lock (`lockfileVersion` 5.4 / 6.0) included, whose CRLF, override-conflict and entry refusals are not lock-text gated — still previews `would_vendor` even when the wet takeover refuses it (the wet refusal keeps the hosted pin), except for the gem preflight below. **Gem preflight before the takeover**: `scan` / `get --mode vendored` ask the gem vendored backend's own refusals BEFORE the upstream restore — the manifest gate (`gemfile_not_loaded`: a `gems.rb` twin or a `BUNDLE_GEMFILE`-configured manifest) and the Gemfile declaration gate evaluated on the Gemfile and Gemfile.lock text the restore would leave (`gemfile_declaration_not_editable`: a declaration inside a `group` / `platforms` / conditional block, a parenthesized or duplicate declaration, …) — so a hosted gem vendored mode cannot wire is reported `failed ` with the hosted `Gemfile` / `Gemfile.lock` byte-untouched (exit 1 / `partialFailure`), and their `--dry-run` preview reports that gem as `would_refuse` with the same `errorCode` (exit 0, like the Bun / vlt `would_refuse` rows), never `would_vendor`. Pinned against real Bundler by `tests/e2e_redirect_gem_build.rs`. `--dry-run` resolves the same restore without writing (registry lookups included): a pin that would restore reports `vendor_would_revert_redirect`, and one that would be refused surfaces in the preview with the wet run's `redirect_revert_failed` code and detail (for bun, whose hosted rewrite replaces the entry's `name@version` spec, the preview first runs the Bun vendored preflight described below and then stops at the advisory instead of reading the still-hosted lock — a lock the vendored backend would refuse is previewed as the wet run's `failed `, never as `vendor_would_revert_redirect`). A purl whose upstream entry cannot be restored — `--offline`, a registry that does not answer, a lock the restore refuses (see "Hosted unwind coverage"; a hosted binary `bun.lockb` pin IS restored for the takeover — its npm registry record is rebuilt natively — while `rollback` / `remove` refuse it) — fails `redirect_revert_failed` with the detail `cannot vendor over the live hosted pin: cannot restore to its upstream registry entry: ; restore it from version control instead (`git checkout -- `)` (for `bun.lock` / `bun.lockb` the detail also adds `, then run \`bun install --force\` (a plain \`bun install\` keeps the patched copy)`) (exit 1 / `partialFailure`, nothing vendored for it, the hosted wiring left in place). The cargo backend's `hosted_redirect_live` refusal backstops a crate whose hosted residue is still in place when it is reached; its detail names `socket-patch rollback` and `git checkout -- Cargo.toml Cargo.lock`. **Bun vendored preflight before the takeover**: `vendor` — like `scan` / `get --mode vendored`, whose pre-download preflight runs earlier — checks `bun.lock` / `bun.lockb` with the shared Bun vendored preflight BEFORE the upstream restore, so a hosted purl on a lock the vendored backend refuses (a pre-version-2 `workspace:` lock → `vendor_bun_workspace_unsupported`; a malformed or unsupported binary lock → `vendor_bun_lockb_invalid`; an unsupported text-lock version → its code) is reported `failed ` with the hosted wiring and active Bun lock byte-untouched (exit 1 / `partialFailure`): the package stays hosted-patched instead of being un-hosted and then refused. `vendor --dry-run` previews that same `failed` code (exit-code parity with the wet run, nothing written) instead of promising `vendor_would_revert_redirect`. Pinned by `tests/in_process_vendor_bun_takeover.rs` and, against real Bun, `tests/mode_migration_bun.rs`. The npm package-lock backend's lock gate gets the same placement: a hosted pin in a project whose `npm-shrinkwrap.json` / `package-lock.json` is not a v2/v3 lock (npm 6's lockfileVersion 1) is refused `failed vendor_lockfile_version_unsupported` BEFORE the restore, in `vendor`, `scan --mode vendored` and `get --mode vendored` alike, so the package stays hosted-patched; the vendored dry-run preview lists every npm purl of such a project as `would_refuse` with that code. Pinned by `tests/in_process_vendor_npm_v1_takeover.rs`. Hosted → vendored and vendored → hosted (`redirect_takeover_reverted_vendored` in `warnings[]`) both work in place on the locks the target mode accepts. **Removed in v5.0**: the run-level `vendor_supersedes_redirect` warning and its reconcile of the redirect ledger (a live lock that already proved vendored won over a stale hosted ledger record) — once the lock routes a package to `.socket/vendor/`, no hosted state is left to go stale. Which way the live lock points is decided by the same lockfile discovery rules `vex` gates attestations on (see "Manifest-less VEX (lockfile discovery)"), for `redirect_supersedes_vendored` and `hosted_wiring_retained` alike. @@ -335,7 +335,7 @@ The rewriter reads a fixed set of candidate files from the project root: the npm **Non-UTF-8 candidate files (#721)**: the rewriters edit UTF-8 text only. A candidate file that exists but is not UTF-8 (for example a UTF-16 `requirements.txt`, which is what Windows PowerShell 5.1's `pip freeze >` writes and which pip installs from) is never read as absent. When a candidate of its ecosystem could rewrite it, the run is refused with `candidate_file_unreadable` (exit 1, `--dry-run` included), the message names the file, and nothing is written; the remedy is to re-save the file as UTF-8. Two exceptions keep their own refusals. A Gradle build file the Gradle planner reaches gets that planner's per-build refusal (`redirect_gradle_build_file_unreadable`, exit 0) and the rest of the run goes ahead, and with no readable Gradle build a stray Gradle file (a lock, a nested script) is never rewritten, so it does not refuse the run, while a non-UTF-8 root `settings.gradle(.kts)` or `build.gradle(.kts)` still refuses it (it may be the build itself). An unreadable `socket-patch.sbt` is refused with `redirect_sbt_owned_file_unreadable`. A vendored→hosted takeover checks this before it reverts anything, so a refused run leaves the vendored wiring, ledger entry and artifact byte-identical. Vendored mode likewise refuses a non-UTF-8 `requirements.txt` or `-r` include by name (`pypi_no_requirements`) instead of wiring around it. Lock-only discovery reads `requirements.txt` and its in-root `-r` includes the way pip decodes them (a UTF-16 or UTF-32 byte-order mark selects that encoding), so such a project's pins are still found instead of reporting "No packages found". -**Gem stale-install guard (additive warning — the canonical narrative; other mentions point here)**: the gem hosted rewrite is pure Gemfile/lock text, so a gem ALREADY materialized under the project's bundle paths keeps its upstream bytes — the next `bundle install` prints `Using ` and never refetches, on **every** bundler major (live-verified 2026-08-19 on 1.17.3 / 2.7.2 / 4.0.18: bundler 4's CHECKSUMS verify at download time only, and nothing is downloaded; `bundle install --force`/`--redownload` re-install from the stale cached `.gem` instead of re-fetching — bundler 1 silently, bundler 4 with an exit-37 checksum refusal that still leaves the upstream bytes installed; the **verified** remedy is removing the installed dir + cache `.gem` + `specifications` entry, then `bundle install`). After the rewrite, a hosted run therefore probes the installed-gem discovery paths (the same ruby-crawler discovery `apply` uses, honoring `--global`/`--global-prefix` like scan's own discovery, plus — read-only — a `.bundle/config` bundle path refused as a write root because it resolves outside the project, which takes the project-local remedy; the `gem env` homes count only when Bundler uses system gems, i.e. no deployment store under `vendor/bundle`, and the first settings tier (app config, environment, global config) that sets `path`, `path.system` or `disable_shared_gems` doesn't set a non-empty `path` without `path.system: true` or `disable_shared_gems: false`, and, when no tier sets any of those, no truthy `deployment` (the first tier that sets it wins; it makes `vendor/bundle` the path), since with such a `path` `bundle install` fetches non-default gems into it and never reuses a system copy; the `.bundle` default that `default_install_uses_path` (Bundler 2.x) or `simulate_version 5` (Bundler 4.x) selects depends on the Bundler that runs, so those flags keep the `gem env` homes judged) for each confirmed gem redirect and judges the materialization against the patch record's `afterHash` file map. Judgment rules: records are found **by uuid** among this run's fetched records (v5.0: hosted mode persists no records, so a purl whose `/patches/view` fetch failed this run is not judged; the warning re-fires on every re-scan whose fetch succeeds, until the stale materialization is gone); a materialization with every file at `afterHash` is already patched and never warns (an agent→hosted migration stays quiet by construction), and when several confirmed variant purls resolve to one installed dir, ANY of them judging it patched keeps it quiet; staleness needs **positive evidence** — at least one record file whose bytes were actually read and hash to neither state's expectation — so missing or unreadable files never produce a warning. Warnings emit `redirect_gem_stale_install` (JSON `warnings[]` + a code-tagged stderr line) in three flavors: a PROJECT-LOCAL dir (under the project root, compared on absolute paths so the default `--cwd .` counts, or under the project's own refused `.bundle/config` path) gets the verified delete-list remedy (installed dir, cache `.gem`, `specifications` entry — plus the project's committed `/.gem` when present and not proven to be the patched artifact, since bundler installs from its cache dir in preference to fetching); a SHARED gem-env home gets a caveat that the home is shared machine-wide and prefers migrating the project to a local bundle path over deleting shared files; and a committed cache-dir archive whose sha256 differs from the patched artifact's warns standalone even with no installed dir at all (a fresh checkout with a committed stale cache re-materializes the upstream bytes forever). A stale-flagged purl is additionally **excluded from the same run's `--vex` `assume_applied` set** — the envelope must never attest a CVE its own warning says is live; the purl falls back to normal installed-tree verification (a patched install still attests, a stale one is omitted). Standalone `vex` and `apply --check` judge gem copies by the same rule (#1098): when Bundler doesn't use system gems, a copy in a `gem env` home is not one the project loads and is not verified, unless it is a default gem (its spec under `specifications/default/`), which Bundler loads from the system home under any path. The cache dir is bundler's `cache_path` setting (`Bundler.app_cache`), resolved in `Bundler::Settings` priority: `BUNDLE_CACHE_PATH:` in the bundler app config (`$BUNDLE_APP_CONFIG/config`, else `.bundle/config`) first, then the `BUNDLE_CACHE_PATH` environment variable, then `BUNDLE_CACHE_PATH:` in the global config (`bundle config set --global`: `$BUNDLE_CONFIG`, else `$BUNDLE_USER_CONFIG`, else `$BUNDLE_USER_HOME/config`, else `~/.bundle/config`), else `vendor/cache`; a relative value is read against the project root. The same global tier, below the app config and the environment, applies to `BUNDLE_GEMFILE:` and, for agent-mode install-root discovery, to `BUNDLE_PATH:`. A present local or environment `path`, `path.system`, or `disable_shared_gems` setting (including an empty string or false flag) shadows the global path tier, matching the tested Bundler 2.6/4 behavior; Bundler 1.x's legacy global-path shortcut is not modeled. An empty higher-tier `gemfile` setting also shadows the global value but leaves an existing nonempty `BUNDLE_GEMFILE` environment value in effect, or uses default manifest discovery when there is none. With `BUNDLE_IGNORE_CONFIG` set (any value) bundler reads no config file, so the app and global configs are skipped here too and only the environment and the default count — the same holds for the `BUNDLE_GEMFILE:` app-config setting. The probe is read-only (nothing is deleted) and skipped on `--dry-run` — deliberately explicit, since nothing was rewritten. Exit code and `status` are unchanged (warning-only, the hosted-refusal posture); a same-run `--vex` may still fail on "nothing to attest" per the embedded-VEX contract. +**Gem stale-install guard (additive warning — the canonical narrative; other mentions point here)**: the gem hosted rewrite is pure Gemfile/lock text, so a gem ALREADY materialized under the project's bundle paths keeps its upstream bytes — the next `bundle install` prints `Using ` and never refetches, on **every** bundler major (live-verified 2026-08-19 on 1.17.3 / 2.7.2 / 4.0.18: bundler 4's CHECKSUMS verify at download time only, and nothing is downloaded; `bundle install --force`/`--redownload` re-install from the stale cached `.gem` instead of re-fetching — bundler 1 silently, bundler 4 with an exit-37 checksum refusal that still leaves the upstream bytes installed; the **verified** remedy is removing the installed dir + cache `.gem` + `specifications` entry, then `bundle install`). After the rewrite, a hosted run therefore probes the installed-gem discovery paths (the same ruby-crawler discovery `apply` uses, honoring `--global`/`--global-prefix` like scan's own discovery, plus — read-only — a `.bundle/config` bundle path refused as a write root because it resolves outside the project, which takes the project-local remedy; the `gem env` homes count only when Bundler uses system gems, i.e. no deployment store under `vendor/bundle`, and the first settings tier (app config, environment, global config) that sets `path`, `path.system` or `disable_shared_gems` doesn't set a non-empty `path` without `path.system: true` or `disable_shared_gems: false`, and, when no tier sets any of those, no truthy `deployment` (the first tier that sets it wins; it makes `vendor/bundle` the path), since with such a `path` `bundle install` fetches non-default gems into it and never reuses a system copy; the `.bundle` default that `default_install_uses_path` (Bundler 2.x) or `simulate_version 5` (Bundler 4.x) selects depends on the Bundler that runs, so those flags keep the `gem env` homes judged) for each confirmed gem redirect and judges the materialization by the archive RubyGems installed it from, `/cache/.gem`: an archive whose sha256 is a patched artifact's (the grant's `gemChecksumSha256`, else its tarball sha256) is already patched and never warns, any other readable archive is stale, and a missing or unreadable one is never evidence. Hosted mode fetches no patch record for this: the warning re-fires on every re-scan until the stale materialization is gone. An agent-mode manifest record of the same patch judges the dir by its `afterHash` file map instead (an in-place patch keeps the upstream archive), so an agent→hosted migration stays quiet; when several confirmed variant purls resolve to one installed dir, ANY of them judging it patched keeps it quiet; staleness needs **positive evidence** — bytes actually read that are not the patched state — so missing or unreadable files never produce a warning. Warnings emit `redirect_gem_stale_install` (JSON `warnings[]` + a code-tagged stderr line) in three flavors: a PROJECT-LOCAL dir (under the project root, compared on absolute paths so the default `--cwd .` counts, or under the project's own refused `.bundle/config` path) gets the verified delete-list remedy (installed dir, cache `.gem`, `specifications` entry — plus the project's committed `/.gem` when present and not proven to be the patched artifact, since bundler installs from its cache dir in preference to fetching); a SHARED gem-env home gets a caveat that the home is shared machine-wide and prefers migrating the project to a local bundle path over deleting shared files; and a committed cache-dir archive whose sha256 differs from the patched artifact's warns standalone even with no installed dir at all (a fresh checkout with a committed stale cache re-materializes the upstream bytes forever). A stale-flagged purl is additionally **excluded from the same run's `--vex` `assume_applied` set** — the envelope must never attest a CVE its own warning says is live; the purl falls back to normal installed-tree verification (a patched install still attests, a stale one is omitted). Standalone `vex` and `apply --check` judge gem copies by the same rule (#1098): when Bundler doesn't use system gems, a copy in a `gem env` home is not one the project loads and is not verified, unless it is a default gem (its spec under `specifications/default/`), which Bundler loads from the system home under any path. The cache dir is bundler's `cache_path` setting (`Bundler.app_cache`), resolved in `Bundler::Settings` priority: `BUNDLE_CACHE_PATH:` in the bundler app config (`$BUNDLE_APP_CONFIG/config`, else `.bundle/config`) first, then the `BUNDLE_CACHE_PATH` environment variable, then `BUNDLE_CACHE_PATH:` in the global config (`bundle config set --global`: `$BUNDLE_CONFIG`, else `$BUNDLE_USER_CONFIG`, else `$BUNDLE_USER_HOME/config`, else `~/.bundle/config`), else `vendor/cache`; a relative value is read against the project root. The same global tier, below the app config and the environment, applies to `BUNDLE_GEMFILE:` and, for agent-mode install-root discovery, to `BUNDLE_PATH:`. A present local or environment `path`, `path.system`, or `disable_shared_gems` setting (including an empty string or false flag) shadows the global path tier, matching the tested Bundler 2.6/4 behavior; Bundler 1.x's legacy global-path shortcut is not modeled. An empty higher-tier `gemfile` setting also shadows the global value but leaves an existing nonempty `BUNDLE_GEMFILE` environment value in effect, or uses default manifest discovery when there is none. With `BUNDLE_IGNORE_CONFIG` set (any value) bundler reads no config file, so the app and global configs are skipped here too and only the environment and the default count — the same holds for the `BUNDLE_GEMFILE:` app-config setting. The probe is read-only (nothing is deleted) and skipped on `--dry-run` — deliberately explicit, since nothing was rewritten. Exit code and `status` are unchanged (warning-only, the hosted-refusal posture); a same-run `--vex` may still fail on "nothing to attest" per the embedded-VEX contract. **Pipenv hosted redirect (`Pipfile.lock`, pipfile-spec 6)**: every category other than `_meta` (`default`, `develop`, and Pipenv 2022+ named categories) that pins the package at the patched version is rewritten to the hosted reference — `{"file" | "path": "#sha256=", "hashes": ["sha256:"]}` with `markers`/`extras`/`index` kept exactly as Pipenv wrote them (present or absent: whether Pipenv records `index` depends on its release, the Pipfile spelling and the locking environment, so only the entry itself knows) and `version` dropped; `_meta` (the Pipfile content hash) and the Pipfile itself are never touched, so `pipenv install --deploy`/`sync`/`verify` keep passing. The reference KEY depends on the installing Pipenv: releases 7–11 only install `path` references, 2018 and later `file` ones (0–6 write pipfile-spec < 6 and are refused). The release is probed once per command with `pipenv --version`, resolved on ABSOLUTE `PATH` entries only (a relative entry would run a `pipenv` planted in the scanned repository; `.bat`/`.cmd` shims are found through `PATHEXT` on Windows), only when a pypi patch actually targets an entry of the lock, and `SOCKET_PIPENV_MAJOR=` pins the answer without spawning anything. An unknown installer selects `file` and warns `redirect_pipenv_installer_unknown` only when the lock was rewritten. **Refusal scope**: a pin/source CONFLICT (another version pinned, a foreign `file`/`path` source, a VCS/editable dependency) refuses the whole dependency atomically across categories as `redirect_pipenv_refused` AND vetoes the sibling Python rewriters (requirements.txt / uv.lock / pyproject) for that patch — the project's Pipenv install could not pick the patch up, so a half-redirected checkout is refused; anything else (no entry for the package, an old pipfile-spec, an unparseable lock, a digest-less patch) is `redirect_pipenv_skipped` and leaves the siblings alone (a stale Pipfile.lock in a uv/Poetry/requirements project must not block them). The veto applies to a LIVE lock only: a `Pipfile.lock` with no `Pipfile` beside it is abandoned, so its conflict refuses that file but never the siblings. Hash enforcement at install time is split by era — the `#sha256=` URL fragment is what Pipenv 2023+ verifies, the `hashes` list what 2018–2022 verify, Pipenv 11 either — so both are load-bearing. **Pipenv stale-install guard**: Pipenv never reinstalls a release that is already present (`pipenv install`, `install --deploy` and `sync` all exit 0 and keep the installed bytes — measured on 11.10.4, 2018.11.26 and 2026.8.0, hosted and vendored), so after the rewrite the run probes the Python crawler's site-packages (VIRTUAL_ENV, `./.venv`, `./venv`, Pipenv's out-of-tree `WORKON_HOME` venv; `--global`/`--global-prefix` honoured) for each confirmed Pipfile.lock redirect with the same rules as the gem guard (records by uuid from this run's fetch, PATCHED = `verify_patch_record` Ok, STALE needs positive evidence, read-only, skipped on `--dry-run`, stale purls excluded from the same-run `--vex` `assume_applied` set) and the Python stale-install guard (`redirect_pypi_stale_install`, see above) names the site-packages dir and the Pipenv-specific verified remedy: `pipenv run pip uninstall -y && pipenv sync` (or `pipenv --rm && pipenv sync`), with the `sync` arguments following the lock, since plain `pipenv sync` installs only `default`: the targeted form re-syncs the categories that pin the package (`--dev` for `develop`, `--categories ""` for a named category) and the `--rm` form re-syncs every non-empty category — NOT `pipenv uninstall`, which rewrites the Pipfile and re-locks the patch away. The vendored backend emits the twin `pypi_pipenv_stale_install` (`skipped` warning event). **Rollback** (v5.0, upstream restore): each hosted entry gets its registry shape back — `"version": "=="`, the entry's own `index` carried back unchanged (refused unless it — and the Pipfile's explicit `index`, if any — names a PyPI source in `_meta.sources`), and every release file's sha256 from PyPI's JSON API (`SOCKET_PYPI_JSON_API`), sorted by filename as Pipenv records them; an entry that pins another version beside the hosted reference is refused with the `git checkout` remedy (see "Hosted unwind coverage"). A Pipfile names no project, so a same-run `--vex` on a Pipenv project needs `--vex-product` (or a git remote) to detect a product purl. **Discovery**: `Pipfile.lock` is part of the lockfile inventory (every category's `==` pins, with the lock's digest set as `Sha256AnyOf` integrity so a lock-only checkout can be vendored by fetching the pure wheel through PyPI's JSON API — only when `_meta.sources` name the public index; a private-index lock stays discovery-only and never reaches pypi.org), and Socket's own hosted / vendored references stay discoverable as the package they replace, so a re-scan of an already-redirected or already-vendored lock-only checkout re-confirms it (`--vex` attests, vendored reports `already_vendored`) instead of finding nothing. @@ -357,7 +357,7 @@ The rewriter reads a fixed set of candidate files from the project root: the npm v5.0 removed the v4 spellings `scan --apply` (use `--mode agent`), `scan --vendor` (use `--mode vendored`), `get --no-apply` (use `--save-only`), and the `download` (use `get`) and `gc` (use `repair`) subcommand aliases, with no deprecation release. Each is now an ordinary clap usage error (exit 2). -**Python stale-install guard**: after a hosted redirect, `scan` / `get` use the Python crawler to inspect every matching installed package, including Poetry's out-of-tree virtualenvs, the project's Hatch environments (Hatch's data dir / `HATCH_DATA_DIR`, `[dirs.env] virtual`, explicit env `path`s) and `--global-prefix`. A readable file that differs from the patch's `afterHash` emits `redirect_pypi_stale_install` in JSON `warnings[]` and human stderr. The probe changes no installed files, re-runs on idempotent scans, and falls back to persisted patch records when fresh record fetching fails. Missing/unreadable files alone do not prove staleness; lock-only checkouts stay quiet. Dry runs skip the probe. Same-run VEX excludes positively stale Python packages (qualifier-insensitive), even with `--vex-no-verify` or a healthy copy in another interpreter; if nothing remains to attest, the command exits 1 with `no_applicable_patches`. Reinstall from the rewritten lock in the affected interpreter and verify with `socket-patch vex`. A stale Hatch environment instead names `hatch env remove ` / `hatch env prune`: Hatch's pip installer (and uv before Hatch 1.16) keeps a same-version release, so only a recreated env picks up the patch. The same Hatch envs are what agent mode patches and `vex` judges for a Hatch project. +**Python stale-install guard**: after a hosted redirect, `scan` / `get` use the Python crawler to inspect every matching installed package, including Poetry's out-of-tree virtualenvs, the project's Hatch environments (Hatch's data dir / `HATCH_DATA_DIR`, `[dirs.env] virtual`, explicit env `path`s) and `--global-prefix`. Each installed copy is judged against the patched wheel the lock now pins: downloaded through its grant URL, verified against the pinned sha256, and compared with the wheel's own `RECORD` (files outside its `.dist-info` and `.data` dirs), but only when the installed `.dist-info/WHEEL` carries the same tags, since another build's bytes say nothing about this one. An agent-mode manifest record of the same patch judges the copy by its `afterHash` map instead. A readable file that differs from the patched state emits `redirect_pypi_stale_install` in JSON `warnings[]` and human stderr. The probe changes no installed files, re-runs on idempotent scans, fetches no patch record, and downloads a wheel only when a copy is installed. Missing/unreadable files, an unfetchable wheel and another build alone do not prove staleness; lock-only checkouts stay quiet. Dry runs skip the probe. Same-run VEX excludes positively stale Python packages (qualifier-insensitive), even with `--vex-no-verify` or a healthy copy in another interpreter; if nothing remains to attest, the command exits 1 with `no_applicable_patches`. Reinstall from the rewritten lock in the affected interpreter and verify with `socket-patch vex`. A stale Hatch environment instead names `hatch env remove ` / `hatch env prune`: Hatch's pip installer (and uv before Hatch 1.16) keeps a same-version release, so only a recreated env picks up the patch. The same Hatch envs are what agent mode patches and `vex` judges for a Hatch project. ### socket.yml patch policy (v5.0) @@ -1732,7 +1732,7 @@ command's integration tests check its output. | `redirect_unattributable` | `skipped` (hosted event) | scan/get `--mode hosted`: the rewriters would pin the patch, but lockfile discovery over the result reads that pin as contested (another lock or requirements file resolves the same version elsewhere, or the pin is not one the package manager consumes), so `vex`, `rollback`, `remove` and `vendor` would refuse it. The candidate is left out of the rewrite, so nothing is written for it; the detail carries discovery's findings. Exit code unchanged. | | `redirect_copy_unpatched` | `warnings[]` (warning) | scan/get `--mode hosted` (pnpm, yarn): the pin was written, but its own lock also installs an unpatched copy of the same `name@version`, which no rewrite reaches: a pnpm `file:` directory or tarball dependency (#935), or a yarn `file:` tarball / directory, URL tarball or berry `portal:` / `link:` dependency declared under another name (`lp2@file:…`, #939, #1236; not repeated for a copy a yarn classic skip warning already names), or a yarn classic copy installed from a parent's `bundleDependencies` (#758). The detail names the lock entry (for a bundled copy, the parent's entry and the copy's directory). Neither the run's own `--vex` nor a later `vex` attests the package (`vex` warns `patched_ref_unattributable`), but discovery still manages the pin, so `rollback` / `remove` undo it. | | `redirect_pin_lockless` | `warnings[]` (warning) | scan/get `--mode hosted` (nuget, cargo): the pin was written without a lockfile that records its version, so `vex` cannot attest it and `rollback` / `remove` / `vendor` refuse it as unattributable. The detail names the lockfile to create (`dotnet restore --use-lock-file`, `cargo generate-lockfile`) before re-running the hosted scan. | -| `redirect_pypi_stale_install` | `warnings[]` (warning) | Hosted Python redirect: readable installed files differ from patched hashes. Read-only, repeated on re-scan, and excludes the package from same-run VEX. See the "Python stale-install guard" section. | +| `redirect_pypi_stale_install` | `warnings[]` (warning) | Hosted Python redirect: readable installed files differ from the pinned patched wheel's `RECORD`. Read-only, repeated on re-scan, and excludes the package from same-run VEX. See the "Python stale-install guard" section. | | `redirect_nuget_stale_global_package` / `vendor_nuget_stale_global_package` | `warnings[]` (warning, hosted) / `skipped` (warning, vendor advisory event) | scan `--mode hosted` / `vendor` / `scan --mode vendored` (nuget, v5.0 #352): NuGet's global packages folder (`NUGET_PACKAGES`, else `~/.nuget/packages`) already holds the patched package extracted from other bytes (its `.nupkg.metadata` `contentHash` is not the patched one). A patch keeps the upstream id and version and NuGet restores a package already in that folder without asking any source, so `dotnet restore` would keep the upstream bytes (silently without a lock, NU1403 against the re-pinned lock with one). The detail names the directory and the remedy: delete it, then `dotnet restore` (`dotnet nuget locals global-packages --clear` also works but empties the whole machine-wide folder, and the detail says so); CI caches of that folder must drop it too. Read-only like the gem guard (the folder is shared machine-wide), re-fired on every re-run until the copy is gone, skipped on `--dry-run`; a hosted purl it flags is excluded from the same run's `--vex` `assume_applied` and reported stale. A dir without `.nupkg.metadata` (a legacy `packages/` folder) is never judged. | | `redirect_gem_stale_install` | `warnings[]` (warning) | scan `--mode hosted` (gem): a stale UNPATCHED materialization (installed gem, or committed archive in bundler's cache dir — `vendor/cache` unless `cache_path` moves it) that `bundle install` will reuse instead of fetching the redirected patch; the detail carries the verified remedy. Full rules and flavors: the "Gem stale-install guard" section. | | `redirect_gem_version_not_locked` | `warnings[]` (warning) | scan `--mode hosted` (gem): the crawled gem version is installed on the machine but no `GEM` section of the project's lock resolves it (another project's copy in the shared gem home). The gem is skipped and the Gemfile and lock stay byte-identical. | diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index bbbefb94a..54de909fb 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -5,16 +5,12 @@ use std::path::Path; use std::time::Duration; -use futures_util::StreamExt; -use socket_patch_core::api::client::hold_back_debug; use socket_patch_core::api::types::BatchPackagePatches; use socket_patch_core::crawlers::CargoCrawler; use socket_patch_core::patch::apply_lock::LockGuard; use socket_patch_core::patch::redirect::yarnrc::resolve_outer_yarn_mirror_for_process; use socket_patch_core::patch::redirect::DepOverride; -use socket_patch_core::utils::concurrent::{ - api_concurrency, api_concurrency_for, ordered_concurrent, -}; +use socket_patch_core::utils::concurrent::api_concurrency; use socket_patch_core::utils::purl::purl_parts; use crate::commands::agent_download::tag_mode; @@ -282,24 +278,32 @@ fn gem_stale_cache_warning(purl: &str, cache_path: &Path) -> serde_json::Value { /// stale-install guard" section of CLI_CONTRACT.md). Read-only: the remedy /// is prescribed, never applied. /// -/// * Records are found by uuid among this run's fetched records; a uuid -/// whose fetch failed is not judged. +/// * An installed dir is judged by the archive RubyGems installed it from, +/// `/cache/.gem`: one whose sha256 is a patched artifact's is +/// patched, any other readable one is stale. An agent-mode manifest +/// record of the same patch judges the dir by its `afterHash` map instead +/// (an in-place patch keeps the upstream archive). /// * Judgments are grouped by installed dir: platform-variant purls of one -/// gem share a dir, and if any variant's record proves it patched, it is -/// never warned. -/// * Stale needs positive evidence (`stale_evidence`: readable bytes that -/// hash to something other than `afterHash`), never missing or unreadable -/// files. -/// * A committed `vendor/cache/.gem` whose sha256 differs from the -/// patched artifact's is stale too (bundler installs from it first): -/// folded into a project-local warning's delete list, or warned alone. +/// gem share a dir, and if any variant proves it patched, it is never +/// warned. +/// * Stale needs positive evidence (readable bytes that hash to something +/// other than the patched state), never missing or unreadable files. +/// * A committed `vendor/cache/.gem` whose sha256 is not a patched +/// artifact's is stale too (bundler installs from it first): folded into +/// a project-local warning's delete list, or warned alone. async fn gem_stale_install_warnings( cwd: &Path, global: bool, global_prefix: Option, confirmed: &[(String, String)], - records: &std::collections::BTreeMap, - gem_artifact_shas: &std::collections::BTreeMap<(String, String), String>, + local_records: &std::collections::BTreeMap< + String, + socket_patch_core::manifest::schema::PatchRecord, + >, + gem_artifact_shas: &std::collections::BTreeMap< + (String, String), + std::collections::BTreeSet, + >, ) -> StaleInstallOutcome { use socket_patch_core::crawlers::types::CrawlerOptions; use socket_patch_core::crawlers::RubyCrawler; @@ -308,15 +312,20 @@ async fn gem_stale_install_warnings( use socket_patch_core::vex::verify::judge_installed_record; let mut out = StaleInstallOutcome::default(); - let find_record = - |uuid: &str| -> Option<&PatchRecord> { records.values().find(|r| r.uuid == uuid) }; // Return before the crawler (and its `gem env` spawn) when nothing can // be judged. - let candidates: Vec<(&str, &PatchRecord)> = confirmed + let candidates: Vec<(&str, Option<&PatchRecord>)> = confirmed .iter() .filter(|(purl, _)| purl.starts_with("pkg:gem/")) - .filter_map(|(purl, uuid)| find_record(uuid).map(|r| (purl.as_str(), r))) - .filter(|(_, r)| !r.files.is_empty()) + .map(|(purl, uuid)| { + let local = local_records + .values() + .find(|r| &r.uuid == uuid && !r.files.is_empty()); + (purl.as_str(), local) + }) + .filter(|(purl, local)| { + local.is_some() || gem_artifact_shas.contains_key(&gem_sha_key(purl)) + }) .collect(); if candidates.is_empty() { return out; @@ -355,7 +364,7 @@ async fn gem_stale_install_warnings( std::collections::BTreeMap::new(); // Bundler's configured cache dir (`cache_path`, default vendor/cache). let app_cache = socket_patch_core::crawlers::ruby_crawler::bundler_app_cache_dir(cwd).await; - for (index, (purl, record)) in candidates.iter().enumerate() { + for (index, (purl, local)) in candidates.iter().enumerate() { for (found, project_local) in &found_per_home { let Some(pkg) = &found[index] else { continue; @@ -375,7 +384,28 @@ async fn gem_stale_install_warnings( positive: false, project_local: *project_local, }); - let judged = judge_installed_record(&pkg.path, record).await; + let judged = match local { + Some(record) => judge_installed_record(&pkg.path, record).await, + None => { + let archive = pkg + .path + .parent() + .and_then(Path::parent) + .map(|home| home.join("cache").join(format!("{leaf}.gem"))); + let sha = match archive { + Some(archive) => file_sha256_hex(&archive).await, + None => None, + }; + let patched_shas = gem_artifact_shas.get(&gem_sha_key(purl)); + match sha { + Some(sha) => socket_patch_core::vex::verify::InstalledRecordJudgment { + patched: patched_shas.is_some_and(|shas| shas.contains(&sha)), + stale_evidence: patched_shas.is_some_and(|shas| !shas.contains(&sha)), + }, + None => Default::default(), + } + } + }; if judged.patched { entry.patched = true; } else if !entry.positive && judged.stale_evidence { @@ -400,7 +430,7 @@ async fn gem_stale_install_warnings( gem_artifact_shas.get(&gem_sha_key(&j.purl)), file_sha256_hex(&project_cache).await, ) { - (Some(want), Some(got)) => &got == want, + (Some(want), Some(got)) => want.contains(&got), // Unknown sha (or unreadable archive): include it — // removal is safe either way, `bundle install` refetches. _ => false, @@ -431,7 +461,7 @@ async fn gem_stale_install_warnings( if cache_covered.contains(purl) { continue; } - let Some(want_sha) = gem_artifact_shas.get(&gem_sha_key(purl)) else { + let Some(want_shas) = gem_artifact_shas.get(&gem_sha_key(purl)) else { continue; }; let Some((_, name, version)) = purl_parts(purl) else { @@ -445,7 +475,7 @@ async fn gem_stale_install_warnings( let Some(got) = file_sha256_hex(&cache_path).await else { continue; }; - if &got == want_sha { + if want_shas.contains(&got) { continue; // the PATCHED archive — healthy commit, nothing stale } out.warnings @@ -1321,15 +1351,6 @@ async fn run_redirect_selected_untracked( let _lock = lock; let confirmed = done.confirmed.clone(); - // Fetch the full patch view (file hashes + vulnerabilities) for each - // CONFIRMED redirect and persist it so a post-install `socket-patch vex` - // can attest the patch. A fetch failure does not undo the redirect, but - // it leaves the patch unattestable — surface it as a warning (JSON + - // stderr) so CI can detect the attestation gap and re-run. - let mut records: std::collections::BTreeMap = - std::collections::BTreeMap::new(); - let mut record_warnings: Vec = Vec::new(); - // SYMLINK GUARD (see `engine::guard`) — before any write, dry runs // included, so a dry run predicts the refusal. The revert side (the // hosted → upstream restore's staged flush) already refuses linked @@ -1354,44 +1375,6 @@ async fn run_redirect_selected_untracked( ); } - if !common.dry_run { - let total = confirmed.len(); - // The views are fetched concurrently but consumed in `confirmed` - // order, so `records` (newest wins), `record_warnings` and the - // held-back `--debug` lines fold deterministically. Each response is - // reduced to its record inside the window: a view carries every - // file's `blobContent`, which would otherwise be buffered for the - // whole window. - let mut views = std::pin::pin!(ordered_concurrent( - confirmed.iter(), - api_concurrency_for(api_client.uses_public_proxy(), confirmed.len()), - |(_, uuid)| { - hold_back_debug(async move { - api_client.fetch_patch(uuid).await.map(|resp| { - resp.map(|resp| { - socket_patch_core::manifest::records::record_from_patch_response(&resp) - }) - }) - }) - }, - )); - for (i, (purl, _)) in confirmed.iter().enumerate() { - status.set(format!("Fetching patch records... ({}/{total})", i + 1)); - let Some(view) = views.next().await else { - break; - }; - match view.release() { - Ok(Some((rec_purl, record))) => { - records.insert(rec_purl, record); - } - Ok(None) | Err(_) => { - record_warnings.push(engine::record_fetch_failed_warning(purl)); - } - } - } - status.finish(); - } - let rewrite = &done.rewrite; // Paths whose existence the commit below changes as discovery's reads // around the overlaid view (directory listings) see the disk: written @@ -1443,6 +1426,18 @@ async fn run_redirect_selected_untracked( return 1; } + // The agent-mode manifest's records: local proof that an installed copy + // was patched in place, for the stale-install probes below. + let local_records: std::collections::BTreeMap = if common.dry_run { + std::collections::BTreeMap::new() + } else { + socket_patch_core::manifest::operations::read_manifest(common.resolved_manifest_path()) + .await + .ok() + .flatten() + .map(|manifest| manifest.patches.into_iter().collect()) + .unwrap_or_default() + }; // Gem stale-install probe (see `gem_stale_install_warnings`): runs after // the writes so the warning describes the project as this run leaves it. // Idempotent re-scans re-confirm and re-probe, so the warning keeps @@ -1452,28 +1447,32 @@ async fn run_redirect_selected_untracked( let gem_stale: StaleInstallOutcome = if common.dry_run { StaleInstallOutcome::default() } else { - // purl-coordinate → the PATCHED .gem artifact's sha256 (registry - // override identifier, tarball integrity fallback) — judges a - // committed vendor/cache archive. - let gem_artifact_shas: std::collections::BTreeMap<(String, String), String> = done - .overrides - .iter() - .filter(|o| o.ecosystem == "gem") - .filter_map(|o| { - let sha = o - .registry_override - .as_ref() - .and_then(|ro| ro.identifiers.gem_checksum_sha256.clone()) - .or_else(|| o.integrity.sha256.clone())?; - Some(((o.name.clone(), o.version.clone()), sha)) - }) - .collect(); + // purl-coordinate → every PATCHED .gem artifact's sha256 (registry + // override identifier, tarball integrity fallback; one per platform + // variant) — judges the installed and committed archives. + let mut gem_artifact_shas: std::collections::BTreeMap< + (String, String), + std::collections::BTreeSet, + > = std::collections::BTreeMap::new(); + for o in done.overrides.iter().filter(|o| o.ecosystem == "gem") { + let sha = o + .registry_override + .as_ref() + .and_then(|ro| ro.identifiers.gem_checksum_sha256.clone()) + .or_else(|| o.integrity.sha256.clone()); + if let Some(sha) = sha { + gem_artifact_shas + .entry((o.name.clone(), o.version.clone())) + .or_default() + .insert(sha); + } + } gem_stale_install_warnings( &common.cwd, common.global, common.global_prefix.clone(), &confirmed, - &records, + &local_records, &gem_artifact_shas, ) .await @@ -1483,6 +1482,7 @@ async fn run_redirect_selected_untracked( } else { python::stale_install_warnings( common, + api_client, &confirmed, &rewrite.confirmed_pipenv_uuids, rewrite @@ -1490,7 +1490,8 @@ async fn run_redirect_selected_untracked( .get("Pipfile.lock") .or_else(|| done.files.get("Pipfile.lock")) .map(String::as_str), - &records, + &done.overrides, + &local_records, ) .await }; @@ -1520,7 +1521,6 @@ async fn run_redirect_selected_untracked( .get(lock_key) .or_else(|| done.files.get(lock_key)) .map(String::as_str), - records: &records, confirmed: &confirmed, confirmed_vlt: &rewrite.confirmed_vlt_uuids, foreign: &rewrite.vlt_foreign_uuids, @@ -1631,9 +1631,9 @@ async fn run_redirect_selected_untracked( // it was taken with these crawler options). `get --mode hosted` // passes none. params.npm_prior = npm_prior.cloned(); - // v5 keeps no hosted ledger: this run's fetched records are the - // hosted record source of the in-run attestation. - params.hosted_records = records.clone(); + // v5 keeps no hosted ledger: the in-run attestation fetches the + // records of the pins this run confirmed. + params.hosted_uuids = confirmed.iter().map(|(_, uuid)| uuid.clone()).collect(); // The gem intake gate withholds every gem file. VEX independently // rediscovers older pins too, so a candidate-only set would miss some // refused hosted gems. Keep their actual installed-byte verification, @@ -1693,9 +1693,8 @@ async fn run_redirect_selected_untracked( // One merged warning list, in one order, for both channels: the // rewriter's own warnings first (e.g. `no package-lock.json`), then the - // record, package-manager, stale-install, takeover and prune warnings. + // package-manager, stale-install, takeover and prune warnings. let mut engine_warnings = rewrite.warnings.clone(); - engine_warnings.extend(record_warnings); engine_warnings.extend(done.rush_warnings.iter().cloned()); engine_warnings.extend(done.pnpm_warnings.iter().cloned()); engine_warnings.extend(done.npm_warnings.iter().cloned()); @@ -2894,8 +2893,8 @@ mod tests { /// Probe invocation with the default surface (project-local discovery, /// no artifact shas) — tests override the knobs they exercise. - /// `records` is the map production hands over: this run's fetched - /// records, by uuid. + /// `records` is the map production hands over: the agent-mode + /// manifest's records, by uuid. async fn probe( cwd: &std::path::Path, confirmed: &[(String, String)], @@ -3143,6 +3142,46 @@ mod tests { assert!(out.warnings.is_empty()); } + /// Without an agent-mode record, an installed dir is judged by the + /// archive RubyGems installed it from (`/cache/.gem`): a + /// patched artifact's sha is patched and quiet, any other readable + /// archive is stale, and a missing one is never evidence. + #[tokio::test] + async fn gem_stale_probe_judges_the_installed_archive_without_a_record() { + use sha2::{Digest as _, Sha256}; + let tmp = tempfile::tempdir().unwrap(); + materialize_gem(tmp.path(), GEM_UPSTREAM); + let cache = gem_home(tmp.path()) + .join("cache") + .join(format!("{GEM_LEAF}.gem")); + let none = std::collections::BTreeMap::new(); + let key = ("stale-unit".to_string(), "1.0.0".to_string()); + let patched_sha = hex::encode(Sha256::digest(b"patched .gem")); + let shas = std::collections::BTreeMap::from([( + key, + std::collections::BTreeSet::from(["0".repeat(64), patched_sha]), + )]); + let one_confirmed = one_confirmed(); + + let out = + gem_stale_install_warnings(tmp.path(), false, None, &one_confirmed, &none, &shas).await; + assert_eq!(out.warnings.len(), 1, "the upstream archive is stale"); + assert_eq!( + out.stale_purls, + std::collections::BTreeSet::from([GEM_PURL.to_string()]) + ); + + std::fs::write(&cache, b"patched .gem").unwrap(); + let out = + gem_stale_install_warnings(tmp.path(), false, None, &one_confirmed, &none, &shas).await; + assert!(out.warnings.is_empty(), "{:?}", out.warnings); + + std::fs::remove_file(&cache).unwrap(); + let out = + gem_stale_install_warnings(tmp.path(), false, None, &one_confirmed, &none, &shas).await; + assert!(out.warnings.is_empty(), "no archive, no evidence"); + } + /// #709: a `.bundle/config` `path` outside the project is refused as an /// install (write) root, but bundler still installs into and loads /// from it — so the probe must read it, or a stale materialization @@ -3316,7 +3355,7 @@ mod tests { let mut shas = std::collections::BTreeMap::new(); shas.insert( ("stale-unit".to_string(), "1.0.0".to_string()), - "0".repeat(64), // the patched artifact's sha — differs + std::collections::BTreeSet::from(["0".repeat(64)]), // the patched artifact's sha — differs ); let out = gem_stale_install_warnings( tmp.path(), @@ -3355,7 +3394,10 @@ mod tests { std::fs::write(&committed, stale_bytes).unwrap(); let key = ("stale-unit".to_string(), "1.0.0".to_string()); let mut shas = std::collections::BTreeMap::new(); - shas.insert(key.clone(), "0".repeat(64)); + shas.insert( + key.clone(), + std::collections::BTreeSet::from(["0".repeat(64)]), + ); let out = gem_stale_install_warnings( tmp.path(), @@ -3380,7 +3422,10 @@ mod tests { // The PATCHED archive (sha matches) is a healthy commit — quiet. let mut patched_shas = std::collections::BTreeMap::new(); - patched_shas.insert(key, hex::encode(Sha256::digest(stale_bytes))); + patched_shas.insert( + key, + std::collections::BTreeSet::from([hex::encode(Sha256::digest(stale_bytes))]), + ); let out = gem_stale_install_warnings( tmp.path(), false, @@ -3401,7 +3446,7 @@ mod tests { let mut shas = std::collections::BTreeMap::new(); shas.insert( ("stale-unit".to_string(), "1.0.0".to_string()), - "0".repeat(64), + std::collections::BTreeSet::from(["0".repeat(64)]), ); let out = gem_stale_install_warnings( tmp.path(), @@ -3440,7 +3485,7 @@ mod tests { let mut shas = std::collections::BTreeMap::new(); shas.insert( ("stale-unit".to_string(), "1.0.0".to_string()), - "0".repeat(64), // the patched artifact's sha — differs + std::collections::BTreeSet::from(["0".repeat(64)]), // the patched artifact's sha — differs ); // Standalone: a fresh checkout with only the configured cache. @@ -3580,7 +3625,7 @@ mod tests { let mut shas = std::collections::BTreeMap::new(); shas.insert( ("stale-unit".to_string(), "1.0.0".to_string()), - "0".repeat(64), // differs from the archive bytes' sha + std::collections::BTreeSet::from(["0".repeat(64)]), // differs from the archive bytes' sha ); let out = gem_stale_install_warnings( tmp.path(), diff --git a/crates/socket-patch-cli/src/commands/scan/hosted/python.rs b/crates/socket-patch-cli/src/commands/scan/hosted/python.rs index e796bd651..04cbcf1a5 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted/python.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted/python.rs @@ -2,42 +2,110 @@ use std::collections::{BTreeMap, BTreeSet}; +use socket_patch_core::api::client::ApiClient; use socket_patch_core::crawlers::PythonCrawler; +use socket_patch_core::hosted::wheel_record::{ + fetch_patched_wheel_record, judge_installed_wheel, WheelRecord, +}; use socket_patch_core::manifest::schema::PatchRecord; +use socket_patch_core::patch::redirect::DepOverride; use socket_patch_core::utils::purl::{purl_parts, strip_purl_qualifiers}; -use socket_patch_core::vex::verify::judge_installed_record; +use socket_patch_core::vex::verify::{judge_installed_record, InstalledRecordJudgment}; use super::StaleInstallOutcome; +/// What an installed copy of one confirmed patch is judged against. +enum Evidence<'a> { + /// An agent-mode manifest record of the same patch: its `afterHash` + /// map, already local. + Local(&'a PatchRecord), + /// The patched wheel the lock now pins, fetched on the first installed + /// copy found (`Some(None)` once that fetch failed). + Wheel { + dep: &'a DepOverride, + sha256: &'a str, + fetched: Option>, + }, +} + +impl Evidence<'_> { + async fn judge(&mut self, api: &ApiClient, site: &std::path::Path) -> InstalledRecordJudgment { + match self { + Evidence::Local(record) => judge_installed_record(site, record).await, + Evidence::Wheel { + dep, + sha256, + fetched, + } => { + if fetched.is_none() { + *fetched = + Some(fetch_patched_wheel_record(api, &dep.artifact_url, sha256).await); + } + match fetched { + Some(Some(record)) => judge_installed_wheel(site, record).await, + _ => InstalledRecordJudgment::default(), + } + } + } + } +} + /// A lock rewrite cannot prove a warm virtualenv has installed the wheel. /// Use the project's own venv discovery (`--global`/`--global-prefix` use /// the crawler's global paths), and inspect every interpreter rather than /// deduplicating by package name. -/// Missing/unreadable files are not positive evidence of stale bytes. +/// +/// Each installed copy is judged against an agent-mode manifest record of +/// the same patch when one exists, else against the patched wheel's own +/// `RECORD` ([`socket_patch_core::hosted::wheel_record`]), downloaded only +/// when a copy is installed. Missing/unreadable files are not positive +/// evidence of stale bytes. pub(super) async fn stale_install_warnings( common: &crate::args::GlobalArgs, + api_client: &ApiClient, confirmed: &[(String, String)], pipenv_uuids: &BTreeSet, // The run's final Pipfile.lock text: the remedy's `pipenv sync` // arguments follow the categories that pin each package. pipenv_lock: Option<&str>, - // This run's fetched records MERGED with the ledger's persisted ones - // (the caller hands the post-merge ledger map), looked up by uuid. - records: &BTreeMap, + // The overrides the rewrite pinned: each patch's wheel URL and sha256. + overrides: &[DepOverride], + // The agent-mode manifest's records, looked up by uuid. + local_records: &BTreeMap, ) -> StaleInstallOutcome { let mut out = StaleInstallOutcome::default(); let pipenv_lock: Option = pipenv_lock.and_then(|text| { serde_json::from_str(socket_patch_core::formats::text::strip_bom(text)).ok() }); - let candidates: Vec<_> = confirmed + let mut candidates: Vec<(&String, &str, Evidence<'_>)> = confirmed .iter() .filter(|(purl, _)| purl.starts_with("pkg:pypi/")) .filter_map(|(purl, uuid)| { - records + let local = local_records .values() - .find(|record| &record.uuid == uuid) - .filter(|record| !record.files.is_empty()) - .map(|record| (purl, record)) + .find(|record| &record.uuid == uuid && !record.files.is_empty()); + let evidence = match local { + Some(record) => Evidence::Local(record), + // `--offline` downloads nothing, so there is no wheel to judge by. + None if common.offline => return None, + None => { + let dep = overrides.iter().find(|dep| { + dep.ecosystem == "pypi" + && &dep.patch_uuid == uuid + && dep + .artifact_url + .split(['?', '#']) + .next() + .is_some_and(|path| path.ends_with(".whl")) + })?; + Evidence::Wheel { + dep, + sha256: dep.integrity.sha256.as_deref()?, + fetched: None, + } + } + }; + Some((purl, uuid.as_str(), evidence)) }) .collect(); if candidates.is_empty() { @@ -105,15 +173,15 @@ pub(super) async fn stale_install_warnings( // (candidate, site) order. let bases: Vec = candidates .iter() - .map(|(purl, _)| strip_purl_qualifiers(purl).to_string()) + .map(|(purl, _, _)| strip_purl_qualifiers(purl).to_string()) .collect(); let mut found_per_site = Vec::with_capacity(paths.len()); for site in &paths { found_per_site.push(crawler.find_each_by_purl(site, &bases).await); } - for (index, (purl, record)) in candidates.into_iter().enumerate() { - if pipenv_uuids.contains(&record.uuid) { - pipenv_purls.insert(purl.clone()); + for (index, (purl, uuid, evidence)) in candidates.iter_mut().enumerate() { + if pipenv_uuids.contains(*uuid) { + pipenv_purls.insert((*purl).clone()); } for found in &found_per_site { let Some(pkg) = &found[index] else { @@ -122,8 +190,8 @@ pub(super) async fn stale_install_warnings( let judgment: &mut Judgment = judgments .entry((pkg.path.clone(), pkg.name.clone(), pkg.version.clone())) .or_default(); - judgment.purls.insert(purl.clone()); - let judged = judge_installed_record(&pkg.path, record).await; + judgment.purls.insert((*purl).clone()); + let judged = evidence.judge(api_client, &pkg.path).await; if judged.patched { judgment.patched = true; } else if judged.stale_evidence { @@ -189,6 +257,15 @@ mod tests { use socket_patch_core::hash::git_sha256::compute_git_sha256_from_bytes; use socket_patch_core::manifest::schema::PatchFileInfo; + /// A client nothing answers: these tests judge by local records only. + fn offline_api() -> ApiClient { + ApiClient::new(socket_patch_core::api::client::ApiClientOptions { + api_url: "http://127.0.0.1:1".into(), + api_token: None, + route: socket_patch_core::api::client::ApiRoute::Proxy, + }) + } + fn record(uuid: &str, file: &str, patched: &[u8]) -> PatchRecord { PatchRecord { uuid: uuid.to_string(), @@ -233,8 +310,16 @@ mod tests { ("one".into(), record("first-uuid", "first.py", b"patched")), ("two".into(), record("second-uuid", "second.py", b"patched")), ]); - let out = - stale_install_warnings(&common, &confirmed, &BTreeSet::new(), None, &ledger).await; + let out = stale_install_warnings( + &common, + &offline_api(), + &confirmed, + &BTreeSet::new(), + None, + &[], + &ledger, + ) + .await; assert_eq!(out.stale_purls, BTreeSet::from([first.to_string()])); assert_eq!(out.warnings.len(), 1); assert!(out.warnings[0]["detail"] @@ -249,8 +334,16 @@ mod tests { "variant".into(), )); ledger.insert("three".into(), record("variant", "first.py", b"upstream")); - let out = - stale_install_warnings(&common, &confirmed, &BTreeSet::new(), None, &ledger).await; + let out = stale_install_warnings( + &common, + &offline_api(), + &confirmed, + &BTreeSet::new(), + None, + &[], + &ledger, + ) + .await; assert!(out.stale_purls.is_empty()); assert!(out.warnings.is_empty()); } @@ -288,8 +381,16 @@ mod tests { let purl = "pkg:pypi/six@1.16.0"; let confirmed = vec![(purl.to_string(), "six-uuid".to_string())]; let ledger = BTreeMap::from([("k".into(), record("six-uuid", "six.py", b"patched"))]); - let out = - stale_install_warnings(&common, &confirmed, &BTreeSet::new(), None, &ledger).await; + let out = stale_install_warnings( + &common, + &offline_api(), + &confirmed, + &BTreeSet::new(), + None, + &[], + &ledger, + ) + .await; assert_eq!(out.stale_purls, BTreeSet::from([purl.to_string()])); assert_eq!(out.warnings.len(), 1); let detail = out.warnings[0]["detail"].as_str().unwrap(); @@ -301,8 +402,16 @@ mod tests { // Patched in the env: nothing to warn about. std::fs::write(site.join("six.py"), b"patched").unwrap(); - let out = - stale_install_warnings(&common, &confirmed, &BTreeSet::new(), None, &ledger).await; + let out = stale_install_warnings( + &common, + &offline_api(), + &confirmed, + &BTreeSet::new(), + None, + &[], + &ledger, + ) + .await; assert!(out.warnings.is_empty()); } @@ -325,8 +434,16 @@ mod tests { let purl = "pkg:pypi/six@1.16.0"; let confirmed = vec![(purl.to_string(), "six-uuid".to_string())]; let ledger = BTreeMap::from([("k".into(), record("six-uuid", "six.py", b"patched"))]); - let out = - stale_install_warnings(&common, &confirmed, &BTreeSet::new(), None, &ledger).await; + let out = stale_install_warnings( + &common, + &offline_api(), + &confirmed, + &BTreeSet::new(), + None, + &[], + &ledger, + ) + .await; assert_eq!(out.stale_purls, BTreeSet::from([purl.to_string()])); assert_eq!(out.warnings[0]["code"], "redirect_pypi_stale_install"); } @@ -353,7 +470,16 @@ mod tests { let pipenv = BTreeSet::from(["six-uuid".to_string()]); let text = r#"{"_meta": {"pipfile-spec": 6}, "default": {}, "develop": {"six": {"file": "x"}}}"#; - let out = stale_install_warnings(&common, &confirmed, &pipenv, Some(text), &ledger).await; + let out = stale_install_warnings( + &common, + &offline_api(), + &confirmed, + &pipenv, + Some(text), + &[], + &ledger, + ) + .await; assert_eq!(out.warnings.len(), 1); let detail = out.warnings[0]["detail"].as_str().unwrap(); let lock: serde_json::Value = serde_json::from_str(text).unwrap(); @@ -392,7 +518,16 @@ mod tests { // #790: a named category reaches the hosted remedy too. let docs = r#"{"_meta": {"pipfile-spec": 6}, "default": {"requests": {}}, "docs": {"six": {}}}"#; - let out = stale_install_warnings(&common, &confirmed, &pipenv, Some(docs), &ledger).await; + let out = stale_install_warnings( + &common, + &offline_api(), + &confirmed, + &pipenv, + Some(docs), + &[], + &ledger, + ) + .await; let detail = out.warnings[0]["detail"].as_str().unwrap(); assert!( detail.contains("-y six && pipenv sync --categories \"docs\"`"), @@ -457,7 +592,16 @@ mod tests { let ledger = BTreeMap::from([("k".into(), record("six-uuid", "six.py", b"patched"))]); let pipenv = BTreeSet::from(["six-uuid".to_string()]); let lock = r#"{"_meta": {"pipfile-spec": 6}, "default": {}, "develop": {"six": {}}}"#; - let out = stale_install_warnings(&common, &confirmed, &pipenv, Some(lock), &ledger).await; + let out = stale_install_warnings( + &common, + &offline_api(), + &confirmed, + &pipenv, + Some(lock), + &[], + &ledger, + ) + .await; assert_eq!(out.stale_purls, BTreeSet::from([purl.to_string()])); assert_eq!(out.warnings.len(), 1); let detail = out.warnings[0]["detail"].as_str().unwrap(); @@ -487,7 +631,16 @@ mod tests { std::fs::create_dir_all(&dist).unwrap(); std::fs::write(dist.join("METADATA"), "Name: six\nVersion: 1.16.0\n").unwrap(); std::fs::write(venv_site.join("six.py"), b"upstream").unwrap(); - let out = stale_install_warnings(&common, &confirmed, &pipenv, Some(lock), &ledger).await; + let out = stale_install_warnings( + &common, + &offline_api(), + &confirmed, + &pipenv, + Some(lock), + &[], + &ledger, + ) + .await; assert_eq!(out.warnings.len(), 1); let detail = out.warnings[0]["detail"].as_str().unwrap(); assert!( @@ -507,7 +660,16 @@ mod tests { // counts as installed: the usual remedy replaces the venv's copy. let base_dist = base_site.join("six-1.16.0.dist-info"); std::fs::write(base_dist.join("METADATA"), "Name: six\nVersion: 1.15.0\n").unwrap(); - let out = stale_install_warnings(&common, &confirmed, &pipenv, Some(lock), &ledger).await; + let out = stale_install_warnings( + &common, + &offline_api(), + &confirmed, + &pipenv, + Some(lock), + &[], + &ledger, + ) + .await; assert_eq!(out.warnings.len(), 1); let detail = out.warnings[0]["detail"].as_str().unwrap(); assert!( @@ -525,7 +687,16 @@ mod tests { "[project]\nname = \"app\"\nversion = \"0.1.0\"\n\n[tool.hatch.envs.default]\npath = \".venv\"\n", ) .unwrap(); - let out = stale_install_warnings(&common, &confirmed, &pipenv, Some(lock), &ledger).await; + let out = stale_install_warnings( + &common, + &offline_api(), + &confirmed, + &pipenv, + Some(lock), + &[], + &ledger, + ) + .await; assert_eq!(out.warnings.len(), 1); let detail = out.warnings[0]["detail"].as_str().unwrap(); assert!( diff --git a/crates/socket-patch-cli/src/commands/vex.rs b/crates/socket-patch-cli/src/commands/vex.rs index 65acda5d0..536bb9635 100644 --- a/crates/socket-patch-cli/src/commands/vex.rs +++ b/crates/socket-patch-cli/src/commands/vex.rs @@ -21,7 +21,7 @@ use std::path::{Path, PathBuf}; use clap::Args; use socket_patch_core::api::client::ApiClient; -use socket_patch_core::manifest::schema::{PatchManifest, PatchRecord}; +use socket_patch_core::manifest::schema::PatchManifest; use socket_patch_core::telemetry::{track_vex_failed, track_vex_generated, TelemetryAuth}; use socket_patch_core::vex::verify::{npm_manifest_after_hash, DEPENDENCY_NOT_INSTALLED}; use socket_patch_core::vex::{ @@ -178,7 +178,7 @@ impl VexEmbedArgs { dry_run: false, product_flag: "--vex-product", npm_prior: None, - hosted_records: Default::default(), + hosted_uuids: Vec::new(), } } } @@ -232,12 +232,12 @@ pub(crate) struct VexBuildParams { /// choice and order are unchanged. Ignored when taken with other crawler /// options. The standalone `vex` passes `None` and walks the tree. pub npm_prior: Option, - /// Embedded hosted `scan --vex` only: the patch records THIS RUN - /// fetched for the pins it confirmed, keyed by purl. v5 hosted mode - /// keeps no ledger, so these are the in-run attestation's hosted - /// records (the post-install standalone `vex` fetches them from the - /// API instead). Empty everywhere else. - pub hosted_records: std::collections::BTreeMap, + /// Embedded hosted `scan --vex` only: the uuids of the pins THIS RUN + /// confirmed. v5 hosted mode keeps no ledger, so the plan fetches their + /// records as the in-run attestation's hosted records (the post-install + /// standalone `vex` finds them through the lockfiles instead). Empty + /// everywhere else. + pub hosted_uuids: Vec, /// The run's API client (see [`RunApiClient`]): seeded by a host /// command, else built at most once when the plan must fetch records. /// Telemetry reads its route when it exists. @@ -373,7 +373,7 @@ pub async fn run(args: VexArgs) -> i32 { dry_run: args.common.dry_run, product_flag: "--product", npm_prior: None, - hosted_records: Default::default(), + hosted_uuids: Vec::new(), api_client: RunApiClient::new(), }; @@ -1813,21 +1813,13 @@ async fn generate_vex_from_manifest_path_inner( for diag in &discovery.diagnostics { note_warning(warnings, common, diag.code, diag.detail.clone()); } - // This run's hosted records (embedded hosted `scan --vex`) join a - // pre-v5 ledger's as the hosted record source, newest wins. - let redirect = if params.hosted_records.is_empty() { - redirect - } else { - let mut state = - redirect.unwrap_or_else(socket_patch_core::patch::redirect::RedirectState::new); - state.records.extend(params.hosted_records.clone()); - Some(state) - }; let sources = Sources { manifest: manifest_file.unwrap_or_else(PatchManifest::new), vendor, redirect, discovery, + // Their records join a pre-v5 ledger's as the hosted record source. + hosted_uuids: params.hosted_uuids.clone(), }; if sources.is_empty() { // A discovery diagnostic (an unparseable lock, a rejected reference) @@ -2579,7 +2571,7 @@ mod npm_prior_tests { dry_run: false, product_flag: "--vex-product", npm_prior: prior, - hosted_records: Default::default(), + hosted_uuids: Vec::new(), api_client: RunApiClient::new(), }; let manifest_path = common.resolved_manifest_path(); @@ -2653,6 +2645,7 @@ mod npm_prior_tests { mod mirror_refusal_tests { use super::*; use crate::commands::vex_sources::HostedWiring; + use socket_patch_core::manifest::schema::PatchRecord; use std::collections::{BTreeMap, HashSet}; /// The current scan can refuse routing while VEX rediscovers an older pin diff --git a/crates/socket-patch-cli/src/commands/vex_sources.rs b/crates/socket-patch-cli/src/commands/vex_sources.rs index 25e49273e..eae3e2359 100644 --- a/crates/socket-patch-cli/src/commands/vex_sources.rs +++ b/crates/socket-patch-cli/src/commands/vex_sources.rs @@ -127,16 +127,21 @@ pub(crate) struct Sources { pub vendor: VendorState, pub redirect: Option, pub discovery: Discovery, + /// Embedded hosted `scan --vex` only: the uuids that run confirmed. + /// v5 hosted mode keeps no ledger, so [`plan`] fetches their records + /// and files them as hosted records. Empty everywhere else. + pub hosted_uuids: Vec, } impl Sources { /// Nothing anywhere could name a patch: no manifest patches, no ledger - /// entries or records, no lockfile references. + /// entries or records, no lockfile references, no in-run hosted pins. pub(crate) fn is_empty(&self) -> bool { self.manifest.patches.is_empty() && self.vendor.entries.is_empty() && self.redirect.as_ref().is_none_or(|r| r.records.is_empty()) && self.discovery.refs.is_empty() + && self.hosted_uuids.is_empty() } } @@ -359,9 +364,28 @@ pub(crate) async fn plan( let Sources { manifest, vendor, - redirect, + mut redirect, discovery, + hosted_uuids, } = sources; + let mut notes = Vec::new(); + // The in-run hosted pins' records, filed under the API's purl like a + // pre-v5 hosted ledger's, later uuids winning. A uuid fetched here is + // not asked for again below. + let mut prefetched: HashSet = HashSet::new(); + let hosted_uuids: Vec = hosted_uuids + .into_iter() + .filter(|uuid| prefetched.insert(uuid.clone())) + .collect(); + if !hosted_uuids.is_empty() { + let mut fetched = fetch_records(common, api_client, &hosted_uuids, &mut notes).await; + let state = redirect.get_or_insert_with(RedirectState::new); + for uuid in &hosted_uuids { + if let Some((purl, record)) = fetched.remove(uuid) { + state.records.insert(purl, record); + } + } + } let redirect_records: BTreeMap = redirect .as_ref() .map(|r| r.records.clone()) @@ -371,7 +395,6 @@ pub(crate) async fn plan( .map(|p| strip_purl_qualifiers(p)) .collect(); - let mut notes = Vec::new(); let mut gated: Vec = Vec::new(); let mut cands = build_candidates(&manifest, &vendor, redirect.as_ref()); let conflicts = wiring_conflicts(&discovery); @@ -565,7 +588,11 @@ pub(crate) async fn plan( } } let uuids: Vec = { - let mut u: Vec = need_api.iter().map(|&i| based[i].0.uuid.clone()).collect(); + let mut u: Vec = need_api + .iter() + .map(|&i| based[i].0.uuid.clone()) + .filter(|uuid| !prefetched.contains(uuid)) + .collect(); u.sort(); u.dedup(); u @@ -1378,6 +1405,7 @@ mod tests { vendor: VendorState::new(), redirect: Some(redirect), discovery: discovery(vec![hosted_ref("pkg:npm/x@1.0.0", U2, true)]), + hosted_uuids: Vec::new(), }; let plan = plan(&common(tmp.path()), sources, &[], &RunApiClient::new()).await; assert_eq!(plan.view.patches["pkg:npm/x@1.0.0"].uuid, U2); @@ -1401,6 +1429,7 @@ mod tests { vendor: VendorState::new(), redirect: None, discovery: discovery(vec![hosted_ref("pkg:npm/x@1.0.0", U2, true)]), + hosted_uuids: Vec::new(), }; let plan = plan(&common(tmp.path()), sources, &[], &RunApiClient::new()).await; assert!( @@ -1436,6 +1465,7 @@ mod tests { vendor: VendorState::new(), redirect: Some(redirect), discovery: discovery(vec![hosted_ref("pkg:npm/x@1.0.0", U1, true)]), + hosted_uuids: Vec::new(), }; let plan = plan(&common(tmp.path()), sources, &[], &RunApiClient::new()).await; assert!(plan @@ -1478,6 +1508,7 @@ mod tests { shrinkwrap, hosted_ref("pkg:npm/y@1.0.0", U2, true), ]), + hosted_uuids: Vec::new(), }; let plan = plan(&common(tmp.path()), sources, &[], &RunApiClient::new()).await; assert_eq!(plan.gated, vec![failed("pkg:npm/x@1.0.0", WIRING_CONFLICT)]); @@ -1526,6 +1557,7 @@ mod tests { vendor: vendor.clone(), redirect: Some(redirect.clone()), discovery: Discovery::default(), + hosted_uuids: Vec::new(), }; let live = plan( &common(tmp.path()), @@ -1619,6 +1651,7 @@ mod tests { vendor: vendor.clone(), redirect: Some(redirect.clone()), discovery, + hosted_uuids: Vec::new(), }; // Control — nothing recognized: the ledgers' own recorded files @@ -1704,6 +1737,7 @@ mod tests { vendor: VendorState::new(), redirect: Some(redirect), discovery: discovery(vec![hosted_ref("pkg:npm/x@1.0.0", U1, true)]), + hosted_uuids: Vec::new(), }; let plan = plan(&common(tmp.path()), sources, &[], &RunApiClient::new()).await; let wiring = &plan.hosted["pkg:npm/x@1.0.0"]; @@ -1735,6 +1769,7 @@ mod tests { vendor: VendorState::new(), redirect: Some(redirect.clone()), discovery, + hosted_uuids: Vec::new(), }; let gated = plan( &common(tmp.path()), @@ -1787,6 +1822,7 @@ mod tests { vendor: VendorState::new(), redirect: Some(redirect), discovery: Discovery::default(), + hosted_uuids: Vec::new(), }; let plan = plan( &common(tmp.path()), @@ -1863,6 +1899,7 @@ mod tests { // The lock does not mention the uuid, and the Gemfile is // not a discovery input: the ledger fallback decides. discovery: Discovery::default(), + hosted_uuids: Vec::new(), }; let plan = plan(&common(root), sources, &[], &RunApiClient::new()).await; if gemfile_wired { diff --git a/crates/socket-patch-cli/src/commands/vlt_heal.rs b/crates/socket-patch-cli/src/commands/vlt_heal.rs index 48504d129..112df679a 100644 --- a/crates/socket-patch-cli/src/commands/vlt_heal.rs +++ b/crates/socket-patch-cli/src/commands/vlt_heal.rs @@ -8,10 +8,9 @@ use std::collections::{BTreeMap, BTreeSet}; use socket_patch_core::constants::npm_family::{VLT_HIDDEN_LOCK_REL, VLT_LOCK}; -use socket_patch_core::manifest::schema::PatchRecord; use socket_patch_core::patch::redirect::vlt; use socket_patch_core::patch::redirect::vlt_heal::{ - self, classify_target, read_install_state, Expected, LedgerTarget, Target, TargetState, + self, classify_target, read_install_state, LedgerTarget, Target, TargetState, }; /// What the post-rewrite heal reports back to the hosted flow: its @@ -60,13 +59,12 @@ struct HealTally { undeterminable_uuids: BTreeSet, } -/// Classify `targets` against `expected` and invalidate the stale ones +/// Classify `targets` against their pins and invalidate the stale ones /// vlt reinstalls, unless cleanup is off; `(target, uuid or purl)` per /// target. async fn heal_targets( common: &crate::args::GlobalArgs, targets: &[(Target<'_>, &str)], - expected: Expected, ) -> HealTally { let mut tally = HealTally::default(); if targets.is_empty() { @@ -75,7 +73,7 @@ async fn heal_targets( let state = read_install_state(&common.cwd).await; let mut stale: Vec<(String, &str)> = Vec::new(); for (target, uuid) in targets { - match classify_target(&state, &common.cwd, target, expected).await { + match classify_target(&state, &common.cwd, target).await { TargetState::Stale if !vlt_heal::reinstalls_after_removal(target.flags) => { tally.optional_left += 1; tally.stale_uuids.insert((*uuid).to_string()); @@ -256,8 +254,8 @@ fn takeover_detail(tally: &HealTally, vendored: usize) -> Option { fn undeterminable_detail(n: usize) -> String { format!( "vlt-lock.json pins Socket-patched packages, but socket-patch could not check {n} \ - installed copies (node_modules is a link, or no patch record was available); run \ - `vlt ci` to be sure the patched packages are installed." + installed copies (node_modules is a link, or node_modules/.vlt-lock.json is missing \ + or unreadable); run `vlt ci` to be sure the patched packages are installed." ) } @@ -265,8 +263,6 @@ fn undeterminable_detail(n: usize) -> String { pub(crate) struct HealInputs<'a> { /// The final `vlt-lock.json` (as written, or as a dry run would write it). pub(crate) final_lock: Option<&'a str>, - /// This run's fetched records merged with the ledger's, keyed by purl. - pub(crate) records: &'a BTreeMap, pub(crate) confirmed: &'a [(String, String)], pub(crate) confirmed_vlt: &'a BTreeSet, pub(crate) foreign: &'a BTreeSet, @@ -297,20 +293,18 @@ pub(crate) async fn heal_after_rewrite( let targets: Vec<(Target<'_>, &str)> = owned .iter() .map(|i| { - let record = inputs.records.values().find(|r| r.uuid == i.patch_uuid); ( Target { dep_id: &i.dep_id, name: &i.name, lock_sha512: i.sha512.as_deref(), - record, flags: i.flags, }, i.patch_uuid.as_str(), ) }) .collect(); - tally = heal_targets(common, &targets, Expected::Patched).await; + tally = heal_targets(common, &targets).await; healed = true; out.warnings.push(serde_json::json!({ "code": REINSTALL_REQUIRED, @@ -342,7 +336,8 @@ pub(crate) async fn heal_after_rewrite( if healed { out.healed_store = Some(copies.clone()); } - for purl in inputs.records.keys() { + let confirmed: BTreeSet<&String> = inputs.confirmed.iter().map(|(purl, _)| purl).collect(); + for purl in confirmed { // The store's keys are decoded purls: look a `%40scope` record // up the way the attribution gate does. let base = socket_patch_core::utils::purl_key::canonical_base_purl(purl); @@ -403,14 +398,13 @@ pub(crate) async fn rollback_heal( dep_id: &t.dep_id, name: &t.name, lock_sha512: sha.as_deref(), - record: t.record.as_ref(), flags: *flags, }, t.purl.as_str(), ) }) .collect(); - let tally = heal_targets(common, &classified, Expected::Pristine).await; + let tally = heal_targets(common, &classified).await; let restored: BTreeSet<&str> = targets.iter().map(|t| t.purl.as_str()).collect(); rollback_detail(&tally, restored.len()) .map(|detail| (REINSTALL_REQUIRED.to_string(), detail)) @@ -436,14 +430,13 @@ pub(crate) async fn takeover_heal( dep_id: &t.dep_id, name: &t.name, lock_sha512: None, - record: t.record.as_ref(), flags: t.flags, }, t.purl.as_str(), ) }) .collect(); - let tally = heal_targets(common, &classified, Expected::Pristine).await; + let tally = heal_targets(common, &classified).await; let vendored: BTreeSet<&str> = targets.iter().map(|t| t.purl.as_str()).collect(); takeover_detail(&tally, vendored.len()) } diff --git a/crates/socket-patch-cli/tests/common/patched_wheel.rs b/crates/socket-patch-cli/tests/common/patched_wheel.rs new file mode 100644 index 000000000..24e230b99 --- /dev/null +++ b/crates/socket-patch-cli/tests/common/patched_wheel.rs @@ -0,0 +1,104 @@ +//! A real patched wheel for the hosted Python stale-install probe, which +//! judges an installed copy against the wheel's own `RECORD`: the served +//! wheel, the grant that pins it, and the installed `WHEEL` file naming +//! the same build. + +use std::io::Write as _; +use std::path::Path; + +use base64::Engine as _; +use sha2::{Digest, Sha256}; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +/// The `WHEEL` file of a pure `py2.py3-none-any` wheel. +pub const WHEEL: &str = + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py2-none-any\nTag: py3-none-any\n"; + +/// A wheel whose `dist_info` records `files` (paths relative to +/// site-packages) at their bytes. +pub fn build(dist_info: &str, files: &[(&str, &[u8])]) -> Vec { + let b64 = |bytes: &[u8]| { + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(Sha256::digest(bytes)) + }; + let mut record = String::new(); + for (name, bytes) in files { + record.push_str(&format!("{name},sha256={},{}\n", b64(bytes), bytes.len())); + } + record.push_str(&format!( + "{dist_info}/WHEEL,sha256={},{}\n{dist_info}/RECORD,,\n", + b64(WHEEL.as_bytes()), + WHEEL.len() + )); + let mut out = std::io::Cursor::new(Vec::new()); + { + let mut zip = zip::ZipWriter::new(&mut out); + let options = zip::write::SimpleFileOptions::default(); + let members = files + .iter() + .map(|(name, bytes)| (name.to_string(), bytes.to_vec())) + .chain([ + (format!("{dist_info}/WHEEL"), WHEEL.as_bytes().to_vec()), + (format!("{dist_info}/RECORD"), record.into_bytes()), + ]); + for (name, bytes) in members { + zip.start_file(name, options).unwrap(); + zip.write_all(&bytes).unwrap(); + } + zip.finish().unwrap(); + } + out.into_inner() +} + +/// The hex sha256 a grant pins `wheel` with. +pub fn sha256(wheel: &[u8]) -> String { + hex::encode(Sha256::digest(wheel)) +} + +/// `//WHEEL`, as an installer keeps it from the wheel. +pub fn install_wheel_metadata(site: &Path, dist_info: &str) { + let dir = site.join(dist_info); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("WHEEL"), WHEEL).unwrap(); +} + +/// Serve `wheel` at `wheel_path` on `server` and mount a priority-1 grant +/// for `uuid` that pins it, ahead of the suite's own grant. Returns the +/// served URL. +pub async fn mount_grant( + server: &MockServer, + org: &str, + uuid: &str, + purl: &str, + wheel_path: &str, + wheel: Vec, +) -> String { + let url = format!("{}{wheel_path}", server.uri()); + let sha = sha256(&wheel); + Mock::given(method("GET")) + .and(path(wheel_path)) + .respond_with(ResponseTemplate::new(200).set_body_bytes(wheel)) + .mount(server) + .await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{org}/patches/package"))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "results": { + uuid: { + "status": "granted", + "url": url, + "purl": purl, + "artifacts": [{ + "kind": "tarball", + "url": url, + "integrity": { "sha256": sha } + }], + "registryOverride": null + } + } + }))) + .with_priority(1) + .mount(server) + .await; + url +} diff --git a/crates/socket-patch-cli/tests/e2e_redirect_gem_stale_install.rs b/crates/socket-patch-cli/tests/e2e_redirect_gem_stale_install.rs index 697b44a2c..d38ffe379 100644 --- a/crates/socket-patch-cli/tests/e2e_redirect_gem_stale_install.rs +++ b/crates/socket-patch-cli/tests/e2e_redirect_gem_stale_install.rs @@ -15,13 +15,13 @@ //! 3. FRESH checkout (no materialization) → quiet. //! 4. TWO gem homes both stale → one warning per home, each naming its //! own paths. -//! 5. RE-SCAN with a failing record fetch: v5 hosted mode keeps no ledger, -//! so a re-scan whose /patches/view fetch fails has no persisted record -//! to judge from — it surfaces `record_fetch_failed` (the VEX-omission -//! detail) and leaves the committed wiring untouched. +//! 5. RE-SCAN: a hosted scan fetches no patch record — the installed +//! gem is judged by the archive RubyGems cached when installing it, so +//! the warning re-fires on every re-scan while the committed wiring is +//! left untouched. //! 6. Same-run `--vex`: the stale purl is excluded from the attestation -//! (built from this run's fetched records) — the envelope must never -//! attest a CVE its own warning says is live. +//! — the envelope must never attest a CVE its own warning says is +//! live. //! 7. Manifest-less standalone `vex` over the same committed state follows //! the installed tree: stale → nothing discoverable (no ledger, lock not //! yet converged); after the prescribed re-install (the lock bundler @@ -52,11 +52,21 @@ const GHSA: &str = "GHSA-gem-stale-e2e"; const UPSTREAM_LIB: &str = "module StaleProbeGem\n def self.status\n \"VULNERABLE\"\n end\nend\n"; const PATCHED_LIB: &str = "module StaleProbeGem\n def self.status\n \"PATCHED\"\n end\nend\n"; +/// The `.gem` archives a real install leaves in the home's `cache/`: the +/// patched one is the artifact the grant names (its sha256 below). +const UPSTREAM_ARCHIVE: &[u8] = b"upstream-gem-archive-bytes"; +const PATCHED_ARCHIVE: &[u8] = b"patched-gem-archive-bytes"; + +fn patched_archive_sha256() -> String { + use sha2::Digest as _; + hex::encode(sha2::Sha256::digest(PATCHED_ARCHIVE)) +} /// Mount the whole patches API: batch discovery, by-package selection, the -/// granted reference (rubygems-compact-index override), and — unless -/// `view_budget` caps it — the patch view whose file map carries the REAL -/// before/after hashes of the fixture lib. With `view_budget: Some(n)` the +/// granted reference (rubygems-compact-index override, carrying the patched +/// archive's sha256), and — unless `view_budget` caps it — the patch view +/// whose file map carries the REAL before/after hashes of the fixture lib +/// (the standalone `vex` reads it). With `view_budget: Some(n)` the /// view answers 200 exactly n times and 500 afterwards (the transient-fetch /// re-fire arm). async fn mount_api(server: &MockServer, view_budget: Option) { @@ -106,7 +116,7 @@ async fn mount_api(server: &MockServer, view_budget: Option) { "artifacts": [{ "kind": "tarball", "url": hosted_url, - "integrity": { "sha256": "c0ffee".repeat(10) + "abcd" } + "integrity": { "sha256": patched_archive_sha256() } }], "registryOverride": { "kind": "rubygems-compact-index", @@ -114,7 +124,7 @@ async fn mount_api(server: &MockServer, view_budget: Option) { "identifiers": { "name": DEP, "version": DEP_VERSION, - "gemChecksumSha256": "c0ffee".repeat(10) + "abcd", + "gemChecksumSha256": patched_archive_sha256(), } } } @@ -180,7 +190,8 @@ fn write_manifest_pair(proj: &Path) { /// Materialize the gem in bundler's deployment layout under one ruby-version /// home — installed dir + cached `.gem` + specifications entry, exactly what -/// a real `bundle install` leaves. Paths are built COMPONENT-WISE (the same +/// a real `bundle install` leaves (the cached archive is the patched one +/// exactly when the lib is). Paths are built COMPONENT-WISE (the same /// join operations the crawler uses) so their `display()` matches the /// production warning text byte-for-byte on every platform. /// Returns (installed gem dir, cache .gem path, specifications path). @@ -199,7 +210,12 @@ fn materialize_installed_gem( std::fs::write(gem_dir.join("lib").join("stale_probe_gem.rb"), lib_content).unwrap(); let cache = home.join("cache").join(format!("{DEP}-{DEP_VERSION}.gem")); std::fs::create_dir_all(cache.parent().unwrap()).unwrap(); - std::fs::write(&cache, b"upstream-gem-archive-bytes").unwrap(); + let archive = if lib_content == PATCHED_LIB { + PATCHED_ARCHIVE + } else { + UPSTREAM_ARCHIVE + }; + std::fs::write(&cache, archive).unwrap(); let spec = home .join("specifications") .join(format!("{DEP}-{DEP_VERSION}.gemspec")); @@ -208,6 +224,13 @@ fn materialize_installed_gem( (gem_dir, cache, spec) } +/// The archive RubyGems leaves in `home/cache/` for the gem it installed. +fn cache_archive(home: &Path, archive: &[u8]) { + let cache = home.join("cache"); + std::fs::create_dir_all(&cache).unwrap(); + std::fs::write(cache.join(format!("{DEP}-{DEP_VERSION}.gem")), archive).unwrap(); +} + fn hosted_scan_json(proj: &Path, api: &str) -> (i32, String, String) { common::run_with_env( proj, @@ -531,22 +554,21 @@ async fn gem_hosted_redirect_warns_once_per_stale_gem_home() { assert!(b.contains(&cache_b.display().to_string()), "{b}"); } -/// v5 hosted mode keeps no ledger: scan 1 warns (from its fetched record) -/// and persists nothing but the Gemfile/lock edits; scan 2's /patches/view -/// fetch fails transiently (500), so it surfaces `record_fetch_failed` with -/// the VEX-omission detail, exits 0, and leaves the committed wiring -/// byte-identical (the re-run is idempotent) — still with no ledger. +/// v5 hosted mode keeps no ledger and fetches no record: scan 1 warns +/// (from the installed gem's cached archive) and persists nothing but the +/// Gemfile/lock edits; scan 2 asks for no patch view either, warns again, +/// exits 0, and leaves the committed wiring byte-identical (the re-run is +/// idempotent) — still with no ledger. #[tokio::test(flavor = "multi_thread")] -async fn gem_hosted_rescan_with_failing_record_fetch_reports_it_and_keeps_the_wiring() { +async fn gem_hosted_rescan_rewarns_without_fetching_records_and_keeps_the_wiring() { let server = MockServer::start().await; - mount_api(&server, Some(1)).await; // view answers 200 exactly once + mount_api(&server, None).await; let tmp = tempfile::tempdir().unwrap(); let proj = tmp.path().join("proj"); std::fs::create_dir_all(&proj).unwrap(); write_manifest_pair(&proj); let (gem_dir, ..) = materialize_installed_gem(&proj, "3.3.0", UPSTREAM_LIB); - // Scan 1: fresh record, warning fires, no ledger persists anything. let (code, stdout, _) = hosted_scan_json(&proj, &server.uri()); assert_eq!(code, 0); let env = common::parse_json_envelope(&stdout); @@ -558,24 +580,19 @@ async fn gem_hosted_rescan_with_failing_record_fetch_reports_it_and_keeps_the_wi let gemfile = std::fs::read(proj.join("Gemfile")).unwrap(); let lock = std::fs::read(proj.join("Gemfile.lock")).unwrap(); - // Scan 2: view 500s → record_fetch_failed with the VEX-omission detail. let (code, stdout, _) = hosted_scan_json(&proj, &server.uri()); assert_eq!(code, 0, "{stdout}"); let env = common::parse_json_envelope(&stdout); - let failed = env["warnings"] - .as_array() + assert_eq!(stale_warnings(&env).len(), 1, "scan 2 warns again: {env}"); + assert!(!stdout.contains("record_fetch_failed"), "{env}"); + let views = server + .received_requests() + .await + .unwrap_or_default() .into_iter() - .flatten() - .find(|w| w["code"] == "record_fetch_failed") - .unwrap_or_else(|| panic!("the transient fetch failure is surfaced: {env}")); - assert_eq!( - failed["detail"], - format!( - "{PURL} was switched to hosted, but its patch record could not be fetched; this run's VEX \ - attestation omits it (`socket-patch vex` fetches it again once the API answers)" - ), - "{env}" - ); + .filter(|r| r.url.path().contains("/patches/view/")) + .count(); + assert_eq!(views, 0, "a hosted scan never fetches a patch view"); assert_eq!( std::fs::read(proj.join("Gemfile")).unwrap(), gemfile, @@ -671,6 +688,7 @@ async fn gem_hosted_stale_bundler4_standalone_install_warns_and_is_not_attested( let gem_dir = home.join("gems").join(format!("{DEP}-{DEP_VERSION}")); std::fs::create_dir_all(gem_dir.join("lib")).unwrap(); std::fs::write(gem_dir.join("lib").join("stale_probe_gem.rb"), UPSTREAM_LIB).unwrap(); + cache_archive(&home, UPSTREAM_ARCHIVE); std::fs::create_dir_all(proj.join("bundle").join("bundler")).unwrap(); std::fs::write( proj.join("bundle").join("bundler").join("setup.rb"), @@ -750,6 +768,7 @@ async fn gem_hosted_stale_dot_bundle_install_warns_and_is_not_attested() { let gem_dir = home.join("gems").join(format!("{DEP}-{DEP_VERSION}")); std::fs::create_dir_all(gem_dir.join("lib")).unwrap(); std::fs::write(gem_dir.join("lib").join("stale_probe_gem.rb"), UPSTREAM_LIB).unwrap(); + cache_archive(&home, UPSTREAM_ARCHIVE); std::fs::write(proj.join(".bundle").join("config"), config).unwrap(); let vex_path = proj.join("out.vex.json"); @@ -849,7 +868,7 @@ async fn gem_hosted_stale_archive_at_configured_cache_path_warns_and_is_not_atte .join(cache_dir) .join(format!("{DEP}-{DEP_VERSION}.gem")); std::fs::create_dir_all(archive.parent().unwrap()).unwrap(); - std::fs::write(&archive, b"upstream-gem-archive-bytes").unwrap(); + std::fs::write(&archive, UPSTREAM_ARCHIVE).unwrap(); let vex_path = proj.join("out.vex.json"); let (code, stdout, stderr) = common::run_with_env( @@ -960,7 +979,7 @@ async fn gem_hosted_stale_archive_at_global_cache_path_warns_and_is_not_attested .join(cache_dir) .join(format!("{DEP}-{DEP_VERSION}.gem")); std::fs::create_dir_all(archive.parent().unwrap()).unwrap(); - std::fs::write(&archive, b"upstream-gem-archive-bytes").unwrap(); + std::fs::write(&archive, UPSTREAM_ARCHIVE).unwrap(); let global_value = global_value.to_str().unwrap().to_string(); let mut env: Vec<(&str, &str)> = vec![(global_var, &global_value)]; @@ -1320,6 +1339,7 @@ async fn gem_hosted_stale_install_under_out_of_tree_config_path_is_not_attested( .join(format!("{DEP}-{DEP_VERSION}")); std::fs::create_dir_all(gem_dir.join("lib")).unwrap(); std::fs::write(gem_dir.join("lib").join("stale_probe_gem.rb"), UPSTREAM_LIB).unwrap(); + cache_archive(&bundle.join("ruby").join("3.3.0"), UPSTREAM_ARCHIVE); let vex_path = proj.join("out.vex.json"); let (code, stdout, stderr) = common::run_with_env( @@ -1386,7 +1406,7 @@ fn stage_system_home_copy(home: &Path, bin_dir: &Path) -> PathBuf { std::fs::create_dir_all(home.join("cache")).unwrap(); std::fs::write( home.join("cache").join(format!("{DEP}-{DEP_VERSION}.gem")), - b"upstream-gem-archive-bytes", + UPSTREAM_ARCHIVE, ) .unwrap(); std::fs::create_dir_all(home.join("specifications")).unwrap(); @@ -1565,7 +1585,7 @@ async fn gem_hosted_default_cwd_keeps_project_local_remedy() { proj.join("vendor") .join("cache") .join(format!("{DEP}-{DEP_VERSION}.gem")), - b"upstream-gem-archive-bytes", + UPSTREAM_ARCHIVE, ) .unwrap(); diff --git a/crates/socket-patch-cli/tests/hosted_memory_engine.rs b/crates/socket-patch-cli/tests/hosted_memory_engine.rs index 032272599..376810278 100644 --- a/crates/socket-patch-cli/tests/hosted_memory_engine.rs +++ b/crates/socket-patch-cli/tests/hosted_memory_engine.rs @@ -14,9 +14,7 @@ use socket_patch_cli::hosted_memory::{ HostedScanOutput, SelectOptions, SessionBuilder, TreeEntryInput, }; use socket_patch_core::api::client::{ApiError, ApiFuture, PatchApi}; -use socket_patch_core::api::types::{ - BatchSearchResponse, PackageVendorResult, PatchResponse, SearchResponse, -}; +use socket_patch_core::api::types::{BatchSearchResponse, PackageVendorResult, SearchResponse}; use tokio_util::sync::CancellationToken; use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; @@ -79,7 +77,7 @@ async fn roots_sharing_a_purl_share_every_lookup() { assert_eq!(count(&server, "POST", "/patches/batch").await, 1); assert_eq!(count(&server, "GET", "/patches/by-package/").await, 1); assert_eq!(count(&server, "POST", "/patches/package").await, 1); - assert_eq!(count(&server, "GET", "/patches/view/").await, 1); + assert_eq!(count(&server, "GET", "/patches/view/").await, 0); let paths: Vec<&str> = output .changed_files .iter() @@ -363,9 +361,6 @@ impl PatchApi for Stalled { ) -> ApiFuture<'a, std::collections::HashMap> { Box::pin(std::future::pending()) } - fn fetch_patch<'a>(&'a self, _: &'a str) -> ApiFuture<'a, Option> { - Box::pin(std::future::pending()) - } fn download_artifact<'a>(&'a self, _: &'a str, _: u64) -> ApiFuture<'a, Vec> { Box::pin(async { Err(ApiError::Other("unused".into())) }) } @@ -422,9 +417,6 @@ impl PatchApi for CountingStalled { ) -> ApiFuture<'a, std::collections::HashMap> { Box::pin(std::future::pending()) } - fn fetch_patch<'a>(&'a self, _: &'a str) -> ApiFuture<'a, Option> { - Box::pin(std::future::pending()) - } fn download_artifact<'a>(&'a self, _: &'a str, _: u64) -> ApiFuture<'a, Vec> { Box::pin(async { Err(ApiError::Other("unused".into())) }) } @@ -945,7 +937,6 @@ async fn stats_count_inputs_and_calls() { "searchPatchesBatch", "searchPatchesByPackage", "fetchRegistryReferences", - "fetchPatch", ] { assert_eq!( output.stats.provider_calls.get(method_name), @@ -953,6 +944,7 @@ async fn stats_count_inputs_and_calls() { "{method_name}" ); } + assert_eq!(output.stats.provider_calls.get("fetchPatch"), None); } #[test] diff --git a/crates/socket-patch-cli/tests/in_process_redirect.rs b/crates/socket-patch-cli/tests/in_process_redirect.rs index 152fefb22..e02239fd6 100644 --- a/crates/socket-patch-cli/tests/in_process_redirect.rs +++ b/crates/socket-patch-cli/tests/in_process_redirect.rs @@ -2009,9 +2009,6 @@ async fn scan_redirect_heals_digestless_bun_tuple_and_rollback_restores_the_regi let server = MockServer::start().await; mock_discovery(&server).await; mock_reference(&server).await; - // The record fetch too, so the "no warnings at all" assertion below is - // exact (without it every run carries a `record_fetch_failed` advisory). - mock_view(&server).await; let tmp = tempfile::tempdir().unwrap(); write_bun_project(tmp.path(), 1); @@ -3167,8 +3164,8 @@ async fn redirect_human_mode_warnings_are_not_json_quoted() { values; stderr=\n{stderr}" ); - // Leg 2 — a GRANTED reference whose patch record cannot be fetched (no - // `view/{uuid}` mock → 404) produces a `record_fetch_failed` warning. + // Leg 2 — a GRANTED reference redirects without the patch record: the + // hosted scan never asks for `view/{uuid}`, so nothing warns about it. let server = MockServer::start().await; mock_discovery(&server).await; mock_reference(&server).await; @@ -3195,17 +3192,20 @@ async fn redirect_human_mode_warnings_are_not_json_quoted() { let stderr = String::from_utf8_lossy(&out.stderr); assert!( stdout.contains("Switched 1 package to hosted patches; rewrote"), - "anchor: the dep must have been redirected so the record fetch runs; \ - stdout=\n{stdout}\nstderr=\n{stderr}" + "anchor: the dep must have been redirected; stdout=\n{stdout}\nstderr=\n{stderr}" ); assert!( - stderr.contains(&format!( - "Warning: {PURL} was switched to hosted, but its patch record could not \ - be fetched" - )), - "the record-fetch warning must print the bare detail string, not a \ - JSON-quoted one; stderr=\n{stderr}" + !stderr.contains("patch record"), + "a hosted scan fetches no patch record; stderr=\n{stderr}" ); + let views = server + .received_requests() + .await + .unwrap_or_default() + .into_iter() + .filter(|r| r.url.path().contains("/view/")) + .count(); + assert_eq!(views, 0, "a hosted scan never fetches a patch view"); } /// The `redirect_rush_repo_state_stale` warning fires exactly when a Rush lock @@ -4979,8 +4979,7 @@ async fn composer_redirect_is_confirmed_and_recorded() { "dist.shasum must pin the patched artifact's sha1; got:\n{lock}" ); - // The confirmation is what drives the record fetch (held in memory for - // the in-run VEX — v5 persists no ledger): no confirmation, no record. + // A hosted scan fetches no patch record (v5 persists no ledger). let views = server .received_requests() .await @@ -4988,7 +4987,7 @@ async fn composer_redirect_is_confirmed_and_recorded() { .iter() .filter(|r| r.url.path().contains("/patches/view/")) .count(); - assert_eq!(views, 1, "the confirmed redirect fetched its patch record"); + assert_eq!(views, 0, "a hosted scan never fetches a patch view"); vlt_hosted_common::assert_no_ledger(tmp.path()); } diff --git a/crates/socket-patch-cli/tests/in_process_redirect/vlt.rs b/crates/socket-patch-cli/tests/in_process_redirect/vlt.rs index 6d0510734..16c8b5084 100644 --- a/crates/socket-patch-cli/tests/in_process_redirect/vlt.rs +++ b/crates/socket-patch-cli/tests/in_process_redirect/vlt.rs @@ -489,8 +489,10 @@ async fn scan_redirect_vlt_heal_rule_b_hidden_lock_without_node() { assert_eq!(warning_detail(&doc, ADVISORY), advisory_invalidated(1)); } +/// Without a hidden lock nothing records which bytes the store entry +/// holds: it is kept, and the advisory says to run `vlt ci`. #[tokio::test] -async fn scan_redirect_vlt_heal_rule_c_no_hidden_lock_with_record() { +async fn scan_redirect_vlt_heal_rule_c_no_hidden_lock_is_undeterminable() { let server = MockServer::start().await; mock_all(&server).await; let tmp = tempfile::tempdir().unwrap(); @@ -499,30 +501,7 @@ async fn scan_redirect_vlt_heal_rule_c_no_hidden_lock_with_record() { let (_, doc) = scan_hosted(tmp.path(), &server, &[], &[]); - assert!(!store_dir(tmp.path(), TILDE_ID).exists()); - assert_eq!(warning_detail(&doc, ADVISORY), advisory_invalidated(1)); -} - -/// Without a hidden lock or a patch record, the installed copy cannot be -/// judged: it is kept, and the advisory says to run `vlt ci`. -#[tokio::test] -async fn scan_redirect_vlt_heal_rule_c_no_record_is_undeterminable() { - let server = MockServer::start().await; - mock_discovery(&server).await; - mock_reference(&server).await; - mock_artifact(&server).await; - Mock::given(method("GET")) - .and(path(format!("/v0/orgs/{ORG}/patches/view/{UUID}"))) - .respond_with(ResponseTemplate::new(404)) - .mount(&server) - .await; - let tmp = tempfile::tempdir().unwrap(); - write_installed_vlt_project(tmp.path(), PRISTINE); - std::fs::remove_file(tmp.path().join("node_modules/.vlt-lock.json")).unwrap(); - - let (_, doc) = scan_hosted(tmp.path(), &server, &[], &[]); - - assert!(warning_codes(&doc).contains(&"record_fetch_failed".to_string())); + assert!(!warning_codes(&doc).contains(&"record_fetch_failed".to_string())); assert!(store_dir(tmp.path(), TILDE_ID).exists()); assert_eq!(warning_detail(&doc, ADVISORY), advisory_undeterminable(1)); } @@ -692,6 +671,9 @@ fn mkfifo(path: &Path) { assert!(status.success()); } +/// A hidden lock that is a FIFO or a symlink is never read (nor followed), +/// so nothing records which bytes the store entry holds: the entry and the +/// hidden lock are kept, and the advisory says to run `vlt ci`. #[cfg(unix)] #[tokio::test] async fn scan_redirect_vlt_heal_hidden_lock_fifo_or_symlink() { @@ -704,9 +686,9 @@ async fn scan_redirect_vlt_heal_hidden_lock_fifo_or_symlink() { std::fs::remove_file(&hidden).unwrap(); mkfifo(&hidden); let (_, doc) = scan_hosted(fifo.path(), &server, &[], &[]); - assert_eq!(warning_detail(&doc, ADVISORY), advisory_invalidated(1)); - assert!(std::fs::symlink_metadata(&hidden).is_err()); - assert!(!store_dir(fifo.path(), TILDE_ID).exists()); + assert_eq!(warning_detail(&doc, ADVISORY), advisory_undeterminable(1)); + assert!(std::fs::symlink_metadata(&hidden).is_ok()); + assert!(store_dir(fifo.path(), TILDE_ID).exists()); let linked = tempfile::tempdir().unwrap(); write_installed_vlt_project(linked.path(), PRISTINE); @@ -715,9 +697,13 @@ async fn scan_redirect_vlt_heal_hidden_lock_fifo_or_symlink() { std::fs::rename(&hidden, &target).unwrap(); std::os::unix::fs::symlink(&target, &hidden).unwrap(); let (_, doc) = scan_hosted(linked.path(), &server, &[], &[]); - assert_eq!(warning_detail(&doc, ADVISORY), advisory_invalidated(1)); - assert!(std::fs::symlink_metadata(&hidden).is_err()); - assert!(target.exists(), "only the link is removed"); + assert_eq!(warning_detail(&doc, ADVISORY), advisory_undeterminable(1)); + assert!( + std::fs::symlink_metadata(&hidden).is_ok(), + "the link is kept" + ); + assert!(target.exists()); + assert!(store_dir(linked.path(), TILDE_ID).exists()); } #[cfg(unix)] diff --git a/crates/socket-patch-cli/tests/in_process_redirect_pdm.rs b/crates/socket-patch-cli/tests/in_process_redirect_pdm.rs index 4ed40f332..0fdd84b01 100644 --- a/crates/socket-patch-cli/tests/in_process_redirect_pdm.rs +++ b/crates/socket-patch-cli/tests/in_process_redirect_pdm.rs @@ -33,6 +33,8 @@ use wiremock::{Mock, MockServer, ResponseTemplate}; #[allow(clippy::duplicate_mod)] #[path = "common/fs_tree.rs"] mod fs_tree; +#[path = "common/patched_wheel.rs"] +mod patched_wheel; #[path = "common/hermetic.rs"] mod hermetic; @@ -546,17 +548,22 @@ async fn hatchling_build_backend_does_not_veto_the_pdm_lock_redirect() { "pyproject untouched" ); assert_no_ledger(tmp.path()); - // Confirmed despite the hatch backend: the record was fetched. - let views = server - .received_requests() - .await - .unwrap_or_default() - .iter() - .filter(|r| r.url.path().ends_with(&format!("/patches/view/{UUID}"))) + // Confirmed despite the hatch backend: an idempotent re-scan reports + // the hosted pin. + let (code, json) = scan_json(tmp.path(), &server, &[]).await; + assert_eq!(code, Some(0), "{json}"); + let pinned = json["events"] + .as_array() + .into_iter() + .flatten() + .filter(|e| { + e["details"]["mode"] == "hosted" + && (e["action"] == "applied" || e["action"] == "verified") + }) .count(); assert_eq!( - views, 1, - "the pdm redirect must be confirmed despite the hatch backend" + pinned, 1, + "the pdm redirect must be confirmed despite the hatch backend: {json}" ); assert_manifestless_vex(tmp.path(), LOCK); @@ -708,7 +715,7 @@ fn stale_warning(json: &serde_json::Value) -> bool { /// Lay `urllib3 1.26.18` with `bytes` as its `response.py` into `site`. fn install_urllib3(site: &Path, bytes: &[u8]) { - std::fs::create_dir_all(site.join("urllib3-1.26.18.dist-info")).unwrap(); + patched_wheel::install_wheel_metadata(site, "urllib3-1.26.18.dist-info"); std::fs::create_dir_all(site.join("urllib3")).unwrap(); std::fs::write(site.join("urllib3").join("response.py"), bytes).unwrap(); } @@ -725,6 +732,20 @@ async fn pdm_recorded_env_is_probed_for_stale_hosted_installs() { for pep582 in [false, true] { let server = MockServer::start().await; mock_api(&server).await; + // The patched wheel the stale-install probe judges against. + let wheel = patched_wheel::build( + "urllib3-1.26.18.dist-info", + &[("urllib3/response.py", PATCHED)], + ); + patched_wheel::mount_grant( + &server, + ORG, + UUID, + PURL, + "/patch/pypi/urllib3/1.26.18/22222222-2222-4222-8222-222222222222/e828efa5-5c6d-43f3-9909-03f5ac232b98/urllib3-1.26.18-py2.py3-none-any.whl", + wheel, + ) + .await; let tmp = tempfile::tempdir().unwrap(); let root = tmp.path().join("app"); std::fs::create_dir_all(&root).unwrap(); diff --git a/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs b/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs index 4b1682cfb..a4778df83 100644 --- a/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs +++ b/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs @@ -31,6 +31,8 @@ use std::path::Path; #[path = "common/hermetic.rs"] mod hermetic; +#[path = "common/patched_wheel.rs"] +mod patched_wheel; #[path = "vex_e2e_common/mod.rs"] mod vex_e2e_common; #[path = "vex_pipenv_pip_steps/mod.rs"] @@ -111,6 +113,24 @@ async fn mock_api(server: &MockServer) { mock_api_serving(server, HOSTED_URL).await; } +/// Serve the patched wheel the stale-install probe judges against and pin +/// it ahead of [`mock_api`]'s grant; returns its URL. +async fn mock_patched_wheel(server: &MockServer) -> String { + let wheel = patched_wheel::build( + "urllib3-1.26.18.dist-info", + &[("urllib3/response.py", PATCHED)], + ); + patched_wheel::mount_grant( + server, + ORG, + UUID, + PURL, + "/patch/pypi/urllib3/1.26.18/22222222-2222-4222-8222-222222222222/e828efa5-5c6d-43f3-9909-03f5ac232b98/urllib3-1.26.18-py2.py3-none-any.whl", + wheel, + ) + .await +} + /// [`mock_api`] with the grant serving `hosted_url` as the patched artifact. async fn mock_api_serving(server: &MockServer, hosted_url: &str) { Mock::given(method("POST")) @@ -243,8 +263,8 @@ fn write_project(root: &Path) { fn write_project_with_upstream_install(root: &Path) { write_project(root); let site = site_packages(root); + patched_wheel::install_wheel_metadata(&site, "urllib3-1.26.18.dist-info"); let dist_info = site.join("urllib3-1.26.18.dist-info"); - std::fs::create_dir_all(&dist_info).unwrap(); std::fs::write( dist_info.join("METADATA"), "Metadata-Version: 2.1\nName: urllib3\nVersion: 1.26.18\n", @@ -318,14 +338,18 @@ async fn mock_pypi(server: &MockServer) { .await; } -/// In-process `rollback`: the hosted pin (on patch.socket.dev) is restored -/// to its upstream entry, re-resolved from the mocked PyPI JSON API. +/// In-process `rollback`: the hosted pin (on patch.socket.dev, or on the +/// mock's own origin where it serves the patched wheel) is restored to its +/// upstream entry, re-resolved from the mocked PyPI JSON API. async fn roll_back(cwd: &Path, server: &MockServer) { mock_pypi(server).await; std::env::set_var("SOCKET_PYPI_JSON_API", format!("{}/pypi", server.uri())); let code = rollback::run(RollbackArgs { targets: Vec::new(), - common: global(cwd, server.uri()), + common: GlobalArgs { + patch_server_url: Some(server.uri()), + ..global(cwd, server.uri()) + }, preserve_state: false, }) .await; @@ -661,6 +685,7 @@ async fn warm_venv_with_the_upstream_release_is_not_attested() { let _major = MajorGuard::set("2026"); let server = MockServer::start().await; mock_api(&server).await; + let hosted = mock_patched_wheel(&server).await; let tmp = tempfile::tempdir().unwrap(); write_project_with_upstream_install(tmp.path()); let lock_path = tmp.path().join("Pipfile.lock"); @@ -673,7 +698,7 @@ async fn warm_venv_with_the_upstream_release_is_not_attested() { let code = run(hosted_args(tmp.path(), server.uri(), Some(&vex_path))).await; let redirected = read(&lock_path); assert!( - redirected.contains(HOSTED_URL), + redirected.contains(&hosted), "the lock is still repointed: {redirected}" ); let attested = vex_path @@ -701,6 +726,8 @@ async fn warm_venv_with_the_upstream_release_is_not_attested() { // Manifest-less: the installed UPSTREAM copy is the evidence, whatever // the lock says — `not_applied`, online (the record from the API). + // The served wheel's origin is the patch server the lock now names. + let origin = server.uri(); std::thread::scope(|scope| { scope .spawn(|| { @@ -714,6 +741,7 @@ async fn warm_venv_with_the_upstream_release_is_not_attested() { &p, &vex_e2e_common::VexRun { product: Some(VEX_PRODUCT.into()), + patch_server_url: Some(origin.clone()), ..vex_e2e_common::VexRun::online(&api) }, ); @@ -744,6 +772,7 @@ async fn dotenv_selected_pipenv_install_is_checked_before_vex() { ] { let server = MockServer::start().await; mock_api(&server).await; + let hosted = mock_patched_wheel(&server).await; let tmp = tempfile::tempdir().unwrap(); let project = tmp.path().join("project"); std::fs::create_dir_all(&project).unwrap(); @@ -840,7 +869,7 @@ async fn dotenv_selected_pipenv_install_is_checked_before_vex() { "the ambient env is unchanged" ); assert_eq!(read(&project.join("Pipfile")), PIPFILE); - assert!(read(&project.join("Pipfile.lock")).contains(HOSTED_URL)); + assert!(read(&project.join("Pipfile.lock")).contains(&hosted)); } } @@ -852,6 +881,7 @@ async fn legacy_dotenv_workon_install_is_checked_before_vex() { for forward_reference in [true, false] { let server = MockServer::start().await; mock_api(&server).await; + mock_patched_wheel(&server).await; let tmp = tempfile::tempdir().unwrap(); let project = tmp.path().join("project"); let legacy = tmp.path().join("legacy-workon"); diff --git a/crates/socket-patch-cli/tests/in_process_redirect_poetry.rs b/crates/socket-patch-cli/tests/in_process_redirect_poetry.rs index 78248d9e9..c5ffe761b 100644 --- a/crates/socket-patch-cli/tests/in_process_redirect_poetry.rs +++ b/crates/socket-patch-cli/tests/in_process_redirect_poetry.rs @@ -13,6 +13,8 @@ use std::path::Path; #[path = "common/hermetic.rs"] mod hermetic; +#[path = "common/patched_wheel.rs"] +mod patched_wheel; #[path = "vex_e2e_common/mod.rs"] mod vex_e2e_common; @@ -378,7 +380,7 @@ fn install_package(root: &Path, venv: &str, bytes: &[u8]) -> std::path::PathBuf .join("python3.12") .join("site-packages") }; - std::fs::create_dir_all(site.join("urllib3-1.26.18.dist-info")).unwrap(); + patched_wheel::install_wheel_metadata(&site, "urllib3-1.26.18.dist-info"); std::fs::create_dir_all(site.join("urllib3")).unwrap(); let file = site.join("urllib3").join("response.py"); std::fs::write(&file, bytes).unwrap(); @@ -423,6 +425,24 @@ fn envelope(out: &std::process::Output) -> serde_json::Value { }) } +/// Serve the patched wheel the stale-install probe judges against and pin +/// it ahead of [`mock_api`]'s grant. +async fn mock_patched_wheel(server: &MockServer) { + let wheel = patched_wheel::build( + "urllib3-1.26.18.dist-info", + &[("urllib3/response.py", PATCHED)], + ); + patched_wheel::mount_grant( + server, + ORG, + UUID, + PURL, + "/patch/pypi/urllib3/1.26.18/22222222-2222-4222-8222-222222222222/e828efa5-5c6d-43f3-9909-03f5ac232b98/urllib3-1.26.18-py2.py3-none-any.whl", + wheel, + ) + .await; +} + fn stale_warning(value: &serde_json::Value) -> bool { // Top-level `warnings` is omitted when empty. value["warnings"] @@ -439,6 +459,7 @@ async fn stale_python_install_warns_and_cannot_attest_even_on_rescan() { for bytes in [UPSTREAM, b"local modification\n".as_slice()] { let server = MockServer::start().await; mock_api(&server).await; + mock_patched_wheel(&server).await; let tmp = tempfile::tempdir().unwrap(); write_project(tmp.path()); let installed = install_package(tmp.path(), ".venv", bytes); @@ -460,22 +481,18 @@ async fn stale_python_install_warns_and_cannot_attest_even_on_rescan() { assert_eq!(json["error"]["code"], "no_applicable_patches", "{json}"); assert!(!vex.exists(), "stale bytes cannot produce a VEX file"); } - // A failed fresh record fetch leaves the probe no evidence (v5 keeps - // no persisted record): the run still succeeds, says the record - // could not be fetched, and never attests. + // The probe judges by the pinned wheel, never the patch record: a + // failing record fetch changes nothing. Mock::given(method("GET")) .and(path(format!("/v0/orgs/{ORG}/patches/view/{UUID}"))) .respond_with(ResponseTemplate::new(404)) .with_priority(1) .mount(&server) .await; - let out = scan_output(tmp.path(), &server, &[]).await; - let stderr = String::from_utf8_lossy(&out.stderr); - assert!(out.status.success(), "{stderr}"); - assert!( - stderr.contains("was switched to hosted, but its patch record could not be fetched"), - "{stderr}" - ); + let out = scan_output(tmp.path(), &server, &["--json"]).await; + let json = envelope(&out); + assert!(out.status.success(), "{json}"); + assert!(stale_warning(&json), "{json}"); assert_eq!( std::fs::read(installed).unwrap(), bytes, @@ -521,6 +538,7 @@ fn install_behind_system_site_venv(root: &Path, bytes: &[u8]) -> std::path::Path async fn stale_base_copy_behind_system_site_venv_warns_and_cannot_attest() { let server = MockServer::start().await; mock_api(&server).await; + mock_patched_wheel(&server).await; let tmp = tempfile::tempdir().unwrap(); write_project(tmp.path()); let installed = install_behind_system_site_venv(tmp.path(), UPSTREAM); @@ -546,6 +564,7 @@ async fn stale_base_copy_behind_system_site_venv_warns_and_cannot_attest() { async fn venv_copy_shadows_stale_base_copy_behind_system_site_venv() { let server = MockServer::start().await; mock_api(&server).await; + mock_patched_wheel(&server).await; let tmp = tempfile::tempdir().unwrap(); write_project(tmp.path()); install_behind_system_site_venv(tmp.path(), UPSTREAM); @@ -567,6 +586,7 @@ async fn venv_copy_shadows_stale_base_copy_behind_system_site_venv() { async fn patched_python_install_attests_without_a_stale_warning() { let server = MockServer::start().await; mock_api(&server).await; + mock_patched_wheel(&server).await; let tmp = tempfile::tempdir().unwrap(); write_project(tmp.path()); let installed = install_package(tmp.path(), ".venv", PATCHED); @@ -588,6 +608,7 @@ async fn patched_python_install_attests_without_a_stale_warning() { async fn healthy_interpreter_cannot_mask_a_stale_python_install() { let server = MockServer::start().await; mock_api(&server).await; + mock_patched_wheel(&server).await; let tmp = tempfile::tempdir().unwrap(); write_project(tmp.path()); install_package(tmp.path(), ".venv", PATCHED); @@ -609,6 +630,7 @@ async fn healthy_interpreter_cannot_mask_a_stale_python_install() { async fn python_probe_does_not_guess_staleness_or_run_on_dry_run() { let server = MockServer::start().await; mock_api(&server).await; + mock_patched_wheel(&server).await; let tmp = tempfile::tempdir().unwrap(); write_project(tmp.path()); let installed = install_package(tmp.path(), ".venv", UPSTREAM); diff --git a/crates/socket-patch-cli/tests/scan/scan_ordered_concurrency_e2e.rs b/crates/socket-patch-cli/tests/scan/scan_ordered_concurrency_e2e.rs index 605a98613..f0ce46703 100644 --- a/crates/socket-patch-cli/tests/scan/scan_ordered_concurrency_e2e.rs +++ b/crates/socket-patch-cli/tests/scan/scan_ordered_concurrency_e2e.rs @@ -830,13 +830,12 @@ async fn mount_hosted(server: &MockServer, failing: &[usize], delay: fn(usize) - } } -/// A wet hosted run where 2 of 6 record views fail, with reversed -/// latencies: the `record_fetch_failed` warnings keep `confirmed` order, -/// stdout and the rewritten lockfile equal a zero-latency run's, and (v5) -/// neither run writes a redirect ledger — a failed record fetch only drops -/// the purl from this run's in-memory VEX records. +/// A wet hosted run fetches no record views: with 2 of 6 views failing and +/// reversed latencies mounted, no view is requested, nothing warns about a +/// record, stdout and the rewritten lockfile equal a zero-latency run's, +/// and (v5) neither run writes a redirect ledger. #[tokio::test] -async fn hosted_record_fetch_failures_keep_order_and_lock_bytes() { +async fn hosted_scan_fetches_no_record_views() { let failing = [1usize, 3]; let slow: fn(usize) -> Duration = |i| Duration::from_millis(50 * (6 - i as u64)); let fast: fn(usize) -> Duration = |_| Duration::ZERO; @@ -854,6 +853,14 @@ async fn hosted_record_fetch_failures_keep_order_and_lock_bytes() { &["--json", "--mode", "hosted", "--yes"], ); assert_eq!(code, 0, "stdout={stdout} stderr={stderr}"); + let views = server + .received_requests() + .await + .unwrap_or_default() + .into_iter() + .filter(|r| r.url.path().contains("/patches/view/")) + .count(); + assert_eq!(views, 0, "a hosted scan never fetches a patch view"); let lock = std::fs::read_to_string(tmp.path().join("package-lock.json")).unwrap(); assert!( !tmp.path() @@ -865,26 +872,7 @@ async fn hosted_record_fetch_failures_keep_order_and_lock_bytes() { } let (stdout, lock) = &outcomes[0]; - let v: serde_json::Value = serde_json::from_str(stdout).unwrap(); - let warnings: Vec<&str> = v["warnings"] - .as_array() - .unwrap() - .iter() - .filter(|w| w["code"] == "record_fetch_failed") - .map(|w| w["detail"].as_str().unwrap()) - .collect(); - assert_eq!(warnings.len(), 2, "{stdout}"); - for (warning, idx) in warnings.iter().zip([1usize, 3]) { - assert_eq!( - *warning, - format!( - "{} was switched to hosted, but its patch record could not be fetched; this \ - run's VEX attestation omits it (`socket-patch vex` fetches it again once the \ - API answers)", - purl(NAMES[idx]) - ) - ); - } + assert!(!stdout.contains("record_fetch_failed"), "{stdout}"); for idx in 0..NAMES.len() { assert!(lock.contains(&hosted_url(idx)), "{lock}"); } diff --git a/crates/socket-patch-cli/tests/vlt_e2e_common/fixture.rs b/crates/socket-patch-cli/tests/vlt_e2e_common/fixture.rs index 5463e314d..8e03d5222 100644 --- a/crates/socket-patch-cli/tests/vlt_e2e_common/fixture.rs +++ b/crates/socket-patch-cli/tests/vlt_e2e_common/fixture.rs @@ -46,8 +46,8 @@ pub fn advisory_invalidated(n: usize) -> String { pub fn advisory_undeterminable(n: usize) -> String { format!( "vlt-lock.json pins Socket-patched packages, but socket-patch could not check {n} \ - installed copies (node_modules is a link, or no patch record was available); run \ - `vlt ci` to be sure the patched packages are installed." + installed copies (node_modules is a link, or node_modules/.vlt-lock.json is missing \ + or unreadable); run `vlt ci` to be sure the patched packages are installed." ) } diff --git a/crates/socket-patch-cli/tests/vlt_hosted_common/mod.rs b/crates/socket-patch-cli/tests/vlt_hosted_common/mod.rs index 999c27ca3..722f8f0fe 100644 --- a/crates/socket-patch-cli/tests/vlt_hosted_common/mod.rs +++ b/crates/socket-patch-cli/tests/vlt_hosted_common/mod.rs @@ -60,8 +60,8 @@ pub fn advisory_cleanup_skipped(n: usize) -> String { pub fn advisory_undeterminable(n: usize) -> String { format!( "vlt-lock.json pins Socket-patched packages, but socket-patch could not check {n} \ - installed copies (node_modules is a link, or no patch record was available); run \ - `vlt ci` to be sure the patched packages are installed." + installed copies (node_modules is a link, or node_modules/.vlt-lock.json is missing \ + or unreadable); run `vlt ci` to be sure the patched packages are installed." ) } diff --git a/crates/socket-patch-core/src/api/client.rs b/crates/socket-patch-core/src/api/client.rs index 555e6a97b..068f39c09 100644 --- a/crates/socket-patch-core/src/api/client.rs +++ b/crates/socket-patch-core/src/api/client.rs @@ -2770,9 +2770,6 @@ pub trait PatchApi: Send + Sync { uuids: &'a [String], ) -> ApiFuture<'a, std::collections::HashMap>; - /// `GET …/patches/view/`; `None` when not found. - fn fetch_patch<'a>(&'a self, uuid: &'a str) -> ApiFuture<'a, Option>; - /// Download a grant-tokenized artifact URL, refusing bodies over /// `max_bytes`. fn download_artifact<'a>(&'a self, url: &'a str, max_bytes: u64) -> ApiFuture<'a, Vec>; @@ -2801,10 +2798,6 @@ impl PatchApi for ApiClient { Box::pin(ApiClient::fetch_registry_references(self, uuids)) } - fn fetch_patch<'a>(&'a self, uuid: &'a str) -> ApiFuture<'a, Option> { - Box::pin(ApiClient::fetch_patch(self, uuid)) - } - fn download_artifact<'a>(&'a self, url: &'a str, max_bytes: u64) -> ApiFuture<'a, Vec> { Box::pin(async move { let outcome = self diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index 063bfc740..4c8d76169 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -3584,19 +3584,6 @@ pub fn guard( .map(|rel| unreadable_refusal(rel)) } -/// The `record_fetch_failed` warning for a confirmed redirect whose patch -/// record could not be fetched. -pub fn record_fetch_failed_warning(purl: &str) -> RewriteWarning { - warning( - "record_fetch_failed", - format!( - "{purl} was switched to hosted, but its patch record could not be \ - fetched; this run's VEX attestation omits it (`socket-patch vex` \ - fetches it again once the API answers)" - ), - ) -} - /// A `{code, detail}` warning. pub fn warning(code: &str, detail: impl Into) -> RewriteWarning { RewriteWarning { diff --git a/crates/socket-patch-core/src/hosted/memory/discover.rs b/crates/socket-patch-core/src/hosted/memory/discover.rs index 563664ba0..7df2b29c1 100644 --- a/crates/socket-patch-core/src/hosted/memory/discover.rs +++ b/crates/socket-patch-core/src/hosted/memory/discover.rs @@ -14,7 +14,7 @@ use std::time::Duration; use crate::api::client::{ApiError, ApiFuture, PatchApi}; use crate::api::ranking::cmp_search_results; -use crate::api::types::{BatchPackagePatches, PackageVendorResult, PatchResponse, SearchResponse}; +use crate::api::types::{BatchPackagePatches, PackageVendorResult, SearchResponse}; use crate::hosted::npm_manifest::HostedClassicArtifact; use crate::utils::purl::{normalize_purl, strip_purl_qualifiers}; use crate::utils::purl_key::PurlKey; @@ -135,10 +135,6 @@ impl Provider { .await } - pub(crate) async fn fetch_patch(&self, uuid: &str) -> Result, ApiError> { - self.call("fetchPatch", self.api.fetch_patch(uuid)).await - } - pub(crate) async fn download_artifact( &self, url: &str, @@ -423,24 +419,6 @@ pub(crate) async fn fetch_classic_artifacts( .collect() } -/// Patch views for every distinct confirmed uuid (wet runs only). -pub(crate) async fn fetch_records( - provider: &Provider, - uuids: &BTreeSet, -) -> BTreeMap> { - let ordered: Vec<&String> = uuids.iter().collect(); - let futures: Vec> = ordered - .iter() - .map(|uuid| -> BoxFuture<'_, _> { Box::pin(provider.fetch_patch(uuid)) }) - .collect(); - let results = join_bounded(futures, provider.concurrency).await; - ordered - .into_iter() - .cloned() - .zip(results.into_iter().map(|r| r.ok().flatten())) - .collect() -} - #[cfg(test)] mod tests { use super::*; @@ -557,12 +535,6 @@ mod tests { ) -> ApiFuture<'a, HashMap> { Box::pin(async { Err(ApiError::Other("unused".into())) }) } - fn fetch_patch<'a>( - &'a self, - _uuid: &'a str, - ) -> ApiFuture<'a, Option> { - Box::pin(async { Err(ApiError::Other("unused".into())) }) - } fn download_artifact<'a>(&'a self, _url: &'a str, _max: u64) -> ApiFuture<'a, Vec> { Box::pin(async { Err(ApiError::Other("unused".into())) }) } diff --git a/crates/socket-patch-core/src/hosted/memory/mod.rs b/crates/socket-patch-core/src/hosted/memory/mod.rs index 4c40d8845..31e00e52d 100644 --- a/crates/socket-patch-core/src/hosted/memory/mod.rs +++ b/crates/socket-patch-core/src/hosted/memory/mod.rs @@ -51,7 +51,7 @@ use std::sync::Arc; use std::time::Instant; use crate::api::client::PatchApi; -use crate::api::types::{PatchResponse, PatchSearchResult}; +use crate::api::types::PatchSearchResult; use crate::crawlers::Ecosystem; use crate::utils::cargo_workspace::member_manifests_in; use crate::vendor::lock_inventory::{ @@ -1246,21 +1246,6 @@ async fn engine( } phases.mark("rollout"); - let record_uuids: BTreeSet = if options.dry_run { - BTreeSet::new() - } else { - rewritten - .iter() - .flat_map(|(_, r)| r.done.confirmed.iter().map(|(_, u)| u.clone())) - .collect() - }; - let records: BTreeMap> = if record_uuids.is_empty() { - BTreeMap::new() - } else { - discover::fetch_records(&provider, &record_uuids).await - }; - phases.mark("records"); - let mut changed: BTreeMap = BTreeMap::new(); let mut changed_binary: BTreeMap)> = BTreeMap::new(); let mut results: BTreeMap = BTreeMap::new(); @@ -1269,7 +1254,6 @@ async fn engine( let mut result = finish_root( state, done, - &records, options.dry_run, &mut changed, &mut changed_binary, @@ -1452,11 +1436,10 @@ fn select_with_policy( offers } -/// Records → the project's result and changed files. +/// The rewrite → the project's result and changed files. fn finish_root( state: &mut RootState, done: Rewritten, - records: &BTreeMap>, dry_run: bool, changed: &mut BTreeMap, changed_binary: &mut BTreeMap)>, @@ -1479,17 +1462,6 @@ fn finish_root( .. } = done; let root = state.root.clone(); - // No ledger keeps the records; the fetch mirrors the disk flow's, so a - // record the API cannot serve warns the same way. - let mut record_warnings: Vec = Vec::new(); - if !dry_run { - for (purl, uuid) in &confirmed { - match records.get(uuid) { - Some(Some(_)) => {} - _ => record_warnings.push(crate::hosted::engine::record_fetch_failed_warning(purl)), - } - } - } let mut project_changes: Vec<(String, String)> = Vec::new(); for (rel, content) in &rewrite.files { @@ -1562,7 +1534,6 @@ fn finish_root( // One typed list in the envelope's order; JSON only at the boundary. let mut warnings = rewrite.warnings.clone(); - warnings.extend(record_warnings); warnings.extend(rush_warnings); warnings.extend(pnpm_warnings); warnings.extend(npm_warnings); @@ -1660,11 +1631,9 @@ mod tests { let mut changed = BTreeMap::new(); let mut changed_binary = BTreeMap::new(); let mut warnings = Vec::new(); - let records = BTreeMap::new(); let first = finish_root( &mut state("", MemoryProject::new()), rewritten(&[("crates/b/Cargo.toml", "A")]), - &records, false, &mut changed, &mut changed_binary, @@ -1675,7 +1644,6 @@ mod tests { let second = finish_root( &mut state("crates/b", MemoryProject::new()), rewritten(&[("Cargo.toml", "B"), ("Cargo.lock", "L")]), - &records, false, &mut changed, &mut changed_binary, @@ -1693,7 +1661,6 @@ mod tests { let same = finish_root( &mut state("crates/b", MemoryProject::new()), rewritten(&[("Cargo.toml", "A")]), - &records, true, &mut changed, &mut changed_binary, @@ -1716,7 +1683,6 @@ mod tests { let result = finish_root( &mut state("web", MemoryProject::new()), done, - &BTreeMap::new(), true, &mut changed, &mut changed_binary, diff --git a/crates/socket-patch-core/src/hosted/mod.rs b/crates/socket-patch-core/src/hosted/mod.rs index d908c64da..952280811 100644 --- a/crates/socket-patch-core/src/hosted/mod.rs +++ b/crates/socket-patch-core/src/hosted/mod.rs @@ -25,3 +25,4 @@ pub mod render; pub mod sbt_reads; pub mod takeover; pub mod vlt; +pub mod wheel_record; diff --git a/crates/socket-patch-core/src/hosted/wheel_record.rs b/crates/socket-patch-core/src/hosted/wheel_record.rs new file mode 100644 index 000000000..4c6464a7f --- /dev/null +++ b/crates/socket-patch-core/src/hosted/wheel_record.rs @@ -0,0 +1,274 @@ +//! The patched wheel's own `RECORD`: the hosted Python stale-install +//! probe's evidence. A hosted rewrite pins the wheel in the lock and +//! installs nothing, so whether a warm virtualenv already holds the patched +//! bytes is read off the wheel the lock now pins: downloaded through its +//! grant URL, verified against the pinned sha256, and compared file by file +//! with the installed copy. + +use std::collections::BTreeSet; +use std::io::Read as _; +use std::path::Path; + +use base64::Engine as _; +use sha2::{Digest, Sha256}; + +use crate::api::client::ApiClient; +use crate::patch::apply::is_safe_relative_subpath; +use crate::vex::verify::InstalledRecordJudgment; + +/// Largest `RECORD` or `WHEEL` member read out of a wheel. +const MAX_RECORD_BYTES: u64 = 16 * 1024 * 1024; + +/// What the patched wheel says it installs. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WheelRecord { + /// The wheel's `-.dist-info` directory, which installers + /// keep under that name in site-packages. + dist_info: String, + /// The wheel's `Tag:` lines: an installed copy built for other tags is + /// another wheel, whose bytes say nothing about this one. + tags: BTreeSet, + /// `(path relative to site-packages, urlsafe base64 sha256)` for every + /// hashed file outside the dist-info and `.data` directories, which + /// installers rewrite or move. + files: Vec<(String, String)>, +} + +/// The patched wheel at `url` (a grant-tokenized artifact URL), verified +/// against `sha256` (hex). `None` when it cannot be fetched, does not +/// verify, or carries no readable `RECORD`. +pub async fn fetch_patched_wheel_record( + client: &ApiClient, + url: &str, + sha256: &str, +) -> Option { + let bytes = client.download_artifact(url).await.ok()?; + if !crate::utils::digest::sha256_hex_of(&bytes).eq_ignore_ascii_case(sha256) { + return None; + } + crate::utils::fs::run_blocking(move || parse_wheel_record(&bytes)).await +} + +fn read_member( + archive: &mut zip::ZipArchive>, + name: &str, +) -> Option { + let entry = archive.by_name(name).ok()?; + if entry.size() > MAX_RECORD_BYTES { + return None; + } + let mut text = String::new(); + entry + .take(MAX_RECORD_BYTES + 1) + .read_to_string(&mut text) + .ok()?; + Some(text) +} + +/// One `RECORD` row's fields: CSV, with `"`-quoted fields doubling `"`. +fn csv_fields(line: &str) -> Vec { + let mut fields = Vec::new(); + let mut field = String::new(); + let mut quoted = false; + let mut chars = line.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '"' if quoted && chars.peek() == Some(&'"') => { + field.push('"'); + chars.next(); + } + '"' => quoted = !quoted, + ',' if !quoted => fields.push(std::mem::take(&mut field)), + _ => field.push(c), + } + } + fields.push(field); + fields +} + +fn parse_wheel_record(bytes: &[u8]) -> Option { + let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes)).ok()?; + // The wheel's own dist-info sits at the root, never nested. + let dist_info = archive.file_names().find_map(|name| { + let stem = name.strip_suffix("/RECORD")?; + (stem.ends_with(".dist-info") && !stem.contains('/')).then(|| stem.to_string()) + })?; + let record = read_member(&mut archive, &format!("{dist_info}/RECORD"))?; + let wheel = read_member(&mut archive, &format!("{dist_info}/WHEEL"))?; + let data_dir = format!("{}.data/", dist_info.strip_suffix(".dist-info")?); + let dist_info_dir = format!("{dist_info}/"); + let files = record + .lines() + .filter_map(|line| { + let fields = csv_fields(line); + let path = fields.first()?; + let hash = fields.get(1)?.strip_prefix("sha256=")?; + (!hash.is_empty() && !path.starts_with(&dist_info_dir) && !path.starts_with(&data_dir)) + .then(|| (path.clone(), hash.to_string())) + }) + .collect(); + Some(WheelRecord { + tags: wheel_tags(&wheel), + dist_info, + files, + }) +} + +fn wheel_tags(wheel: &str) -> BTreeSet { + wheel + .lines() + .filter_map(|line| line.split_once(':')) + .filter(|(name, _)| name.trim().eq_ignore_ascii_case("Tag")) + .map(|(_, value)| value.trim().to_string()) + .collect() +} + +/// Judge the copy installed in `site` against `record`. +/// +/// * The installed `/WHEEL` must carry the patched wheel's tags; +/// any other copy (another platform's build, an sdist install with no +/// `WHEEL`) is never judged. +/// * `patched`: every recorded file reads back at its recorded hash. +/// * `stale_evidence`: some recorded file was read and hashes to something +/// else. Missing, unreadable and unsafe-path files are never evidence. +pub async fn judge_installed_wheel(site: &Path, record: &WheelRecord) -> InstalledRecordJudgment { + let site = site.to_path_buf(); + let record = record.clone(); + crate::utils::fs::run_blocking(move || { + let installed_wheel = site.join(&record.dist_info).join("WHEEL"); + let same_build = std::fs::read_to_string(installed_wheel) + .is_ok_and(|text| wheel_tags(&text) == record.tags && !record.tags.is_empty()); + if !same_build || record.files.is_empty() { + return InstalledRecordJudgment::default(); + } + let mut out = InstalledRecordJudgment { + patched: true, + stale_evidence: false, + }; + for (path, want) in &record.files { + if !is_safe_relative_subpath(path) { + out.patched = false; + continue; + } + let Ok(bytes) = std::fs::read(site.join(path)) else { + out.patched = false; + continue; + }; + let got = + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(Sha256::digest(&bytes)); + if got != want.trim_end_matches('=') { + return InstalledRecordJudgment { + patched: false, + stale_evidence: true, + }; + } + } + out + }) + .await +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write as _; + + fn b64(bytes: &[u8]) -> String { + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(Sha256::digest(bytes)) + } + + const PATCHED: &[u8] = b"print('patched')\n"; + const UPSTREAM: &[u8] = b"print('upstream')\n"; + const WHEEL: &str = "Wheel-Version: 1.0\nTag: py3-none-any\n"; + + fn wheel() -> Vec { + let mut out = std::io::Cursor::new(Vec::new()); + { + let mut zip = zip::ZipWriter::new(&mut out); + let options = zip::write::SimpleFileOptions::default(); + let record = format!( + "pkg/mod.py,sha256={},{}\n\"pkg/a,b.py\",sha256={},1\n\ + pkg-1.0.dist-info/METADATA,sha256={},1\npkg-1.0.data/scripts/x,sha256={},1\n\ + pkg-1.0.dist-info/RECORD,,\n", + b64(PATCHED), + PATCHED.len(), + b64(b"x"), + b64(b"m"), + b64(b"s"), + ); + for (name, body) in [ + ("pkg/mod.py", PATCHED.to_vec()), + ("pkg-1.0.dist-info/WHEEL", WHEEL.as_bytes().to_vec()), + ("pkg-1.0.dist-info/RECORD", record.into_bytes()), + ] { + zip.start_file(name, options).unwrap(); + zip.write_all(&body).unwrap(); + } + zip.finish().unwrap(); + } + out.into_inner() + } + + fn install(site: &Path, module: &[u8], wheel: &str) { + std::fs::create_dir_all(site.join("pkg")).unwrap(); + std::fs::create_dir_all(site.join("pkg-1.0.dist-info")).unwrap(); + std::fs::write(site.join("pkg/mod.py"), module).unwrap(); + std::fs::write(site.join("pkg/a,b.py"), b"x").unwrap(); + std::fs::write(site.join("pkg-1.0.dist-info/WHEEL"), wheel).unwrap(); + } + + #[test] + fn record_skips_dist_info_and_data_rows_and_reads_quoted_paths() { + let record = parse_wheel_record(&wheel()).unwrap(); + assert_eq!(record.dist_info, "pkg-1.0.dist-info"); + assert_eq!(record.tags, BTreeSet::from(["py3-none-any".to_string()])); + assert_eq!( + record.files, + [ + ("pkg/mod.py".to_string(), b64(PATCHED)), + ("pkg/a,b.py".to_string(), b64(b"x")), + ] + ); + } + + #[tokio::test] + async fn patched_stale_and_other_builds() { + let record = parse_wheel_record(&wheel()).unwrap(); + let tmp = tempfile::tempdir().unwrap(); + install(tmp.path(), PATCHED, WHEEL); + let judged = judge_installed_wheel(tmp.path(), &record).await; + assert!(judged.patched && !judged.stale_evidence, "{judged:?}"); + + install(tmp.path(), UPSTREAM, WHEEL); + let judged = judge_installed_wheel(tmp.path(), &record).await; + assert!(!judged.patched && judged.stale_evidence, "{judged:?}"); + + install( + tmp.path(), + UPSTREAM, + "Wheel-Version: 1.0\nTag: cp312-cp312-macosx_11_0_arm64\n", + ); + assert_eq!( + judge_installed_wheel(tmp.path(), &record).await, + InstalledRecordJudgment::default(), + "another build's bytes are never evidence" + ); + } + + #[tokio::test] + async fn missing_files_are_not_evidence() { + let record = parse_wheel_record(&wheel()).unwrap(); + let tmp = tempfile::tempdir().unwrap(); + install(tmp.path(), PATCHED, WHEEL); + std::fs::remove_file(tmp.path().join("pkg/mod.py")).unwrap(); + assert_eq!( + judge_installed_wheel(tmp.path(), &record).await, + InstalledRecordJudgment::default() + ); + } + + #[test] + fn a_wheel_without_record_or_wheel_metadata_has_no_record() { + assert!(parse_wheel_record(b"not a zip").is_none()); + } +} diff --git a/crates/socket-patch-core/src/patch/redirect/vlt_heal.rs b/crates/socket-patch-core/src/patch/redirect/vlt_heal.rs index cfa4fe362..a1696c74f 100644 --- a/crates/socket-patch-core/src/patch/redirect/vlt_heal.rs +++ b/crates/socket-patch-core/src/patch/redirect/vlt_heal.rs @@ -2,12 +2,12 @@ //! //! vlt never refreshes a store entry it already holds: after the lock is //! repointed, `vlt install` keeps `node_modules/.vlt/` and the -//! hidden lock as they are. A store entry whose bytes are not the ones the -//! lock now pins is invalidated (the entry plus `node_modules/.vlt-lock.json`) -//! so the next install extracts the pinned artifact. An entry whose state -//! cannot be determined is never removed, nor is one vlt would not -//! reinstall (an optional node), and nothing outside the project root is -//! ever touched. +//! hidden lock as they are. A store entry the hidden lock does not record +//! at the integrity the lock now pins is invalidated (the entry plus +//! `node_modules/.vlt-lock.json`) so the next install extracts the pinned +//! artifact. An entry whose state cannot be determined is never removed, +//! nor is one vlt would not reinstall (an optional node), and nothing +//! outside the project root is ever touched. use std::collections::BTreeSet; use std::path::{Path, PathBuf}; @@ -16,22 +16,11 @@ use serde_json::{Map, Value}; use super::hosted_patch_uuid; use crate::constants::npm_family::{VLT_HIDDEN_LOCK_REL, VLT_STORE_DIR}; -use crate::manifest::schema::PatchRecord; -use crate::patch::apply::{is_safe_relative_subpath, normalize_file_path}; -use crate::patch::file_hash::compute_file_git_sha256; use crate::utils::purl_key::PurlKey; use crate::vendor::vlt_lock_text::{ is_default_registry, is_registry_package_name, sniff_lock, split_dep_id, DepIdKind, LockSniff, }; -/// The installed state a target should be in: patched after a redirect, -/// pristine after a rollback or remove. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Expected { - Patched, - Pristine, -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum TargetState { Healthy, @@ -39,13 +28,6 @@ pub enum TargetState { Undeterminable, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ByteCheck { - Match, - Mismatch, - Unknown, -} - #[derive(Debug, Clone, PartialEq, Eq)] enum StoreState { /// No `node_modules` or no `node_modules/.vlt`: nothing is installed. @@ -92,7 +74,6 @@ pub struct Target<'a> { /// Slot [2] of the node in the final `vlt-lock.json` (`None` when the /// node is gone or the slot is null). pub lock_sha512: Option<&'a str>, - pub record: Option<&'a PatchRecord>, /// Slot [0] of the node, see [`reinstalls_after_removal`]. pub flags: Option, } @@ -104,9 +85,6 @@ pub struct LedgerTarget { pub purl: String, pub dep_id: String, pub name: String, - /// Always `None`: v5 keeps no hosted ledger to read a record from, so - /// the heal judges the installed copy against the lock's own pins. - pub record: Option, /// Slot [0] of the node in the pre-restore lock. pub flags: Option, } @@ -273,58 +251,16 @@ fn package_dir(root: &Path, dep_id: &str, name: &str) -> PathBuf { dir } -async fn file_hash(path: &Path) -> Result, ()> { - match compute_file_git_sha256(path).await { - Ok(hash) => Ok(Some(hash)), - Err(e) if crate::patch::apply::is_missing_path(&e) => Ok(None), - Err(_) => Err(()), - } -} - -async fn record_check(dir: &Path, record: &PatchRecord, expected: Expected) -> ByteCheck { - let mut unknown = false; - for (file, info) in &record.files { - let rel = normalize_file_path(file); - if !is_safe_relative_subpath(rel) { - return ByteCheck::Unknown; - } - let Ok(current) = file_hash(&dir.join(rel)).await else { - unknown = true; - continue; - }; - let want = match expected { - Expected::Patched => &info.after_hash, - Expected::Pristine => &info.before_hash, - }; - let matches = match current.as_deref() { - None => want.is_empty(), - Some(hash) => hash == want.as_str(), - }; - if !matches { - return ByteCheck::Mismatch; - } - } - if unknown { - ByteCheck::Unknown - } else { - ByteCheck::Match - } -} - -async fn bytes_check(dir: &Path, target: &Target<'_>, expected: Expected) -> ByteCheck { - match target.record.filter(|r| !r.files.is_empty()) { - Some(record) => record_check(dir, record, expected).await, - None => ByteCheck::Unknown, - } -} - -/// Is `target`'s store entry stale against `expected`, healthy, or -/// impossible to judge? The module doc states the rules. +/// Is `target`'s store entry stale against the pin `target` names (the +/// lock's slot [2]), healthy, or impossible to judge? The hidden lock +/// records the integrity of what vlt installed for each DepID, so it +/// decides: a missing node or another integrity is stale, the pinned one +/// healthy. Without a readable hidden lock nothing proves which bytes the +/// entry holds. pub async fn classify_target( state: &InstallState, root: &Path, target: &Target<'_>, - expected: Expected, ) -> TargetState { match state.store { StoreState::Absent => return TargetState::Healthy, @@ -338,24 +274,15 @@ pub async fn classify_target( if !tokio::fs::metadata(&dir).await.is_ok_and(|m| m.is_dir()) { return TargetState::Healthy; } - let bytes = bytes_check(&dir, target, expected).await; - if let HiddenLock::Parsed(nodes) = &state.hidden { - match nodes.get(target.dep_id) { - None => return TargetState::Stale, - Some(node) => { - if node.get(2).and_then(Value::as_str) != target.lock_sha512 { - return TargetState::Stale; - } - } - } - } - if bytes == ByteCheck::Mismatch { - return TargetState::Stale; - } - if !matches!(state.hidden, HiddenLock::Parsed(_)) && bytes == ByteCheck::Unknown { + let HiddenLock::Parsed(nodes) = &state.hidden else { return TargetState::Undeterminable; + }; + match nodes.get(target.dep_id) { + Some(node) if node.get(2).and_then(Value::as_str) == target.lock_sha512 => { + TargetState::Healthy + } + _ => TargetState::Stale, } - TargetState::Healthy } async fn remove_entry(path: &Path) -> std::io::Result<()> { @@ -409,8 +336,7 @@ pub async fn invalidate(root: &Path, state: &InstallState, stale: &[String]) -> /// The Socket-hosted vlt nodes of `lock` (read BEFORE a restore rewrites /// it) that stand for one of `purls` — the ledger-free rollback heal's -/// targets. No patch record rides along (v5 keeps no hosted ledger), so the -/// heal judges each installed copy against the lock's own pins. +/// targets, each judged against the lock's own pins. pub fn lock_targets(lock: &str, origins: &[String], purls: &[String]) -> Vec { let wanted: Vec = purls.iter().map(|p| PurlKey::new(p)).collect(); socket_owned_instances(lock, origins) @@ -423,7 +349,6 @@ pub fn lock_targets(lock: &str, origins: &[String], purls: &[String]) -> Vec Vec PatchRecord { - PatchRecord { - uuid: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa".into(), - exported_at: String::new(), - files: HashMap::from([( - "package/index.js".to_string(), - PatchFileInfo { - before_hash: compute_git_sha256_from_bytes(PRISTINE), - after_hash: compute_git_sha256_from_bytes(PATCHED), - }, - )]), - vulnerabilities: HashMap::new(), - description: String::new(), - license: String::new(), - tier: String::new(), - } - } - fn store(root: &Path, id: &str, index: &[u8]) -> PathBuf { let dir = package_dir(root, id, "left-pad"); std::fs::create_dir_all(&dir).unwrap(); @@ -480,19 +384,18 @@ mod tests { .unwrap(); } - fn target(record: Option<&PatchRecord>) -> Target<'_> { + fn target() -> Target<'static> { Target { dep_id: ID, name: "left-pad", lock_sha512: Some(LOCK_SHA), - record, flags: Some(0), } } - async fn classify(root: &Path, target: &Target<'_>, expected: Expected) -> TargetState { + async fn classify(root: &Path, target: &Target<'_>) -> TargetState { let state = read_install_state(root).await; - classify_target(&state, root, target, expected).await + classify_target(&state, root, target).await } #[tokio::test] @@ -500,11 +403,7 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); store(tmp.path(), ID, PATCHED); hidden(tmp.path(), Some("sha512-UPSTREAM")); - let rec = record(); - assert_eq!( - classify(tmp.path(), &target(Some(&rec)), Expected::Patched).await, - TargetState::Stale - ); + assert_eq!(classify(tmp.path(), &target()).await, TargetState::Stale); } #[tokio::test] @@ -512,75 +411,31 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); store(tmp.path(), ID, PATCHED); hidden(tmp.path(), None); - let rec = record(); - assert_eq!( - classify(tmp.path(), &target(Some(&rec)), Expected::Patched).await, - TargetState::Stale - ); + assert_eq!(classify(tmp.path(), &target()).await, TargetState::Stale); } #[tokio::test] - async fn rule_c_bytes_mismatch_whatever_the_hidden_lock_says() { - let tmp = tempfile::tempdir().unwrap(); - store(tmp.path(), ID, PRISTINE); - hidden(tmp.path(), Some(LOCK_SHA)); - let rec = record(); - assert_eq!( - classify(tmp.path(), &target(Some(&rec)), Expected::Patched).await, - TargetState::Stale - ); - std::fs::remove_file(tmp.path().join(VLT_HIDDEN_LOCK_REL)).unwrap(); - assert_eq!( - classify(tmp.path(), &target(Some(&rec)), Expected::Patched).await, - TargetState::Stale - ); - } - - #[tokio::test] - async fn healthy_patched_and_pristine_trees() { + async fn a_hidden_lock_at_the_pin_is_healthy_and_none_is_undeterminable() { let tmp = tempfile::tempdir().unwrap(); store(tmp.path(), ID, PATCHED); - hidden(tmp.path(), Some(LOCK_SHA)); - let rec = record(); assert_eq!( - classify(tmp.path(), &target(Some(&rec)), Expected::Patched).await, - TargetState::Healthy - ); - std::fs::remove_file(tmp.path().join(VLT_HIDDEN_LOCK_REL)).unwrap(); - assert_eq!( - classify(tmp.path(), &target(Some(&rec)), Expected::Patched).await, - TargetState::Healthy, - "a tree without a hidden lock is judged by its bytes" - ); - assert_eq!( - classify(tmp.path(), &target(Some(&rec)), Expected::Pristine).await, - TargetState::Stale, - "patched bytes are stale once the pin is restored" - ); - store(tmp.path(), ID, PRISTINE); - assert_eq!( - classify(tmp.path(), &target(Some(&rec)), Expected::Pristine).await, - TargetState::Healthy - ); - } - - #[tokio::test] - async fn undeterminable_without_hidden_lock_or_record() { - let tmp = tempfile::tempdir().unwrap(); - store(tmp.path(), ID, PATCHED); - assert_eq!( - classify(tmp.path(), &target(None), Expected::Patched).await, + classify(tmp.path(), &target()).await, TargetState::Undeterminable ); hidden(tmp.path(), Some(LOCK_SHA)); + assert_eq!(classify(tmp.path(), &target()).await, TargetState::Healthy); + let pristine = Target { + lock_sha512: Some("sha512-UPSTREAM"), + ..target() + }; assert_eq!( - classify(tmp.path(), &target(None), Expected::Patched).await, - TargetState::Healthy, - "a parsed hidden lock that agrees decides alone" + classify(tmp.path(), &pristine).await, + TargetState::Stale, + "the patched entry is stale once the pin is restored" ); std::fs::write(tmp.path().join(VLT_HIDDEN_LOCK_REL), b"{not json").unwrap(); assert_eq!( - classify(tmp.path(), &target(None), Expected::Pristine).await, + classify(tmp.path(), &target()).await, TargetState::Undeterminable ); } @@ -588,16 +443,9 @@ mod tests { #[tokio::test] async fn an_uninstalled_target_is_healthy() { let tmp = tempfile::tempdir().unwrap(); - let rec = record(); - assert_eq!( - classify(tmp.path(), &target(Some(&rec)), Expected::Patched).await, - TargetState::Healthy - ); + assert_eq!(classify(tmp.path(), &target()).await, TargetState::Healthy); std::fs::create_dir_all(tmp.path().join(VLT_STORE_DIR)).unwrap(); - assert_eq!( - classify(tmp.path(), &target(Some(&rec)), Expected::Patched).await, - TargetState::Healthy - ); + assert_eq!(classify(tmp.path(), &target()).await, TargetState::Healthy); } #[cfg(unix)] @@ -619,10 +467,9 @@ mod tests { .unwrap(); } hidden(&real_root, Some("sha512-UPSTREAM")); - let rec = record(); let state = read_install_state(&project).await; assert_eq!( - classify_target(&state, &project, &target(Some(&rec)), Expected::Patched).await, + classify_target(&state, &project, &target()).await, TargetState::Undeterminable, "{linked}" ); @@ -799,7 +646,7 @@ mod tests { /// The rollback heal's only target source (the ledger-driven /// `ledger_targets` is gone): every Socket-owned instance of a wanted - /// purl, peer contexts included, with its slot [0] flags and no record. + /// purl, peer contexts included, with its slot [0] flags. #[test] fn lock_targets_name_every_owned_instance_of_the_purls() { let lock = r#"{ @@ -834,9 +681,7 @@ mod tests { ), ] ); - assert!(targets - .iter() - .all(|t| t.name == "left-pad" && t.record.is_none())); + assert!(targets.iter().all(|t| t.name == "left-pad")); assert!(lock_targets("\u{feff}{}", &[], &purls).is_empty()); } diff --git a/crates/socket-patch-node/npm/index.d.ts b/crates/socket-patch-node/npm/index.d.ts index 655b33ff1..8327b0696 100644 --- a/crates/socket-patch-node/npm/index.d.ts +++ b/crates/socket-patch-node/npm/index.d.ts @@ -35,11 +35,12 @@ export interface PatchProvider { searchPatchesBatch(request: { components: { purl: string }[] }): Promise> searchPatchesByPackage(request: { purl: string }): Promise> fetchRegistryReferences(request: { uuids: string[] }): Promise }>> - fetchPatch(request: { uuid: string }): Promise> + // Ignored: the engine no longer fetches patch views. Optional so existing providers still type-check. + fetchPatch?(request: { uuid: string }): Promise> downloadArtifact(request: { url: string; maxBytes: number }): Promise> } // Providers MUST resolve (never reject); the JS loader wraps provider methods so a thrown/rejected call becomes {ok:false, kind:'other'}. -// A 'not_found' failure mirrors the HTTP API's 404 per method: searchPatchesByPackage => no patches, fetchRegistryReferences => no references, fetchPatch => null, searchPatchesBatch and downloadArtifact => error. +// A 'not_found' failure mirrors the HTTP API's 404 per method: searchPatchesByPackage => no patches, fetchRegistryReferences => no references, searchPatchesBatch and downloadArtifact => error. export interface HostedScanLimits { maxFileBytes?: number /*20 MiB*/; maxTotalBytes?: number /*64 MiB*/; maxFiles?: number /*2000*/; maxPurls?: number /*20000*/; maxProjects?: number /*200*/; maxArtifactBytes?: number /*32 MiB*/ } export interface HostedScanSessionOptions { diff --git a/crates/socket-patch-node/npm/index.js b/crates/socket-patch-node/npm/index.js index 65d695e9e..9bb5401f1 100644 --- a/crates/socket-patch-node/npm/index.js +++ b/crates/socket-patch-node/npm/index.js @@ -17,7 +17,6 @@ const JSON_PROVIDER_METHODS = [ 'searchPatchesBatch', 'searchPatchesByPackage', 'fetchRegistryReferences', - 'fetchPatch', ] const MAX_PROVIDER_MESSAGE = 2000 const REQUIRED_EXPORTS = [ diff --git a/crates/socket-patch-node/npm/test/smoke.mjs b/crates/socket-patch-node/npm/test/smoke.mjs index 0009f955e..425f35ce0 100644 --- a/crates/socket-patch-node/npm/test/smoke.mjs +++ b/crates/socket-patch-node/npm/test/smoke.mjs @@ -219,7 +219,7 @@ test('streamed session redirects the package-lock fixture', async () => { assert.equal(result.stats.patchesRedirected, 1) assert.equal(calls.searchPatchesBatch, 1) assert.equal(calls.fetchRegistryReferences, 1) - assert.ok(calls.fetchPatch >= 1) + assert.equal(calls.fetchPatch, 0, 'patch views are never fetched') }) test('dry run previews the lockfile without patch fetches', async () => { @@ -433,9 +433,6 @@ test('not_found failures map per method like the HTTP API 404', async () => { assert.equal(references.project.error, undefined) assert.deepEqual(references.project.redirected, []) - const patch = await run({ fetchPatch: notFound }) - assert.ok(patch.calls.fetchPatch >= 1) - assert.equal(patch.project.error, undefined) const batch = await run({ searchPatchesBatch: notFound }) assert.equal(batch.project.error?.code, 'patch_lookup_failed') diff --git a/crates/socket-patch-node/src/lib.rs b/crates/socket-patch-node/src/lib.rs index 29fa8e6ae..ff5d59452 100644 --- a/crates/socket-patch-node/src/lib.rs +++ b/crates/socket-patch-node/src/lib.rs @@ -217,9 +217,6 @@ impl NativeHostedScanSession { fetch_registry_references: provider_function(&provider, "fetchRegistryReferences")? .create_ref() .map_err(napi_js_error)?, - fetch_patch: provider_function(&provider, "fetchPatch")? - .create_ref() - .map_err(napi_js_error)?, download_artifact: provider_function(&provider, "downloadArtifact")? .create_ref() .map_err(napi_js_error)?, diff --git a/crates/socket-patch-node/src/provider.rs b/crates/socket-patch-node/src/provider.rs index 1215354cc..3fb3552ed 100644 --- a/crates/socket-patch-node/src/provider.rs +++ b/crates/socket-patch-node/src/provider.rs @@ -17,9 +17,7 @@ use napi_derive::napi; use serde::de::DeserializeOwned; use serde::Deserialize; use socket_patch_core::api::client::{sort_batch_response, ApiError, ApiFuture, PatchApi}; -use socket_patch_core::api::types::{ - BatchSearchResponse, PackageVendorResult, PatchResponse, SearchResponse, -}; +use socket_patch_core::api::types::{BatchSearchResponse, PackageVendorResult, SearchResponse}; type JsonCall = ThreadsafeFunction, String, Status, false>; type DownloadCall = ThreadsafeFunction, String, Status, false>; @@ -42,7 +40,6 @@ pub(crate) struct ProviderRefs { pub(crate) search_patches_batch: JsonRef, pub(crate) search_patches_by_package: JsonRef, pub(crate) fetch_registry_references: JsonRef, - pub(crate) fetch_patch: JsonRef, pub(crate) download_artifact: DownloadRef, } @@ -50,7 +47,6 @@ pub(crate) struct JsPatchApi { search_patches_batch: JsonCall, search_patches_by_package: JsonCall, fetch_registry_references: JsonCall, - fetch_patch: JsonCall, download_artifact: DownloadCall, } @@ -71,7 +67,6 @@ impl JsPatchApi { search_patches_batch: json_call(env, &refs.search_patches_batch)?, search_patches_by_package: json_call(env, &refs.search_patches_by_package)?, fetch_registry_references: json_call(env, &refs.fetch_registry_references)?, - fetch_patch: json_call(env, &refs.fetch_patch)?, download_artifact: refs .download_artifact .borrow_back(env)? @@ -237,16 +232,6 @@ impl PatchApi for JsPatchApi { }) } - fn fetch_patch<'a>(&'a self, uuid: &'a str) -> ApiFuture<'a, Option> { - Box::pin(async move { - let request = serde_json::json!({ "uuid": uuid }).to_string(); - match call_json("fetchPatch", &self.fetch_patch, request).await? { - Reply::Value(value) => Ok(value), - Reply::NotFound(_) => Ok(None), - } - }) - } - fn download_artifact<'a>(&'a self, url: &'a str, max_bytes: u64) -> ApiFuture<'a, Vec> { Box::pin(async move { let method = "downloadArtifact"; diff --git a/docs/testing/poetry-compatibility.md b/docs/testing/poetry-compatibility.md index 7152c4a33..0d3139ba1 100644 --- a/docs/testing/poetry-compatibility.md +++ b/docs/testing/poetry-compatibility.md @@ -95,7 +95,7 @@ Two consequences for Poetry releases before 1.4: Hosted mode also checks the installed package's bytes, independently of the lock's writer version. `redirect_pypi_stale_install` means readable files in a discovered -interpreter still differ from the patched hashes. This read-only check repeats on +interpreter still differ from the pinned patched wheel's `RECORD`. This read-only check repeats on re-scans and excludes the package from same-run VEX, even with `--vex-no-verify` or a healthy copy in another interpreter. If no patches remain to attest, `--vex` fails with `no_applicable_patches`. Reinstall in the affected From fb3d4183852150c00139b671240b93019ba88655 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 19:25:11 +0000 Subject: [PATCH 2/2] Read wheel probe files through the regular-file guard; fix vlt rule (c) e2e judge_installed_wheel read the installed WHEEL and every RECORD path with bare std::fs reads. Site-packages is user-writable, so a FIFO there would block the hosted scan's blocking pool. Read through read_regular_*_sync so a FIFO or device counts as unreadable, and add a regression test for both the WHEEL and a RECORD path. The real-vlt e2e for heal rule (c) without a hidden lock still expected the old record-based heal (copies invalidated). Hosted scans no longer fetch patch records, so that case is now undeterminable: the copies are kept and the advisory says to run `vlt ci`. Match the in-process test and check that `vlt ci` then installs the patched package. Co-Authored-By: Claude --- .../tests/e2e_redirect_vlt_build.rs | 17 +++++++----- .../src/hosted/wheel_record.rs | 27 +++++++++++++++++-- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/crates/socket-patch-cli/tests/e2e_redirect_vlt_build.rs b/crates/socket-patch-cli/tests/e2e_redirect_vlt_build.rs index cbea59cc2..49800005d 100644 --- a/crates/socket-patch-cli/tests/e2e_redirect_vlt_build.rs +++ b/crates/socket-patch-cli/tests/e2e_redirect_vlt_build.rs @@ -343,7 +343,10 @@ async fn vlt_pinned_matrix_hosted_heal_rule_b_hidden_lock_without_node() { fx.leg.ran(); } -/// Heal rule (c): no hidden lock, the record's afterHash decides. +/// Heal rule (c): no hidden lock. Hosted scans no longer fetch patch +/// records, so nothing records which bytes the installed copies hold: they +/// are kept, and the advisory says to run `vlt ci`, which installs the +/// patched packages. #[tokio::test(flavor = "multi_thread")] #[ignore = "real vlt: SOCKET_PATCH_VLT_E2E_JS"] async fn vlt_pinned_matrix_hosted_heal_rule_c_no_hidden_lock() { @@ -353,19 +356,19 @@ async fn vlt_pinned_matrix_hosted_heal_rule_c_no_hidden_lock() { let fx = Fixture::build(leg, Shape::with_bystander().warm()).await; let _ = std::fs::remove_file(fx.proj.join(HIDDEN_LOCK)); let ids = fx.store_ids(fx.t()); - let snap = fx.snapshot(&fx.proj, &ids); let doc = fx.scan(&[]); - fx.assert_advisory(&doc, &advisory_invalidated(ids.len())); + fx.assert_advisory(&doc, &advisory_undeterminable(ids.len())); for id in &ids { - assert!(!store_entry(&fx.proj, id).exists(), "{id} invalidated"); + assert!(store_entry(&fx.proj, id).exists(), "{id} kept"); } - snap.assert_same(&fx.snapshot(&fx.proj, &ids), "the rule (c) heal"); assert!( store_entry(&fx.proj, &node_id(&read_lock(&fx.proj), MS.0, MS.1)).exists(), "the bystander is healthy" ); - fx.vlt_ok(&fx.proj, &["install"]); - assert_eq!(state(&fx.proj, fx.t()), State::Patched); + if fx.leg.at_least(HAS_CI_FROM) { + fx.vlt_ok(&fx.proj, &["ci"]); + assert_eq!(state(&fx.proj, fx.t()), State::Patched); + } fx.leg.ran(); } diff --git a/crates/socket-patch-core/src/hosted/wheel_record.rs b/crates/socket-patch-core/src/hosted/wheel_record.rs index 4c6464a7f..4bf782f91 100644 --- a/crates/socket-patch-core/src/hosted/wheel_record.rs +++ b/crates/socket-patch-core/src/hosted/wheel_record.rs @@ -136,7 +136,9 @@ pub async fn judge_installed_wheel(site: &Path, record: &WheelRecord) -> Install let record = record.clone(); crate::utils::fs::run_blocking(move || { let installed_wheel = site.join(&record.dist_info).join("WHEEL"); - let same_build = std::fs::read_to_string(installed_wheel) + // Site-packages is user-writable: read through the regular-file + // guard so a FIFO or device counts as unreadable instead of blocking. + let same_build = crate::utils::fs::read_regular_to_string_sync(&installed_wheel) .is_ok_and(|text| wheel_tags(&text) == record.tags && !record.tags.is_empty()); if !same_build || record.files.is_empty() { return InstalledRecordJudgment::default(); @@ -150,7 +152,7 @@ pub async fn judge_installed_wheel(site: &Path, record: &WheelRecord) -> Install out.patched = false; continue; } - let Ok(bytes) = std::fs::read(site.join(path)) else { + let Ok(bytes) = crate::utils::fs::read_regular_to_bytes_sync(&site.join(path)) else { out.patched = false; continue; }; @@ -255,6 +257,27 @@ mod tests { ); } + #[cfg(unix)] + #[tokio::test] + async fn a_fifo_in_site_packages_is_unreadable_not_a_hang() { + let record = parse_wheel_record(&wheel()).unwrap(); + for target in ["pkg/mod.py", "pkg-1.0.dist-info/WHEEL"] { + let tmp = tempfile::tempdir().unwrap(); + install(tmp.path(), PATCHED, WHEEL); + let fifo = tmp.path().join(target); + std::fs::remove_file(&fifo).unwrap(); + let c = std::ffi::CString::new(fifo.as_os_str().as_encoded_bytes()).unwrap(); + assert_eq!(unsafe { libc::mkfifo(c.as_ptr(), 0o600) }, 0); + let judged = tokio::time::timeout( + std::time::Duration::from_secs(10), + judge_installed_wheel(tmp.path(), &record), + ) + .await + .expect("a FIFO must not block the probe"); + assert_eq!(judged, InstalledRecordJudgment::default(), "{target}"); + } + } + #[tokio::test] async fn missing_files_are_not_evidence() { let record = parse_wheel_record(&wheel()).unwrap();