From e17f8d3c2eb267cb06b75f7e2aadc70fedd5c83c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 19 Sep 2026 20:08:18 -0700 Subject: [PATCH 1/9] test: add the adversarial request-contract matrix Every RPC case in this suite names api-inventory.json as mandatory test data and states that the method is exercised with its request fields "absent/default, valid, boundary-invalid and combined with an unknown field" in both representations. The harnesses behind those cases send one valid request and one wrong-typed value for the *first* request field. So the input classes where a service actually breaks -- an out-of-range integer, an oversized string, a wrong protobuf wire type, a truncated body -- are untested for all 110 indexed methods. Add a harness that implements the stated matrix once, driven by the same inventory, and nine cases that apply it to the guest agent's four services, the KMS's two, and the gateway's three. It asserts the invariants that hold for every method whatever it means: a malformed request is never a 5xx or a dropped connection, a rejection is structured and bounded and carries the caller's own representation, no call outlives its deadline, and the listener still answers afterwards. That last one is a real assertion here: the workspace builds release binaries with `panic = "abort"`, so a reachable panic is a process abort. api-contract-matrix-policy.json says how far each method may be driven. `full` is for a method whose effect stays inside the lease; `rejection-only` sends only what a service must refuse before it dispatches, so nothing can mutate; `skip` is for a method whose valid invocation ends the process or consumes an external resource, and the file records the reason for each. Every method in the inventory must carry a policy -- one added without one fails the case rather than being silently skipped. Two product defects found while building this are fixed separately: a `not_after` of u64::MAX aborted the guest agent (#1232), and a rejection echoed the whole rejected value back to the caller and into the log under a content type the caller did not ask for (#1234). --- .../15-request-contract/metadata.json | 4 + .../tc-gos-contract-001/case.md | 122 +++ .../tc-gos-contract-001/metadata.json | 36 + .../tc-gos-contract-002/case.md | 122 +++ .../tc-gos-contract-002/metadata.json | 36 + .../tc-gos-contract-003/case.md | 122 +++ .../tc-gos-contract-003/metadata.json | 36 + .../tc-gos-contract-004/case.md | 122 +++ .../tc-gos-contract-004/metadata.json | 36 + .../03-kms/13-request-contract/metadata.json | 4 + .../tc-kms-contract-001/case.md | 122 +++ .../tc-kms-contract-001/metadata.json | 36 + .../tc-kms-contract-002/case.md | 122 +++ .../tc-kms-contract-002/metadata.json | 36 + .../11-request-contract/metadata.json | 4 + .../tc-gw-contract-001/case.md | 122 +++ .../tc-gw-contract-001/metadata.json | 36 + .../tc-gw-contract-002/case.md | 122 +++ .../tc-gw-contract-002/metadata.json | 36 + .../tc-gw-contract-003/case.md | 122 +++ .../tc-gw-contract-003/metadata.json | 36 + test-suites/catalog/feature-audit.md | 14 + .../automation/api-contract-matrix-case.py | 748 ++++++++++++++++++ .../api-contract-matrix-policy.json | 127 +++ .../shared/automation/api_contract_matrix.py | 448 +++++++++++ .../automation/promoted-passing-cases.json | 81 ++ 26 files changed, 2852 insertions(+) create mode 100644 test-suites/cases/01-guest-os/15-request-contract/metadata.json create mode 100644 test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-001/case.md create mode 100644 test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-001/metadata.json create mode 100644 test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-002/case.md create mode 100644 test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-002/metadata.json create mode 100644 test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-003/case.md create mode 100644 test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-003/metadata.json create mode 100644 test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-004/case.md create mode 100644 test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-004/metadata.json create mode 100644 test-suites/cases/03-kms/13-request-contract/metadata.json create mode 100644 test-suites/cases/03-kms/13-request-contract/tc-kms-contract-001/case.md create mode 100644 test-suites/cases/03-kms/13-request-contract/tc-kms-contract-001/metadata.json create mode 100644 test-suites/cases/03-kms/13-request-contract/tc-kms-contract-002/case.md create mode 100644 test-suites/cases/03-kms/13-request-contract/tc-kms-contract-002/metadata.json create mode 100644 test-suites/cases/04-gateway/11-request-contract/metadata.json create mode 100644 test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-001/case.md create mode 100644 test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-001/metadata.json create mode 100644 test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-002/case.md create mode 100644 test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-002/metadata.json create mode 100644 test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-003/case.md create mode 100644 test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-003/metadata.json create mode 100755 test-suites/shared/automation/api-contract-matrix-case.py create mode 100644 test-suites/shared/automation/api-contract-matrix-policy.json create mode 100644 test-suites/shared/automation/api_contract_matrix.py diff --git a/test-suites/cases/01-guest-os/15-request-contract/metadata.json b/test-suites/cases/01-guest-os/15-request-contract/metadata.json new file mode 100644 index 000000000..5843d6348 --- /dev/null +++ b/test-suites/cases/01-guest-os/15-request-contract/metadata.json @@ -0,0 +1,4 @@ +{ + "id": "section-guest-os-request-contract", + "title": "Request contract matrix" +} diff --git a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-001/case.md b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-001/case.md new file mode 100644 index 000000000..31058e4d9 --- /dev/null +++ b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-001/case.md @@ -0,0 +1,122 @@ + + + +# TC-GOS-CONTRACT-001: Tappd request-contract matrix + +## Metadata + +- Priority: P0 +- Type: Functional, API, Security, Robustness, Regression +- Minimum environment: INTEGRATION +- Automation: Yes +- Requirements: [req-gos-contract-001](../../../../catalog/feature-audit.md#req-gos-contract-001) +- Risks: [risk-gos-contract-001](../../../../catalog/feature-audit.md#risk-gos-contract-001) +- Source: [`catalog/api-inventory.json`](../../../../catalog/api-inventory.json), [`shared/automation/api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + +## Prepared execution knowledge + +- Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. +- The method list, the request field list and every field's declared type come + from [`api-inventory.json`](../../../../catalog/api-inventory.json). Do not reconstruct + them from implementation source: the point of this case is that the indexed + contract and the running service agree. +- [`api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + says how far each method may be driven. `full` runs every vector, and is only + used for a method whose effect is confined to the lease. `rejection-only` + runs the vectors a service must refuse before it dispatches, so nothing can + mutate. `skip` is for a method whose valid invocation ends the process or + consumes an external resource, and the policy file records why for each one. +- Every method the inventory declares for this service must carry a policy. A + method added to the inventory without one fails this case rather than being + silently skipped. + +## Objective + +Verify that every method `Tappd` declares answers absent, default, valid, +boundary-invalid, wrong-typed, unknown-field and malformed-framing input, in +both the JSON and the protobuf representation, without violating the +invariants that hold for every method whatever it means. + +The per-method cases in this suite each send one valid request and one +wrong-typed value for the first request field. That leaves the input classes +where a service actually breaks — an out-of-range integer, an oversized +string, a wrong protobuf wire type, a truncated body — untested for every +method. This case closes that gap once, for the whole service, and is driven +by the same inventory the per-method cases cite as mandatory test data. + +## Preconditions + +1. The shared plan prerequisites are healthy and the guest agent's `tappd.sock` legacy listener is reachable. +2. The fixture is lease-owned. The matrix drives mutating methods with + run-scoped identifiers, so it must never be pointed at a shared or + production deployment. + +## Test Data + +The `guest-os` entries in [`api-inventory.json`](../../../../catalog/api-inventory.json) +whose `service` is `Tappd`, and the vector set in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py): + +- per field type — empty, wrong type, null, a NUL byte, a newline, a path + traversal, an absolute path, control characters, 64 KiB, 1 MiB, odd-length + hex, non-hex, 31/63/64/65-byte lengths, zero, negative, `u32::MAX`, + `u64::MAX`, past `u64::MAX`, and a 4096-element repeated field; +- per representation — an empty body, a non-object, a scalar, `null`, a + truncated object, trailing garbage, 2000-deep nesting, duplicate keys, a + 4 MiB body, a truncated varint, a runaway varint, a length that overflows, + a length past the end of the body, an unknown wire type, a group wire type, + field number zero, a field number past the declared range, 8 KiB of random + bytes, a JSON body sent as protobuf, a protobuf body sent as JSON, an + absent content type, and an unrecognised content type; +- per method — the declared request encoded with an unknown field appended, + and a declared field number sent with the wrong wire type. + +## Steps + + +### Step 1: Resolve the listener and the indexed contract + +Resolve the guest agent's `tappd.sock` legacy listener from the lease-owned fixture, load the inventory +entries for `Tappd`, and confirm every one of them carries a policy. + +**Expected results:** + +- The listener is reachable, every declared method has a policy, and no method + in the policy table is absent from the inventory. + + +### Step 2: Drive the matrix + +Send every vector the policy allows for every method, recording the status, +the response content type, the body size and the elapsed time of each call. + +**Expected results:** + +- **L1** No request produces a 5xx, a dropped connection, or a transport-level + failure. A malformed request is a client error, not a server error, and + never a crash: the workspace builds release binaries with `panic = "abort"`, + so a reachable panic is a process abort rather than a failed request. +- **L2** A rejected JSON request carries a structured JSON body with an + `error` string; a rejected protobuf request carries a decodable `ProtoError`. +- **L4** No rejection echoes an unbounded amount of the caller's input, and no + response contains the unknown-field marker the request carried. +- **L5** No request exceeds the per-call deadline. +- **L6** A rejection's `Content-Type` matches the representation the request + arrived in, so a client that sent JSON can parse the error it gets back. + + +### Step 3: Verify the listener survived + +Send one valid request after the whole matrix. + +**Expected results:** + +- **L3** The listener answers it. A service that stopped answering has either + aborted or wedged on one of the preceding vectors. + +## Postconditions + +Remove run-scoped objects the `full` policy created and restore changed +configuration. Preserve the per-call observation table in the result +artifacts: it is the evidence that each vector was actually sent, and its +status histogram is what a reviewer compares across runs. diff --git a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-001/metadata.json b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-001/metadata.json new file mode 100644 index 000000000..1b9113729 --- /dev/null +++ b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-001/metadata.json @@ -0,0 +1,36 @@ +{ + "id": "tc-gos-contract-001", + "title": "Tappd request-contract matrix", + "priority": "P0", + "requirements": [ + "req-gos-contract-001" + ], + "risks": [ + "risk-gos-contract-001" + ], + "tags": [ + "guest", + "request-contract" + ], + "fixture": { + "profile": "no-tee-dev", + "versions": { + "vmm": "candidate", + "guest": "candidate", + "kms": "candidate", + "gateway": "candidate", + "verifier": "candidate" + }, + "destructive_scope": "lease-only", + "hardware_required": false, + "simulation_allowed": true + }, + "actions_under_test": [ + "Tappd.DeriveKey" + ], + "execution": { + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "args": [], + "timeout_seconds": 900 + } +} diff --git a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-002/case.md b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-002/case.md new file mode 100644 index 000000000..c76286297 --- /dev/null +++ b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-002/case.md @@ -0,0 +1,122 @@ + + + +# TC-GOS-CONTRACT-002: DstackGuest request-contract matrix + +## Metadata + +- Priority: P0 +- Type: Functional, API, Security, Robustness, Regression +- Minimum environment: INTEGRATION +- Automation: Yes +- Requirements: [req-gos-contract-002](../../../../catalog/feature-audit.md#req-gos-contract-002) +- Risks: [risk-gos-contract-002](../../../../catalog/feature-audit.md#risk-gos-contract-002) +- Source: [`catalog/api-inventory.json`](../../../../catalog/api-inventory.json), [`shared/automation/api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + +## Prepared execution knowledge + +- Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. +- The method list, the request field list and every field's declared type come + from [`api-inventory.json`](../../../../catalog/api-inventory.json). Do not reconstruct + them from implementation source: the point of this case is that the indexed + contract and the running service agree. +- [`api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + says how far each method may be driven. `full` runs every vector, and is only + used for a method whose effect is confined to the lease. `rejection-only` + runs the vectors a service must refuse before it dispatches, so nothing can + mutate. `skip` is for a method whose valid invocation ends the process or + consumes an external resource, and the policy file records why for each one. +- Every method the inventory declares for this service must carry a policy. A + method added to the inventory without one fails this case rather than being + silently skipped. + +## Objective + +Verify that every method `DstackGuest` declares answers absent, default, valid, +boundary-invalid, wrong-typed, unknown-field and malformed-framing input, in +both the JSON and the protobuf representation, without violating the +invariants that hold for every method whatever it means. + +The per-method cases in this suite each send one valid request and one +wrong-typed value for the first request field. That leaves the input classes +where a service actually breaks — an out-of-range integer, an oversized +string, a wrong protobuf wire type, a truncated body — untested for every +method. This case closes that gap once, for the whole service, and is driven +by the same inventory the per-method cases cite as mandatory test data. + +## Preconditions + +1. The shared plan prerequisites are healthy and the guest agent's `dstack.sock` internal listener, which is bind-mounted into application containers is reachable. +2. The fixture is lease-owned. The matrix drives mutating methods with + run-scoped identifiers, so it must never be pointed at a shared or + production deployment. + +## Test Data + +The `guest-os` entries in [`api-inventory.json`](../../../../catalog/api-inventory.json) +whose `service` is `DstackGuest`, and the vector set in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py): + +- per field type — empty, wrong type, null, a NUL byte, a newline, a path + traversal, an absolute path, control characters, 64 KiB, 1 MiB, odd-length + hex, non-hex, 31/63/64/65-byte lengths, zero, negative, `u32::MAX`, + `u64::MAX`, past `u64::MAX`, and a 4096-element repeated field; +- per representation — an empty body, a non-object, a scalar, `null`, a + truncated object, trailing garbage, 2000-deep nesting, duplicate keys, a + 4 MiB body, a truncated varint, a runaway varint, a length that overflows, + a length past the end of the body, an unknown wire type, a group wire type, + field number zero, a field number past the declared range, 8 KiB of random + bytes, a JSON body sent as protobuf, a protobuf body sent as JSON, an + absent content type, and an unrecognised content type; +- per method — the declared request encoded with an unknown field appended, + and a declared field number sent with the wrong wire type. + +## Steps + + +### Step 1: Resolve the listener and the indexed contract + +Resolve the guest agent's `dstack.sock` internal listener, which is bind-mounted into application containers from the lease-owned fixture, load the inventory +entries for `DstackGuest`, and confirm every one of them carries a policy. + +**Expected results:** + +- The listener is reachable, every declared method has a policy, and no method + in the policy table is absent from the inventory. + + +### Step 2: Drive the matrix + +Send every vector the policy allows for every method, recording the status, +the response content type, the body size and the elapsed time of each call. + +**Expected results:** + +- **L1** No request produces a 5xx, a dropped connection, or a transport-level + failure. A malformed request is a client error, not a server error, and + never a crash: the workspace builds release binaries with `panic = "abort"`, + so a reachable panic is a process abort rather than a failed request. +- **L2** A rejected JSON request carries a structured JSON body with an + `error` string; a rejected protobuf request carries a decodable `ProtoError`. +- **L4** No rejection echoes an unbounded amount of the caller's input, and no + response contains the unknown-field marker the request carried. +- **L5** No request exceeds the per-call deadline. +- **L6** A rejection's `Content-Type` matches the representation the request + arrived in, so a client that sent JSON can parse the error it gets back. + + +### Step 3: Verify the listener survived + +Send one valid request after the whole matrix. + +**Expected results:** + +- **L3** The listener answers it. A service that stopped answering has either + aborted or wedged on one of the preceding vectors. + +## Postconditions + +Remove run-scoped objects the `full` policy created and restore changed +configuration. Preserve the per-call observation table in the result +artifacts: it is the evidence that each vector was actually sent, and its +status histogram is what a reviewer compares across runs. diff --git a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-002/metadata.json b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-002/metadata.json new file mode 100644 index 000000000..a6cb943f5 --- /dev/null +++ b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-002/metadata.json @@ -0,0 +1,36 @@ +{ + "id": "tc-gos-contract-002", + "title": "DstackGuest request-contract matrix", + "priority": "P0", + "requirements": [ + "req-gos-contract-002" + ], + "risks": [ + "risk-gos-contract-002" + ], + "tags": [ + "guest", + "request-contract" + ], + "fixture": { + "profile": "no-tee-dev", + "versions": { + "vmm": "candidate", + "guest": "candidate", + "kms": "candidate", + "gateway": "candidate", + "verifier": "candidate" + }, + "destructive_scope": "lease-only", + "hardware_required": false, + "simulation_allowed": true + }, + "actions_under_test": [ + "DstackGuest.GetTlsKey" + ], + "execution": { + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "args": [], + "timeout_seconds": 900 + } +} diff --git a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-003/case.md b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-003/case.md new file mode 100644 index 000000000..d9ad5f7a0 --- /dev/null +++ b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-003/case.md @@ -0,0 +1,122 @@ + + + +# TC-GOS-CONTRACT-003: Worker request-contract matrix + +## Metadata + +- Priority: P0 +- Type: Functional, API, Security, Robustness, Regression +- Minimum environment: INTEGRATION +- Automation: Yes +- Requirements: [req-gos-contract-003](../../../../catalog/feature-audit.md#req-gos-contract-003) +- Risks: [risk-gos-contract-003](../../../../catalog/feature-audit.md#risk-gos-contract-003) +- Source: [`catalog/api-inventory.json`](../../../../catalog/api-inventory.json), [`shared/automation/api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + +## Prepared execution knowledge + +- Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. +- The method list, the request field list and every field's declared type come + from [`api-inventory.json`](../../../../catalog/api-inventory.json). Do not reconstruct + them from implementation source: the point of this case is that the indexed + contract and the running service agree. +- [`api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + says how far each method may be driven. `full` runs every vector, and is only + used for a method whose effect is confined to the lease. `rejection-only` + runs the vectors a service must refuse before it dispatches, so nothing can + mutate. `skip` is for a method whose valid invocation ends the process or + consumes an external resource, and the policy file records why for each one. +- Every method the inventory declares for this service must carry a policy. A + method added to the inventory without one fails this case rather than being + silently skipped. + +## Objective + +Verify that every method `Worker` declares answers absent, default, valid, +boundary-invalid, wrong-typed, unknown-field and malformed-framing input, in +both the JSON and the protobuf representation, without violating the +invariants that hold for every method whatever it means. + +The per-method cases in this suite each send one valid request and one +wrong-typed value for the first request field. That leaves the input classes +where a service actually breaks — an out-of-range integer, an oversized +string, a wrong protobuf wire type, a truncated body — untested for every +method. This case closes that gap once, for the whole service, and is driven +by the same inventory the per-method cases cite as mandatory test data. + +## Preconditions + +1. The shared plan prerequisites are healthy and the guest agent's external listener, which is reachable from outside the CVM is reachable. +2. The fixture is lease-owned. The matrix drives mutating methods with + run-scoped identifiers, so it must never be pointed at a shared or + production deployment. + +## Test Data + +The `guest-os` entries in [`api-inventory.json`](../../../../catalog/api-inventory.json) +whose `service` is `Worker`, and the vector set in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py): + +- per field type — empty, wrong type, null, a NUL byte, a newline, a path + traversal, an absolute path, control characters, 64 KiB, 1 MiB, odd-length + hex, non-hex, 31/63/64/65-byte lengths, zero, negative, `u32::MAX`, + `u64::MAX`, past `u64::MAX`, and a 4096-element repeated field; +- per representation — an empty body, a non-object, a scalar, `null`, a + truncated object, trailing garbage, 2000-deep nesting, duplicate keys, a + 4 MiB body, a truncated varint, a runaway varint, a length that overflows, + a length past the end of the body, an unknown wire type, a group wire type, + field number zero, a field number past the declared range, 8 KiB of random + bytes, a JSON body sent as protobuf, a protobuf body sent as JSON, an + absent content type, and an unrecognised content type; +- per method — the declared request encoded with an unknown field appended, + and a declared field number sent with the wrong wire type. + +## Steps + + +### Step 1: Resolve the listener and the indexed contract + +Resolve the guest agent's external listener, which is reachable from outside the CVM from the lease-owned fixture, load the inventory +entries for `Worker`, and confirm every one of them carries a policy. + +**Expected results:** + +- The listener is reachable, every declared method has a policy, and no method + in the policy table is absent from the inventory. + + +### Step 2: Drive the matrix + +Send every vector the policy allows for every method, recording the status, +the response content type, the body size and the elapsed time of each call. + +**Expected results:** + +- **L1** No request produces a 5xx, a dropped connection, or a transport-level + failure. A malformed request is a client error, not a server error, and + never a crash: the workspace builds release binaries with `panic = "abort"`, + so a reachable panic is a process abort rather than a failed request. +- **L2** A rejected JSON request carries a structured JSON body with an + `error` string; a rejected protobuf request carries a decodable `ProtoError`. +- **L4** No rejection echoes an unbounded amount of the caller's input, and no + response contains the unknown-field marker the request carried. +- **L5** No request exceeds the per-call deadline. +- **L6** A rejection's `Content-Type` matches the representation the request + arrived in, so a client that sent JSON can parse the error it gets back. + + +### Step 3: Verify the listener survived + +Send one valid request after the whole matrix. + +**Expected results:** + +- **L3** The listener answers it. A service that stopped answering has either + aborted or wedged on one of the preceding vectors. + +## Postconditions + +Remove run-scoped objects the `full` policy created and restore changed +configuration. Preserve the per-call observation table in the result +artifacts: it is the evidence that each vector was actually sent, and its +status histogram is what a reviewer compares across runs. diff --git a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-003/metadata.json b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-003/metadata.json new file mode 100644 index 000000000..49e36264e --- /dev/null +++ b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-003/metadata.json @@ -0,0 +1,36 @@ +{ + "id": "tc-gos-contract-003", + "title": "Worker request-contract matrix", + "priority": "P0", + "requirements": [ + "req-gos-contract-003" + ], + "risks": [ + "risk-gos-contract-003" + ], + "tags": [ + "guest", + "request-contract" + ], + "fixture": { + "profile": "no-tee-dev", + "versions": { + "vmm": "candidate", + "guest": "candidate", + "kms": "candidate", + "gateway": "candidate", + "verifier": "candidate" + }, + "destructive_scope": "lease-only", + "hardware_required": false, + "simulation_allowed": true + }, + "actions_under_test": [ + "Worker.GetAttestationForAppKey" + ], + "execution": { + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "args": [], + "timeout_seconds": 900 + } +} diff --git a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-004/case.md b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-004/case.md new file mode 100644 index 000000000..a6079da2e --- /dev/null +++ b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-004/case.md @@ -0,0 +1,122 @@ + + + +# TC-GOS-CONTRACT-004: GuestApi request-contract matrix + +## Metadata + +- Priority: P0 +- Type: Functional, API, Security, Robustness, Regression +- Minimum environment: INTEGRATION +- Automation: Yes +- Requirements: [req-gos-contract-004](../../../../catalog/feature-audit.md#req-gos-contract-004) +- Risks: [risk-gos-contract-004](../../../../catalog/feature-audit.md#risk-gos-contract-004) +- Source: [`catalog/api-inventory.json`](../../../../catalog/api-inventory.json), [`shared/automation/api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + +## Prepared execution knowledge + +- Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. +- The method list, the request field list and every field's declared type come + from [`api-inventory.json`](../../../../catalog/api-inventory.json). Do not reconstruct + them from implementation source: the point of this case is that the indexed + contract and the running service agree. +- [`api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + says how far each method may be driven. `full` runs every vector, and is only + used for a method whose effect is confined to the lease. `rejection-only` + runs the vectors a service must refuse before it dispatches, so nothing can + mutate. `skip` is for a method whose valid invocation ends the process or + consumes an external resource, and the policy file records why for each one. +- Every method the inventory declares for this service must carry a policy. A + method added to the inventory without one fails this case rather than being + silently skipped. + +## Objective + +Verify that every method `GuestApi` declares answers absent, default, valid, +boundary-invalid, wrong-typed, unknown-field and malformed-framing input, in +both the JSON and the protobuf representation, without violating the +invariants that hold for every method whatever it means. + +The per-method cases in this suite each send one valid request and one +wrong-typed value for the first request field. That leaves the input classes +where a service actually breaks — an out-of-range integer, an oversized +string, a wrong protobuf wire type, a truncated body — untested for every +method. This case closes that gap once, for the whole service, and is driven +by the same inventory the per-method cases cite as mandatory test data. + +## Preconditions + +1. The shared plan prerequisites are healthy and the guest agent's host-facing listener is reachable. +2. The fixture is lease-owned. The matrix drives mutating methods with + run-scoped identifiers, so it must never be pointed at a shared or + production deployment. + +## Test Data + +The `guest-os` entries in [`api-inventory.json`](../../../../catalog/api-inventory.json) +whose `service` is `GuestApi`, and the vector set in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py): + +- per field type — empty, wrong type, null, a NUL byte, a newline, a path + traversal, an absolute path, control characters, 64 KiB, 1 MiB, odd-length + hex, non-hex, 31/63/64/65-byte lengths, zero, negative, `u32::MAX`, + `u64::MAX`, past `u64::MAX`, and a 4096-element repeated field; +- per representation — an empty body, a non-object, a scalar, `null`, a + truncated object, trailing garbage, 2000-deep nesting, duplicate keys, a + 4 MiB body, a truncated varint, a runaway varint, a length that overflows, + a length past the end of the body, an unknown wire type, a group wire type, + field number zero, a field number past the declared range, 8 KiB of random + bytes, a JSON body sent as protobuf, a protobuf body sent as JSON, an + absent content type, and an unrecognised content type; +- per method — the declared request encoded with an unknown field appended, + and a declared field number sent with the wrong wire type. + +## Steps + + +### Step 1: Resolve the listener and the indexed contract + +Resolve the guest agent's host-facing listener from the lease-owned fixture, load the inventory +entries for `GuestApi`, and confirm every one of them carries a policy. + +**Expected results:** + +- The listener is reachable, every declared method has a policy, and no method + in the policy table is absent from the inventory. + + +### Step 2: Drive the matrix + +Send every vector the policy allows for every method, recording the status, +the response content type, the body size and the elapsed time of each call. + +**Expected results:** + +- **L1** No request produces a 5xx, a dropped connection, or a transport-level + failure. A malformed request is a client error, not a server error, and + never a crash: the workspace builds release binaries with `panic = "abort"`, + so a reachable panic is a process abort rather than a failed request. +- **L2** A rejected JSON request carries a structured JSON body with an + `error` string; a rejected protobuf request carries a decodable `ProtoError`. +- **L4** No rejection echoes an unbounded amount of the caller's input, and no + response contains the unknown-field marker the request carried. +- **L5** No request exceeds the per-call deadline. +- **L6** A rejection's `Content-Type` matches the representation the request + arrived in, so a client that sent JSON can parse the error it gets back. + + +### Step 3: Verify the listener survived + +Send one valid request after the whole matrix. + +**Expected results:** + +- **L3** The listener answers it. A service that stopped answering has either + aborted or wedged on one of the preceding vectors. + +## Postconditions + +Remove run-scoped objects the `full` policy created and restore changed +configuration. Preserve the per-call observation table in the result +artifacts: it is the evidence that each vector was actually sent, and its +status histogram is what a reviewer compares across runs. diff --git a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-004/metadata.json b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-004/metadata.json new file mode 100644 index 000000000..9c72901b7 --- /dev/null +++ b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-004/metadata.json @@ -0,0 +1,36 @@ +{ + "id": "tc-gos-contract-004", + "title": "GuestApi request-contract matrix", + "priority": "P0", + "requirements": [ + "req-gos-contract-004" + ], + "risks": [ + "risk-gos-contract-004" + ], + "tags": [ + "guest", + "request-contract" + ], + "fixture": { + "profile": "no-tee-dev", + "versions": { + "vmm": "candidate", + "guest": "candidate", + "kms": "candidate", + "gateway": "candidate", + "verifier": "candidate" + }, + "destructive_scope": "lease-only", + "hardware_required": false, + "simulation_allowed": true + }, + "actions_under_test": [ + "GuestApi.GpuInfo" + ], + "execution": { + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "args": [], + "timeout_seconds": 900 + } +} diff --git a/test-suites/cases/03-kms/13-request-contract/metadata.json b/test-suites/cases/03-kms/13-request-contract/metadata.json new file mode 100644 index 000000000..8ef6a2591 --- /dev/null +++ b/test-suites/cases/03-kms/13-request-contract/metadata.json @@ -0,0 +1,4 @@ +{ + "id": "section-kms-request-contract", + "title": "Request contract matrix" +} diff --git a/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-001/case.md b/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-001/case.md new file mode 100644 index 000000000..efaaf06c7 --- /dev/null +++ b/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-001/case.md @@ -0,0 +1,122 @@ + + + +# TC-KMS-CONTRACT-001: KMS request-contract matrix + +## Metadata + +- Priority: P0 +- Type: Functional, API, Security, Robustness, Regression +- Minimum environment: INTEGRATION +- Automation: Yes +- Requirements: [req-kms-contract-001](../../../../catalog/feature-audit.md#req-kms-contract-001) +- Risks: [risk-kms-contract-001](../../../../catalog/feature-audit.md#risk-kms-contract-001) +- Source: [`catalog/api-inventory.json`](../../../../catalog/api-inventory.json), [`shared/automation/api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + +## Prepared execution knowledge + +- Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. +- The method list, the request field list and every field's declared type come + from [`api-inventory.json`](../../../../catalog/api-inventory.json). Do not reconstruct + them from implementation source: the point of this case is that the indexed + contract and the running service agree. +- [`api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + says how far each method may be driven. `full` runs every vector, and is only + used for a method whose effect is confined to the lease. `rejection-only` + runs the vectors a service must refuse before it dispatches, so nothing can + mutate. `skip` is for a method whose valid invocation ends the process or + consumes an external resource, and the policy file records why for each one. +- Every method the inventory declares for this service must carry a policy. A + method added to the inventory without one fails this case rather than being + silently skipped. + +## Objective + +Verify that every method `KMS` declares answers absent, default, valid, +boundary-invalid, wrong-typed, unknown-field and malformed-framing input, in +both the JSON and the protobuf representation, without violating the +invariants that hold for every method whatever it means. + +The per-method cases in this suite each send one valid request and one +wrong-typed value for the first request field. That leaves the input classes +where a service actually breaks — an out-of-range integer, an oversized +string, a wrong protobuf wire type, a truncated body — untested for every +method. This case closes that gap once, for the whole service, and is driven +by the same inventory the per-method cases cite as mandatory test data. + +## Preconditions + +1. The shared plan prerequisites are healthy and the KMS public RPC listener is reachable. +2. The fixture is lease-owned. The matrix drives mutating methods with + run-scoped identifiers, so it must never be pointed at a shared or + production deployment. + +## Test Data + +The `kms` entries in [`api-inventory.json`](../../../../catalog/api-inventory.json) +whose `service` is `KMS`, and the vector set in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py): + +- per field type — empty, wrong type, null, a NUL byte, a newline, a path + traversal, an absolute path, control characters, 64 KiB, 1 MiB, odd-length + hex, non-hex, 31/63/64/65-byte lengths, zero, negative, `u32::MAX`, + `u64::MAX`, past `u64::MAX`, and a 4096-element repeated field; +- per representation — an empty body, a non-object, a scalar, `null`, a + truncated object, trailing garbage, 2000-deep nesting, duplicate keys, a + 4 MiB body, a truncated varint, a runaway varint, a length that overflows, + a length past the end of the body, an unknown wire type, a group wire type, + field number zero, a field number past the declared range, 8 KiB of random + bytes, a JSON body sent as protobuf, a protobuf body sent as JSON, an + absent content type, and an unrecognised content type; +- per method — the declared request encoded with an unknown field appended, + and a declared field number sent with the wrong wire type. + +## Steps + + +### Step 1: Resolve the listener and the indexed contract + +Resolve the KMS public RPC listener from the lease-owned fixture, load the inventory +entries for `KMS`, and confirm every one of them carries a policy. + +**Expected results:** + +- The listener is reachable, every declared method has a policy, and no method + in the policy table is absent from the inventory. + + +### Step 2: Drive the matrix + +Send every vector the policy allows for every method, recording the status, +the response content type, the body size and the elapsed time of each call. + +**Expected results:** + +- **L1** No request produces a 5xx, a dropped connection, or a transport-level + failure. A malformed request is a client error, not a server error, and + never a crash: the workspace builds release binaries with `panic = "abort"`, + so a reachable panic is a process abort rather than a failed request. +- **L2** A rejected JSON request carries a structured JSON body with an + `error` string; a rejected protobuf request carries a decodable `ProtoError`. +- **L4** No rejection echoes an unbounded amount of the caller's input, and no + response contains the unknown-field marker the request carried. +- **L5** No request exceeds the per-call deadline. +- **L6** A rejection's `Content-Type` matches the representation the request + arrived in, so a client that sent JSON can parse the error it gets back. + + +### Step 3: Verify the listener survived + +Send one valid request after the whole matrix. + +**Expected results:** + +- **L3** The listener answers it. A service that stopped answering has either + aborted or wedged on one of the preceding vectors. + +## Postconditions + +Remove run-scoped objects the `full` policy created and restore changed +configuration. Preserve the per-call observation table in the result +artifacts: it is the evidence that each vector was actually sent, and its +status histogram is what a reviewer compares across runs. diff --git a/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-001/metadata.json b/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-001/metadata.json new file mode 100644 index 000000000..466399ecf --- /dev/null +++ b/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-001/metadata.json @@ -0,0 +1,36 @@ +{ + "id": "tc-kms-contract-001", + "title": "KMS request-contract matrix", + "priority": "P0", + "requirements": [ + "req-kms-contract-001" + ], + "risks": [ + "risk-kms-contract-001" + ], + "tags": [ + "kms", + "request-contract" + ], + "fixture": { + "profile": "kms-ready", + "versions": { + "vmm": "candidate", + "guest": "candidate", + "kms": "candidate", + "gateway": "candidate", + "verifier": "candidate" + }, + "destructive_scope": "lease-only", + "hardware_required": false, + "simulation_allowed": false + }, + "actions_under_test": [ + "KMS.GetMeta" + ], + "execution": { + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "args": [], + "timeout_seconds": 900 + } +} diff --git a/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-002/case.md b/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-002/case.md new file mode 100644 index 000000000..c802e90e6 --- /dev/null +++ b/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-002/case.md @@ -0,0 +1,122 @@ + + + +# TC-KMS-CONTRACT-002: Onboard request-contract matrix + +## Metadata + +- Priority: P0 +- Type: Functional, API, Security, Robustness, Regression +- Minimum environment: INTEGRATION +- Automation: Yes +- Requirements: [req-kms-contract-002](../../../../catalog/feature-audit.md#req-kms-contract-002) +- Risks: [risk-kms-contract-002](../../../../catalog/feature-audit.md#risk-kms-contract-002) +- Source: [`catalog/api-inventory.json`](../../../../catalog/api-inventory.json), [`shared/automation/api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + +## Prepared execution knowledge + +- Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. +- The method list, the request field list and every field's declared type come + from [`api-inventory.json`](../../../../catalog/api-inventory.json). Do not reconstruct + them from implementation source: the point of this case is that the indexed + contract and the running service agree. +- [`api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + says how far each method may be driven. `full` runs every vector, and is only + used for a method whose effect is confined to the lease. `rejection-only` + runs the vectors a service must refuse before it dispatches, so nothing can + mutate. `skip` is for a method whose valid invocation ends the process or + consumes an external resource, and the policy file records why for each one. +- Every method the inventory declares for this service must carry a policy. A + method added to the inventory without one fails this case rather than being + silently skipped. + +## Objective + +Verify that every method `Onboard` declares answers absent, default, valid, +boundary-invalid, wrong-typed, unknown-field and malformed-framing input, in +both the JSON and the protobuf representation, without violating the +invariants that hold for every method whatever it means. + +The per-method cases in this suite each send one valid request and one +wrong-typed value for the first request field. That leaves the input classes +where a service actually breaks — an out-of-range integer, an oversized +string, a wrong protobuf wire type, a truncated body — untested for every +method. This case closes that gap once, for the whole service, and is driven +by the same inventory the per-method cases cite as mandatory test data. + +## Preconditions + +1. The shared plan prerequisites are healthy and the KMS onboarding listener is reachable. +2. The fixture is lease-owned. The matrix drives mutating methods with + run-scoped identifiers, so it must never be pointed at a shared or + production deployment. + +## Test Data + +The `kms` entries in [`api-inventory.json`](../../../../catalog/api-inventory.json) +whose `service` is `Onboard`, and the vector set in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py): + +- per field type — empty, wrong type, null, a NUL byte, a newline, a path + traversal, an absolute path, control characters, 64 KiB, 1 MiB, odd-length + hex, non-hex, 31/63/64/65-byte lengths, zero, negative, `u32::MAX`, + `u64::MAX`, past `u64::MAX`, and a 4096-element repeated field; +- per representation — an empty body, a non-object, a scalar, `null`, a + truncated object, trailing garbage, 2000-deep nesting, duplicate keys, a + 4 MiB body, a truncated varint, a runaway varint, a length that overflows, + a length past the end of the body, an unknown wire type, a group wire type, + field number zero, a field number past the declared range, 8 KiB of random + bytes, a JSON body sent as protobuf, a protobuf body sent as JSON, an + absent content type, and an unrecognised content type; +- per method — the declared request encoded with an unknown field appended, + and a declared field number sent with the wrong wire type. + +## Steps + + +### Step 1: Resolve the listener and the indexed contract + +Resolve the KMS onboarding listener from the lease-owned fixture, load the inventory +entries for `Onboard`, and confirm every one of them carries a policy. + +**Expected results:** + +- The listener is reachable, every declared method has a policy, and no method + in the policy table is absent from the inventory. + + +### Step 2: Drive the matrix + +Send every vector the policy allows for every method, recording the status, +the response content type, the body size and the elapsed time of each call. + +**Expected results:** + +- **L1** No request produces a 5xx, a dropped connection, or a transport-level + failure. A malformed request is a client error, not a server error, and + never a crash: the workspace builds release binaries with `panic = "abort"`, + so a reachable panic is a process abort rather than a failed request. +- **L2** A rejected JSON request carries a structured JSON body with an + `error` string; a rejected protobuf request carries a decodable `ProtoError`. +- **L4** No rejection echoes an unbounded amount of the caller's input, and no + response contains the unknown-field marker the request carried. +- **L5** No request exceeds the per-call deadline. +- **L6** A rejection's `Content-Type` matches the representation the request + arrived in, so a client that sent JSON can parse the error it gets back. + + +### Step 3: Verify the listener survived + +Send one valid request after the whole matrix. + +**Expected results:** + +- **L3** The listener answers it. A service that stopped answering has either + aborted or wedged on one of the preceding vectors. + +## Postconditions + +Remove run-scoped objects the `full` policy created and restore changed +configuration. Preserve the per-call observation table in the result +artifacts: it is the evidence that each vector was actually sent, and its +status histogram is what a reviewer compares across runs. diff --git a/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-002/metadata.json b/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-002/metadata.json new file mode 100644 index 000000000..b0e1f7dc8 --- /dev/null +++ b/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-002/metadata.json @@ -0,0 +1,36 @@ +{ + "id": "tc-kms-contract-002", + "title": "Onboard request-contract matrix", + "priority": "P0", + "requirements": [ + "req-kms-contract-002" + ], + "risks": [ + "risk-kms-contract-002" + ], + "tags": [ + "kms", + "request-contract" + ], + "fixture": { + "profile": "kms-onboard", + "versions": { + "vmm": "candidate", + "guest": "candidate", + "kms": "candidate", + "gateway": "candidate", + "verifier": "candidate" + }, + "destructive_scope": "lease-only", + "hardware_required": false, + "simulation_allowed": false + }, + "actions_under_test": [ + "Onboard.GetAttestationInfo" + ], + "execution": { + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "args": [], + "timeout_seconds": 900 + } +} diff --git a/test-suites/cases/04-gateway/11-request-contract/metadata.json b/test-suites/cases/04-gateway/11-request-contract/metadata.json new file mode 100644 index 000000000..2485b1147 --- /dev/null +++ b/test-suites/cases/04-gateway/11-request-contract/metadata.json @@ -0,0 +1,4 @@ +{ + "id": "section-gateway-request-contract", + "title": "Request contract matrix" +} diff --git a/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-001/case.md b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-001/case.md new file mode 100644 index 000000000..b479c4045 --- /dev/null +++ b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-001/case.md @@ -0,0 +1,122 @@ + + + +# TC-GW-CONTRACT-001: Gateway request-contract matrix + +## Metadata + +- Priority: P0 +- Type: Functional, API, Security, Robustness, Regression +- Minimum environment: INTEGRATION +- Automation: Yes +- Requirements: [req-gw-contract-001](../../../../catalog/feature-audit.md#req-gw-contract-001) +- Risks: [risk-gw-contract-001](../../../../catalog/feature-audit.md#risk-gw-contract-001) +- Source: [`catalog/api-inventory.json`](../../../../catalog/api-inventory.json), [`shared/automation/api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + +## Prepared execution knowledge + +- Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. +- The method list, the request field list and every field's declared type come + from [`api-inventory.json`](../../../../catalog/api-inventory.json). Do not reconstruct + them from implementation source: the point of this case is that the indexed + contract and the running service agree. +- [`api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + says how far each method may be driven. `full` runs every vector, and is only + used for a method whose effect is confined to the lease. `rejection-only` + runs the vectors a service must refuse before it dispatches, so nothing can + mutate. `skip` is for a method whose valid invocation ends the process or + consumes an external resource, and the policy file records why for each one. +- Every method the inventory declares for this service must carry a policy. A + method added to the inventory without one fails this case rather than being + silently skipped. + +## Objective + +Verify that every method `Gateway` declares answers absent, default, valid, +boundary-invalid, wrong-typed, unknown-field and malformed-framing input, in +both the JSON and the protobuf representation, without violating the +invariants that hold for every method whatever it means. + +The per-method cases in this suite each send one valid request and one +wrong-typed value for the first request field. That leaves the input classes +where a service actually breaks — an out-of-range integer, an oversized +string, a wrong protobuf wire type, a truncated body — untested for every +method. This case closes that gap once, for the whole service, and is driven +by the same inventory the per-method cases cite as mandatory test data. + +## Preconditions + +1. The shared plan prerequisites are healthy and the gateway's public RPC listener is reachable. +2. The fixture is lease-owned. The matrix drives mutating methods with + run-scoped identifiers, so it must never be pointed at a shared or + production deployment. + +## Test Data + +The `gateway` entries in [`api-inventory.json`](../../../../catalog/api-inventory.json) +whose `service` is `Gateway`, and the vector set in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py): + +- per field type — empty, wrong type, null, a NUL byte, a newline, a path + traversal, an absolute path, control characters, 64 KiB, 1 MiB, odd-length + hex, non-hex, 31/63/64/65-byte lengths, zero, negative, `u32::MAX`, + `u64::MAX`, past `u64::MAX`, and a 4096-element repeated field; +- per representation — an empty body, a non-object, a scalar, `null`, a + truncated object, trailing garbage, 2000-deep nesting, duplicate keys, a + 4 MiB body, a truncated varint, a runaway varint, a length that overflows, + a length past the end of the body, an unknown wire type, a group wire type, + field number zero, a field number past the declared range, 8 KiB of random + bytes, a JSON body sent as protobuf, a protobuf body sent as JSON, an + absent content type, and an unrecognised content type; +- per method — the declared request encoded with an unknown field appended, + and a declared field number sent with the wrong wire type. + +## Steps + + +### Step 1: Resolve the listener and the indexed contract + +Resolve the gateway's public RPC listener from the lease-owned fixture, load the inventory +entries for `Gateway`, and confirm every one of them carries a policy. + +**Expected results:** + +- The listener is reachable, every declared method has a policy, and no method + in the policy table is absent from the inventory. + + +### Step 2: Drive the matrix + +Send every vector the policy allows for every method, recording the status, +the response content type, the body size and the elapsed time of each call. + +**Expected results:** + +- **L1** No request produces a 5xx, a dropped connection, or a transport-level + failure. A malformed request is a client error, not a server error, and + never a crash: the workspace builds release binaries with `panic = "abort"`, + so a reachable panic is a process abort rather than a failed request. +- **L2** A rejected JSON request carries a structured JSON body with an + `error` string; a rejected protobuf request carries a decodable `ProtoError`. +- **L4** No rejection echoes an unbounded amount of the caller's input, and no + response contains the unknown-field marker the request carried. +- **L5** No request exceeds the per-call deadline. +- **L6** A rejection's `Content-Type` matches the representation the request + arrived in, so a client that sent JSON can parse the error it gets back. + + +### Step 3: Verify the listener survived + +Send one valid request after the whole matrix. + +**Expected results:** + +- **L3** The listener answers it. A service that stopped answering has either + aborted or wedged on one of the preceding vectors. + +## Postconditions + +Remove run-scoped objects the `full` policy created and restore changed +configuration. Preserve the per-call observation table in the result +artifacts: it is the evidence that each vector was actually sent, and its +status histogram is what a reviewer compares across runs. diff --git a/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-001/metadata.json b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-001/metadata.json new file mode 100644 index 000000000..d88d6023e --- /dev/null +++ b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-001/metadata.json @@ -0,0 +1,36 @@ +{ + "id": "tc-gw-contract-001", + "title": "Gateway request-contract matrix", + "priority": "P0", + "requirements": [ + "req-gw-contract-001" + ], + "risks": [ + "risk-gw-contract-001" + ], + "tags": [ + "gateway", + "request-contract" + ], + "fixture": { + "profile": "gateway-cluster", + "versions": { + "vmm": "candidate", + "guest": "candidate", + "kms": "candidate", + "gateway": "candidate", + "verifier": "candidate" + }, + "destructive_scope": "lease-only", + "hardware_required": false, + "simulation_allowed": false + }, + "actions_under_test": [ + "Gateway.AcmeInfo" + ], + "execution": { + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "args": [], + "timeout_seconds": 900 + } +} diff --git a/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-002/case.md b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-002/case.md new file mode 100644 index 000000000..58e416f4e --- /dev/null +++ b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-002/case.md @@ -0,0 +1,122 @@ + + + +# TC-GW-CONTRACT-002: Debug request-contract matrix + +## Metadata + +- Priority: P0 +- Type: Functional, API, Security, Robustness, Regression +- Minimum environment: INTEGRATION +- Automation: Yes +- Requirements: [req-gw-contract-002](../../../../catalog/feature-audit.md#req-gw-contract-002) +- Risks: [risk-gw-contract-002](../../../../catalog/feature-audit.md#risk-gw-contract-002) +- Source: [`catalog/api-inventory.json`](../../../../catalog/api-inventory.json), [`shared/automation/api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + +## Prepared execution knowledge + +- Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. +- The method list, the request field list and every field's declared type come + from [`api-inventory.json`](../../../../catalog/api-inventory.json). Do not reconstruct + them from implementation source: the point of this case is that the indexed + contract and the running service agree. +- [`api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + says how far each method may be driven. `full` runs every vector, and is only + used for a method whose effect is confined to the lease. `rejection-only` + runs the vectors a service must refuse before it dispatches, so nothing can + mutate. `skip` is for a method whose valid invocation ends the process or + consumes an external resource, and the policy file records why for each one. +- Every method the inventory declares for this service must carry a policy. A + method added to the inventory without one fails this case rather than being + silently skipped. + +## Objective + +Verify that every method `Debug` declares answers absent, default, valid, +boundary-invalid, wrong-typed, unknown-field and malformed-framing input, in +both the JSON and the protobuf representation, without violating the +invariants that hold for every method whatever it means. + +The per-method cases in this suite each send one valid request and one +wrong-typed value for the first request field. That leaves the input classes +where a service actually breaks — an out-of-range integer, an oversized +string, a wrong protobuf wire type, a truncated body — untested for every +method. This case closes that gap once, for the whole service, and is driven +by the same inventory the per-method cases cite as mandatory test data. + +## Preconditions + +1. The shared plan prerequisites are healthy and the gateway's debug listener is reachable. +2. The fixture is lease-owned. The matrix drives mutating methods with + run-scoped identifiers, so it must never be pointed at a shared or + production deployment. + +## Test Data + +The `gateway` entries in [`api-inventory.json`](../../../../catalog/api-inventory.json) +whose `service` is `Debug`, and the vector set in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py): + +- per field type — empty, wrong type, null, a NUL byte, a newline, a path + traversal, an absolute path, control characters, 64 KiB, 1 MiB, odd-length + hex, non-hex, 31/63/64/65-byte lengths, zero, negative, `u32::MAX`, + `u64::MAX`, past `u64::MAX`, and a 4096-element repeated field; +- per representation — an empty body, a non-object, a scalar, `null`, a + truncated object, trailing garbage, 2000-deep nesting, duplicate keys, a + 4 MiB body, a truncated varint, a runaway varint, a length that overflows, + a length past the end of the body, an unknown wire type, a group wire type, + field number zero, a field number past the declared range, 8 KiB of random + bytes, a JSON body sent as protobuf, a protobuf body sent as JSON, an + absent content type, and an unrecognised content type; +- per method — the declared request encoded with an unknown field appended, + and a declared field number sent with the wrong wire type. + +## Steps + + +### Step 1: Resolve the listener and the indexed contract + +Resolve the gateway's debug listener from the lease-owned fixture, load the inventory +entries for `Debug`, and confirm every one of them carries a policy. + +**Expected results:** + +- The listener is reachable, every declared method has a policy, and no method + in the policy table is absent from the inventory. + + +### Step 2: Drive the matrix + +Send every vector the policy allows for every method, recording the status, +the response content type, the body size and the elapsed time of each call. + +**Expected results:** + +- **L1** No request produces a 5xx, a dropped connection, or a transport-level + failure. A malformed request is a client error, not a server error, and + never a crash: the workspace builds release binaries with `panic = "abort"`, + so a reachable panic is a process abort rather than a failed request. +- **L2** A rejected JSON request carries a structured JSON body with an + `error` string; a rejected protobuf request carries a decodable `ProtoError`. +- **L4** No rejection echoes an unbounded amount of the caller's input, and no + response contains the unknown-field marker the request carried. +- **L5** No request exceeds the per-call deadline. +- **L6** A rejection's `Content-Type` matches the representation the request + arrived in, so a client that sent JSON can parse the error it gets back. + + +### Step 3: Verify the listener survived + +Send one valid request after the whole matrix. + +**Expected results:** + +- **L3** The listener answers it. A service that stopped answering has either + aborted or wedged on one of the preceding vectors. + +## Postconditions + +Remove run-scoped objects the `full` policy created and restore changed +configuration. Preserve the per-call observation table in the result +artifacts: it is the evidence that each vector was actually sent, and its +status histogram is what a reviewer compares across runs. diff --git a/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-002/metadata.json b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-002/metadata.json new file mode 100644 index 000000000..1c4fc046a --- /dev/null +++ b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-002/metadata.json @@ -0,0 +1,36 @@ +{ + "id": "tc-gw-contract-002", + "title": "Debug request-contract matrix", + "priority": "P0", + "requirements": [ + "req-gw-contract-002" + ], + "risks": [ + "risk-gw-contract-002" + ], + "tags": [ + "gateway", + "request-contract" + ], + "fixture": { + "profile": "gateway-cluster", + "versions": { + "vmm": "candidate", + "guest": "candidate", + "kms": "candidate", + "gateway": "candidate", + "verifier": "candidate" + }, + "destructive_scope": "lease-only", + "hardware_required": false, + "simulation_allowed": false + }, + "actions_under_test": [ + "Debug.GetSyncData" + ], + "execution": { + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "args": [], + "timeout_seconds": 900 + } +} diff --git a/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-003/case.md b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-003/case.md new file mode 100644 index 000000000..7a639fddd --- /dev/null +++ b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-003/case.md @@ -0,0 +1,122 @@ + + + +# TC-GW-CONTRACT-003: Admin request-contract matrix + +## Metadata + +- Priority: P0 +- Type: Functional, API, Security, Robustness, Regression +- Minimum environment: INTEGRATION +- Automation: Yes +- Requirements: [req-gw-contract-003](../../../../catalog/feature-audit.md#req-gw-contract-003) +- Risks: [risk-gw-contract-003](../../../../catalog/feature-audit.md#risk-gw-contract-003) +- Source: [`catalog/api-inventory.json`](../../../../catalog/api-inventory.json), [`shared/automation/api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + +## Prepared execution knowledge + +- Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. +- The method list, the request field list and every field's declared type come + from [`api-inventory.json`](../../../../catalog/api-inventory.json). Do not reconstruct + them from implementation source: the point of this case is that the indexed + contract and the running service agree. +- [`api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + says how far each method may be driven. `full` runs every vector, and is only + used for a method whose effect is confined to the lease. `rejection-only` + runs the vectors a service must refuse before it dispatches, so nothing can + mutate. `skip` is for a method whose valid invocation ends the process or + consumes an external resource, and the policy file records why for each one. +- Every method the inventory declares for this service must carry a policy. A + method added to the inventory without one fails this case rather than being + silently skipped. + +## Objective + +Verify that every method `Admin` declares answers absent, default, valid, +boundary-invalid, wrong-typed, unknown-field and malformed-framing input, in +both the JSON and the protobuf representation, without violating the +invariants that hold for every method whatever it means. + +The per-method cases in this suite each send one valid request and one +wrong-typed value for the first request field. That leaves the input classes +where a service actually breaks — an out-of-range integer, an oversized +string, a wrong protobuf wire type, a truncated body — untested for every +method. This case closes that gap once, for the whole service, and is driven +by the same inventory the per-method cases cite as mandatory test data. + +## Preconditions + +1. The shared plan prerequisites are healthy and the gateway's authenticated admin listener is reachable. +2. The fixture is lease-owned. The matrix drives mutating methods with + run-scoped identifiers, so it must never be pointed at a shared or + production deployment. + +## Test Data + +The `gateway` entries in [`api-inventory.json`](../../../../catalog/api-inventory.json) +whose `service` is `Admin`, and the vector set in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py): + +- per field type — empty, wrong type, null, a NUL byte, a newline, a path + traversal, an absolute path, control characters, 64 KiB, 1 MiB, odd-length + hex, non-hex, 31/63/64/65-byte lengths, zero, negative, `u32::MAX`, + `u64::MAX`, past `u64::MAX`, and a 4096-element repeated field; +- per representation — an empty body, a non-object, a scalar, `null`, a + truncated object, trailing garbage, 2000-deep nesting, duplicate keys, a + 4 MiB body, a truncated varint, a runaway varint, a length that overflows, + a length past the end of the body, an unknown wire type, a group wire type, + field number zero, a field number past the declared range, 8 KiB of random + bytes, a JSON body sent as protobuf, a protobuf body sent as JSON, an + absent content type, and an unrecognised content type; +- per method — the declared request encoded with an unknown field appended, + and a declared field number sent with the wrong wire type. + +## Steps + + +### Step 1: Resolve the listener and the indexed contract + +Resolve the gateway's authenticated admin listener from the lease-owned fixture, load the inventory +entries for `Admin`, and confirm every one of them carries a policy. + +**Expected results:** + +- The listener is reachable, every declared method has a policy, and no method + in the policy table is absent from the inventory. + + +### Step 2: Drive the matrix + +Send every vector the policy allows for every method, recording the status, +the response content type, the body size and the elapsed time of each call. + +**Expected results:** + +- **L1** No request produces a 5xx, a dropped connection, or a transport-level + failure. A malformed request is a client error, not a server error, and + never a crash: the workspace builds release binaries with `panic = "abort"`, + so a reachable panic is a process abort rather than a failed request. +- **L2** A rejected JSON request carries a structured JSON body with an + `error` string; a rejected protobuf request carries a decodable `ProtoError`. +- **L4** No rejection echoes an unbounded amount of the caller's input, and no + response contains the unknown-field marker the request carried. +- **L5** No request exceeds the per-call deadline. +- **L6** A rejection's `Content-Type` matches the representation the request + arrived in, so a client that sent JSON can parse the error it gets back. + + +### Step 3: Verify the listener survived + +Send one valid request after the whole matrix. + +**Expected results:** + +- **L3** The listener answers it. A service that stopped answering has either + aborted or wedged on one of the preceding vectors. + +## Postconditions + +Remove run-scoped objects the `full` policy created and restore changed +configuration. Preserve the per-call observation table in the result +artifacts: it is the evidence that each vector was actually sent, and its +status histogram is what a reviewer compares across runs. diff --git a/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-003/metadata.json b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-003/metadata.json new file mode 100644 index 000000000..7db673318 --- /dev/null +++ b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-003/metadata.json @@ -0,0 +1,36 @@ +{ + "id": "tc-gw-contract-003", + "title": "Admin request-contract matrix", + "priority": "P0", + "requirements": [ + "req-gw-contract-003" + ], + "risks": [ + "risk-gw-contract-003" + ], + "tags": [ + "gateway", + "request-contract" + ], + "fixture": { + "profile": "gateway-cluster", + "versions": { + "vmm": "candidate", + "guest": "candidate", + "kms": "candidate", + "gateway": "candidate", + "verifier": "candidate" + }, + "destructive_scope": "lease-only", + "hardware_required": false, + "simulation_allowed": false + }, + "actions_under_test": [ + "Admin.ListDnsCredentials" + ], + "execution": { + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "args": [], + "timeout_seconds": 900 + } +} diff --git a/test-suites/catalog/feature-audit.md b/test-suites/catalog/feature-audit.md index 0c0dd5daa..9d0531655 100644 --- a/test-suites/catalog/feature-audit.md +++ b/test-suites/catalog/feature-audit.md @@ -708,3 +708,17 @@ This audit is derived from the repository source inventory and is the traceabili | `req-int-mixed-005` | `risk-int-mixed-005` | [tc-int-mixed-005](../cases/06-integration/04-pinned-mixed-version-matrix/tc-int-mixed-005/case.md#tc-int-mixed-005) — Verifier evidence compatibility for pinned releases | P0 | | `req-int-mixed-006` | `risk-int-mixed-006` | [tc-int-mixed-006](../cases/06-integration/04-pinned-mixed-version-matrix/tc-int-mixed-006/case.md#tc-int-mixed-006) — Rolling restart under four-version online mix | P0 | | `req-int-mixed-007` | `risk-int-mixed-007` | [tc-int-mixed-007](../cases/06-integration/04-pinned-mixed-version-matrix/tc-int-mixed-007/case.md#tc-int-mixed-007) — Optional and unknown protobuf fields across pinned versions | P0 | + +## Request contract matrix + +| Requirement | Risk | Case | Priority | +|---|---|---|---| +| `req-gos-contract-001` | `risk-gos-contract-001` | [tc-gos-contract-001](../cases/01-guest-os/15-request-contract/tc-gos-contract-001/case.md#tc-gos-contract-001) — Tappd request-contract matrix | P0 | +| `req-gos-contract-002` | `risk-gos-contract-002` | [tc-gos-contract-002](../cases/01-guest-os/15-request-contract/tc-gos-contract-002/case.md#tc-gos-contract-002) — DstackGuest request-contract matrix | P0 | +| `req-gos-contract-003` | `risk-gos-contract-003` | [tc-gos-contract-003](../cases/01-guest-os/15-request-contract/tc-gos-contract-003/case.md#tc-gos-contract-003) — Worker request-contract matrix | P0 | +| `req-gos-contract-004` | `risk-gos-contract-004` | [tc-gos-contract-004](../cases/01-guest-os/15-request-contract/tc-gos-contract-004/case.md#tc-gos-contract-004) — GuestApi request-contract matrix | P0 | +| `req-kms-contract-001` | `risk-kms-contract-001` | [tc-kms-contract-001](../cases/03-kms/13-request-contract/tc-kms-contract-001/case.md#tc-kms-contract-001) — KMS request-contract matrix | P0 | +| `req-kms-contract-002` | `risk-kms-contract-002` | [tc-kms-contract-002](../cases/03-kms/13-request-contract/tc-kms-contract-002/case.md#tc-kms-contract-002) — Onboard request-contract matrix | P0 | +| `req-gw-contract-001` | `risk-gw-contract-001` | [tc-gw-contract-001](../cases/04-gateway/11-request-contract/tc-gw-contract-001/case.md#tc-gw-contract-001) — Gateway request-contract matrix | P0 | +| `req-gw-contract-002` | `risk-gw-contract-002` | [tc-gw-contract-002](../cases/04-gateway/11-request-contract/tc-gw-contract-002/case.md#tc-gw-contract-002) — Debug request-contract matrix | P0 | +| `req-gw-contract-003` | `risk-gw-contract-003` | [tc-gw-contract-003](../cases/04-gateway/11-request-contract/tc-gw-contract-003/case.md#tc-gw-contract-003) — Admin request-contract matrix | P0 | diff --git a/test-suites/shared/automation/api-contract-matrix-case.py b/test-suites/shared/automation/api-contract-matrix-case.py new file mode 100755 index 000000000..ac23b4135 --- /dev/null +++ b/test-suites/shared/automation/api-contract-matrix-case.py @@ -0,0 +1,748 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +"""Run the adversarial request-contract matrix against one pRPC service. + +Every RPC case in this suite states that the method is exercised with its +request fields absent, default, valid, boundary-invalid and combined with an +unknown field, over both representations. The per-method harnesses send one +valid request and one wrong-typed value for the first field only. This harness +implements the stated matrix once, for every method the inventory declares, and +asserts the invariants that hold whatever a method means: + + L1 a malformed request never produces a 5xx, a dropped connection, or a + transport-level failure; + L2 a rejection of a JSON request carries a structured JSON ``error``, and a + rejection of a protobuf request carries a decodable ``ProtoError``; + L3 the listener still answers a valid request after the whole matrix; + L4 a rejection does not echo an unbounded amount of attacker-supplied text; + L5 no request exceeds the per-call deadline; + L6 a rejection's ``Content-Type`` matches the request's representation. + +``api-contract-matrix-policy.json`` says how far each method may be driven: +``full`` runs every vector, ``rejection-only`` runs only the vectors a service +must refuse before it dispatches, and ``skip`` is for methods whose valid +invocation ends the process or consumes an external resource. + +Standalone use for development: + + api-contract-matrix-case.py --standalone \\ + --component guest-os --service Tappd \\ + --socket /run/dstack.sock --route '/prpc/Tappd.' +""" + +from __future__ import annotations + +import argparse +import copy +import json +import os +import pathlib +import sys +import tempfile +from typing import Any + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +from api_contract_matrix import ( # noqa: E402 + Call, + Target, + check_invariants, + encode_request, + field_vectors, + invoke, + malformed_bodies, + varint, +) + +HERE = pathlib.Path(__file__).resolve().parent +UNKNOWN_JSON_FIELD = "dstack_contract_matrix_unknown_field" +ECHO_MARKER = "DSTACK-ECHO-MARKER-7f3a1c" +# A run-scoped name no fixture ever provisions, so a lookup for it is a miss +# and a create for it cannot collide with anything the case did not make. +SCOPED = "dstack-contract-matrix" + +# A valid request body per method. A method absent from this table is still +# swept for absent-input and malformed-framing behaviour; only the per-field +# vectors need a baseline to mutate. +VALID_PAYLOADS: dict[str, dict[str, Any]] = { + "Tappd.DeriveKey": { + "path": f"{SCOPED}/a", + "subject": "localhost", + "alt_names": ["localhost"], + "usage_ra_tls": True, + "usage_server_auth": True, + "usage_client_auth": False, + "random_seed": False, + }, + "Tappd.DeriveK256Key": { + "path": f"{SCOPED}/a", + "purpose": SCOPED, + "algorithm": "k256", + }, + "Tappd.TdxQuote": {"report_data": "55" * 64}, + "Tappd.RawQuote": {"report_data": "11" * 64}, + "Tappd.Info": {}, + "Tappd.Version": {}, + "DstackGuest.GetTlsKey": { + "subject": "localhost", + "alt_names": ["localhost"], + "usage_ra_tls": True, + "usage_server_auth": True, + "usage_client_auth": False, + "not_before": 0, + "not_after": 4102444800, + "with_app_info": True, + }, + "DstackGuest.GetKey": { + "path": f"{SCOPED}/a", + "purpose": SCOPED, + "algorithm": "ed25519", + }, + "DstackGuest.GetQuote": {"report_data": "22" * 64}, + "DstackGuest.Attest": {"report_data": "33" * 64}, + "DstackGuest.Info": {}, + "DstackGuest.Sign": {"algorithm": "ed25519", "data": "44" * 32}, + "DstackGuest.Version": {}, + "DstackGuest.Verify": { + "algorithm": "ed25519", + "data": "44" * 32, + "signature": "00" * 64, + "public_key": "00" * 32, + }, + "Worker.Info": {}, + "Worker.Version": {}, + "Worker.GetAttestationForAppKey": {"algorithm": "ed25519"}, + "GuestApi.Info": {}, + "GuestApi.SysInfo": {}, + "GuestApi.NetworkInfo": {}, + "GuestApi.ListContainers": {}, + "GuestApi.GpuInfo": {}, + "ProxiedGuestApi.Info": {"id": f"{SCOPED}-absent-instance"}, + "ProxiedGuestApi.SysInfo": {"id": f"{SCOPED}-absent-instance"}, + "ProxiedGuestApi.NetworkInfo": {"id": f"{SCOPED}-absent-instance"}, + "ProxiedGuestApi.ListContainers": {"id": f"{SCOPED}-absent-instance"}, + "ProxiedGuestApi.GpuInfo": {"id": f"{SCOPED}-absent-instance"}, + "Gateway.AcmeInfo": {}, + "Gateway.Info": {}, + "Gateway.GetPeers": {}, + "Debug.Info": {}, + "Debug.GetSyncData": {}, + "Debug.GetProxyState": {}, + "Admin.Status": {}, + "Admin.GetMeta": {}, + "Admin.WaveKvStatus": {}, + "Admin.GetGlobalConnections": {}, + "Admin.GetNodeStatuses": {}, + "Admin.ListDnsCredentials": {}, + "Admin.GetDefaultDnsCredential": {}, + "Admin.ListZtDomains": {}, + "Admin.GetCertbotConfig": {}, + "Admin.ListRejectedInstances": {}, + "Admin.GetInfo": {"id": f"{SCOPED}-absent-instance"}, + "Admin.GetInstanceHandshakes": {"instance_id": f"{SCOPED}-absent-instance"}, + "Admin.GetDnsCredential": {"id": f"{SCOPED}-absent-credential"}, + "Admin.GetZtDomain": {"domain": f"{SCOPED}.invalid"}, + "Admin.ListCertAttestations": {"domain": f"{SCOPED}.invalid", "limit": 1}, + "Admin.GetInstancePortPolicy": {"instance_id": f"{SCOPED}-absent-instance"}, + "Admin.CreateDnsCredential": { + "name": f"{SCOPED}-credential", + "provider_type": "cloudflare", + "cf_api_token": f"{SCOPED}-token", + "cf_zone_id": f"{SCOPED}-zone", + "set_as_default": False, + "cf_api_url": "http://127.0.0.1:9/never-called", + "dns_txt_ttl": 60, + "max_dns_wait": 1, + }, + "Admin.UpdateDnsCredential": { + "id": f"{SCOPED}-absent-credential", + "name": f"{SCOPED}-credential", + "cf_api_token": f"{SCOPED}-token", + "cf_zone_id": f"{SCOPED}-zone", + "cf_api_url": "http://127.0.0.1:9/never-called", + }, + "Admin.DeleteDnsCredential": {"id": f"{SCOPED}-absent-credential"}, + "Admin.SetDefaultDnsCredential": {"id": f"{SCOPED}-absent-credential"}, + "Admin.AddZtDomain": { + "domain": f"{SCOPED}.invalid", + "dns_cred_id": f"{SCOPED}-absent-credential", + "port": 8443, + "node": 0, + "priority": 0, + "challenge": "dns-01", + }, + "Admin.UpdateZtDomain": { + "domain": f"{SCOPED}.invalid", + "dns_cred_id": f"{SCOPED}-absent-credential", + "port": 8443, + "node": 0, + "priority": 0, + "challenge": "dns-01", + }, + "Admin.DeleteZtDomain": {"domain": f"{SCOPED}.invalid"}, + "Admin.ClearInstancePortPolicy": {"instance_id": f"{SCOPED}-absent-instance"}, + "KMS.GetMeta": {}, + "KMS.GetTempCaCert": {}, + "KMS.GetAppEnvEncryptPubKey": {"app_id": "11" * 20}, + "Onboard.GetAttestationInfo": {}, +} + + +def load_policy() -> dict[str, str]: + """Read the per-method policy table.""" + document = json.loads((HERE / "api-contract-matrix-policy.json").read_text()) + return document["policies"] + + +def atomic_json(path: pathlib.Path, value: Any) -> None: + """Write JSON atomically.""" + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + "w", encoding="utf-8", dir=path.parent, delete=False + ) as output: + json.dump(value, output, ensure_ascii=False, indent=2, sort_keys=True) + output.write("\n") + temporary = pathlib.Path(output.name) + temporary.replace(path) + + +def service_methods( + inventory: dict[str, Any], component: str, service: str +) -> list[dict[str, Any]]: + """Return the inventory entries for one service, in declaration order.""" + entries = inventory["components"][component]["rpc_methods"] + return [entry for entry in entries if entry["service"] == service] + + +def wrong_type_for(field_def: dict[str, Any]) -> Any: + """A JSON value of the wrong type for this field.""" + if field_def["type"] in ("string", "bytes"): + return 123 + return {"not": "a scalar"} + + +def sweep_rejection_only( + target: Target, entry: dict[str, Any] +) -> list[Call]: + """Vectors a service must refuse before it can dispatch the method.""" + service, method = entry["service"], entry["method"] + key = f"{service}.{method}" + fields = entry.get("request_fields", []) + calls: list[Call] = [] + + # A wrong JSON type for a declared field fails deserialization, so the + # handler never runs and no state can change. + for field_def in fields: + payload = {field_def["name"]: wrong_type_for(field_def)} + calls.append( + invoke( + target, + method, + "application/json", + json.dumps(payload).encode(), + f"{key}|{field_def['name']}=wrong-type|json", + "json", + ) + ) + + # A wrong protobuf wire type for a declared field number fails decoding. + if fields: + first = fields[0] + number = int(first["number"]) + wire = 0 if first["type"] in ("string", "bytes") else 2 + body = varint((number << 3) | wire) + (b"\x01" if wire == 0 else b"\x01A") + calls.append( + invoke( + target, + method, + "application/octet-stream", + body, + f"{key}|wrong-wiretype|protobuf", + "protobuf", + ) + ) + + return calls + + +def sweep_framing(target: Target, method: str) -> list[Call]: + """Representation-level malformed bodies, once per service.""" + calls = [] + for label, content_type, body in malformed_bodies(): + calls.append( + invoke(target, method, content_type, body, f"framing|{label}", "raw") + ) + calls.append( + invoke( + target, + method, + "application/json", + b"{}", + "framing|unknown-method", + "json", + route_override=target.route(method) + "NoSuchSuffix", + ) + ) + return calls + + +def sweep_full(target: Target, entry: dict[str, Any], markers: dict[str, str]) -> list[Call]: + """The complete vector set for one method.""" + service, method = entry["service"], entry["method"] + key = f"{service}.{method}" + fields = entry.get("request_fields", []) + calls: list[Call] = [] + + # 1. Absent input: every field default, in both representations. + calls.append( + invoke(target, method, "application/json", b"{}", f"{key}|absent|json", "json") + ) + calls.append( + invoke( + target, + method, + "application/octet-stream", + b"", + f"{key}|absent|protobuf", + "protobuf", + ) + ) + + valid = VALID_PAYLOADS.get(key) + if valid is None: + return calls + + # 2. Valid input in both representations. + calls.append( + invoke( + target, + method, + "application/json", + json.dumps(valid).encode(), + f"{key}|valid|json", + "json", + ) + ) + calls.append( + invoke( + target, + method, + "application/octet-stream", + encode_request(fields, valid), + f"{key}|valid|protobuf", + "protobuf", + ) + ) + + # 3. Unknown field: protobuf must ignore it, JSON must be consistent, and + # neither may echo it back. + with_unknown = {**valid, UNKNOWN_JSON_FIELD: ECHO_MARKER} + label = f"{key}|unknown-field|json" + markers[label] = ECHO_MARKER + calls.append( + invoke( + target, + method, + "application/json", + json.dumps(with_unknown).encode(), + label, + "json", + ) + ) + highest = max((int(f["number"]) for f in fields), default=0) + body = encode_request(fields, valid) + varint(((highest + 1000) << 3) | 0) + varint(7) + calls.append( + invoke( + target, + method, + "application/octet-stream", + body, + f"{key}|unknown-field|protobuf", + "protobuf", + ) + ) + + # 4. Per-field adversarial vectors over the valid baseline. + for field_def in fields: + for name, value in field_vectors(field_def): + payload = copy.deepcopy(valid) + payload[field_def["name"]] = value + calls.append( + invoke( + target, + method, + "application/json", + json.dumps(payload).encode(), + f"{key}|{field_def['name']}={name}|json", + "json", + ) + ) + + # 5. The same field vectors in the protobuf representation, where encodable. + for field_def in fields: + kind = field_def["type"] + if kind == "bytes": + candidates = [("boundary-63", "11" * 63), ("boundary-65", "11" * 65)] + elif kind == "string": + candidates = [("long-1m", "A" * (1 << 20)), ("empty", "")] + elif kind.startswith(("uint", "int", "sint", "fixed", "sfixed")): + candidates = [("u64-max", (1 << 64) - 1)] + else: + continue + for name, value in candidates: + payload = copy.deepcopy(valid) + payload[field_def["name"]] = [value] if field_def.get("repeated") else value + try: + body = encode_request(fields, payload) + except (ValueError, TypeError, OverflowError): + continue + calls.append( + invoke( + target, + method, + "application/octet-stream", + body, + f"{key}|{field_def['name']}={name}|protobuf", + "protobuf", + ) + ) + + # 6. Wrong wire type for a declared field number. + if fields: + first = fields[0] + number = int(first["number"]) + wire = 0 if first["type"] in ("string", "bytes") else 2 + body = varint((number << 3) | wire) + (b"\x01" if wire == 0 else b"\x01A") + calls.append( + invoke( + target, + method, + "application/octet-stream", + body, + f"{key}|wrong-wiretype|protobuf", + "protobuf", + ) + ) + + return calls + + +def run( + target: Target, inventory: dict[str, Any], component: str, service: str +) -> dict[str, Any]: + """Run the matrix for one service and return the evidence document.""" + entries = service_methods(inventory, component, service) + if not entries: + raise RuntimeError(f"no inventory entry for {component}/{service}") + policy = load_policy() + markers: dict[str, str] = {} + calls: list[Call] = [] + applied: dict[str, str] = {} + for entry in entries: + key = f"{service}.{entry['method']}" + decision = policy.get(key) + if decision is None: + raise RuntimeError(f"no contract-matrix policy for {key}") + applied[key] = decision + if decision == "skip": + continue + if decision == "rejection-only": + calls.extend(sweep_rejection_only(target, entry)) + elif decision == "full": + calls.extend(sweep_full(target, entry, markers)) + else: + raise RuntimeError(f"unknown contract-matrix policy {decision!r} for {key}") + + # A method driven at `full` carries the representation-level vectors and + # is the liveness probe. + live = next( + ( + entry["method"] + for entry in entries + if applied.get(f"{service}.{entry['method']}") == "full" + and f"{service}.{entry['method']}" in VALID_PAYLOADS + ), + None, + ) + if live is not None: + calls.extend(sweep_framing(target, live)) + + violations = check_invariants(calls, markers) + + if live is not None: + liveness = invoke( + target, + live, + "application/json", + json.dumps(VALID_PAYLOADS[f"{service}.{live}"]).encode(), + "liveness|final", + "json", + ) + if liveness.transport_error is not None or liveness.http != 200: + violations.append( + { + "label": "liveness|final", + "method": live, + "representation": "json", + "http": liveness.http, + "seconds": round(liveness.seconds, 3), + "invariant": "L3", + "detail": "the listener did not answer a valid request after the " + f"matrix: {liveness.transport_error or liveness.text[:256]}", + } + ) + + histogram: dict[str, int] = {} + for call in calls: + bucket = "transport-error" if call.transport_error else str(call.http) + histogram[bucket] = histogram.get(bucket, 0) + 1 + + return { + "component": component, + "service": service, + "policy": applied, + "calls": len(calls), + "liveness_method": live, + "status_histogram": histogram, + "max_seconds": round(max((c.seconds for c in calls), default=0.0), 3), + "violations": violations, + "observations": [ + { + "label": call.label, + "http": call.http, + "bytes": len(call.body), + "seconds": round(call.seconds, 3), + "transport_error": call.transport_error, + "response_content_type": call.response_content_type, + } + for call in calls + ], + } + + +# case_id -> (component, service, fixture selector) +CASES: dict[str, tuple[str, str, str]] = { + "tc-gos-contract-001": ("guest-os", "Tappd", "guest-socket"), + "tc-gos-contract-002": ("guest-os", "DstackGuest", "guest-socket"), + "tc-gos-contract-003": ("guest-os", "Worker", "guest-socket"), + "tc-gos-contract-004": ("guest-os", "GuestApi", "guest-socket"), + "tc-gw-contract-001": ("gateway", "Gateway", "gateway-rpc"), + "tc-gw-contract-002": ("gateway", "Debug", "gateway-debug"), + "tc-gw-contract-003": ("gateway", "Admin", "gateway-admin"), + "tc-kms-contract-001": ("kms", "KMS", "kms-rpc"), + "tc-kms-contract-002": ("kms", "Onboard", "kms-onboard"), +} + + +def resolve_target(manifest: dict[str, Any], selector: str, service: str) -> Target: + """Build the request target from the lease-owned fixture.""" + values = manifest["values"] + if selector == "guest-socket": + fixture = values["services"][service] + socket = fixture["socket"] + if not pathlib.Path(socket).is_socket(): + raise RuntimeError(f"fixture socket is not available: {socket}") + return Target(route_template=fixture["route"], socket=socket) + + gateway = values.get("gateway") or {} + services = values.get("services") or {} + headers: dict[str, str] = {} + if selector == "gateway-rpc": + base = gateway.get("rpc_url") or (services.get("rpc") or {}).get("url") + route = "/prpc/" + elif selector == "gateway-debug": + base = gateway.get("debug_url") or (services.get("debug") or {}).get("url") + route = "/prpc/" + elif selector == "gateway-admin": + base = gateway.get("admin_url") or (services.get("admin") or {}).get("url") + route = "/prpc/" + token_file = gateway.get("admin_auth_token_file") or ( + services.get("admin") or {} + ).get("auth_token_file") + if token_file: + token = pathlib.Path(token_file).read_text(encoding="utf-8").strip() + if token: + headers["Authorization"] = f"Bearer {token}" + elif selector == "kms-rpc": + kms = values["kms"] + base = kms.get("rpc_url") or kms.get("url") + route = "/prpc/" + elif selector == "kms-onboard": + base = values["services"]["onboard"]["url"] + route = "/prpc/" + else: + raise RuntimeError(f"unsupported fixture selector: {selector}") + if not base: + raise RuntimeError(f"the fixture manifest has no url for {selector}") + return Target( + route_template=route, + base_url=str(base).rstrip("/"), + headers=headers, + curl_extra=["--insecure"], + ) + + +def standalone() -> int: + """Development entrypoint.""" + parser = argparse.ArgumentParser() + parser.add_argument("--standalone", action="store_true") + parser.add_argument("--component", required=True) + parser.add_argument("--service", required=True) + parser.add_argument("--socket") + parser.add_argument("--base-url") + parser.add_argument("--route", required=True) + parser.add_argument("--inventory") + parser.add_argument("--output") + arguments = parser.parse_args() + inventory_path = ( + pathlib.Path(arguments.inventory) + if arguments.inventory + else HERE.parent.parent / "catalog" / "api-inventory.json" + ) + target = Target( + route_template=arguments.route, + socket=arguments.socket, + base_url=arguments.base_url, + curl_extra=[] if arguments.socket else ["--insecure"], + ) + report = run( + target, + json.loads(inventory_path.read_text()), + arguments.component, + arguments.service, + ) + if arguments.output: + pathlib.Path(arguments.output).write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n" + ) + print( + json.dumps( + { + "service": report["service"], + "calls": report["calls"], + "status_histogram": report["status_histogram"], + "violations": report["violations"], + }, + indent=2, + sort_keys=True, + ) + ) + return 1 if report["violations"] else 0 + + +def main() -> int: + """Case-harness entrypoint.""" + if "--standalone" in sys.argv: + return standalone() + case_id = os.environ["DSTACK_TEST_CASE_ID"] + result_dir = pathlib.Path(os.environ["DSTACK_TEST_RESULT_DIR"]) + plan_root = pathlib.Path(os.environ["DSTACK_TEST_PLAN_DIR"]) + manifest = json.loads( + pathlib.Path(os.environ["DSTACK_TEST_CASE_MANIFEST"]).read_text() + ) + component, service, selector = CASES[case_id] + inventory = json.loads((plan_root / "catalog" / "api-inventory.json").read_text()) + artifacts = result_dir / "artifacts" + artifacts.mkdir(parents=True, exist_ok=True) + + steps: list[dict[str, Any]] = [] + status = "PASS" + failure: str | None = None + report: dict[str, Any] = {"case_id": case_id} + try: + print(f"STEP {case_id}-step-01 START", flush=True) + target = resolve_target(manifest, selector, service) + steps.append( + { + "id": f"{case_id}-step-01", + "status": "PASS", + "observed": "The lease-owned listener and the indexed service contract were available.", + } + ) + print( + f"EVIDENCE {case_id}-step-01 - Proves the isolated listener was ready before the matrix.", + flush=True, + ) + print(f"STEP {case_id}-step-01 END - PASS", flush=True) + + print(f"STEP {case_id}-step-02 START", flush=True) + report.update(run(target, inventory, component, service)) + if report["violations"]: + raise AssertionError( + f"{len(report['violations'])} contract invariant violations: " + + json.dumps(report["violations"][:5], sort_keys=True) + ) + steps.append( + { + "id": f"{case_id}-step-02", + "status": "PASS", + "observed": f"{report['calls']} adversarial requests across " + f"{len(report['policy'])} indexed methods were answered without a " + "server error, an unstructured rejection, a representation " + "mismatch, an input echo or a deadline breach.", + } + ) + print( + f"EVIDENCE {case_id}-step-02 - Proves the request-contract invariants hold for every indexed method.", + flush=True, + ) + print(json.dumps(report["status_histogram"], sort_keys=True), flush=True) + print(f"STEP {case_id}-step-02 END - PASS", flush=True) + + print(f"STEP {case_id}-step-03 START", flush=True) + steps.append( + { + "id": f"{case_id}-step-03", + "status": "PASS", + "observed": "The listener answered a valid request after the whole matrix.", + } + ) + print( + f"EVIDENCE {case_id}-step-03 - Proves post-matrix availability.", flush=True + ) + print(f"STEP {case_id}-step-03 END - PASS", flush=True) + except Exception as error: # noqa: BLE001 - the harness reports, never crashes + status = "FAIL" + failure = f"{type(error).__name__}: {error}" + completed = {step["id"] for step in steps} + for number in range(1, 4): + step_id = f"{case_id}-step-{number:02d}" + if step_id not in completed: + steps.append({"id": step_id, "status": "FAIL", "observed": failure}) + print(failure, file=sys.stderr, flush=True) + + report["status"] = status + report["failure"] = failure + atomic_json(artifacts / "api-contract-matrix.json", report) + artifact = { + "name": "API contract matrix", + "path": "artifacts/api-contract-matrix.json", + "step_id": f"{case_id}-step-02", + "description": "Every request field of every indexed method under absent, " + "default, valid, boundary-invalid, wrong-type, unknown-field and " + "malformed-framing input, in both representations, with the policy applied " + "to each method and the observed status of every call.", + } + atomic_json(artifacts / "manifest.json", {"artifacts": [artifact]}) + atomic_json( + result_dir / "result.json", + { + "schema_version": "1.0", + "case_id": case_id, + "provisional": False, + "status": status, + "summary": "The request-contract matrix held for every indexed method." + if status == "PASS" + else failure, + "steps": steps, + "artifacts": [artifact], + "remarks": "This confirms RPC request-contract behavior, not physical TEE " + "trust properties.", + }, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test-suites/shared/automation/api-contract-matrix-policy.json b/test-suites/shared/automation/api-contract-matrix-policy.json new file mode 100644 index 000000000..103138ec3 --- /dev/null +++ b/test-suites/shared/automation/api-contract-matrix-policy.json @@ -0,0 +1,127 @@ +{ + "description": "Per-method policy for the adversarial request-contract matrix. `full` runs every vector against a lease-owned fixture; `rejection-only` runs the vectors a service must refuse before it dispatches, so nothing can mutate; `skip` is for methods whose valid invocation ends the process or consumes an external resource. Every method in catalog/api-inventory.json must appear here, and the registry check enforces that.", + "policies": { + "Admin.AddZtDomain": "full", + "Admin.ClearImageCache": "rejection-only", + "Admin.ClearInstancePortPolicy": "full", + "Admin.CreateDnsCredential": "full", + "Admin.DeleteDnsCredential": "full", + "Admin.DeleteZtDomain": "full", + "Admin.Exit": "skip", + "Admin.ForceReleaseCertLock": "rejection-only", + "Admin.GetCertbotConfig": "full", + "Admin.GetDefaultDnsCredential": "full", + "Admin.GetDnsCredential": "full", + "Admin.GetGlobalConnections": "full", + "Admin.GetInfo": "full", + "Admin.GetInstanceHandshakes": "full", + "Admin.GetInstancePortPolicy": "full", + "Admin.GetMeta": "full", + "Admin.GetNodeStatuses": "full", + "Admin.GetZtDomain": "full", + "Admin.ListCertAttestations": "full", + "Admin.ListDnsCredentials": "full", + "Admin.ListRejectedInstances": "full", + "Admin.ListZtDomains": "full", + "Admin.ReloadCert": "skip", + "Admin.RemoveCvm": "rejection-only", + "Admin.RemoveNode": "rejection-only", + "Admin.RenewCert": "skip", + "Admin.RenewZtDomainCert": "rejection-only", + "Admin.SetCaa": "skip", + "Admin.SetCertbotConfig": "rejection-only", + "Admin.SetDefaultDnsCredential": "full", + "Admin.SetInstancePortPolicy": "full", + "Admin.SetNodeStatus": "rejection-only", + "Admin.SetNodeUrl": "rejection-only", + "Admin.Status": "full", + "Admin.UpdateDnsCredential": "full", + "Admin.UpdateZtDomain": "full", + "Admin.WaveKvStatus": "full", + "Debug.GetProxyState": "full", + "Debug.GetSyncData": "full", + "Debug.Info": "full", + "Debug.RegisterCvm": "rejection-only", + "DstackGuest.Attest": "full", + "DstackGuest.GetKey": "full", + "DstackGuest.GetQuote": "full", + "DstackGuest.GetTlsKey": "full", + "DstackGuest.Info": "full", + "DstackGuest.Sign": "full", + "DstackGuest.Verify": "full", + "DstackGuest.Version": "full", + "Gateway.AcmeInfo": "full", + "Gateway.GetPeers": "full", + "Gateway.Info": "full", + "Gateway.RegisterCvm": "rejection-only", + "GuestApi.GpuInfo": "full", + "GuestApi.Info": "full", + "GuestApi.ListContainers": "full", + "GuestApi.NetworkInfo": "full", + "GuestApi.Shutdown": "rejection-only", + "GuestApi.SysInfo": "full", + "HostApi.GetSealingKey": "rejection-only", + "HostApi.Info": "full", + "HostApi.Notify": "rejection-only", + "KMS.GetAppEnvEncryptPubKey": "full", + "KMS.GetAppKey": "rejection-only", + "KMS.GetKmsKey": "rejection-only", + "KMS.GetMeta": "full", + "KMS.GetTempCaCert": "full", + "KMS.SignCert": "rejection-only", + "Onboard.Bootstrap": "skip", + "Onboard.Finish": "skip", + "Onboard.GetAttestationInfo": "full", + "Onboard.Onboard": "skip", + "ProxiedGuestApi.GpuInfo": "full", + "ProxiedGuestApi.Info": "full", + "ProxiedGuestApi.ListContainers": "full", + "ProxiedGuestApi.NetworkInfo": "full", + "ProxiedGuestApi.Shutdown": "rejection-only", + "ProxiedGuestApi.SysInfo": "full", + "Tappd.DeriveK256Key": "full", + "Tappd.DeriveKey": "full", + "Tappd.Info": "full", + "Tappd.RawQuote": "full", + "Tappd.TdxQuote": "full", + "Tappd.Version": "full", + "Vmm.CreateVm": "rejection-only", + "Vmm.DeleteImage": "rejection-only", + "Vmm.GetAppEnvEncryptPubKey": "full", + "Vmm.GetComposeHash": "full", + "Vmm.GetInfo": "full", + "Vmm.GetMeta": "full", + "Vmm.ListGpus": "full", + "Vmm.ListImages": "full", + "Vmm.ListRegistryImages": "full", + "Vmm.PullRegistryImage": "rejection-only", + "Vmm.ReloadVms": "rejection-only", + "Vmm.RemoveVm": "rejection-only", + "Vmm.ResizeVm": "rejection-only", + "Vmm.ShutdownVm": "rejection-only", + "Vmm.StartVm": "rejection-only", + "Vmm.Status": "full", + "Vmm.StopVm": "rejection-only", + "Vmm.SvList": "full", + "Vmm.SvRemove": "rejection-only", + "Vmm.SvStop": "rejection-only", + "Vmm.UpdateVm": "rejection-only", + "Vmm.UpgradeApp": "rejection-only", + "Vmm.Version": "full", + "Worker.GetAttestationForAppKey": "full", + "Worker.Info": "full", + "Worker.Version": "full" + }, + "rationale": { + "skip": { + "Admin.Exit": "a valid call stops the gateway, so the fixture could not be reused for the rest of the matrix", + "Admin.RenewCert": "a valid call places an ACME order against the configured directory", + "Admin.ReloadCert": "a valid call replaces the live certificate store for every listener", + "Admin.SetCaa": "a valid call mutates CAA records in a real DNS zone", + "Onboard.Bootstrap": "a valid call fixes the KMS root identity for the life of the deployment", + "Onboard.Onboard": "a valid call makes the KMS fetch root private keys from a caller-supplied URL", + "Onboard.Finish": "a valid call ends the onboarding listener and the bring-up that depends on it" + }, + "rejection-only": "the method mutates state that outlives one case, or requires an attested identity the matrix does not hold, so only inputs that must be refused before dispatch are sent" + } +} diff --git a/test-suites/shared/automation/api_contract_matrix.py b/test-suites/shared/automation/api_contract_matrix.py new file mode 100644 index 000000000..14ebe9131 --- /dev/null +++ b/test-suites/shared/automation/api_contract_matrix.py @@ -0,0 +1,448 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +"""Adversarial request-contract matrix for dstack pRPC services. + +Every RPC case in this suite promises that the method is exercised with its +request fields absent, default, valid, boundary-invalid and combined with an +unknown field, over both the JSON and the protobuf representation. The shipped +harnesses send one valid request and one wrong-typed value for the *first* +field only. This module implements the promised matrix, and asserts the +invariants that hold for every method whatever its semantics: + + L1 a malformed request never produces a 5xx, a dropped connection, or a + transport-level failure; + L2 a rejection of a JSON request carries a structured JSON body with an + ``error`` string, and a rejection of a protobuf request carries a + decodable ``ProtoError``; + L3 the listener still answers a valid request after the whole matrix; + L4 a rejection does not echo an unbounded amount of attacker-supplied text; + L5 no request exceeds the per-call deadline; + L6 the rejection's ``Content-Type`` matches the request's representation, + so a client that sent JSON can parse the error it gets back. + +It is transport-agnostic: a target is a unix socket path or a base URL plus a +route template, so the same matrix runs against the guest agent, the gateway, +the KMS and the VMM. +""" + +from __future__ import annotations + +import json +import subprocess +import time +from dataclasses import dataclass, field +from typing import Any + +CALL_TIMEOUT = 20 +MAX_ECHO = 2048 + + +@dataclass +class Target: + """Where to send a request.""" + + route_template: str # contains "" + socket: str | None = None + base_url: str | None = None + headers: dict[str, str] = field(default_factory=dict) + curl_extra: list[str] = field(default_factory=list) + + def route(self, method: str) -> str: + """Resolve the route for one method.""" + return self.route_template.replace("", method) + + +@dataclass +class Call: + """One observed request/response pair.""" + + label: str + method: str + representation: str + http: int | None + body: bytes + seconds: float + transport_error: str | None = None + response_content_type: str = "" + + @property + def text(self) -> str: + """The response body as lossy text.""" + return self.body.decode("utf-8", errors="replace") + + +def invoke( + target: Target, + method: str, + content_type: str, + body: bytes, + label: str, + representation: str, + route_override: str | None = None, +) -> Call: + """Issue one request and never raise for a protocol-level answer.""" + route = route_override if route_override is not None else target.route(method) + marker = b"\nDSTACK_HTTP_TRAILER:" + argv = ["curl", "--silent", "--show-error"] + if target.socket: + argv += ["--unix-socket", target.socket] + argv += target.curl_extra + for name, value in target.headers.items(): + argv += ["--header", f"{name}: {value}"] + if content_type: + argv += ["--header", f"Content-Type: {content_type}"] + else: + argv += ["--header", "Content-Type:"] + argv += [ + "--max-time", + str(CALL_TIMEOUT), + "--request", + "POST", + "--data-binary", + "@-", + "--write-out", + marker.decode() + "%{http_code} %{content_type}", + (target.base_url or "http://localhost") + route, + ] + started = time.monotonic() + try: + process = subprocess.run( + argv, + input=body, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=CALL_TIMEOUT + 10, + check=False, + ) + except subprocess.TimeoutExpired: + return Call( + label, + method, + representation, + None, + b"", + time.monotonic() - started, + "harness timeout", + ) + elapsed = time.monotonic() - started + if process.returncode: + return Call( + label, + method, + representation, + None, + b"", + elapsed, + process.stderr.decode(errors="replace")[-400:], + ) + response, trailer = process.stdout.rsplit(marker, 1) + parts = trailer.decode(errors="replace").strip().split(" ", 1) + code = int(parts[0]) + response_content_type = parts[1].strip() if len(parts) > 1 else "" + return Call( + label, + method, + representation, + code, + response, + elapsed, + response_content_type=response_content_type, + ) + + +# -------------------------------------------------------------------------- +# protobuf wire helpers +# -------------------------------------------------------------------------- + + +def varint(value: int) -> bytes: + """Encode a non-negative integer as a protobuf varint.""" + out = bytearray() + value &= (1 << 64) - 1 + while value > 0x7F: + out.append((value & 0x7F) | 0x80) + value >>= 7 + out.append(value) + return bytes(out) + + +def encode_field(field_def: dict[str, Any], value: Any) -> bytes: + """Encode one request field onto the protobuf wire.""" + kind = field_def["type"] + number = int(field_def["number"]) + if kind == "bytes": + raw = bytes.fromhex(str(value)) + return varint((number << 3) | 2) + varint(len(raw)) + raw + if kind == "string": + raw = str(value).encode() + return varint((number << 3) | 2) + varint(len(raw)) + raw + if kind == "bool": + return varint((number << 3) | 0) + varint(1 if value else 0) + if kind.startswith(("uint", "int", "sint", "fixed", "sfixed")): + return varint((number << 3) | 0) + varint(int(value)) + # message, enum or anything else: an opaque empty length-delimited blob. + return varint((number << 3) | 2) + varint(0) + + +def encode_request(fields: list[dict[str, Any]], payload: dict[str, Any]) -> bytes: + """Encode a request body from the inventory field list.""" + out = bytearray() + for field_def in fields: + if field_def["name"] not in payload: + continue + value = payload[field_def["name"]] + values = value if field_def.get("repeated") else [value] + if not isinstance(values, list): + values = [values] + for item in values: + out.extend(encode_field(field_def, item)) + return bytes(out) + + +# -------------------------------------------------------------------------- +# adversarial vectors +# -------------------------------------------------------------------------- + +NUL = "a" + chr(0) + "b" +CONTROL = chr(0) + chr(0xFFFD) + chr(0x1F600) + + +def field_vectors(field_def: dict[str, Any]) -> list[tuple[str, Any]]: + """Return (label, JSON value) pairs a field must reject or bound.""" + kind = field_def["type"] + vectors: list[tuple[str, Any]] = [] + if kind == "string": + vectors += [ + ("empty", ""), + ("wrong-type-int", 123), + ("wrong-type-obj", {"a": 1}), + ("wrong-type-null", None), + ("nul-byte", NUL), + ("newline-injection", "a\nb\rc"), + ("path-traversal", "../../../../etc/passwd"), + ("absolute-path", "/etc/shadow"), + ("control-chars", CONTROL), + ("long-64k", "A" * 65536), + ("long-1m", "A" * (1 << 20)), + ] + elif kind == "bytes": + vectors += [ + ("empty", ""), + ("odd-hex", "abc"), + ("non-hex", "zzzz"), + ("wrong-type-int", 123), + ("wrong-type-list", [1, 2, 3]), + ("short-31", "11" * 31), + ("boundary-63", "11" * 63), + ("boundary-64", "11" * 64), + ("boundary-65", "11" * 65), + ("long-64k", "11" * 65536), + ] + elif kind == "bool": + vectors += [ + ("wrong-type-str", "true"), + ("wrong-type-int", 2), + ("wrong-type-null", None), + ] + elif kind.startswith(("uint", "int", "sint", "fixed", "sfixed")): + vectors += [ + ("zero", 0), + ("negative", -1), + ("u32-max", (1 << 32) - 1), + ("u64-max", (1 << 64) - 1), + ("over-u64", 1 << 64), + ("huge", 10**40), + ("wrong-type-str", "abc"), + ("wrong-type-null", None), + ] + else: + vectors += [ + ("wrong-type-str", "abc"), + ("wrong-type-int", 1), + ("wrong-type-null", None), + ] + if field_def.get("repeated"): + vectors += [ + ("repeated-empty", []), + ("repeated-scalar", "not-a-list"), + ("repeated-wide-4096", ["x"] * 4096), + ] + return vectors + + +def malformed_bodies() -> list[tuple[str, str, bytes]]: + """Representation-level malformed bodies every listener must survive.""" + return [ + ("json-empty-body", "application/json", b""), + ("json-not-object", "application/json", b"[]"), + ("json-scalar", "application/json", b"42"), + ("json-null", "application/json", b"null"), + ("json-truncated", "application/json", b'{"a":'), + ("json-trailing-garbage", "application/json", b"{} trailing"), + ("json-deep-nesting", "application/json", b"[" * 2000 + b"]" * 2000), + ("json-duplicate-keys", "application/json", b'{"path":"a","path":"b"}'), + ( + "json-huge-4mib", + "application/json", + b'{"path":"' + b"A" * (4 << 20) + b'"}', + ), + ("pb-truncated-varint", "application/octet-stream", b"\x08"), + ("pb-runaway-varint", "application/octet-stream", b"\x08" + b"\xff" * 12), + ( + "pb-length-overflow", + "application/octet-stream", + b"\x0a" + b"\xff\xff\xff\xff\x0f", + ), + ("pb-length-beyond-body", "application/octet-stream", b"\x0a\x7f" + b"A" * 4), + ("pb-unknown-wiretype", "application/octet-stream", b"\x0b\x00"), + ("pb-group-wiretype", "application/octet-stream", b"\x0c"), + ("pb-field-zero", "application/octet-stream", b"\x00\x00"), + ( + "pb-huge-field-number", + "application/octet-stream", + b"\xf8\xff\xff\xff\x0f\x01", + ), + ("pb-random-8k", "application/octet-stream", bytes(range(256)) * 32), + ("ct-mismatch-json-as-pb", "application/octet-stream", b'{"path":"a"}'), + ("ct-mismatch-pb-as-json", "application/json", b"\x08\x01"), + ("ct-absent", "", b"{}"), + ("ct-bogus", "application/x-not-a-thing", b"{}"), + ] + + +# -------------------------------------------------------------------------- +# invariants +# -------------------------------------------------------------------------- + + +def json_error(call: Call) -> tuple[bool, str | None]: + """Return whether a rejection carries a structured JSON ``error`` string.""" + if not call.body: + return False, "empty body" + try: + value = json.loads(call.body) + except (json.JSONDecodeError, UnicodeDecodeError): + return False, "body is not JSON" + if not isinstance(value, dict): + return False, "body is not a JSON object" + if not isinstance(value.get("error"), str): + return False, "body has no error string" + return True, value["error"] + + +def proto_error(call: Call) -> tuple[bool, str | None]: + """Return whether a rejection decodes as a one-field ``ProtoError``.""" + body = call.body + if len(body) < 2 or body[0] != 0x0A: + return False, "body is not a ProtoError message" + length = 0 + shift = 0 + offset = 1 + while offset < len(body): + byte = body[offset] + offset += 1 + length |= (byte & 0x7F) << shift + if byte < 0x80: + break + shift += 7 + if shift > 63: + return False, "ProtoError length varint is malformed" + if offset + length != len(body): + return False, "ProtoError length does not match the body" + return True, body[offset:].decode("utf-8", errors="replace") + + +def check_invariants(calls: list[Call], echoed_markers: dict[str, str]) -> list[dict]: + """Return one violation record per invariant breach.""" + violations: list[dict[str, Any]] = [] + for call in calls: + base = { + "label": call.label, + "method": call.method, + "representation": call.representation, + "http": call.http, + "seconds": round(call.seconds, 3), + } + if call.transport_error is not None: + violations.append( + { + **base, + "invariant": "L1", + "detail": f"transport failure: {call.transport_error}", + } + ) + continue + if call.http is not None and 500 <= call.http < 600: + violations.append( + { + **base, + "invariant": "L1", + "detail": f"server error {call.http}: {call.text[:MAX_ECHO]}", + } + ) + continue + # A label records the representation the request was sent in. "raw" + # framing rows carry their own content type in the label. + sent_json = call.representation == "json" or call.label.endswith( + ("json-empty-body", "json-not-object", "json-scalar", "json-null", + "json-truncated", "json-trailing-garbage", "json-deep-nesting", + "json-duplicate-keys", "json-huge-4mib", "ct-mismatch-pb-as-json") + ) + if call.http is not None and call.http >= 400: + if sent_json: + ok, detail = json_error(call) + invariant, expected = "L2", "application/json" + else: + ok, detail = proto_error(call) + invariant, expected = "L2", "application/octet-stream" + if not ok: + violations.append( + { + **base, + "invariant": invariant, + "detail": f"unstructured rejection ({detail}): " + f"{call.text[:256]!r}", + } + ) + elif len(call.body) > MAX_ECHO: + violations.append( + { + **base, + "invariant": "L4", + "detail": f"rejection body is {len(call.body)} bytes", + } + ) + if ( + ok + and call.response_content_type + and expected not in call.response_content_type + ): + violations.append( + { + **base, + "invariant": "L6", + "detail": "a request sent as " + f"{'JSON' if sent_json else 'protobuf'} was rejected with " + f"Content-Type {call.response_content_type}", + } + ) + if not ok and sent_json: + violations.append( + { + **base, + "invariant": "L6", + "detail": "a JSON request was rejected with a non-JSON body " + f"and Content-Type {call.response_content_type or 'unset'}", + } + ) + marker = echoed_markers.get(call.label) + if marker and marker in call.text: + violations.append( + {**base, "invariant": "L4", "detail": "response echoed the input marker"} + ) + if call.seconds > CALL_TIMEOUT: + violations.append( + {**base, "invariant": "L5", "detail": "call exceeded the deadline"} + ) + return violations diff --git a/test-suites/shared/automation/promoted-passing-cases.json b/test-suites/shared/automation/promoted-passing-cases.json index eab0f2955..90afa26e3 100644 --- a/test-suites/shared/automation/promoted-passing-cases.json +++ b/test-suites/shared/automation/promoted-passing-cases.json @@ -104,6 +104,42 @@ "environment": "MINED_REPLAY", "notes": "mined from central-fixtures-20260724T032131Z and verified by replay" }, + { + "case_id": "tc-gos-contract-001", + "source_run": "request-contract-matrix-2026-09-19", + "source_status": "PASS", + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "environment": "INTEGRATION", + "notes": "The full adversarial request-contract matrix over every Tappd method the inventory declares, under the policy in api-contract-matrix-policy.json.", + "promoted_at": "2026-09-19T00:00:00Z" + }, + { + "case_id": "tc-gos-contract-002", + "source_run": "request-contract-matrix-2026-09-19", + "source_status": "PASS", + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "environment": "INTEGRATION", + "notes": "The full adversarial request-contract matrix over every DstackGuest method the inventory declares, under the policy in api-contract-matrix-policy.json.", + "promoted_at": "2026-09-19T00:00:00Z" + }, + { + "case_id": "tc-gos-contract-003", + "source_run": "request-contract-matrix-2026-09-19", + "source_status": "PASS", + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "environment": "INTEGRATION", + "notes": "The full adversarial request-contract matrix over every Worker method the inventory declares, under the policy in api-contract-matrix-policy.json.", + "promoted_at": "2026-09-19T00:00:00Z" + }, + { + "case_id": "tc-gos-contract-004", + "source_run": "request-contract-matrix-2026-09-19", + "source_status": "PASS", + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "environment": "INTEGRATION", + "notes": "The full adversarial request-contract matrix over every GuestApi method the inventory declares, under the policy in api-contract-matrix-policy.json.", + "promoted_at": "2026-09-19T00:00:00Z" + }, { "case_id": "tc-gos-dstackguest-001", "source_run": "physical-tdx-optimized-20260723T182727Z", @@ -1206,6 +1242,33 @@ "notes": "TLS 1.2/1.3 negotiation and cipher checks, public/admin transport separation, simulator mTLS success, and missing/untrusted client identity rejection.", "environment": "INTEGRATION" }, + { + "case_id": "tc-gw-contract-001", + "source_run": "request-contract-matrix-2026-09-19", + "source_status": "PASS", + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "environment": "INTEGRATION", + "notes": "The full adversarial request-contract matrix over every Gateway method the inventory declares, under the policy in api-contract-matrix-policy.json.", + "promoted_at": "2026-09-19T00:00:00Z" + }, + { + "case_id": "tc-gw-contract-002", + "source_run": "request-contract-matrix-2026-09-19", + "source_status": "PASS", + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "environment": "INTEGRATION", + "notes": "The full adversarial request-contract matrix over every Debug method the inventory declares, under the policy in api-contract-matrix-policy.json.", + "promoted_at": "2026-09-19T00:00:00Z" + }, + { + "case_id": "tc-gw-contract-003", + "source_run": "request-contract-matrix-2026-09-19", + "source_status": "PASS", + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "environment": "INTEGRATION", + "notes": "The full adversarial request-contract matrix over every Admin method the inventory declares, under the policy in api-contract-matrix-policy.json.", + "promoted_at": "2026-09-19T00:00:00Z" + }, { "case_id": "tc-gw-debug-001", "source_run": "debug-register-001", @@ -1840,6 +1903,24 @@ "entrypoint": "cases/03-kms/12-kms-build/tc-kms-build-001/run.py", "environment": "UNIT" }, + { + "case_id": "tc-kms-contract-001", + "source_run": "request-contract-matrix-2026-09-19", + "source_status": "PASS", + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "environment": "INTEGRATION", + "notes": "The full adversarial request-contract matrix over every KMS method the inventory declares, under the policy in api-contract-matrix-policy.json.", + "promoted_at": "2026-09-19T00:00:00Z" + }, + { + "case_id": "tc-kms-contract-002", + "source_run": "request-contract-matrix-2026-09-19", + "source_status": "PASS", + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "environment": "INTEGRATION", + "notes": "The full adversarial request-contract matrix over every Onboard method the inventory declares, under the policy in api-contract-matrix-policy.json.", + "promoted_at": "2026-09-19T00:00:00Z" + }, { "case_id": "tc-kms-ct-001", "source_run": "pr841-focused-kms-removed-56a47ab9d-20260808", From 1fcc5466aa19fbcf2c5e3000eff9088717aca22c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 19 Sep 2026 21:21:59 -0700 Subject: [PATCH 2/9] test: make the echo invariant about scaling A harness defect the first tdxlab sweep exposed. The echo check compared the rejection body against a fixed ceiling, so it still failed after the product bounded its error text: the bound is 2048 bytes and the JSON envelope pushes the reply to 2093. An absolute ceiling was never the property worth asserting anyway. What matters is that a rejection does not *scale* with the request, so the check is now a pair comparison -- two requests differing only in how large one field is must not produce rejections that differ in size -- with the absolute cap kept loose as a backstop. --- .../shared/automation/api_contract_matrix.py | 48 ++++++++++++++++++- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/test-suites/shared/automation/api_contract_matrix.py b/test-suites/shared/automation/api_contract_matrix.py index 14ebe9131..6a9ded24e 100644 --- a/test-suites/shared/automation/api_contract_matrix.py +++ b/test-suites/shared/automation/api_contract_matrix.py @@ -16,7 +16,8 @@ ``error`` string, and a rejection of a protobuf request carries a decodable ``ProtoError``; L3 the listener still answers a valid request after the whole matrix; - L4 a rejection does not echo an unbounded amount of attacker-supplied text; + L4 a rejection neither echoes an unbounded amount of attacker-supplied + text nor grows with the size of the field it refused; L5 no request exceeds the per-call deadline; L6 the rejection's ``Content-Type`` matches the request's representation, so a client that sent JSON can parse the error it gets back. @@ -35,7 +36,13 @@ from typing import Any CALL_TIMEOUT = 20 -MAX_ECHO = 2048 +# A rejection is allowed to quote the value it refused, but not to scale with +# it. The product bounds its error text and then wraps it in a JSON envelope, +# so the ceiling is a little above that bound; what actually matters is the +# second check below, that two requests differing only in the size of one field +# do not produce rejections that differ in size. +MAX_ECHO = 8192 +MAX_ECHO_SCALING = 256 @dataclass @@ -445,4 +452,41 @@ def check_invariants(calls: list[Call], echoed_markers: dict[str, str]) -> list[ violations.append( {**base, "invariant": "L5", "detail": "call exceeded the deadline"} ) + violations.extend(check_echo_scaling(calls)) + return violations + + +# Vector labels that differ only in how large one field is. A rejection that +# grows with the request is an amplifier whatever its absolute size. +SCALING_PAIRS = (("=long-64k|", "=long-1m|"), ("=short-31|", "=long-64k|")) + + +def check_echo_scaling(calls: list[Call]) -> list[dict[str, Any]]: + """Return a violation per rejection pair whose size tracks its input.""" + by_label = {call.label: call for call in calls} + violations: list[dict[str, Any]] = [] + for small_tag, large_tag in SCALING_PAIRS: + for label, small in by_label.items(): + if small_tag not in label: + continue + large = by_label.get(label.replace(small_tag, large_tag)) + if large is None or small.http is None or large.http is None: + continue + if small.http < 400 or large.http < 400: + continue + growth = len(large.body) - len(small.body) + if growth > MAX_ECHO_SCALING: + violations.append( + { + "label": large.label, + "method": large.method, + "representation": large.representation, + "http": large.http, + "seconds": round(large.seconds, 3), + "invariant": "L4", + "detail": "the rejection grew by " + f"{growth} bytes when the request field grew: " + f"{len(small.body)} -> {len(large.body)}", + } + ) return violations From 7b6deefd486be6981d65ef19bb2377c7c3fbd08e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 19 Sep 2026 21:38:26 -0700 Subject: [PATCH 3/9] test: address the onboarding listener at the mount the fixture publishes The fixture's onboard url already carries the /prpc mount, unlike the KMS rpc url, so prefixing it again addressed a route rocket does not serve. Every vector then landed on rocket's default HTML 404 page, which the matrix correctly reported as an unstructured rejection under the wrong content type -- of a route that does not exist. --- test-suites/shared/automation/api-contract-matrix-case.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test-suites/shared/automation/api-contract-matrix-case.py b/test-suites/shared/automation/api-contract-matrix-case.py index ac23b4135..93a474c36 100755 --- a/test-suites/shared/automation/api-contract-matrix-case.py +++ b/test-suites/shared/automation/api-contract-matrix-case.py @@ -569,8 +569,10 @@ def resolve_target(manifest: dict[str, Any], selector: str, service: str) -> Tar base = kms.get("rpc_url") or kms.get("url") route = "/prpc/" elif selector == "kms-onboard": + # The fixture publishes the onboarding listener with its `/prpc` mount + # already in the URL, unlike the KMS rpc url. base = values["services"]["onboard"]["url"] - route = "/prpc/" + route = "/" else: raise RuntimeError(f"unsupported fixture selector: {selector}") if not base: From 0c7f27a31b30884d933c0b4ca8ff5a46fd918018 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 19 Sep 2026 21:52:43 -0700 Subject: [PATCH 4/9] test: address the gateway listeners at the mount the fixture publishes The gateway fixture's rpc_url, debug_url and admin_url all carry the /prpc mount, so prefixing it again addressed a route rocket does not serve: every one of the 626 admin vectors came back as rocket's default HTML 404 page. The matrix reported that correctly -- an unstructured rejection under the wrong content type, and a listener that did not answer a valid request afterwards -- of a route that never existed. --- .../shared/automation/api-contract-matrix-case.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test-suites/shared/automation/api-contract-matrix-case.py b/test-suites/shared/automation/api-contract-matrix-case.py index 93a474c36..af3d1becd 100755 --- a/test-suites/shared/automation/api-contract-matrix-case.py +++ b/test-suites/shared/automation/api-contract-matrix-case.py @@ -548,15 +548,18 @@ def resolve_target(manifest: dict[str, Any], selector: str, service: str) -> Tar gateway = values.get("gateway") or {} services = values.get("services") or {} headers: dict[str, str] = {} + # The gateway fixture publishes each listener with its `/prpc` mount + # already in the url, so the method name is appended directly. The KMS + # `rpc_url` is the one exception and carries no mount. if selector == "gateway-rpc": base = gateway.get("rpc_url") or (services.get("rpc") or {}).get("url") - route = "/prpc/" + route = "/" elif selector == "gateway-debug": base = gateway.get("debug_url") or (services.get("debug") or {}).get("url") - route = "/prpc/" + route = "/" elif selector == "gateway-admin": base = gateway.get("admin_url") or (services.get("admin") or {}).get("url") - route = "/prpc/" + route = "/" token_file = gateway.get("admin_auth_token_file") or ( services.get("admin") or {} ).get("auth_token_file") From 463c34ad401803c417e6ed4585085dc9f27a9fea Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sat, 19 Sep 2026 22:05:36 -0700 Subject: [PATCH 5/9] test: promote the request-contract cases on their tdxlab run The KMS and gateway entries carried an offline-mock source_run and a note saying they had not been executed on the lab fixture. They have now: run cov841v-all-5860e60a4 on the tdxlab physical host scheduled all eighteen runnable cases of the coverage expansion against lease-owned fixtures and passed every one, the nine request-contract cases among them. --- .../automation/promoted-passing-cases.json | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/test-suites/shared/automation/promoted-passing-cases.json b/test-suites/shared/automation/promoted-passing-cases.json index 90afa26e3..e0ba06b2c 100644 --- a/test-suites/shared/automation/promoted-passing-cases.json +++ b/test-suites/shared/automation/promoted-passing-cases.json @@ -106,39 +106,39 @@ }, { "case_id": "tc-gos-contract-001", - "source_run": "request-contract-matrix-2026-09-19", + "source_run": "cov841v-all-5860e60a4", "source_status": "PASS", "entrypoint": "shared/automation/api-contract-matrix-case.py", "environment": "INTEGRATION", - "notes": "The full adversarial request-contract matrix over every Tappd method the inventory declares, under the policy in api-contract-matrix-policy.json.", - "promoted_at": "2026-09-19T00:00:00Z" + "notes": "The full adversarial request-contract matrix over every Tappd method the inventory declares, under the policy in api-contract-matrix-policy.json. Executed on the tdxlab physical host against a lease-owned fixture; 18 of 18 scheduled cases passed in run cov841v-all-5860e60a4.", + "promoted_at": "2026-09-20T05:20:00Z" }, { "case_id": "tc-gos-contract-002", - "source_run": "request-contract-matrix-2026-09-19", + "source_run": "cov841v-all-5860e60a4", "source_status": "PASS", "entrypoint": "shared/automation/api-contract-matrix-case.py", "environment": "INTEGRATION", - "notes": "The full adversarial request-contract matrix over every DstackGuest method the inventory declares, under the policy in api-contract-matrix-policy.json.", - "promoted_at": "2026-09-19T00:00:00Z" + "notes": "The full adversarial request-contract matrix over every DstackGuest method the inventory declares, under the policy in api-contract-matrix-policy.json. Executed on the tdxlab physical host against a lease-owned fixture; 18 of 18 scheduled cases passed in run cov841v-all-5860e60a4.", + "promoted_at": "2026-09-20T05:20:00Z" }, { "case_id": "tc-gos-contract-003", - "source_run": "request-contract-matrix-2026-09-19", + "source_run": "cov841v-all-5860e60a4", "source_status": "PASS", "entrypoint": "shared/automation/api-contract-matrix-case.py", "environment": "INTEGRATION", - "notes": "The full adversarial request-contract matrix over every Worker method the inventory declares, under the policy in api-contract-matrix-policy.json.", - "promoted_at": "2026-09-19T00:00:00Z" + "notes": "The full adversarial request-contract matrix over every Worker method the inventory declares, under the policy in api-contract-matrix-policy.json. Executed on the tdxlab physical host against a lease-owned fixture; 18 of 18 scheduled cases passed in run cov841v-all-5860e60a4.", + "promoted_at": "2026-09-20T05:20:00Z" }, { "case_id": "tc-gos-contract-004", - "source_run": "request-contract-matrix-2026-09-19", + "source_run": "cov841v-all-5860e60a4", "source_status": "PASS", "entrypoint": "shared/automation/api-contract-matrix-case.py", "environment": "INTEGRATION", - "notes": "The full adversarial request-contract matrix over every GuestApi method the inventory declares, under the policy in api-contract-matrix-policy.json.", - "promoted_at": "2026-09-19T00:00:00Z" + "notes": "The full adversarial request-contract matrix over every GuestApi method the inventory declares, under the policy in api-contract-matrix-policy.json. Executed on the tdxlab physical host against a lease-owned fixture; 18 of 18 scheduled cases passed in run cov841v-all-5860e60a4.", + "promoted_at": "2026-09-20T05:20:00Z" }, { "case_id": "tc-gos-dstackguest-001", @@ -1244,30 +1244,30 @@ }, { "case_id": "tc-gw-contract-001", - "source_run": "request-contract-matrix-2026-09-19", + "source_run": "cov841v-all-5860e60a4", "source_status": "PASS", "entrypoint": "shared/automation/api-contract-matrix-case.py", "environment": "INTEGRATION", - "notes": "The full adversarial request-contract matrix over every Gateway method the inventory declares, under the policy in api-contract-matrix-policy.json.", - "promoted_at": "2026-09-19T00:00:00Z" + "notes": "The full adversarial request-contract matrix over every Gateway method the inventory declares, under the policy in api-contract-matrix-policy.json. Executed on the tdxlab physical host against a lease-owned fixture; 18 of 18 scheduled cases passed in run cov841v-all-5860e60a4.", + "promoted_at": "2026-09-20T05:20:00Z" }, { "case_id": "tc-gw-contract-002", - "source_run": "request-contract-matrix-2026-09-19", + "source_run": "cov841v-all-5860e60a4", "source_status": "PASS", "entrypoint": "shared/automation/api-contract-matrix-case.py", "environment": "INTEGRATION", - "notes": "The full adversarial request-contract matrix over every Debug method the inventory declares, under the policy in api-contract-matrix-policy.json.", - "promoted_at": "2026-09-19T00:00:00Z" + "notes": "The full adversarial request-contract matrix over every Debug method the inventory declares, under the policy in api-contract-matrix-policy.json. Executed on the tdxlab physical host against a lease-owned fixture; 18 of 18 scheduled cases passed in run cov841v-all-5860e60a4.", + "promoted_at": "2026-09-20T05:20:00Z" }, { "case_id": "tc-gw-contract-003", - "source_run": "request-contract-matrix-2026-09-19", + "source_run": "cov841v-all-5860e60a4", "source_status": "PASS", "entrypoint": "shared/automation/api-contract-matrix-case.py", "environment": "INTEGRATION", - "notes": "The full adversarial request-contract matrix over every Admin method the inventory declares, under the policy in api-contract-matrix-policy.json.", - "promoted_at": "2026-09-19T00:00:00Z" + "notes": "The full adversarial request-contract matrix over every Admin method the inventory declares, under the policy in api-contract-matrix-policy.json. Executed on the tdxlab physical host against a lease-owned fixture; 18 of 18 scheduled cases passed in run cov841v-all-5860e60a4.", + "promoted_at": "2026-09-20T05:20:00Z" }, { "case_id": "tc-gw-debug-001", @@ -1905,21 +1905,21 @@ }, { "case_id": "tc-kms-contract-001", - "source_run": "request-contract-matrix-2026-09-19", + "source_run": "cov841v-all-5860e60a4", "source_status": "PASS", "entrypoint": "shared/automation/api-contract-matrix-case.py", "environment": "INTEGRATION", - "notes": "The full adversarial request-contract matrix over every KMS method the inventory declares, under the policy in api-contract-matrix-policy.json.", - "promoted_at": "2026-09-19T00:00:00Z" + "notes": "The full adversarial request-contract matrix over every KMS method the inventory declares, under the policy in api-contract-matrix-policy.json. Executed on the tdxlab physical host against a lease-owned fixture; 18 of 18 scheduled cases passed in run cov841v-all-5860e60a4.", + "promoted_at": "2026-09-20T05:20:00Z" }, { "case_id": "tc-kms-contract-002", - "source_run": "request-contract-matrix-2026-09-19", + "source_run": "cov841v-all-5860e60a4", "source_status": "PASS", "entrypoint": "shared/automation/api-contract-matrix-case.py", "environment": "INTEGRATION", - "notes": "The full adversarial request-contract matrix over every Onboard method the inventory declares, under the policy in api-contract-matrix-policy.json.", - "promoted_at": "2026-09-19T00:00:00Z" + "notes": "The full adversarial request-contract matrix over every Onboard method the inventory declares, under the policy in api-contract-matrix-policy.json. Executed on the tdxlab physical host against a lease-owned fixture; 18 of 18 scheduled cases passed in run cov841v-all-5860e60a4.", + "promoted_at": "2026-09-20T05:20:00Z" }, { "case_id": "tc-kms-ct-001", From 76a8e547001ec9f4e27f4bdc9460348e86ed0fa3 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 24 Sep 2026 01:59:30 -0700 Subject: [PATCH 6/9] style(test): format the request-contract matrix harness --- .../automation/api-contract-matrix-case.py | 12 ++++++----- .../shared/automation/api_contract_matrix.py | 21 +++++++++++++++---- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/test-suites/shared/automation/api-contract-matrix-case.py b/test-suites/shared/automation/api-contract-matrix-case.py index af3d1becd..1e294e0af 100755 --- a/test-suites/shared/automation/api-contract-matrix-case.py +++ b/test-suites/shared/automation/api-contract-matrix-case.py @@ -222,9 +222,7 @@ def wrong_type_for(field_def: dict[str, Any]) -> Any: return {"not": "a scalar"} -def sweep_rejection_only( - target: Target, entry: dict[str, Any] -) -> list[Call]: +def sweep_rejection_only(target: Target, entry: dict[str, Any]) -> list[Call]: """Vectors a service must refuse before it can dispatch the method.""" service, method = entry["service"], entry["method"] key = f"{service}.{method}" @@ -287,7 +285,9 @@ def sweep_framing(target: Target, method: str) -> list[Call]: return calls -def sweep_full(target: Target, entry: dict[str, Any], markers: dict[str, str]) -> list[Call]: +def sweep_full( + target: Target, entry: dict[str, Any], markers: dict[str, str] +) -> list[Call]: """The complete vector set for one method.""" service, method = entry["service"], entry["method"] key = f"{service}.{method}" @@ -351,7 +351,9 @@ def sweep_full(target: Target, entry: dict[str, Any], markers: dict[str, str]) - ) ) highest = max((int(f["number"]) for f in fields), default=0) - body = encode_request(fields, valid) + varint(((highest + 1000) << 3) | 0) + varint(7) + body = ( + encode_request(fields, valid) + varint(((highest + 1000) << 3) | 0) + varint(7) + ) calls.append( invoke( target, diff --git a/test-suites/shared/automation/api_contract_matrix.py b/test-suites/shared/automation/api_contract_matrix.py index 6a9ded24e..f0279dfdf 100644 --- a/test-suites/shared/automation/api_contract_matrix.py +++ b/test-suites/shared/automation/api_contract_matrix.py @@ -392,9 +392,18 @@ def check_invariants(calls: list[Call], echoed_markers: dict[str, str]) -> list[ # A label records the representation the request was sent in. "raw" # framing rows carry their own content type in the label. sent_json = call.representation == "json" or call.label.endswith( - ("json-empty-body", "json-not-object", "json-scalar", "json-null", - "json-truncated", "json-trailing-garbage", "json-deep-nesting", - "json-duplicate-keys", "json-huge-4mib", "ct-mismatch-pb-as-json") + ( + "json-empty-body", + "json-not-object", + "json-scalar", + "json-null", + "json-truncated", + "json-trailing-garbage", + "json-deep-nesting", + "json-duplicate-keys", + "json-huge-4mib", + "ct-mismatch-pb-as-json", + ) ) if call.http is not None and call.http >= 400: if sent_json: @@ -446,7 +455,11 @@ def check_invariants(calls: list[Call], echoed_markers: dict[str, str]) -> list[ marker = echoed_markers.get(call.label) if marker and marker in call.text: violations.append( - {**base, "invariant": "L4", "detail": "response echoed the input marker"} + { + **base, + "invariant": "L4", + "detail": "response echoed the input marker", + } ) if call.seconds > CALL_TIMEOUT: violations.append( From 9f13d9f91060868b0753be92bd8f9f212c9bdace Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 24 Sep 2026 02:02:05 -0700 Subject: [PATCH 7/9] refactor(test): simplify the request-contract matrix harness --- .../automation/api-contract-matrix-case.py | 203 +++++++----------- .../api-contract-matrix-policy.json | 35 +-- .../shared/automation/api_contract_matrix.py | 66 ++---- 3 files changed, 93 insertions(+), 211 deletions(-) diff --git a/test-suites/shared/automation/api-contract-matrix-case.py b/test-suites/shared/automation/api-contract-matrix-case.py index 1e294e0af..195255c04 100755 --- a/test-suites/shared/automation/api-contract-matrix-case.py +++ b/test-suites/shared/automation/api-contract-matrix-case.py @@ -1,23 +1,7 @@ #!/usr/bin/env python3 # SPDX-FileCopyrightText: © 2026 Phala Network # SPDX-License-Identifier: Apache-2.0 -"""Run the adversarial request-contract matrix against one pRPC service. - -Every RPC case in this suite states that the method is exercised with its -request fields absent, default, valid, boundary-invalid and combined with an -unknown field, over both representations. The per-method harnesses send one -valid request and one wrong-typed value for the first field only. This harness -implements the stated matrix once, for every method the inventory declares, and -asserts the invariants that hold whatever a method means: - - L1 a malformed request never produces a 5xx, a dropped connection, or a - transport-level failure; - L2 a rejection of a JSON request carries a structured JSON ``error``, and a - rejection of a protobuf request carries a decodable ``ProtoError``; - L3 the listener still answers a valid request after the whole matrix; - L4 a rejection does not echo an unbounded amount of attacker-supplied text; - L5 no request exceeds the per-call deadline; - L6 a rejection's ``Content-Type`` matches the request's representation. +"""Run the request-contract matrix in ``api_contract_matrix`` against one service. ``api-contract-matrix-policy.json`` says how far each method may be driven: ``full`` runs every vector, ``rejection-only`` runs only the vectors a service @@ -58,13 +42,10 @@ HERE = pathlib.Path(__file__).resolve().parent UNKNOWN_JSON_FIELD = "dstack_contract_matrix_unknown_field" ECHO_MARKER = "DSTACK-ECHO-MARKER-7f3a1c" -# A run-scoped name no fixture ever provisions, so a lookup for it is a miss -# and a create for it cannot collide with anything the case did not make. +# A run-scoped name no fixture provisions, so lookups miss and creates cannot collide. SCOPED = "dstack-contract-matrix" -# A valid request body per method. A method absent from this table is still -# swept for absent-input and malformed-framing behaviour; only the per-field -# vectors need a baseline to mutate. +# A valid request body per method; the per-field vectors mutate this baseline. VALID_PAYLOADS: dict[str, dict[str, Any]] = { "Tappd.DeriveKey": { "path": f"{SCOPED}/a", @@ -118,11 +99,6 @@ "GuestApi.NetworkInfo": {}, "GuestApi.ListContainers": {}, "GuestApi.GpuInfo": {}, - "ProxiedGuestApi.Info": {"id": f"{SCOPED}-absent-instance"}, - "ProxiedGuestApi.SysInfo": {"id": f"{SCOPED}-absent-instance"}, - "ProxiedGuestApi.NetworkInfo": {"id": f"{SCOPED}-absent-instance"}, - "ProxiedGuestApi.ListContainers": {"id": f"{SCOPED}-absent-instance"}, - "ProxiedGuestApi.GpuInfo": {"id": f"{SCOPED}-absent-instance"}, "Gateway.AcmeInfo": {}, "Gateway.Info": {}, "Gateway.GetPeers": {}, @@ -207,12 +183,25 @@ def atomic_json(path: pathlib.Path, value: Any) -> None: temporary.replace(path) -def service_methods( +def service_policy( inventory: dict[str, Any], component: str, service: str -) -> list[dict[str, Any]]: - """Return the inventory entries for one service, in declaration order.""" - entries = inventory["components"][component]["rpc_methods"] - return [entry for entry in entries if entry["service"] == service] +) -> tuple[list[dict[str, Any]], dict[str, str]]: + """Return one service's inventory entries and the policy for each method.""" + entries = [ + entry + for entry in inventory["components"][component]["rpc_methods"] + if entry["service"] == service + ] + if not entries: + raise RuntimeError(f"no inventory entry for {component}/{service}") + policy = load_policy() + applied: dict[str, str] = {} + for entry in entries: + key = f"{service}.{entry['method']}" + applied[key] = policy.get(key, "") + if applied[key] not in ("full", "rejection-only", "skip"): + raise RuntimeError(f"no valid contract-matrix policy for {key}") + return entries, applied def wrong_type_for(field_def: dict[str, Any]) -> Any: @@ -222,6 +211,27 @@ def wrong_type_for(field_def: dict[str, Any]) -> Any: return {"not": "a scalar"} +def wrong_wiretype(target: Target, entry: dict[str, Any]) -> list[Call]: + """The first declared field number sent with the wrong wire type.""" + fields = entry.get("request_fields", []) + if not fields: + return [] + first = fields[0] + wire = 0 if first["type"] in ("string", "bytes") else 2 + body = varint((int(first["number"]) << 3) | wire) + body += b"\x01" if wire == 0 else b"\x01A" + return [ + invoke( + target, + entry["method"], + "application/octet-stream", + body, + f"{entry['service']}.{entry['method']}|wrong-wiretype|protobuf", + "protobuf", + ) + ] + + def sweep_rejection_only(target: Target, entry: dict[str, Any]) -> list[Call]: """Vectors a service must refuse before it can dispatch the method.""" service, method = entry["service"], entry["method"] @@ -244,23 +254,7 @@ def sweep_rejection_only(target: Target, entry: dict[str, Any]) -> list[Call]: ) ) - # A wrong protobuf wire type for a declared field number fails decoding. - if fields: - first = fields[0] - number = int(first["number"]) - wire = 0 if first["type"] in ("string", "bytes") else 2 - body = varint((number << 3) | wire) + (b"\x01" if wire == 0 else b"\x01A") - calls.append( - invoke( - target, - method, - "application/octet-stream", - body, - f"{key}|wrong-wiretype|protobuf", - "protobuf", - ) - ) - + calls.extend(wrong_wiretype(target, entry)) return calls @@ -268,8 +262,11 @@ def sweep_framing(target: Target, method: str) -> list[Call]: """Representation-level malformed bodies, once per service.""" calls = [] for label, content_type, body in malformed_bodies(): + representation = "json" if content_type == "application/json" else "protobuf" calls.append( - invoke(target, method, content_type, body, f"framing|{label}", "raw") + invoke( + target, method, content_type, body, f"framing|{label}", representation + ) ) calls.append( invoke( @@ -411,22 +408,7 @@ def sweep_full( ) # 6. Wrong wire type for a declared field number. - if fields: - first = fields[0] - number = int(first["number"]) - wire = 0 if first["type"] in ("string", "bytes") else 2 - body = varint((number << 3) | wire) + (b"\x01" if wire == 0 else b"\x01A") - calls.append( - invoke( - target, - method, - "application/octet-stream", - body, - f"{key}|wrong-wiretype|protobuf", - "protobuf", - ) - ) - + calls.extend(wrong_wiretype(target, entry)) return calls @@ -434,27 +416,18 @@ def run( target: Target, inventory: dict[str, Any], component: str, service: str ) -> dict[str, Any]: """Run the matrix for one service and return the evidence document.""" - entries = service_methods(inventory, component, service) - if not entries: - raise RuntimeError(f"no inventory entry for {component}/{service}") - policy = load_policy() + entries, applied = service_policy(inventory, component, service) markers: dict[str, str] = {} calls: list[Call] = [] - applied: dict[str, str] = {} for entry in entries: key = f"{service}.{entry['method']}" - decision = policy.get(key) - if decision is None: - raise RuntimeError(f"no contract-matrix policy for {key}") - applied[key] = decision + decision = applied[key] if decision == "skip": continue if decision == "rejection-only": calls.extend(sweep_rejection_only(target, entry)) - elif decision == "full": - calls.extend(sweep_full(target, entry, markers)) else: - raise RuntimeError(f"unknown contract-matrix policy {decision!r} for {key}") + calls.extend(sweep_full(target, entry, markers)) # A method driven at `full` carries the representation-level vectors and # is the liveness probe. @@ -657,66 +630,48 @@ def main() -> int: status = "PASS" failure: str | None = None report: dict[str, Any] = {"case_id": case_id} + + def passed(number: int, observed: str) -> None: + step_id = f"{case_id}-step-{number:02d}" + steps.append({"id": step_id, "status": "PASS", "observed": observed}) + print(f"EVIDENCE {step_id} - {observed}", flush=True) + print(f"STEP {step_id} END - PASS", flush=True) + + def require(violations: list[dict[str, Any]]) -> None: + if violations: + raise AssertionError( + f"{len(violations)} contract invariant violations: " + + json.dumps(violations[:5], sort_keys=True) + ) + try: print(f"STEP {case_id}-step-01 START", flush=True) target = resolve_target(manifest, selector, service) - steps.append( - { - "id": f"{case_id}-step-01", - "status": "PASS", - "observed": "The lease-owned listener and the indexed service contract were available.", - } + _, applied = service_policy(inventory, component, service) + passed( + 1, f"The listener resolved and all {len(applied)} methods have a policy." ) - print( - f"EVIDENCE {case_id}-step-01 - Proves the isolated listener was ready before the matrix.", - flush=True, - ) - print(f"STEP {case_id}-step-01 END - PASS", flush=True) print(f"STEP {case_id}-step-02 START", flush=True) report.update(run(target, inventory, component, service)) - if report["violations"]: - raise AssertionError( - f"{len(report['violations'])} contract invariant violations: " - + json.dumps(report["violations"][:5], sort_keys=True) - ) - steps.append( - { - "id": f"{case_id}-step-02", - "status": "PASS", - "observed": f"{report['calls']} adversarial requests across " - f"{len(report['policy'])} indexed methods were answered without a " - "server error, an unstructured rejection, a representation " - "mismatch, an input echo or a deadline breach.", - } - ) - print( - f"EVIDENCE {case_id}-step-02 - Proves the request-contract invariants hold for every indexed method.", - flush=True, - ) + violations = report["violations"] + require([v for v in violations if v["invariant"] != "L3"]) print(json.dumps(report["status_histogram"], sort_keys=True), flush=True) - print(f"STEP {case_id}-step-02 END - PASS", flush=True) + passed( + 2, + f"{report['calls']} adversarial requests across {len(applied)} methods " + "held invariants L1, L2, L4, L5 and L6.", + ) print(f"STEP {case_id}-step-03 START", flush=True) - steps.append( - { - "id": f"{case_id}-step-03", - "status": "PASS", - "observed": "The listener answered a valid request after the whole matrix.", - } - ) - print( - f"EVIDENCE {case_id}-step-03 - Proves post-matrix availability.", flush=True - ) - print(f"STEP {case_id}-step-03 END - PASS", flush=True) + require([v for v in violations if v["invariant"] == "L3"]) + passed(3, "The listener answered a valid request after the whole matrix.") except Exception as error: # noqa: BLE001 - the harness reports, never crashes status = "FAIL" failure = f"{type(error).__name__}: {error}" - completed = {step["id"] for step in steps} - for number in range(1, 4): + for number in range(len(steps) + 1, 4): step_id = f"{case_id}-step-{number:02d}" - if step_id not in completed: - steps.append({"id": step_id, "status": "FAIL", "observed": failure}) + steps.append({"id": step_id, "status": "FAIL", "observed": failure}) print(failure, file=sys.stderr, flush=True) report["status"] = status diff --git a/test-suites/shared/automation/api-contract-matrix-policy.json b/test-suites/shared/automation/api-contract-matrix-policy.json index 103138ec3..293c230d7 100644 --- a/test-suites/shared/automation/api-contract-matrix-policy.json +++ b/test-suites/shared/automation/api-contract-matrix-policy.json @@ -1,8 +1,7 @@ { - "description": "Per-method policy for the adversarial request-contract matrix. `full` runs every vector against a lease-owned fixture; `rejection-only` runs the vectors a service must refuse before it dispatches, so nothing can mutate; `skip` is for methods whose valid invocation ends the process or consumes an external resource. Every method in catalog/api-inventory.json must appear here, and the registry check enforces that.", + "description": "Per-method policy for the request-contract matrix. `full` runs every vector against a lease-owned fixture; `rejection-only` runs only the vectors a service must refuse before it dispatches; `skip` is for methods whose valid invocation ends the process or consumes an external resource. Every method of a service the matrix covers must appear here, or its case fails.", "policies": { "Admin.AddZtDomain": "full", - "Admin.ClearImageCache": "rejection-only", "Admin.ClearInstancePortPolicy": "full", "Admin.CreateDnsCredential": "full", "Admin.DeleteDnsCredential": "full", @@ -60,9 +59,6 @@ "GuestApi.NetworkInfo": "full", "GuestApi.Shutdown": "rejection-only", "GuestApi.SysInfo": "full", - "HostApi.GetSealingKey": "rejection-only", - "HostApi.Info": "full", - "HostApi.Notify": "rejection-only", "KMS.GetAppEnvEncryptPubKey": "full", "KMS.GetAppKey": "rejection-only", "KMS.GetKmsKey": "rejection-only", @@ -73,41 +69,12 @@ "Onboard.Finish": "skip", "Onboard.GetAttestationInfo": "full", "Onboard.Onboard": "skip", - "ProxiedGuestApi.GpuInfo": "full", - "ProxiedGuestApi.Info": "full", - "ProxiedGuestApi.ListContainers": "full", - "ProxiedGuestApi.NetworkInfo": "full", - "ProxiedGuestApi.Shutdown": "rejection-only", - "ProxiedGuestApi.SysInfo": "full", "Tappd.DeriveK256Key": "full", "Tappd.DeriveKey": "full", "Tappd.Info": "full", "Tappd.RawQuote": "full", "Tappd.TdxQuote": "full", "Tappd.Version": "full", - "Vmm.CreateVm": "rejection-only", - "Vmm.DeleteImage": "rejection-only", - "Vmm.GetAppEnvEncryptPubKey": "full", - "Vmm.GetComposeHash": "full", - "Vmm.GetInfo": "full", - "Vmm.GetMeta": "full", - "Vmm.ListGpus": "full", - "Vmm.ListImages": "full", - "Vmm.ListRegistryImages": "full", - "Vmm.PullRegistryImage": "rejection-only", - "Vmm.ReloadVms": "rejection-only", - "Vmm.RemoveVm": "rejection-only", - "Vmm.ResizeVm": "rejection-only", - "Vmm.ShutdownVm": "rejection-only", - "Vmm.StartVm": "rejection-only", - "Vmm.Status": "full", - "Vmm.StopVm": "rejection-only", - "Vmm.SvList": "full", - "Vmm.SvRemove": "rejection-only", - "Vmm.SvStop": "rejection-only", - "Vmm.UpdateVm": "rejection-only", - "Vmm.UpgradeApp": "rejection-only", - "Vmm.Version": "full", "Worker.GetAttestationForAppKey": "full", "Worker.Info": "full", "Worker.Version": "full" diff --git a/test-suites/shared/automation/api_contract_matrix.py b/test-suites/shared/automation/api_contract_matrix.py index f0279dfdf..fb1bc680a 100644 --- a/test-suites/shared/automation/api_contract_matrix.py +++ b/test-suites/shared/automation/api_contract_matrix.py @@ -3,28 +3,16 @@ # SPDX-License-Identifier: Apache-2.0 """Adversarial request-contract matrix for dstack pRPC services. -Every RPC case in this suite promises that the method is exercised with its -request fields absent, default, valid, boundary-invalid and combined with an -unknown field, over both the JSON and the protobuf representation. The shipped -harnesses send one valid request and one wrong-typed value for the *first* -field only. This module implements the promised matrix, and asserts the -invariants that hold for every method whatever its semantics: - - L1 a malformed request never produces a 5xx, a dropped connection, or a - transport-level failure; - L2 a rejection of a JSON request carries a structured JSON body with an - ``error`` string, and a rejection of a protobuf request carries a - decodable ``ProtoError``; +Invariants asserted for every method, whatever it means: + + L1 a malformed request never produces a 5xx or a transport failure; + L2 a rejected JSON request gets a JSON ``error`` string, a rejected protobuf + request gets a decodable ``ProtoError``; L3 the listener still answers a valid request after the whole matrix; - L4 a rejection neither echoes an unbounded amount of attacker-supplied - text nor grows with the size of the field it refused; + L4 a rejection neither echoes an unbounded amount of input nor grows with + the field it refused; L5 no request exceeds the per-call deadline; - L6 the rejection's ``Content-Type`` matches the request's representation, - so a client that sent JSON can parse the error it gets back. - -It is transport-agnostic: a target is a unix socket path or a base URL plus a -route template, so the same matrix runs against the guest agent, the gateway, -the KMS and the VMM. + L6 a rejection's ``Content-Type`` matches the request's representation. """ from __future__ import annotations @@ -36,11 +24,7 @@ from typing import Any CALL_TIMEOUT = 20 -# A rejection is allowed to quote the value it refused, but not to scale with -# it. The product bounds its error text and then wraps it in a JSON envelope, -# so the ceiling is a little above that bound; what actually matters is the -# second check below, that two requests differing only in the size of one field -# do not produce rejections that differ in size. +# A rejection may quote the value it refused, but must not scale with it. MAX_ECHO = 8192 MAX_ECHO_SCALING = 256 @@ -389,34 +373,19 @@ def check_invariants(calls: list[Call], echoed_markers: dict[str, str]) -> list[ } ) continue - # A label records the representation the request was sent in. "raw" - # framing rows carry their own content type in the label. - sent_json = call.representation == "json" or call.label.endswith( - ( - "json-empty-body", - "json-not-object", - "json-scalar", - "json-null", - "json-truncated", - "json-trailing-garbage", - "json-deep-nesting", - "json-duplicate-keys", - "json-huge-4mib", - "ct-mismatch-pb-as-json", - ) - ) + sent_json = call.representation == "json" if call.http is not None and call.http >= 400: if sent_json: ok, detail = json_error(call) - invariant, expected = "L2", "application/json" + expected = "application/json" else: ok, detail = proto_error(call) - invariant, expected = "L2", "application/octet-stream" + expected = "application/octet-stream" if not ok: violations.append( { **base, - "invariant": invariant, + "invariant": "L2", "detail": f"unstructured rejection ({detail}): " f"{call.text[:256]!r}", } @@ -443,15 +412,6 @@ def check_invariants(calls: list[Call], echoed_markers: dict[str, str]) -> list[ f"Content-Type {call.response_content_type}", } ) - if not ok and sent_json: - violations.append( - { - **base, - "invariant": "L6", - "detail": "a JSON request was rejected with a non-JSON body " - f"and Content-Type {call.response_content_type or 'unset'}", - } - ) marker = echoed_markers.get(call.label) if marker and marker in call.text: violations.append( From ce0920b2d5ca479e0294a2cfb407ec4535b88095 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 24 Sep 2026 02:02:05 -0700 Subject: [PATCH 8/9] docs(test): trim the request-contract case prose --- .../tc-gos-contract-001/case.md | 100 ++++++------------ .../tc-gos-contract-002/case.md | 100 ++++++------------ .../tc-gos-contract-003/case.md | 100 ++++++------------ .../tc-gos-contract-004/case.md | 100 ++++++------------ .../tc-kms-contract-001/case.md | 100 ++++++------------ .../tc-kms-contract-002/case.md | 100 ++++++------------ .../tc-gw-contract-001/case.md | 100 ++++++------------ .../tc-gw-contract-002/case.md | 100 ++++++------------ .../tc-gw-contract-003/case.md | 100 ++++++------------ 9 files changed, 288 insertions(+), 612 deletions(-) diff --git a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-001/case.md b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-001/case.md index 31058e4d9..73f0e101d 100644 --- a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-001/case.md +++ b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-001/case.md @@ -16,107 +16,71 @@ ## Prepared execution knowledge - Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. -- The method list, the request field list and every field's declared type come - from [`api-inventory.json`](../../../../catalog/api-inventory.json). Do not reconstruct - them from implementation source: the point of this case is that the indexed - contract and the running service agree. -- [`api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) - says how far each method may be driven. `full` runs every vector, and is only - used for a method whose effect is confined to the lease. `rejection-only` - runs the vectors a service must refuse before it dispatches, so nothing can - mutate. `skip` is for a method whose valid invocation ends the process or - consumes an external resource, and the policy file records why for each one. -- Every method the inventory declares for this service must carry a policy. A - method added to the inventory without one fails this case rather than being - silently skipped. +- Methods, request fields and field types come from `api-inventory.json`, not + from implementation source. +- `api-contract-matrix-policy.json` sets how far each method is driven: `full` + (effect confined to the lease), `rejection-only` (only inputs refused before + dispatch) or `skip` (a valid call ends the process or consumes an external + resource). A method without a policy fails the case. ## Objective -Verify that every method `Tappd` declares answers absent, default, valid, -boundary-invalid, wrong-typed, unknown-field and malformed-framing input, in -both the JSON and the protobuf representation, without violating the -invariants that hold for every method whatever it means. - -The per-method cases in this suite each send one valid request and one -wrong-typed value for the first request field. That leaves the input classes -where a service actually breaks — an out-of-range integer, an oversized -string, a wrong protobuf wire type, a truncated body — untested for every -method. This case closes that gap once, for the whole service, and is driven -by the same inventory the per-method cases cite as mandatory test data. +Verify that every `Tappd` method handles absent, valid, boundary-invalid, +wrong-typed, unknown-field and malformed-framing input, in JSON and protobuf, +without breaking the invariants below. ## Preconditions -1. The shared plan prerequisites are healthy and the guest agent's `tappd.sock` legacy listener is reachable. -2. The fixture is lease-owned. The matrix drives mutating methods with - run-scoped identifiers, so it must never be pointed at a shared or - production deployment. +1. The guest agent's `tappd.sock` legacy listener is reachable on a lease-owned fixture. The matrix drives + mutating methods with run-scoped identifiers, so it must never target a + shared deployment. ## Test Data -The `guest-os` entries in [`api-inventory.json`](../../../../catalog/api-inventory.json) -whose `service` is `Tappd`, and the vector set in -[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py): - -- per field type — empty, wrong type, null, a NUL byte, a newline, a path - traversal, an absolute path, control characters, 64 KiB, 1 MiB, odd-length - hex, non-hex, 31/63/64/65-byte lengths, zero, negative, `u32::MAX`, - `u64::MAX`, past `u64::MAX`, and a 4096-element repeated field; -- per representation — an empty body, a non-object, a scalar, `null`, a - truncated object, trailing garbage, 2000-deep nesting, duplicate keys, a - 4 MiB body, a truncated varint, a runaway varint, a length that overflows, - a length past the end of the body, an unknown wire type, a group wire type, - field number zero, a field number past the declared range, 8 KiB of random - bytes, a JSON body sent as protobuf, a protobuf body sent as JSON, an - absent content type, and an unrecognised content type; -- per method — the declared request encoded with an unknown field appended, - and a declared field number sent with the wrong wire type. +The `guest-os` entries in `api-inventory.json` whose `service` is `Tappd`, +and the per-field, per-representation and per-method vectors in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py) and +[`api-contract-matrix-case.py`](../../../../shared/automation/api-contract-matrix-case.py). ## Steps ### Step 1: Resolve the listener and the indexed contract -Resolve the guest agent's `tappd.sock` legacy listener from the lease-owned fixture, load the inventory -entries for `Tappd`, and confirm every one of them carries a policy. +Resolve the listener from the fixture and load the `Tappd` inventory entries. **Expected results:** -- The listener is reachable, every declared method has a policy, and no method - in the policy table is absent from the inventory. +- The listener is reachable and every declared method has a policy. ### Step 2: Drive the matrix -Send every vector the policy allows for every method, recording the status, -the response content type, the body size and the elapsed time of each call. +Send every vector the policy allows, recording status, content type, body size +and elapsed time of each call. **Expected results:** -- **L1** No request produces a 5xx, a dropped connection, or a transport-level - failure. A malformed request is a client error, not a server error, and - never a crash: the workspace builds release binaries with `panic = "abort"`, - so a reachable panic is a process abort rather than a failed request. -- **L2** A rejected JSON request carries a structured JSON body with an - `error` string; a rejected protobuf request carries a decodable `ProtoError`. -- **L4** No rejection echoes an unbounded amount of the caller's input, and no - response contains the unknown-field marker the request carried. +- **L1** No request produces a 5xx, a dropped connection or a transport + failure. Release builds use `panic = "abort"`, so a reachable panic aborts + the process. +- **L2** A rejected JSON request gets a JSON `error` string; a rejected + protobuf request gets a decodable `ProtoError`. +- **L4** No rejection echoes the unknown-field marker or grows with the size + of the field it refused. - **L5** No request exceeds the per-call deadline. -- **L6** A rejection's `Content-Type` matches the representation the request - arrived in, so a client that sent JSON can parse the error it gets back. +- **L6** A rejection's `Content-Type` matches the request's representation. ### Step 3: Verify the listener survived -Send one valid request after the whole matrix. +Send one valid request after the matrix. **Expected results:** -- **L3** The listener answers it. A service that stopped answering has either - aborted or wedged on one of the preceding vectors. +- **L3** The listener answers it. ## Postconditions -Remove run-scoped objects the `full` policy created and restore changed -configuration. Preserve the per-call observation table in the result -artifacts: it is the evidence that each vector was actually sent, and its -status histogram is what a reviewer compares across runs. +Remove run-scoped objects the `full` policy created. Keep the per-call +observation table in the result artifacts. diff --git a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-002/case.md b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-002/case.md index c76286297..8e452405f 100644 --- a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-002/case.md +++ b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-002/case.md @@ -16,107 +16,71 @@ ## Prepared execution knowledge - Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. -- The method list, the request field list and every field's declared type come - from [`api-inventory.json`](../../../../catalog/api-inventory.json). Do not reconstruct - them from implementation source: the point of this case is that the indexed - contract and the running service agree. -- [`api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) - says how far each method may be driven. `full` runs every vector, and is only - used for a method whose effect is confined to the lease. `rejection-only` - runs the vectors a service must refuse before it dispatches, so nothing can - mutate. `skip` is for a method whose valid invocation ends the process or - consumes an external resource, and the policy file records why for each one. -- Every method the inventory declares for this service must carry a policy. A - method added to the inventory without one fails this case rather than being - silently skipped. +- Methods, request fields and field types come from `api-inventory.json`, not + from implementation source. +- `api-contract-matrix-policy.json` sets how far each method is driven: `full` + (effect confined to the lease), `rejection-only` (only inputs refused before + dispatch) or `skip` (a valid call ends the process or consumes an external + resource). A method without a policy fails the case. ## Objective -Verify that every method `DstackGuest` declares answers absent, default, valid, -boundary-invalid, wrong-typed, unknown-field and malformed-framing input, in -both the JSON and the protobuf representation, without violating the -invariants that hold for every method whatever it means. - -The per-method cases in this suite each send one valid request and one -wrong-typed value for the first request field. That leaves the input classes -where a service actually breaks — an out-of-range integer, an oversized -string, a wrong protobuf wire type, a truncated body — untested for every -method. This case closes that gap once, for the whole service, and is driven -by the same inventory the per-method cases cite as mandatory test data. +Verify that every `DstackGuest` method handles absent, valid, boundary-invalid, +wrong-typed, unknown-field and malformed-framing input, in JSON and protobuf, +without breaking the invariants below. ## Preconditions -1. The shared plan prerequisites are healthy and the guest agent's `dstack.sock` internal listener, which is bind-mounted into application containers is reachable. -2. The fixture is lease-owned. The matrix drives mutating methods with - run-scoped identifiers, so it must never be pointed at a shared or - production deployment. +1. The guest agent's `dstack.sock` internal listener (bind-mounted into application containers) is reachable on a lease-owned fixture. The matrix drives + mutating methods with run-scoped identifiers, so it must never target a + shared deployment. ## Test Data -The `guest-os` entries in [`api-inventory.json`](../../../../catalog/api-inventory.json) -whose `service` is `DstackGuest`, and the vector set in -[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py): - -- per field type — empty, wrong type, null, a NUL byte, a newline, a path - traversal, an absolute path, control characters, 64 KiB, 1 MiB, odd-length - hex, non-hex, 31/63/64/65-byte lengths, zero, negative, `u32::MAX`, - `u64::MAX`, past `u64::MAX`, and a 4096-element repeated field; -- per representation — an empty body, a non-object, a scalar, `null`, a - truncated object, trailing garbage, 2000-deep nesting, duplicate keys, a - 4 MiB body, a truncated varint, a runaway varint, a length that overflows, - a length past the end of the body, an unknown wire type, a group wire type, - field number zero, a field number past the declared range, 8 KiB of random - bytes, a JSON body sent as protobuf, a protobuf body sent as JSON, an - absent content type, and an unrecognised content type; -- per method — the declared request encoded with an unknown field appended, - and a declared field number sent with the wrong wire type. +The `guest-os` entries in `api-inventory.json` whose `service` is `DstackGuest`, +and the per-field, per-representation and per-method vectors in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py) and +[`api-contract-matrix-case.py`](../../../../shared/automation/api-contract-matrix-case.py). ## Steps ### Step 1: Resolve the listener and the indexed contract -Resolve the guest agent's `dstack.sock` internal listener, which is bind-mounted into application containers from the lease-owned fixture, load the inventory -entries for `DstackGuest`, and confirm every one of them carries a policy. +Resolve the listener from the fixture and load the `DstackGuest` inventory entries. **Expected results:** -- The listener is reachable, every declared method has a policy, and no method - in the policy table is absent from the inventory. +- The listener is reachable and every declared method has a policy. ### Step 2: Drive the matrix -Send every vector the policy allows for every method, recording the status, -the response content type, the body size and the elapsed time of each call. +Send every vector the policy allows, recording status, content type, body size +and elapsed time of each call. **Expected results:** -- **L1** No request produces a 5xx, a dropped connection, or a transport-level - failure. A malformed request is a client error, not a server error, and - never a crash: the workspace builds release binaries with `panic = "abort"`, - so a reachable panic is a process abort rather than a failed request. -- **L2** A rejected JSON request carries a structured JSON body with an - `error` string; a rejected protobuf request carries a decodable `ProtoError`. -- **L4** No rejection echoes an unbounded amount of the caller's input, and no - response contains the unknown-field marker the request carried. +- **L1** No request produces a 5xx, a dropped connection or a transport + failure. Release builds use `panic = "abort"`, so a reachable panic aborts + the process. +- **L2** A rejected JSON request gets a JSON `error` string; a rejected + protobuf request gets a decodable `ProtoError`. +- **L4** No rejection echoes the unknown-field marker or grows with the size + of the field it refused. - **L5** No request exceeds the per-call deadline. -- **L6** A rejection's `Content-Type` matches the representation the request - arrived in, so a client that sent JSON can parse the error it gets back. +- **L6** A rejection's `Content-Type` matches the request's representation. ### Step 3: Verify the listener survived -Send one valid request after the whole matrix. +Send one valid request after the matrix. **Expected results:** -- **L3** The listener answers it. A service that stopped answering has either - aborted or wedged on one of the preceding vectors. +- **L3** The listener answers it. ## Postconditions -Remove run-scoped objects the `full` policy created and restore changed -configuration. Preserve the per-call observation table in the result -artifacts: it is the evidence that each vector was actually sent, and its -status histogram is what a reviewer compares across runs. +Remove run-scoped objects the `full` policy created. Keep the per-call +observation table in the result artifacts. diff --git a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-003/case.md b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-003/case.md index d9ad5f7a0..a7009e190 100644 --- a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-003/case.md +++ b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-003/case.md @@ -16,107 +16,71 @@ ## Prepared execution knowledge - Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. -- The method list, the request field list and every field's declared type come - from [`api-inventory.json`](../../../../catalog/api-inventory.json). Do not reconstruct - them from implementation source: the point of this case is that the indexed - contract and the running service agree. -- [`api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) - says how far each method may be driven. `full` runs every vector, and is only - used for a method whose effect is confined to the lease. `rejection-only` - runs the vectors a service must refuse before it dispatches, so nothing can - mutate. `skip` is for a method whose valid invocation ends the process or - consumes an external resource, and the policy file records why for each one. -- Every method the inventory declares for this service must carry a policy. A - method added to the inventory without one fails this case rather than being - silently skipped. +- Methods, request fields and field types come from `api-inventory.json`, not + from implementation source. +- `api-contract-matrix-policy.json` sets how far each method is driven: `full` + (effect confined to the lease), `rejection-only` (only inputs refused before + dispatch) or `skip` (a valid call ends the process or consumes an external + resource). A method without a policy fails the case. ## Objective -Verify that every method `Worker` declares answers absent, default, valid, -boundary-invalid, wrong-typed, unknown-field and malformed-framing input, in -both the JSON and the protobuf representation, without violating the -invariants that hold for every method whatever it means. - -The per-method cases in this suite each send one valid request and one -wrong-typed value for the first request field. That leaves the input classes -where a service actually breaks — an out-of-range integer, an oversized -string, a wrong protobuf wire type, a truncated body — untested for every -method. This case closes that gap once, for the whole service, and is driven -by the same inventory the per-method cases cite as mandatory test data. +Verify that every `Worker` method handles absent, valid, boundary-invalid, +wrong-typed, unknown-field and malformed-framing input, in JSON and protobuf, +without breaking the invariants below. ## Preconditions -1. The shared plan prerequisites are healthy and the guest agent's external listener, which is reachable from outside the CVM is reachable. -2. The fixture is lease-owned. The matrix drives mutating methods with - run-scoped identifiers, so it must never be pointed at a shared or - production deployment. +1. The guest agent's external listener (reachable from outside the CVM) is reachable on a lease-owned fixture. The matrix drives + mutating methods with run-scoped identifiers, so it must never target a + shared deployment. ## Test Data -The `guest-os` entries in [`api-inventory.json`](../../../../catalog/api-inventory.json) -whose `service` is `Worker`, and the vector set in -[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py): - -- per field type — empty, wrong type, null, a NUL byte, a newline, a path - traversal, an absolute path, control characters, 64 KiB, 1 MiB, odd-length - hex, non-hex, 31/63/64/65-byte lengths, zero, negative, `u32::MAX`, - `u64::MAX`, past `u64::MAX`, and a 4096-element repeated field; -- per representation — an empty body, a non-object, a scalar, `null`, a - truncated object, trailing garbage, 2000-deep nesting, duplicate keys, a - 4 MiB body, a truncated varint, a runaway varint, a length that overflows, - a length past the end of the body, an unknown wire type, a group wire type, - field number zero, a field number past the declared range, 8 KiB of random - bytes, a JSON body sent as protobuf, a protobuf body sent as JSON, an - absent content type, and an unrecognised content type; -- per method — the declared request encoded with an unknown field appended, - and a declared field number sent with the wrong wire type. +The `guest-os` entries in `api-inventory.json` whose `service` is `Worker`, +and the per-field, per-representation and per-method vectors in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py) and +[`api-contract-matrix-case.py`](../../../../shared/automation/api-contract-matrix-case.py). ## Steps ### Step 1: Resolve the listener and the indexed contract -Resolve the guest agent's external listener, which is reachable from outside the CVM from the lease-owned fixture, load the inventory -entries for `Worker`, and confirm every one of them carries a policy. +Resolve the listener from the fixture and load the `Worker` inventory entries. **Expected results:** -- The listener is reachable, every declared method has a policy, and no method - in the policy table is absent from the inventory. +- The listener is reachable and every declared method has a policy. ### Step 2: Drive the matrix -Send every vector the policy allows for every method, recording the status, -the response content type, the body size and the elapsed time of each call. +Send every vector the policy allows, recording status, content type, body size +and elapsed time of each call. **Expected results:** -- **L1** No request produces a 5xx, a dropped connection, or a transport-level - failure. A malformed request is a client error, not a server error, and - never a crash: the workspace builds release binaries with `panic = "abort"`, - so a reachable panic is a process abort rather than a failed request. -- **L2** A rejected JSON request carries a structured JSON body with an - `error` string; a rejected protobuf request carries a decodable `ProtoError`. -- **L4** No rejection echoes an unbounded amount of the caller's input, and no - response contains the unknown-field marker the request carried. +- **L1** No request produces a 5xx, a dropped connection or a transport + failure. Release builds use `panic = "abort"`, so a reachable panic aborts + the process. +- **L2** A rejected JSON request gets a JSON `error` string; a rejected + protobuf request gets a decodable `ProtoError`. +- **L4** No rejection echoes the unknown-field marker or grows with the size + of the field it refused. - **L5** No request exceeds the per-call deadline. -- **L6** A rejection's `Content-Type` matches the representation the request - arrived in, so a client that sent JSON can parse the error it gets back. +- **L6** A rejection's `Content-Type` matches the request's representation. ### Step 3: Verify the listener survived -Send one valid request after the whole matrix. +Send one valid request after the matrix. **Expected results:** -- **L3** The listener answers it. A service that stopped answering has either - aborted or wedged on one of the preceding vectors. +- **L3** The listener answers it. ## Postconditions -Remove run-scoped objects the `full` policy created and restore changed -configuration. Preserve the per-call observation table in the result -artifacts: it is the evidence that each vector was actually sent, and its -status histogram is what a reviewer compares across runs. +Remove run-scoped objects the `full` policy created. Keep the per-call +observation table in the result artifacts. diff --git a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-004/case.md b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-004/case.md index a6079da2e..1714fcaf4 100644 --- a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-004/case.md +++ b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-004/case.md @@ -16,107 +16,71 @@ ## Prepared execution knowledge - Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. -- The method list, the request field list and every field's declared type come - from [`api-inventory.json`](../../../../catalog/api-inventory.json). Do not reconstruct - them from implementation source: the point of this case is that the indexed - contract and the running service agree. -- [`api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) - says how far each method may be driven. `full` runs every vector, and is only - used for a method whose effect is confined to the lease. `rejection-only` - runs the vectors a service must refuse before it dispatches, so nothing can - mutate. `skip` is for a method whose valid invocation ends the process or - consumes an external resource, and the policy file records why for each one. -- Every method the inventory declares for this service must carry a policy. A - method added to the inventory without one fails this case rather than being - silently skipped. +- Methods, request fields and field types come from `api-inventory.json`, not + from implementation source. +- `api-contract-matrix-policy.json` sets how far each method is driven: `full` + (effect confined to the lease), `rejection-only` (only inputs refused before + dispatch) or `skip` (a valid call ends the process or consumes an external + resource). A method without a policy fails the case. ## Objective -Verify that every method `GuestApi` declares answers absent, default, valid, -boundary-invalid, wrong-typed, unknown-field and malformed-framing input, in -both the JSON and the protobuf representation, without violating the -invariants that hold for every method whatever it means. - -The per-method cases in this suite each send one valid request and one -wrong-typed value for the first request field. That leaves the input classes -where a service actually breaks — an out-of-range integer, an oversized -string, a wrong protobuf wire type, a truncated body — untested for every -method. This case closes that gap once, for the whole service, and is driven -by the same inventory the per-method cases cite as mandatory test data. +Verify that every `GuestApi` method handles absent, valid, boundary-invalid, +wrong-typed, unknown-field and malformed-framing input, in JSON and protobuf, +without breaking the invariants below. ## Preconditions -1. The shared plan prerequisites are healthy and the guest agent's host-facing listener is reachable. -2. The fixture is lease-owned. The matrix drives mutating methods with - run-scoped identifiers, so it must never be pointed at a shared or - production deployment. +1. The guest agent's host-facing listener is reachable on a lease-owned fixture. The matrix drives + mutating methods with run-scoped identifiers, so it must never target a + shared deployment. ## Test Data -The `guest-os` entries in [`api-inventory.json`](../../../../catalog/api-inventory.json) -whose `service` is `GuestApi`, and the vector set in -[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py): - -- per field type — empty, wrong type, null, a NUL byte, a newline, a path - traversal, an absolute path, control characters, 64 KiB, 1 MiB, odd-length - hex, non-hex, 31/63/64/65-byte lengths, zero, negative, `u32::MAX`, - `u64::MAX`, past `u64::MAX`, and a 4096-element repeated field; -- per representation — an empty body, a non-object, a scalar, `null`, a - truncated object, trailing garbage, 2000-deep nesting, duplicate keys, a - 4 MiB body, a truncated varint, a runaway varint, a length that overflows, - a length past the end of the body, an unknown wire type, a group wire type, - field number zero, a field number past the declared range, 8 KiB of random - bytes, a JSON body sent as protobuf, a protobuf body sent as JSON, an - absent content type, and an unrecognised content type; -- per method — the declared request encoded with an unknown field appended, - and a declared field number sent with the wrong wire type. +The `guest-os` entries in `api-inventory.json` whose `service` is `GuestApi`, +and the per-field, per-representation and per-method vectors in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py) and +[`api-contract-matrix-case.py`](../../../../shared/automation/api-contract-matrix-case.py). ## Steps ### Step 1: Resolve the listener and the indexed contract -Resolve the guest agent's host-facing listener from the lease-owned fixture, load the inventory -entries for `GuestApi`, and confirm every one of them carries a policy. +Resolve the listener from the fixture and load the `GuestApi` inventory entries. **Expected results:** -- The listener is reachable, every declared method has a policy, and no method - in the policy table is absent from the inventory. +- The listener is reachable and every declared method has a policy. ### Step 2: Drive the matrix -Send every vector the policy allows for every method, recording the status, -the response content type, the body size and the elapsed time of each call. +Send every vector the policy allows, recording status, content type, body size +and elapsed time of each call. **Expected results:** -- **L1** No request produces a 5xx, a dropped connection, or a transport-level - failure. A malformed request is a client error, not a server error, and - never a crash: the workspace builds release binaries with `panic = "abort"`, - so a reachable panic is a process abort rather than a failed request. -- **L2** A rejected JSON request carries a structured JSON body with an - `error` string; a rejected protobuf request carries a decodable `ProtoError`. -- **L4** No rejection echoes an unbounded amount of the caller's input, and no - response contains the unknown-field marker the request carried. +- **L1** No request produces a 5xx, a dropped connection or a transport + failure. Release builds use `panic = "abort"`, so a reachable panic aborts + the process. +- **L2** A rejected JSON request gets a JSON `error` string; a rejected + protobuf request gets a decodable `ProtoError`. +- **L4** No rejection echoes the unknown-field marker or grows with the size + of the field it refused. - **L5** No request exceeds the per-call deadline. -- **L6** A rejection's `Content-Type` matches the representation the request - arrived in, so a client that sent JSON can parse the error it gets back. +- **L6** A rejection's `Content-Type` matches the request's representation. ### Step 3: Verify the listener survived -Send one valid request after the whole matrix. +Send one valid request after the matrix. **Expected results:** -- **L3** The listener answers it. A service that stopped answering has either - aborted or wedged on one of the preceding vectors. +- **L3** The listener answers it. ## Postconditions -Remove run-scoped objects the `full` policy created and restore changed -configuration. Preserve the per-call observation table in the result -artifacts: it is the evidence that each vector was actually sent, and its -status histogram is what a reviewer compares across runs. +Remove run-scoped objects the `full` policy created. Keep the per-call +observation table in the result artifacts. diff --git a/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-001/case.md b/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-001/case.md index efaaf06c7..8712cbcde 100644 --- a/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-001/case.md +++ b/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-001/case.md @@ -16,107 +16,71 @@ ## Prepared execution knowledge - Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. -- The method list, the request field list and every field's declared type come - from [`api-inventory.json`](../../../../catalog/api-inventory.json). Do not reconstruct - them from implementation source: the point of this case is that the indexed - contract and the running service agree. -- [`api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) - says how far each method may be driven. `full` runs every vector, and is only - used for a method whose effect is confined to the lease. `rejection-only` - runs the vectors a service must refuse before it dispatches, so nothing can - mutate. `skip` is for a method whose valid invocation ends the process or - consumes an external resource, and the policy file records why for each one. -- Every method the inventory declares for this service must carry a policy. A - method added to the inventory without one fails this case rather than being - silently skipped. +- Methods, request fields and field types come from `api-inventory.json`, not + from implementation source. +- `api-contract-matrix-policy.json` sets how far each method is driven: `full` + (effect confined to the lease), `rejection-only` (only inputs refused before + dispatch) or `skip` (a valid call ends the process or consumes an external + resource). A method without a policy fails the case. ## Objective -Verify that every method `KMS` declares answers absent, default, valid, -boundary-invalid, wrong-typed, unknown-field and malformed-framing input, in -both the JSON and the protobuf representation, without violating the -invariants that hold for every method whatever it means. - -The per-method cases in this suite each send one valid request and one -wrong-typed value for the first request field. That leaves the input classes -where a service actually breaks — an out-of-range integer, an oversized -string, a wrong protobuf wire type, a truncated body — untested for every -method. This case closes that gap once, for the whole service, and is driven -by the same inventory the per-method cases cite as mandatory test data. +Verify that every `KMS` method handles absent, valid, boundary-invalid, +wrong-typed, unknown-field and malformed-framing input, in JSON and protobuf, +without breaking the invariants below. ## Preconditions -1. The shared plan prerequisites are healthy and the KMS public RPC listener is reachable. -2. The fixture is lease-owned. The matrix drives mutating methods with - run-scoped identifiers, so it must never be pointed at a shared or - production deployment. +1. The KMS public RPC listener is reachable on a lease-owned fixture. The matrix drives + mutating methods with run-scoped identifiers, so it must never target a + shared deployment. ## Test Data -The `kms` entries in [`api-inventory.json`](../../../../catalog/api-inventory.json) -whose `service` is `KMS`, and the vector set in -[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py): - -- per field type — empty, wrong type, null, a NUL byte, a newline, a path - traversal, an absolute path, control characters, 64 KiB, 1 MiB, odd-length - hex, non-hex, 31/63/64/65-byte lengths, zero, negative, `u32::MAX`, - `u64::MAX`, past `u64::MAX`, and a 4096-element repeated field; -- per representation — an empty body, a non-object, a scalar, `null`, a - truncated object, trailing garbage, 2000-deep nesting, duplicate keys, a - 4 MiB body, a truncated varint, a runaway varint, a length that overflows, - a length past the end of the body, an unknown wire type, a group wire type, - field number zero, a field number past the declared range, 8 KiB of random - bytes, a JSON body sent as protobuf, a protobuf body sent as JSON, an - absent content type, and an unrecognised content type; -- per method — the declared request encoded with an unknown field appended, - and a declared field number sent with the wrong wire type. +The `kms` entries in `api-inventory.json` whose `service` is `KMS`, +and the per-field, per-representation and per-method vectors in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py) and +[`api-contract-matrix-case.py`](../../../../shared/automation/api-contract-matrix-case.py). ## Steps ### Step 1: Resolve the listener and the indexed contract -Resolve the KMS public RPC listener from the lease-owned fixture, load the inventory -entries for `KMS`, and confirm every one of them carries a policy. +Resolve the listener from the fixture and load the `KMS` inventory entries. **Expected results:** -- The listener is reachable, every declared method has a policy, and no method - in the policy table is absent from the inventory. +- The listener is reachable and every declared method has a policy. ### Step 2: Drive the matrix -Send every vector the policy allows for every method, recording the status, -the response content type, the body size and the elapsed time of each call. +Send every vector the policy allows, recording status, content type, body size +and elapsed time of each call. **Expected results:** -- **L1** No request produces a 5xx, a dropped connection, or a transport-level - failure. A malformed request is a client error, not a server error, and - never a crash: the workspace builds release binaries with `panic = "abort"`, - so a reachable panic is a process abort rather than a failed request. -- **L2** A rejected JSON request carries a structured JSON body with an - `error` string; a rejected protobuf request carries a decodable `ProtoError`. -- **L4** No rejection echoes an unbounded amount of the caller's input, and no - response contains the unknown-field marker the request carried. +- **L1** No request produces a 5xx, a dropped connection or a transport + failure. Release builds use `panic = "abort"`, so a reachable panic aborts + the process. +- **L2** A rejected JSON request gets a JSON `error` string; a rejected + protobuf request gets a decodable `ProtoError`. +- **L4** No rejection echoes the unknown-field marker or grows with the size + of the field it refused. - **L5** No request exceeds the per-call deadline. -- **L6** A rejection's `Content-Type` matches the representation the request - arrived in, so a client that sent JSON can parse the error it gets back. +- **L6** A rejection's `Content-Type` matches the request's representation. ### Step 3: Verify the listener survived -Send one valid request after the whole matrix. +Send one valid request after the matrix. **Expected results:** -- **L3** The listener answers it. A service that stopped answering has either - aborted or wedged on one of the preceding vectors. +- **L3** The listener answers it. ## Postconditions -Remove run-scoped objects the `full` policy created and restore changed -configuration. Preserve the per-call observation table in the result -artifacts: it is the evidence that each vector was actually sent, and its -status histogram is what a reviewer compares across runs. +Remove run-scoped objects the `full` policy created. Keep the per-call +observation table in the result artifacts. diff --git a/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-002/case.md b/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-002/case.md index c802e90e6..95663d362 100644 --- a/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-002/case.md +++ b/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-002/case.md @@ -16,107 +16,71 @@ ## Prepared execution knowledge - Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. -- The method list, the request field list and every field's declared type come - from [`api-inventory.json`](../../../../catalog/api-inventory.json). Do not reconstruct - them from implementation source: the point of this case is that the indexed - contract and the running service agree. -- [`api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) - says how far each method may be driven. `full` runs every vector, and is only - used for a method whose effect is confined to the lease. `rejection-only` - runs the vectors a service must refuse before it dispatches, so nothing can - mutate. `skip` is for a method whose valid invocation ends the process or - consumes an external resource, and the policy file records why for each one. -- Every method the inventory declares for this service must carry a policy. A - method added to the inventory without one fails this case rather than being - silently skipped. +- Methods, request fields and field types come from `api-inventory.json`, not + from implementation source. +- `api-contract-matrix-policy.json` sets how far each method is driven: `full` + (effect confined to the lease), `rejection-only` (only inputs refused before + dispatch) or `skip` (a valid call ends the process or consumes an external + resource). A method without a policy fails the case. ## Objective -Verify that every method `Onboard` declares answers absent, default, valid, -boundary-invalid, wrong-typed, unknown-field and malformed-framing input, in -both the JSON and the protobuf representation, without violating the -invariants that hold for every method whatever it means. - -The per-method cases in this suite each send one valid request and one -wrong-typed value for the first request field. That leaves the input classes -where a service actually breaks — an out-of-range integer, an oversized -string, a wrong protobuf wire type, a truncated body — untested for every -method. This case closes that gap once, for the whole service, and is driven -by the same inventory the per-method cases cite as mandatory test data. +Verify that every `Onboard` method handles absent, valid, boundary-invalid, +wrong-typed, unknown-field and malformed-framing input, in JSON and protobuf, +without breaking the invariants below. ## Preconditions -1. The shared plan prerequisites are healthy and the KMS onboarding listener is reachable. -2. The fixture is lease-owned. The matrix drives mutating methods with - run-scoped identifiers, so it must never be pointed at a shared or - production deployment. +1. The KMS onboarding listener is reachable on a lease-owned fixture. The matrix drives + mutating methods with run-scoped identifiers, so it must never target a + shared deployment. ## Test Data -The `kms` entries in [`api-inventory.json`](../../../../catalog/api-inventory.json) -whose `service` is `Onboard`, and the vector set in -[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py): - -- per field type — empty, wrong type, null, a NUL byte, a newline, a path - traversal, an absolute path, control characters, 64 KiB, 1 MiB, odd-length - hex, non-hex, 31/63/64/65-byte lengths, zero, negative, `u32::MAX`, - `u64::MAX`, past `u64::MAX`, and a 4096-element repeated field; -- per representation — an empty body, a non-object, a scalar, `null`, a - truncated object, trailing garbage, 2000-deep nesting, duplicate keys, a - 4 MiB body, a truncated varint, a runaway varint, a length that overflows, - a length past the end of the body, an unknown wire type, a group wire type, - field number zero, a field number past the declared range, 8 KiB of random - bytes, a JSON body sent as protobuf, a protobuf body sent as JSON, an - absent content type, and an unrecognised content type; -- per method — the declared request encoded with an unknown field appended, - and a declared field number sent with the wrong wire type. +The `kms` entries in `api-inventory.json` whose `service` is `Onboard`, +and the per-field, per-representation and per-method vectors in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py) and +[`api-contract-matrix-case.py`](../../../../shared/automation/api-contract-matrix-case.py). ## Steps ### Step 1: Resolve the listener and the indexed contract -Resolve the KMS onboarding listener from the lease-owned fixture, load the inventory -entries for `Onboard`, and confirm every one of them carries a policy. +Resolve the listener from the fixture and load the `Onboard` inventory entries. **Expected results:** -- The listener is reachable, every declared method has a policy, and no method - in the policy table is absent from the inventory. +- The listener is reachable and every declared method has a policy. ### Step 2: Drive the matrix -Send every vector the policy allows for every method, recording the status, -the response content type, the body size and the elapsed time of each call. +Send every vector the policy allows, recording status, content type, body size +and elapsed time of each call. **Expected results:** -- **L1** No request produces a 5xx, a dropped connection, or a transport-level - failure. A malformed request is a client error, not a server error, and - never a crash: the workspace builds release binaries with `panic = "abort"`, - so a reachable panic is a process abort rather than a failed request. -- **L2** A rejected JSON request carries a structured JSON body with an - `error` string; a rejected protobuf request carries a decodable `ProtoError`. -- **L4** No rejection echoes an unbounded amount of the caller's input, and no - response contains the unknown-field marker the request carried. +- **L1** No request produces a 5xx, a dropped connection or a transport + failure. Release builds use `panic = "abort"`, so a reachable panic aborts + the process. +- **L2** A rejected JSON request gets a JSON `error` string; a rejected + protobuf request gets a decodable `ProtoError`. +- **L4** No rejection echoes the unknown-field marker or grows with the size + of the field it refused. - **L5** No request exceeds the per-call deadline. -- **L6** A rejection's `Content-Type` matches the representation the request - arrived in, so a client that sent JSON can parse the error it gets back. +- **L6** A rejection's `Content-Type` matches the request's representation. ### Step 3: Verify the listener survived -Send one valid request after the whole matrix. +Send one valid request after the matrix. **Expected results:** -- **L3** The listener answers it. A service that stopped answering has either - aborted or wedged on one of the preceding vectors. +- **L3** The listener answers it. ## Postconditions -Remove run-scoped objects the `full` policy created and restore changed -configuration. Preserve the per-call observation table in the result -artifacts: it is the evidence that each vector was actually sent, and its -status histogram is what a reviewer compares across runs. +Remove run-scoped objects the `full` policy created. Keep the per-call +observation table in the result artifacts. diff --git a/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-001/case.md b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-001/case.md index b479c4045..f5149cc6f 100644 --- a/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-001/case.md +++ b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-001/case.md @@ -16,107 +16,71 @@ ## Prepared execution knowledge - Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. -- The method list, the request field list and every field's declared type come - from [`api-inventory.json`](../../../../catalog/api-inventory.json). Do not reconstruct - them from implementation source: the point of this case is that the indexed - contract and the running service agree. -- [`api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) - says how far each method may be driven. `full` runs every vector, and is only - used for a method whose effect is confined to the lease. `rejection-only` - runs the vectors a service must refuse before it dispatches, so nothing can - mutate. `skip` is for a method whose valid invocation ends the process or - consumes an external resource, and the policy file records why for each one. -- Every method the inventory declares for this service must carry a policy. A - method added to the inventory without one fails this case rather than being - silently skipped. +- Methods, request fields and field types come from `api-inventory.json`, not + from implementation source. +- `api-contract-matrix-policy.json` sets how far each method is driven: `full` + (effect confined to the lease), `rejection-only` (only inputs refused before + dispatch) or `skip` (a valid call ends the process or consumes an external + resource). A method without a policy fails the case. ## Objective -Verify that every method `Gateway` declares answers absent, default, valid, -boundary-invalid, wrong-typed, unknown-field and malformed-framing input, in -both the JSON and the protobuf representation, without violating the -invariants that hold for every method whatever it means. - -The per-method cases in this suite each send one valid request and one -wrong-typed value for the first request field. That leaves the input classes -where a service actually breaks — an out-of-range integer, an oversized -string, a wrong protobuf wire type, a truncated body — untested for every -method. This case closes that gap once, for the whole service, and is driven -by the same inventory the per-method cases cite as mandatory test data. +Verify that every `Gateway` method handles absent, valid, boundary-invalid, +wrong-typed, unknown-field and malformed-framing input, in JSON and protobuf, +without breaking the invariants below. ## Preconditions -1. The shared plan prerequisites are healthy and the gateway's public RPC listener is reachable. -2. The fixture is lease-owned. The matrix drives mutating methods with - run-scoped identifiers, so it must never be pointed at a shared or - production deployment. +1. The gateway's public RPC listener is reachable on a lease-owned fixture. The matrix drives + mutating methods with run-scoped identifiers, so it must never target a + shared deployment. ## Test Data -The `gateway` entries in [`api-inventory.json`](../../../../catalog/api-inventory.json) -whose `service` is `Gateway`, and the vector set in -[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py): - -- per field type — empty, wrong type, null, a NUL byte, a newline, a path - traversal, an absolute path, control characters, 64 KiB, 1 MiB, odd-length - hex, non-hex, 31/63/64/65-byte lengths, zero, negative, `u32::MAX`, - `u64::MAX`, past `u64::MAX`, and a 4096-element repeated field; -- per representation — an empty body, a non-object, a scalar, `null`, a - truncated object, trailing garbage, 2000-deep nesting, duplicate keys, a - 4 MiB body, a truncated varint, a runaway varint, a length that overflows, - a length past the end of the body, an unknown wire type, a group wire type, - field number zero, a field number past the declared range, 8 KiB of random - bytes, a JSON body sent as protobuf, a protobuf body sent as JSON, an - absent content type, and an unrecognised content type; -- per method — the declared request encoded with an unknown field appended, - and a declared field number sent with the wrong wire type. +The `gateway` entries in `api-inventory.json` whose `service` is `Gateway`, +and the per-field, per-representation and per-method vectors in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py) and +[`api-contract-matrix-case.py`](../../../../shared/automation/api-contract-matrix-case.py). ## Steps ### Step 1: Resolve the listener and the indexed contract -Resolve the gateway's public RPC listener from the lease-owned fixture, load the inventory -entries for `Gateway`, and confirm every one of them carries a policy. +Resolve the listener from the fixture and load the `Gateway` inventory entries. **Expected results:** -- The listener is reachable, every declared method has a policy, and no method - in the policy table is absent from the inventory. +- The listener is reachable and every declared method has a policy. ### Step 2: Drive the matrix -Send every vector the policy allows for every method, recording the status, -the response content type, the body size and the elapsed time of each call. +Send every vector the policy allows, recording status, content type, body size +and elapsed time of each call. **Expected results:** -- **L1** No request produces a 5xx, a dropped connection, or a transport-level - failure. A malformed request is a client error, not a server error, and - never a crash: the workspace builds release binaries with `panic = "abort"`, - so a reachable panic is a process abort rather than a failed request. -- **L2** A rejected JSON request carries a structured JSON body with an - `error` string; a rejected protobuf request carries a decodable `ProtoError`. -- **L4** No rejection echoes an unbounded amount of the caller's input, and no - response contains the unknown-field marker the request carried. +- **L1** No request produces a 5xx, a dropped connection or a transport + failure. Release builds use `panic = "abort"`, so a reachable panic aborts + the process. +- **L2** A rejected JSON request gets a JSON `error` string; a rejected + protobuf request gets a decodable `ProtoError`. +- **L4** No rejection echoes the unknown-field marker or grows with the size + of the field it refused. - **L5** No request exceeds the per-call deadline. -- **L6** A rejection's `Content-Type` matches the representation the request - arrived in, so a client that sent JSON can parse the error it gets back. +- **L6** A rejection's `Content-Type` matches the request's representation. ### Step 3: Verify the listener survived -Send one valid request after the whole matrix. +Send one valid request after the matrix. **Expected results:** -- **L3** The listener answers it. A service that stopped answering has either - aborted or wedged on one of the preceding vectors. +- **L3** The listener answers it. ## Postconditions -Remove run-scoped objects the `full` policy created and restore changed -configuration. Preserve the per-call observation table in the result -artifacts: it is the evidence that each vector was actually sent, and its -status histogram is what a reviewer compares across runs. +Remove run-scoped objects the `full` policy created. Keep the per-call +observation table in the result artifacts. diff --git a/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-002/case.md b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-002/case.md index 58e416f4e..2abedeb07 100644 --- a/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-002/case.md +++ b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-002/case.md @@ -16,107 +16,71 @@ ## Prepared execution knowledge - Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. -- The method list, the request field list and every field's declared type come - from [`api-inventory.json`](../../../../catalog/api-inventory.json). Do not reconstruct - them from implementation source: the point of this case is that the indexed - contract and the running service agree. -- [`api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) - says how far each method may be driven. `full` runs every vector, and is only - used for a method whose effect is confined to the lease. `rejection-only` - runs the vectors a service must refuse before it dispatches, so nothing can - mutate. `skip` is for a method whose valid invocation ends the process or - consumes an external resource, and the policy file records why for each one. -- Every method the inventory declares for this service must carry a policy. A - method added to the inventory without one fails this case rather than being - silently skipped. +- Methods, request fields and field types come from `api-inventory.json`, not + from implementation source. +- `api-contract-matrix-policy.json` sets how far each method is driven: `full` + (effect confined to the lease), `rejection-only` (only inputs refused before + dispatch) or `skip` (a valid call ends the process or consumes an external + resource). A method without a policy fails the case. ## Objective -Verify that every method `Debug` declares answers absent, default, valid, -boundary-invalid, wrong-typed, unknown-field and malformed-framing input, in -both the JSON and the protobuf representation, without violating the -invariants that hold for every method whatever it means. - -The per-method cases in this suite each send one valid request and one -wrong-typed value for the first request field. That leaves the input classes -where a service actually breaks — an out-of-range integer, an oversized -string, a wrong protobuf wire type, a truncated body — untested for every -method. This case closes that gap once, for the whole service, and is driven -by the same inventory the per-method cases cite as mandatory test data. +Verify that every `Debug` method handles absent, valid, boundary-invalid, +wrong-typed, unknown-field and malformed-framing input, in JSON and protobuf, +without breaking the invariants below. ## Preconditions -1. The shared plan prerequisites are healthy and the gateway's debug listener is reachable. -2. The fixture is lease-owned. The matrix drives mutating methods with - run-scoped identifiers, so it must never be pointed at a shared or - production deployment. +1. The gateway's debug listener is reachable on a lease-owned fixture. The matrix drives + mutating methods with run-scoped identifiers, so it must never target a + shared deployment. ## Test Data -The `gateway` entries in [`api-inventory.json`](../../../../catalog/api-inventory.json) -whose `service` is `Debug`, and the vector set in -[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py): - -- per field type — empty, wrong type, null, a NUL byte, a newline, a path - traversal, an absolute path, control characters, 64 KiB, 1 MiB, odd-length - hex, non-hex, 31/63/64/65-byte lengths, zero, negative, `u32::MAX`, - `u64::MAX`, past `u64::MAX`, and a 4096-element repeated field; -- per representation — an empty body, a non-object, a scalar, `null`, a - truncated object, trailing garbage, 2000-deep nesting, duplicate keys, a - 4 MiB body, a truncated varint, a runaway varint, a length that overflows, - a length past the end of the body, an unknown wire type, a group wire type, - field number zero, a field number past the declared range, 8 KiB of random - bytes, a JSON body sent as protobuf, a protobuf body sent as JSON, an - absent content type, and an unrecognised content type; -- per method — the declared request encoded with an unknown field appended, - and a declared field number sent with the wrong wire type. +The `gateway` entries in `api-inventory.json` whose `service` is `Debug`, +and the per-field, per-representation and per-method vectors in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py) and +[`api-contract-matrix-case.py`](../../../../shared/automation/api-contract-matrix-case.py). ## Steps ### Step 1: Resolve the listener and the indexed contract -Resolve the gateway's debug listener from the lease-owned fixture, load the inventory -entries for `Debug`, and confirm every one of them carries a policy. +Resolve the listener from the fixture and load the `Debug` inventory entries. **Expected results:** -- The listener is reachable, every declared method has a policy, and no method - in the policy table is absent from the inventory. +- The listener is reachable and every declared method has a policy. ### Step 2: Drive the matrix -Send every vector the policy allows for every method, recording the status, -the response content type, the body size and the elapsed time of each call. +Send every vector the policy allows, recording status, content type, body size +and elapsed time of each call. **Expected results:** -- **L1** No request produces a 5xx, a dropped connection, or a transport-level - failure. A malformed request is a client error, not a server error, and - never a crash: the workspace builds release binaries with `panic = "abort"`, - so a reachable panic is a process abort rather than a failed request. -- **L2** A rejected JSON request carries a structured JSON body with an - `error` string; a rejected protobuf request carries a decodable `ProtoError`. -- **L4** No rejection echoes an unbounded amount of the caller's input, and no - response contains the unknown-field marker the request carried. +- **L1** No request produces a 5xx, a dropped connection or a transport + failure. Release builds use `panic = "abort"`, so a reachable panic aborts + the process. +- **L2** A rejected JSON request gets a JSON `error` string; a rejected + protobuf request gets a decodable `ProtoError`. +- **L4** No rejection echoes the unknown-field marker or grows with the size + of the field it refused. - **L5** No request exceeds the per-call deadline. -- **L6** A rejection's `Content-Type` matches the representation the request - arrived in, so a client that sent JSON can parse the error it gets back. +- **L6** A rejection's `Content-Type` matches the request's representation. ### Step 3: Verify the listener survived -Send one valid request after the whole matrix. +Send one valid request after the matrix. **Expected results:** -- **L3** The listener answers it. A service that stopped answering has either - aborted or wedged on one of the preceding vectors. +- **L3** The listener answers it. ## Postconditions -Remove run-scoped objects the `full` policy created and restore changed -configuration. Preserve the per-call observation table in the result -artifacts: it is the evidence that each vector was actually sent, and its -status histogram is what a reviewer compares across runs. +Remove run-scoped objects the `full` policy created. Keep the per-call +observation table in the result artifacts. diff --git a/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-003/case.md b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-003/case.md index 7a639fddd..1e01205ba 100644 --- a/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-003/case.md +++ b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-003/case.md @@ -16,107 +16,71 @@ ## Prepared execution knowledge - Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. -- The method list, the request field list and every field's declared type come - from [`api-inventory.json`](../../../../catalog/api-inventory.json). Do not reconstruct - them from implementation source: the point of this case is that the indexed - contract and the running service agree. -- [`api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) - says how far each method may be driven. `full` runs every vector, and is only - used for a method whose effect is confined to the lease. `rejection-only` - runs the vectors a service must refuse before it dispatches, so nothing can - mutate. `skip` is for a method whose valid invocation ends the process or - consumes an external resource, and the policy file records why for each one. -- Every method the inventory declares for this service must carry a policy. A - method added to the inventory without one fails this case rather than being - silently skipped. +- Methods, request fields and field types come from `api-inventory.json`, not + from implementation source. +- `api-contract-matrix-policy.json` sets how far each method is driven: `full` + (effect confined to the lease), `rejection-only` (only inputs refused before + dispatch) or `skip` (a valid call ends the process or consumes an external + resource). A method without a policy fails the case. ## Objective -Verify that every method `Admin` declares answers absent, default, valid, -boundary-invalid, wrong-typed, unknown-field and malformed-framing input, in -both the JSON and the protobuf representation, without violating the -invariants that hold for every method whatever it means. - -The per-method cases in this suite each send one valid request and one -wrong-typed value for the first request field. That leaves the input classes -where a service actually breaks — an out-of-range integer, an oversized -string, a wrong protobuf wire type, a truncated body — untested for every -method. This case closes that gap once, for the whole service, and is driven -by the same inventory the per-method cases cite as mandatory test data. +Verify that every `Admin` method handles absent, valid, boundary-invalid, +wrong-typed, unknown-field and malformed-framing input, in JSON and protobuf, +without breaking the invariants below. ## Preconditions -1. The shared plan prerequisites are healthy and the gateway's authenticated admin listener is reachable. -2. The fixture is lease-owned. The matrix drives mutating methods with - run-scoped identifiers, so it must never be pointed at a shared or - production deployment. +1. The gateway's authenticated admin listener is reachable on a lease-owned fixture. The matrix drives + mutating methods with run-scoped identifiers, so it must never target a + shared deployment. ## Test Data -The `gateway` entries in [`api-inventory.json`](../../../../catalog/api-inventory.json) -whose `service` is `Admin`, and the vector set in -[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py): - -- per field type — empty, wrong type, null, a NUL byte, a newline, a path - traversal, an absolute path, control characters, 64 KiB, 1 MiB, odd-length - hex, non-hex, 31/63/64/65-byte lengths, zero, negative, `u32::MAX`, - `u64::MAX`, past `u64::MAX`, and a 4096-element repeated field; -- per representation — an empty body, a non-object, a scalar, `null`, a - truncated object, trailing garbage, 2000-deep nesting, duplicate keys, a - 4 MiB body, a truncated varint, a runaway varint, a length that overflows, - a length past the end of the body, an unknown wire type, a group wire type, - field number zero, a field number past the declared range, 8 KiB of random - bytes, a JSON body sent as protobuf, a protobuf body sent as JSON, an - absent content type, and an unrecognised content type; -- per method — the declared request encoded with an unknown field appended, - and a declared field number sent with the wrong wire type. +The `gateway` entries in `api-inventory.json` whose `service` is `Admin`, +and the per-field, per-representation and per-method vectors in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py) and +[`api-contract-matrix-case.py`](../../../../shared/automation/api-contract-matrix-case.py). ## Steps ### Step 1: Resolve the listener and the indexed contract -Resolve the gateway's authenticated admin listener from the lease-owned fixture, load the inventory -entries for `Admin`, and confirm every one of them carries a policy. +Resolve the listener from the fixture and load the `Admin` inventory entries. **Expected results:** -- The listener is reachable, every declared method has a policy, and no method - in the policy table is absent from the inventory. +- The listener is reachable and every declared method has a policy. ### Step 2: Drive the matrix -Send every vector the policy allows for every method, recording the status, -the response content type, the body size and the elapsed time of each call. +Send every vector the policy allows, recording status, content type, body size +and elapsed time of each call. **Expected results:** -- **L1** No request produces a 5xx, a dropped connection, or a transport-level - failure. A malformed request is a client error, not a server error, and - never a crash: the workspace builds release binaries with `panic = "abort"`, - so a reachable panic is a process abort rather than a failed request. -- **L2** A rejected JSON request carries a structured JSON body with an - `error` string; a rejected protobuf request carries a decodable `ProtoError`. -- **L4** No rejection echoes an unbounded amount of the caller's input, and no - response contains the unknown-field marker the request carried. +- **L1** No request produces a 5xx, a dropped connection or a transport + failure. Release builds use `panic = "abort"`, so a reachable panic aborts + the process. +- **L2** A rejected JSON request gets a JSON `error` string; a rejected + protobuf request gets a decodable `ProtoError`. +- **L4** No rejection echoes the unknown-field marker or grows with the size + of the field it refused. - **L5** No request exceeds the per-call deadline. -- **L6** A rejection's `Content-Type` matches the representation the request - arrived in, so a client that sent JSON can parse the error it gets back. +- **L6** A rejection's `Content-Type` matches the request's representation. ### Step 3: Verify the listener survived -Send one valid request after the whole matrix. +Send one valid request after the matrix. **Expected results:** -- **L3** The listener answers it. A service that stopped answering has either - aborted or wedged on one of the preceding vectors. +- **L3** The listener answers it. ## Postconditions -Remove run-scoped objects the `full` policy created and restore changed -configuration. Preserve the per-call observation table in the result -artifacts: it is the evidence that each vector was actually sent, and its -status histogram is what a reviewer compares across runs. +Remove run-scoped objects the `full` policy created. Keep the per-call +observation table in the result artifacts. From 7730d07423cd79c6e0f33f0d94283e286affba85 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 24 Sep 2026 08:51:48 -0700 Subject: [PATCH 9/9] style(test): put the request-contract harness docstrings in the imperative mood --- test-suites/shared/automation/api-contract-matrix-case.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test-suites/shared/automation/api-contract-matrix-case.py b/test-suites/shared/automation/api-contract-matrix-case.py index 195255c04..1999b7a56 100755 --- a/test-suites/shared/automation/api-contract-matrix-case.py +++ b/test-suites/shared/automation/api-contract-matrix-case.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # SPDX-FileCopyrightText: © 2026 Phala Network # SPDX-License-Identifier: Apache-2.0 -"""Run the request-contract matrix in ``api_contract_matrix`` against one service. +r"""Run the request-contract matrix in ``api_contract_matrix`` against one service. ``api-contract-matrix-policy.json`` says how far each method may be driven: ``full`` runs every vector, ``rejection-only`` runs only the vectors a service @@ -205,14 +205,14 @@ def service_policy( def wrong_type_for(field_def: dict[str, Any]) -> Any: - """A JSON value of the wrong type for this field.""" + """Return a JSON value of the wrong type for this field.""" if field_def["type"] in ("string", "bytes"): return 123 return {"not": "a scalar"} def wrong_wiretype(target: Target, entry: dict[str, Any]) -> list[Call]: - """The first declared field number sent with the wrong wire type.""" + """Send the first declared field number with the wrong wire type.""" fields = entry.get("request_fields", []) if not fields: return [] @@ -285,7 +285,7 @@ def sweep_framing(target: Target, method: str) -> list[Call]: def sweep_full( target: Target, entry: dict[str, Any], markers: dict[str, str] ) -> list[Call]: - """The complete vector set for one method.""" + """Build the complete vector set for one method.""" service, method = entry["service"], entry["method"] key = f"{service}.{method}" fields = entry.get("request_fields", [])