diff --git a/test-suites/cases/01-guest-os/15-request-contract/metadata.json b/test-suites/cases/01-guest-os/15-request-contract/metadata.json new file mode 100644 index 000000000..5843d6348 --- /dev/null +++ b/test-suites/cases/01-guest-os/15-request-contract/metadata.json @@ -0,0 +1,4 @@ +{ + "id": "section-guest-os-request-contract", + "title": "Request contract matrix" +} diff --git a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-001/case.md b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-001/case.md new file mode 100644 index 000000000..73f0e101d --- /dev/null +++ b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-001/case.md @@ -0,0 +1,86 @@ + + + +# TC-GOS-CONTRACT-001: Tappd request-contract matrix + +## Metadata + +- Priority: P0 +- Type: Functional, API, Security, Robustness, Regression +- Minimum environment: INTEGRATION +- Automation: Yes +- Requirements: [req-gos-contract-001](../../../../catalog/feature-audit.md#req-gos-contract-001) +- Risks: [risk-gos-contract-001](../../../../catalog/feature-audit.md#risk-gos-contract-001) +- Source: [`catalog/api-inventory.json`](../../../../catalog/api-inventory.json), [`shared/automation/api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + +## Prepared execution knowledge + +- Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. +- Methods, request fields and field types come from `api-inventory.json`, not + from implementation source. +- `api-contract-matrix-policy.json` sets how far each method is driven: `full` + (effect confined to the lease), `rejection-only` (only inputs refused before + dispatch) or `skip` (a valid call ends the process or consumes an external + resource). A method without a policy fails the case. + +## Objective + +Verify that every `Tappd` method handles absent, valid, boundary-invalid, +wrong-typed, unknown-field and malformed-framing input, in JSON and protobuf, +without breaking the invariants below. + +## Preconditions + +1. The guest agent's `tappd.sock` legacy listener is reachable on a lease-owned fixture. The matrix drives + mutating methods with run-scoped identifiers, so it must never target a + shared deployment. + +## Test Data + +The `guest-os` entries in `api-inventory.json` whose `service` is `Tappd`, +and the per-field, per-representation and per-method vectors in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py) and +[`api-contract-matrix-case.py`](../../../../shared/automation/api-contract-matrix-case.py). + +## Steps + + +### Step 1: Resolve the listener and the indexed contract + +Resolve the listener from the fixture and load the `Tappd` inventory entries. + +**Expected results:** + +- The listener is reachable and every declared method has a policy. + + +### Step 2: Drive the matrix + +Send every vector the policy allows, recording status, content type, body size +and elapsed time of each call. + +**Expected results:** + +- **L1** No request produces a 5xx, a dropped connection or a transport + failure. Release builds use `panic = "abort"`, so a reachable panic aborts + the process. +- **L2** A rejected JSON request gets a JSON `error` string; a rejected + protobuf request gets a decodable `ProtoError`. +- **L4** No rejection echoes the unknown-field marker or grows with the size + of the field it refused. +- **L5** No request exceeds the per-call deadline. +- **L6** A rejection's `Content-Type` matches the request's representation. + + +### Step 3: Verify the listener survived + +Send one valid request after the matrix. + +**Expected results:** + +- **L3** The listener answers it. + +## Postconditions + +Remove run-scoped objects the `full` policy created. Keep the per-call +observation table in the result artifacts. diff --git a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-001/metadata.json b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-001/metadata.json new file mode 100644 index 000000000..1b9113729 --- /dev/null +++ b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-001/metadata.json @@ -0,0 +1,36 @@ +{ + "id": "tc-gos-contract-001", + "title": "Tappd request-contract matrix", + "priority": "P0", + "requirements": [ + "req-gos-contract-001" + ], + "risks": [ + "risk-gos-contract-001" + ], + "tags": [ + "guest", + "request-contract" + ], + "fixture": { + "profile": "no-tee-dev", + "versions": { + "vmm": "candidate", + "guest": "candidate", + "kms": "candidate", + "gateway": "candidate", + "verifier": "candidate" + }, + "destructive_scope": "lease-only", + "hardware_required": false, + "simulation_allowed": true + }, + "actions_under_test": [ + "Tappd.DeriveKey" + ], + "execution": { + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "args": [], + "timeout_seconds": 900 + } +} diff --git a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-002/case.md b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-002/case.md new file mode 100644 index 000000000..8e452405f --- /dev/null +++ b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-002/case.md @@ -0,0 +1,86 @@ + + + +# TC-GOS-CONTRACT-002: DstackGuest request-contract matrix + +## Metadata + +- Priority: P0 +- Type: Functional, API, Security, Robustness, Regression +- Minimum environment: INTEGRATION +- Automation: Yes +- Requirements: [req-gos-contract-002](../../../../catalog/feature-audit.md#req-gos-contract-002) +- Risks: [risk-gos-contract-002](../../../../catalog/feature-audit.md#risk-gos-contract-002) +- Source: [`catalog/api-inventory.json`](../../../../catalog/api-inventory.json), [`shared/automation/api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + +## Prepared execution knowledge + +- Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. +- Methods, request fields and field types come from `api-inventory.json`, not + from implementation source. +- `api-contract-matrix-policy.json` sets how far each method is driven: `full` + (effect confined to the lease), `rejection-only` (only inputs refused before + dispatch) or `skip` (a valid call ends the process or consumes an external + resource). A method without a policy fails the case. + +## Objective + +Verify that every `DstackGuest` method handles absent, valid, boundary-invalid, +wrong-typed, unknown-field and malformed-framing input, in JSON and protobuf, +without breaking the invariants below. + +## Preconditions + +1. The guest agent's `dstack.sock` internal listener (bind-mounted into application containers) is reachable on a lease-owned fixture. The matrix drives + mutating methods with run-scoped identifiers, so it must never target a + shared deployment. + +## Test Data + +The `guest-os` entries in `api-inventory.json` whose `service` is `DstackGuest`, +and the per-field, per-representation and per-method vectors in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py) and +[`api-contract-matrix-case.py`](../../../../shared/automation/api-contract-matrix-case.py). + +## Steps + + +### Step 1: Resolve the listener and the indexed contract + +Resolve the listener from the fixture and load the `DstackGuest` inventory entries. + +**Expected results:** + +- The listener is reachable and every declared method has a policy. + + +### Step 2: Drive the matrix + +Send every vector the policy allows, recording status, content type, body size +and elapsed time of each call. + +**Expected results:** + +- **L1** No request produces a 5xx, a dropped connection or a transport + failure. Release builds use `panic = "abort"`, so a reachable panic aborts + the process. +- **L2** A rejected JSON request gets a JSON `error` string; a rejected + protobuf request gets a decodable `ProtoError`. +- **L4** No rejection echoes the unknown-field marker or grows with the size + of the field it refused. +- **L5** No request exceeds the per-call deadline. +- **L6** A rejection's `Content-Type` matches the request's representation. + + +### Step 3: Verify the listener survived + +Send one valid request after the matrix. + +**Expected results:** + +- **L3** The listener answers it. + +## Postconditions + +Remove run-scoped objects the `full` policy created. Keep the per-call +observation table in the result artifacts. diff --git a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-002/metadata.json b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-002/metadata.json new file mode 100644 index 000000000..a6cb943f5 --- /dev/null +++ b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-002/metadata.json @@ -0,0 +1,36 @@ +{ + "id": "tc-gos-contract-002", + "title": "DstackGuest request-contract matrix", + "priority": "P0", + "requirements": [ + "req-gos-contract-002" + ], + "risks": [ + "risk-gos-contract-002" + ], + "tags": [ + "guest", + "request-contract" + ], + "fixture": { + "profile": "no-tee-dev", + "versions": { + "vmm": "candidate", + "guest": "candidate", + "kms": "candidate", + "gateway": "candidate", + "verifier": "candidate" + }, + "destructive_scope": "lease-only", + "hardware_required": false, + "simulation_allowed": true + }, + "actions_under_test": [ + "DstackGuest.GetTlsKey" + ], + "execution": { + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "args": [], + "timeout_seconds": 900 + } +} diff --git a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-003/case.md b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-003/case.md new file mode 100644 index 000000000..a7009e190 --- /dev/null +++ b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-003/case.md @@ -0,0 +1,86 @@ + + + +# TC-GOS-CONTRACT-003: Worker request-contract matrix + +## Metadata + +- Priority: P0 +- Type: Functional, API, Security, Robustness, Regression +- Minimum environment: INTEGRATION +- Automation: Yes +- Requirements: [req-gos-contract-003](../../../../catalog/feature-audit.md#req-gos-contract-003) +- Risks: [risk-gos-contract-003](../../../../catalog/feature-audit.md#risk-gos-contract-003) +- Source: [`catalog/api-inventory.json`](../../../../catalog/api-inventory.json), [`shared/automation/api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + +## Prepared execution knowledge + +- Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. +- Methods, request fields and field types come from `api-inventory.json`, not + from implementation source. +- `api-contract-matrix-policy.json` sets how far each method is driven: `full` + (effect confined to the lease), `rejection-only` (only inputs refused before + dispatch) or `skip` (a valid call ends the process or consumes an external + resource). A method without a policy fails the case. + +## Objective + +Verify that every `Worker` method handles absent, valid, boundary-invalid, +wrong-typed, unknown-field and malformed-framing input, in JSON and protobuf, +without breaking the invariants below. + +## Preconditions + +1. The guest agent's external listener (reachable from outside the CVM) is reachable on a lease-owned fixture. The matrix drives + mutating methods with run-scoped identifiers, so it must never target a + shared deployment. + +## Test Data + +The `guest-os` entries in `api-inventory.json` whose `service` is `Worker`, +and the per-field, per-representation and per-method vectors in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py) and +[`api-contract-matrix-case.py`](../../../../shared/automation/api-contract-matrix-case.py). + +## Steps + + +### Step 1: Resolve the listener and the indexed contract + +Resolve the listener from the fixture and load the `Worker` inventory entries. + +**Expected results:** + +- The listener is reachable and every declared method has a policy. + + +### Step 2: Drive the matrix + +Send every vector the policy allows, recording status, content type, body size +and elapsed time of each call. + +**Expected results:** + +- **L1** No request produces a 5xx, a dropped connection or a transport + failure. Release builds use `panic = "abort"`, so a reachable panic aborts + the process. +- **L2** A rejected JSON request gets a JSON `error` string; a rejected + protobuf request gets a decodable `ProtoError`. +- **L4** No rejection echoes the unknown-field marker or grows with the size + of the field it refused. +- **L5** No request exceeds the per-call deadline. +- **L6** A rejection's `Content-Type` matches the request's representation. + + +### Step 3: Verify the listener survived + +Send one valid request after the matrix. + +**Expected results:** + +- **L3** The listener answers it. + +## Postconditions + +Remove run-scoped objects the `full` policy created. Keep the per-call +observation table in the result artifacts. diff --git a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-003/metadata.json b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-003/metadata.json new file mode 100644 index 000000000..49e36264e --- /dev/null +++ b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-003/metadata.json @@ -0,0 +1,36 @@ +{ + "id": "tc-gos-contract-003", + "title": "Worker request-contract matrix", + "priority": "P0", + "requirements": [ + "req-gos-contract-003" + ], + "risks": [ + "risk-gos-contract-003" + ], + "tags": [ + "guest", + "request-contract" + ], + "fixture": { + "profile": "no-tee-dev", + "versions": { + "vmm": "candidate", + "guest": "candidate", + "kms": "candidate", + "gateway": "candidate", + "verifier": "candidate" + }, + "destructive_scope": "lease-only", + "hardware_required": false, + "simulation_allowed": true + }, + "actions_under_test": [ + "Worker.GetAttestationForAppKey" + ], + "execution": { + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "args": [], + "timeout_seconds": 900 + } +} diff --git a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-004/case.md b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-004/case.md new file mode 100644 index 000000000..1714fcaf4 --- /dev/null +++ b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-004/case.md @@ -0,0 +1,86 @@ + + + +# TC-GOS-CONTRACT-004: GuestApi request-contract matrix + +## Metadata + +- Priority: P0 +- Type: Functional, API, Security, Robustness, Regression +- Minimum environment: INTEGRATION +- Automation: Yes +- Requirements: [req-gos-contract-004](../../../../catalog/feature-audit.md#req-gos-contract-004) +- Risks: [risk-gos-contract-004](../../../../catalog/feature-audit.md#risk-gos-contract-004) +- Source: [`catalog/api-inventory.json`](../../../../catalog/api-inventory.json), [`shared/automation/api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + +## Prepared execution knowledge + +- Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. +- Methods, request fields and field types come from `api-inventory.json`, not + from implementation source. +- `api-contract-matrix-policy.json` sets how far each method is driven: `full` + (effect confined to the lease), `rejection-only` (only inputs refused before + dispatch) or `skip` (a valid call ends the process or consumes an external + resource). A method without a policy fails the case. + +## Objective + +Verify that every `GuestApi` method handles absent, valid, boundary-invalid, +wrong-typed, unknown-field and malformed-framing input, in JSON and protobuf, +without breaking the invariants below. + +## Preconditions + +1. The guest agent's host-facing listener is reachable on a lease-owned fixture. The matrix drives + mutating methods with run-scoped identifiers, so it must never target a + shared deployment. + +## Test Data + +The `guest-os` entries in `api-inventory.json` whose `service` is `GuestApi`, +and the per-field, per-representation and per-method vectors in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py) and +[`api-contract-matrix-case.py`](../../../../shared/automation/api-contract-matrix-case.py). + +## Steps + + +### Step 1: Resolve the listener and the indexed contract + +Resolve the listener from the fixture and load the `GuestApi` inventory entries. + +**Expected results:** + +- The listener is reachable and every declared method has a policy. + + +### Step 2: Drive the matrix + +Send every vector the policy allows, recording status, content type, body size +and elapsed time of each call. + +**Expected results:** + +- **L1** No request produces a 5xx, a dropped connection or a transport + failure. Release builds use `panic = "abort"`, so a reachable panic aborts + the process. +- **L2** A rejected JSON request gets a JSON `error` string; a rejected + protobuf request gets a decodable `ProtoError`. +- **L4** No rejection echoes the unknown-field marker or grows with the size + of the field it refused. +- **L5** No request exceeds the per-call deadline. +- **L6** A rejection's `Content-Type` matches the request's representation. + + +### Step 3: Verify the listener survived + +Send one valid request after the matrix. + +**Expected results:** + +- **L3** The listener answers it. + +## Postconditions + +Remove run-scoped objects the `full` policy created. Keep the per-call +observation table in the result artifacts. diff --git a/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-004/metadata.json b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-004/metadata.json new file mode 100644 index 000000000..9c72901b7 --- /dev/null +++ b/test-suites/cases/01-guest-os/15-request-contract/tc-gos-contract-004/metadata.json @@ -0,0 +1,36 @@ +{ + "id": "tc-gos-contract-004", + "title": "GuestApi request-contract matrix", + "priority": "P0", + "requirements": [ + "req-gos-contract-004" + ], + "risks": [ + "risk-gos-contract-004" + ], + "tags": [ + "guest", + "request-contract" + ], + "fixture": { + "profile": "no-tee-dev", + "versions": { + "vmm": "candidate", + "guest": "candidate", + "kms": "candidate", + "gateway": "candidate", + "verifier": "candidate" + }, + "destructive_scope": "lease-only", + "hardware_required": false, + "simulation_allowed": true + }, + "actions_under_test": [ + "GuestApi.GpuInfo" + ], + "execution": { + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "args": [], + "timeout_seconds": 900 + } +} diff --git a/test-suites/cases/03-kms/13-request-contract/metadata.json b/test-suites/cases/03-kms/13-request-contract/metadata.json new file mode 100644 index 000000000..8ef6a2591 --- /dev/null +++ b/test-suites/cases/03-kms/13-request-contract/metadata.json @@ -0,0 +1,4 @@ +{ + "id": "section-kms-request-contract", + "title": "Request contract matrix" +} diff --git a/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-001/case.md b/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-001/case.md new file mode 100644 index 000000000..8712cbcde --- /dev/null +++ b/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-001/case.md @@ -0,0 +1,86 @@ + + + +# TC-KMS-CONTRACT-001: KMS request-contract matrix + +## Metadata + +- Priority: P0 +- Type: Functional, API, Security, Robustness, Regression +- Minimum environment: INTEGRATION +- Automation: Yes +- Requirements: [req-kms-contract-001](../../../../catalog/feature-audit.md#req-kms-contract-001) +- Risks: [risk-kms-contract-001](../../../../catalog/feature-audit.md#risk-kms-contract-001) +- Source: [`catalog/api-inventory.json`](../../../../catalog/api-inventory.json), [`shared/automation/api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + +## Prepared execution knowledge + +- Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. +- Methods, request fields and field types come from `api-inventory.json`, not + from implementation source. +- `api-contract-matrix-policy.json` sets how far each method is driven: `full` + (effect confined to the lease), `rejection-only` (only inputs refused before + dispatch) or `skip` (a valid call ends the process or consumes an external + resource). A method without a policy fails the case. + +## Objective + +Verify that every `KMS` method handles absent, valid, boundary-invalid, +wrong-typed, unknown-field and malformed-framing input, in JSON and protobuf, +without breaking the invariants below. + +## Preconditions + +1. The KMS public RPC listener is reachable on a lease-owned fixture. The matrix drives + mutating methods with run-scoped identifiers, so it must never target a + shared deployment. + +## Test Data + +The `kms` entries in `api-inventory.json` whose `service` is `KMS`, +and the per-field, per-representation and per-method vectors in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py) and +[`api-contract-matrix-case.py`](../../../../shared/automation/api-contract-matrix-case.py). + +## Steps + + +### Step 1: Resolve the listener and the indexed contract + +Resolve the listener from the fixture and load the `KMS` inventory entries. + +**Expected results:** + +- The listener is reachable and every declared method has a policy. + + +### Step 2: Drive the matrix + +Send every vector the policy allows, recording status, content type, body size +and elapsed time of each call. + +**Expected results:** + +- **L1** No request produces a 5xx, a dropped connection or a transport + failure. Release builds use `panic = "abort"`, so a reachable panic aborts + the process. +- **L2** A rejected JSON request gets a JSON `error` string; a rejected + protobuf request gets a decodable `ProtoError`. +- **L4** No rejection echoes the unknown-field marker or grows with the size + of the field it refused. +- **L5** No request exceeds the per-call deadline. +- **L6** A rejection's `Content-Type` matches the request's representation. + + +### Step 3: Verify the listener survived + +Send one valid request after the matrix. + +**Expected results:** + +- **L3** The listener answers it. + +## Postconditions + +Remove run-scoped objects the `full` policy created. Keep the per-call +observation table in the result artifacts. diff --git a/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-001/metadata.json b/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-001/metadata.json new file mode 100644 index 000000000..466399ecf --- /dev/null +++ b/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-001/metadata.json @@ -0,0 +1,36 @@ +{ + "id": "tc-kms-contract-001", + "title": "KMS request-contract matrix", + "priority": "P0", + "requirements": [ + "req-kms-contract-001" + ], + "risks": [ + "risk-kms-contract-001" + ], + "tags": [ + "kms", + "request-contract" + ], + "fixture": { + "profile": "kms-ready", + "versions": { + "vmm": "candidate", + "guest": "candidate", + "kms": "candidate", + "gateway": "candidate", + "verifier": "candidate" + }, + "destructive_scope": "lease-only", + "hardware_required": false, + "simulation_allowed": false + }, + "actions_under_test": [ + "KMS.GetMeta" + ], + "execution": { + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "args": [], + "timeout_seconds": 900 + } +} diff --git a/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-002/case.md b/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-002/case.md new file mode 100644 index 000000000..95663d362 --- /dev/null +++ b/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-002/case.md @@ -0,0 +1,86 @@ + + + +# TC-KMS-CONTRACT-002: Onboard request-contract matrix + +## Metadata + +- Priority: P0 +- Type: Functional, API, Security, Robustness, Regression +- Minimum environment: INTEGRATION +- Automation: Yes +- Requirements: [req-kms-contract-002](../../../../catalog/feature-audit.md#req-kms-contract-002) +- Risks: [risk-kms-contract-002](../../../../catalog/feature-audit.md#risk-kms-contract-002) +- Source: [`catalog/api-inventory.json`](../../../../catalog/api-inventory.json), [`shared/automation/api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + +## Prepared execution knowledge + +- Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. +- Methods, request fields and field types come from `api-inventory.json`, not + from implementation source. +- `api-contract-matrix-policy.json` sets how far each method is driven: `full` + (effect confined to the lease), `rejection-only` (only inputs refused before + dispatch) or `skip` (a valid call ends the process or consumes an external + resource). A method without a policy fails the case. + +## Objective + +Verify that every `Onboard` method handles absent, valid, boundary-invalid, +wrong-typed, unknown-field and malformed-framing input, in JSON and protobuf, +without breaking the invariants below. + +## Preconditions + +1. The KMS onboarding listener is reachable on a lease-owned fixture. The matrix drives + mutating methods with run-scoped identifiers, so it must never target a + shared deployment. + +## Test Data + +The `kms` entries in `api-inventory.json` whose `service` is `Onboard`, +and the per-field, per-representation and per-method vectors in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py) and +[`api-contract-matrix-case.py`](../../../../shared/automation/api-contract-matrix-case.py). + +## Steps + + +### Step 1: Resolve the listener and the indexed contract + +Resolve the listener from the fixture and load the `Onboard` inventory entries. + +**Expected results:** + +- The listener is reachable and every declared method has a policy. + + +### Step 2: Drive the matrix + +Send every vector the policy allows, recording status, content type, body size +and elapsed time of each call. + +**Expected results:** + +- **L1** No request produces a 5xx, a dropped connection or a transport + failure. Release builds use `panic = "abort"`, so a reachable panic aborts + the process. +- **L2** A rejected JSON request gets a JSON `error` string; a rejected + protobuf request gets a decodable `ProtoError`. +- **L4** No rejection echoes the unknown-field marker or grows with the size + of the field it refused. +- **L5** No request exceeds the per-call deadline. +- **L6** A rejection's `Content-Type` matches the request's representation. + + +### Step 3: Verify the listener survived + +Send one valid request after the matrix. + +**Expected results:** + +- **L3** The listener answers it. + +## Postconditions + +Remove run-scoped objects the `full` policy created. Keep the per-call +observation table in the result artifacts. diff --git a/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-002/metadata.json b/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-002/metadata.json new file mode 100644 index 000000000..b0e1f7dc8 --- /dev/null +++ b/test-suites/cases/03-kms/13-request-contract/tc-kms-contract-002/metadata.json @@ -0,0 +1,36 @@ +{ + "id": "tc-kms-contract-002", + "title": "Onboard request-contract matrix", + "priority": "P0", + "requirements": [ + "req-kms-contract-002" + ], + "risks": [ + "risk-kms-contract-002" + ], + "tags": [ + "kms", + "request-contract" + ], + "fixture": { + "profile": "kms-onboard", + "versions": { + "vmm": "candidate", + "guest": "candidate", + "kms": "candidate", + "gateway": "candidate", + "verifier": "candidate" + }, + "destructive_scope": "lease-only", + "hardware_required": false, + "simulation_allowed": false + }, + "actions_under_test": [ + "Onboard.GetAttestationInfo" + ], + "execution": { + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "args": [], + "timeout_seconds": 900 + } +} diff --git a/test-suites/cases/04-gateway/11-request-contract/metadata.json b/test-suites/cases/04-gateway/11-request-contract/metadata.json new file mode 100644 index 000000000..2485b1147 --- /dev/null +++ b/test-suites/cases/04-gateway/11-request-contract/metadata.json @@ -0,0 +1,4 @@ +{ + "id": "section-gateway-request-contract", + "title": "Request contract matrix" +} diff --git a/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-001/case.md b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-001/case.md new file mode 100644 index 000000000..f5149cc6f --- /dev/null +++ b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-001/case.md @@ -0,0 +1,86 @@ + + + +# TC-GW-CONTRACT-001: Gateway request-contract matrix + +## Metadata + +- Priority: P0 +- Type: Functional, API, Security, Robustness, Regression +- Minimum environment: INTEGRATION +- Automation: Yes +- Requirements: [req-gw-contract-001](../../../../catalog/feature-audit.md#req-gw-contract-001) +- Risks: [risk-gw-contract-001](../../../../catalog/feature-audit.md#risk-gw-contract-001) +- Source: [`catalog/api-inventory.json`](../../../../catalog/api-inventory.json), [`shared/automation/api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + +## Prepared execution knowledge + +- Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. +- Methods, request fields and field types come from `api-inventory.json`, not + from implementation source. +- `api-contract-matrix-policy.json` sets how far each method is driven: `full` + (effect confined to the lease), `rejection-only` (only inputs refused before + dispatch) or `skip` (a valid call ends the process or consumes an external + resource). A method without a policy fails the case. + +## Objective + +Verify that every `Gateway` method handles absent, valid, boundary-invalid, +wrong-typed, unknown-field and malformed-framing input, in JSON and protobuf, +without breaking the invariants below. + +## Preconditions + +1. The gateway's public RPC listener is reachable on a lease-owned fixture. The matrix drives + mutating methods with run-scoped identifiers, so it must never target a + shared deployment. + +## Test Data + +The `gateway` entries in `api-inventory.json` whose `service` is `Gateway`, +and the per-field, per-representation and per-method vectors in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py) and +[`api-contract-matrix-case.py`](../../../../shared/automation/api-contract-matrix-case.py). + +## Steps + + +### Step 1: Resolve the listener and the indexed contract + +Resolve the listener from the fixture and load the `Gateway` inventory entries. + +**Expected results:** + +- The listener is reachable and every declared method has a policy. + + +### Step 2: Drive the matrix + +Send every vector the policy allows, recording status, content type, body size +and elapsed time of each call. + +**Expected results:** + +- **L1** No request produces a 5xx, a dropped connection or a transport + failure. Release builds use `panic = "abort"`, so a reachable panic aborts + the process. +- **L2** A rejected JSON request gets a JSON `error` string; a rejected + protobuf request gets a decodable `ProtoError`. +- **L4** No rejection echoes the unknown-field marker or grows with the size + of the field it refused. +- **L5** No request exceeds the per-call deadline. +- **L6** A rejection's `Content-Type` matches the request's representation. + + +### Step 3: Verify the listener survived + +Send one valid request after the matrix. + +**Expected results:** + +- **L3** The listener answers it. + +## Postconditions + +Remove run-scoped objects the `full` policy created. Keep the per-call +observation table in the result artifacts. diff --git a/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-001/metadata.json b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-001/metadata.json new file mode 100644 index 000000000..d88d6023e --- /dev/null +++ b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-001/metadata.json @@ -0,0 +1,36 @@ +{ + "id": "tc-gw-contract-001", + "title": "Gateway request-contract matrix", + "priority": "P0", + "requirements": [ + "req-gw-contract-001" + ], + "risks": [ + "risk-gw-contract-001" + ], + "tags": [ + "gateway", + "request-contract" + ], + "fixture": { + "profile": "gateway-cluster", + "versions": { + "vmm": "candidate", + "guest": "candidate", + "kms": "candidate", + "gateway": "candidate", + "verifier": "candidate" + }, + "destructive_scope": "lease-only", + "hardware_required": false, + "simulation_allowed": false + }, + "actions_under_test": [ + "Gateway.AcmeInfo" + ], + "execution": { + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "args": [], + "timeout_seconds": 900 + } +} diff --git a/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-002/case.md b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-002/case.md new file mode 100644 index 000000000..2abedeb07 --- /dev/null +++ b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-002/case.md @@ -0,0 +1,86 @@ + + + +# TC-GW-CONTRACT-002: Debug request-contract matrix + +## Metadata + +- Priority: P0 +- Type: Functional, API, Security, Robustness, Regression +- Minimum environment: INTEGRATION +- Automation: Yes +- Requirements: [req-gw-contract-002](../../../../catalog/feature-audit.md#req-gw-contract-002) +- Risks: [risk-gw-contract-002](../../../../catalog/feature-audit.md#risk-gw-contract-002) +- Source: [`catalog/api-inventory.json`](../../../../catalog/api-inventory.json), [`shared/automation/api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + +## Prepared execution knowledge + +- Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. +- Methods, request fields and field types come from `api-inventory.json`, not + from implementation source. +- `api-contract-matrix-policy.json` sets how far each method is driven: `full` + (effect confined to the lease), `rejection-only` (only inputs refused before + dispatch) or `skip` (a valid call ends the process or consumes an external + resource). A method without a policy fails the case. + +## Objective + +Verify that every `Debug` method handles absent, valid, boundary-invalid, +wrong-typed, unknown-field and malformed-framing input, in JSON and protobuf, +without breaking the invariants below. + +## Preconditions + +1. The gateway's debug listener is reachable on a lease-owned fixture. The matrix drives + mutating methods with run-scoped identifiers, so it must never target a + shared deployment. + +## Test Data + +The `gateway` entries in `api-inventory.json` whose `service` is `Debug`, +and the per-field, per-representation and per-method vectors in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py) and +[`api-contract-matrix-case.py`](../../../../shared/automation/api-contract-matrix-case.py). + +## Steps + + +### Step 1: Resolve the listener and the indexed contract + +Resolve the listener from the fixture and load the `Debug` inventory entries. + +**Expected results:** + +- The listener is reachable and every declared method has a policy. + + +### Step 2: Drive the matrix + +Send every vector the policy allows, recording status, content type, body size +and elapsed time of each call. + +**Expected results:** + +- **L1** No request produces a 5xx, a dropped connection or a transport + failure. Release builds use `panic = "abort"`, so a reachable panic aborts + the process. +- **L2** A rejected JSON request gets a JSON `error` string; a rejected + protobuf request gets a decodable `ProtoError`. +- **L4** No rejection echoes the unknown-field marker or grows with the size + of the field it refused. +- **L5** No request exceeds the per-call deadline. +- **L6** A rejection's `Content-Type` matches the request's representation. + + +### Step 3: Verify the listener survived + +Send one valid request after the matrix. + +**Expected results:** + +- **L3** The listener answers it. + +## Postconditions + +Remove run-scoped objects the `full` policy created. Keep the per-call +observation table in the result artifacts. diff --git a/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-002/metadata.json b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-002/metadata.json new file mode 100644 index 000000000..1c4fc046a --- /dev/null +++ b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-002/metadata.json @@ -0,0 +1,36 @@ +{ + "id": "tc-gw-contract-002", + "title": "Debug request-contract matrix", + "priority": "P0", + "requirements": [ + "req-gw-contract-002" + ], + "risks": [ + "risk-gw-contract-002" + ], + "tags": [ + "gateway", + "request-contract" + ], + "fixture": { + "profile": "gateway-cluster", + "versions": { + "vmm": "candidate", + "guest": "candidate", + "kms": "candidate", + "gateway": "candidate", + "verifier": "candidate" + }, + "destructive_scope": "lease-only", + "hardware_required": false, + "simulation_allowed": false + }, + "actions_under_test": [ + "Debug.GetSyncData" + ], + "execution": { + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "args": [], + "timeout_seconds": 900 + } +} diff --git a/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-003/case.md b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-003/case.md new file mode 100644 index 000000000..1e01205ba --- /dev/null +++ b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-003/case.md @@ -0,0 +1,86 @@ + + + +# TC-GW-CONTRACT-003: Admin request-contract matrix + +## Metadata + +- Priority: P0 +- Type: Functional, API, Security, Robustness, Regression +- Minimum environment: INTEGRATION +- Automation: Yes +- Requirements: [req-gw-contract-003](../../../../catalog/feature-audit.md#req-gw-contract-003) +- Risks: [risk-gw-contract-003](../../../../catalog/feature-audit.md#risk-gw-contract-003) +- Source: [`catalog/api-inventory.json`](../../../../catalog/api-inventory.json), [`shared/automation/api-contract-matrix-policy.json`](../../../../shared/automation/api-contract-matrix-policy.json) + +## Prepared execution knowledge + +- Read and obey [`shared/automation/execution-guide.md`](../../../../shared/automation/execution-guide.md) before executing Step 1. +- Methods, request fields and field types come from `api-inventory.json`, not + from implementation source. +- `api-contract-matrix-policy.json` sets how far each method is driven: `full` + (effect confined to the lease), `rejection-only` (only inputs refused before + dispatch) or `skip` (a valid call ends the process or consumes an external + resource). A method without a policy fails the case. + +## Objective + +Verify that every `Admin` method handles absent, valid, boundary-invalid, +wrong-typed, unknown-field and malformed-framing input, in JSON and protobuf, +without breaking the invariants below. + +## Preconditions + +1. The gateway's authenticated admin listener is reachable on a lease-owned fixture. The matrix drives + mutating methods with run-scoped identifiers, so it must never target a + shared deployment. + +## Test Data + +The `gateway` entries in `api-inventory.json` whose `service` is `Admin`, +and the per-field, per-representation and per-method vectors in +[`api_contract_matrix.py`](../../../../shared/automation/api_contract_matrix.py) and +[`api-contract-matrix-case.py`](../../../../shared/automation/api-contract-matrix-case.py). + +## Steps + + +### Step 1: Resolve the listener and the indexed contract + +Resolve the listener from the fixture and load the `Admin` inventory entries. + +**Expected results:** + +- The listener is reachable and every declared method has a policy. + + +### Step 2: Drive the matrix + +Send every vector the policy allows, recording status, content type, body size +and elapsed time of each call. + +**Expected results:** + +- **L1** No request produces a 5xx, a dropped connection or a transport + failure. Release builds use `panic = "abort"`, so a reachable panic aborts + the process. +- **L2** A rejected JSON request gets a JSON `error` string; a rejected + protobuf request gets a decodable `ProtoError`. +- **L4** No rejection echoes the unknown-field marker or grows with the size + of the field it refused. +- **L5** No request exceeds the per-call deadline. +- **L6** A rejection's `Content-Type` matches the request's representation. + + +### Step 3: Verify the listener survived + +Send one valid request after the matrix. + +**Expected results:** + +- **L3** The listener answers it. + +## Postconditions + +Remove run-scoped objects the `full` policy created. Keep the per-call +observation table in the result artifacts. diff --git a/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-003/metadata.json b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-003/metadata.json new file mode 100644 index 000000000..7db673318 --- /dev/null +++ b/test-suites/cases/04-gateway/11-request-contract/tc-gw-contract-003/metadata.json @@ -0,0 +1,36 @@ +{ + "id": "tc-gw-contract-003", + "title": "Admin request-contract matrix", + "priority": "P0", + "requirements": [ + "req-gw-contract-003" + ], + "risks": [ + "risk-gw-contract-003" + ], + "tags": [ + "gateway", + "request-contract" + ], + "fixture": { + "profile": "gateway-cluster", + "versions": { + "vmm": "candidate", + "guest": "candidate", + "kms": "candidate", + "gateway": "candidate", + "verifier": "candidate" + }, + "destructive_scope": "lease-only", + "hardware_required": false, + "simulation_allowed": false + }, + "actions_under_test": [ + "Admin.ListDnsCredentials" + ], + "execution": { + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "args": [], + "timeout_seconds": 900 + } +} diff --git a/test-suites/catalog/feature-audit.md b/test-suites/catalog/feature-audit.md index ca4357e0f..371372019 100644 --- a/test-suites/catalog/feature-audit.md +++ b/test-suites/catalog/feature-audit.md @@ -709,6 +709,20 @@ This audit is derived from the repository source inventory and is the traceabili | `req-int-mixed-006` | `risk-int-mixed-006` | [tc-int-mixed-006](../cases/06-integration/04-pinned-mixed-version-matrix/tc-int-mixed-006/case.md#tc-int-mixed-006) — Rolling restart under four-version online mix | P0 | | `req-int-mixed-007` | `risk-int-mixed-007` | [tc-int-mixed-007](../cases/06-integration/04-pinned-mixed-version-matrix/tc-int-mixed-007/case.md#tc-int-mixed-007) — Optional and unknown protobuf fields across pinned versions | P0 | +## Request contract matrix + +| Requirement | Risk | Case | Priority | +|---|---|---|---| +| `req-gos-contract-001` | `risk-gos-contract-001` | [tc-gos-contract-001](../cases/01-guest-os/15-request-contract/tc-gos-contract-001/case.md#tc-gos-contract-001) — Tappd request-contract matrix | P0 | +| `req-gos-contract-002` | `risk-gos-contract-002` | [tc-gos-contract-002](../cases/01-guest-os/15-request-contract/tc-gos-contract-002/case.md#tc-gos-contract-002) — DstackGuest request-contract matrix | P0 | +| `req-gos-contract-003` | `risk-gos-contract-003` | [tc-gos-contract-003](../cases/01-guest-os/15-request-contract/tc-gos-contract-003/case.md#tc-gos-contract-003) — Worker request-contract matrix | P0 | +| `req-gos-contract-004` | `risk-gos-contract-004` | [tc-gos-contract-004](../cases/01-guest-os/15-request-contract/tc-gos-contract-004/case.md#tc-gos-contract-004) — GuestApi request-contract matrix | P0 | +| `req-kms-contract-001` | `risk-kms-contract-001` | [tc-kms-contract-001](../cases/03-kms/13-request-contract/tc-kms-contract-001/case.md#tc-kms-contract-001) — KMS request-contract matrix | P0 | +| `req-kms-contract-002` | `risk-kms-contract-002` | [tc-kms-contract-002](../cases/03-kms/13-request-contract/tc-kms-contract-002/case.md#tc-kms-contract-002) — Onboard request-contract matrix | P0 | +| `req-gw-contract-001` | `risk-gw-contract-001` | [tc-gw-contract-001](../cases/04-gateway/11-request-contract/tc-gw-contract-001/case.md#tc-gw-contract-001) — Gateway request-contract matrix | P0 | +| `req-gw-contract-002` | `risk-gw-contract-002` | [tc-gw-contract-002](../cases/04-gateway/11-request-contract/tc-gw-contract-002/case.md#tc-gw-contract-002) — Debug request-contract matrix | P0 | +| `req-gw-contract-003` | `risk-gw-contract-003` | [tc-gw-contract-003](../cases/04-gateway/11-request-contract/tc-gw-contract-003/case.md#tc-gw-contract-003) — Admin request-contract matrix | P0 | + ## Guest-agent concurrency and robustness | Requirement | Risk | Case | Priority | diff --git a/test-suites/shared/automation/api-contract-matrix-case.py b/test-suites/shared/automation/api-contract-matrix-case.py new file mode 100755 index 000000000..1999b7a56 --- /dev/null +++ b/test-suites/shared/automation/api-contract-matrix-case.py @@ -0,0 +1,710 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +r"""Run the request-contract matrix in ``api_contract_matrix`` against one service. + +``api-contract-matrix-policy.json`` says how far each method may be driven: +``full`` runs every vector, ``rejection-only`` runs only the vectors a service +must refuse before it dispatches, and ``skip`` is for methods whose valid +invocation ends the process or consumes an external resource. + +Standalone use for development: + + api-contract-matrix-case.py --standalone \\ + --component guest-os --service Tappd \\ + --socket /run/dstack.sock --route '/prpc/Tappd.' +""" + +from __future__ import annotations + +import argparse +import copy +import json +import os +import pathlib +import sys +import tempfile +from typing import Any + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +from api_contract_matrix import ( # noqa: E402 + Call, + Target, + check_invariants, + encode_request, + field_vectors, + invoke, + malformed_bodies, + varint, +) + +HERE = pathlib.Path(__file__).resolve().parent +UNKNOWN_JSON_FIELD = "dstack_contract_matrix_unknown_field" +ECHO_MARKER = "DSTACK-ECHO-MARKER-7f3a1c" +# A run-scoped name no fixture provisions, so lookups miss and creates cannot collide. +SCOPED = "dstack-contract-matrix" + +# A valid request body per method; the per-field vectors mutate this baseline. +VALID_PAYLOADS: dict[str, dict[str, Any]] = { + "Tappd.DeriveKey": { + "path": f"{SCOPED}/a", + "subject": "localhost", + "alt_names": ["localhost"], + "usage_ra_tls": True, + "usage_server_auth": True, + "usage_client_auth": False, + "random_seed": False, + }, + "Tappd.DeriveK256Key": { + "path": f"{SCOPED}/a", + "purpose": SCOPED, + "algorithm": "k256", + }, + "Tappd.TdxQuote": {"report_data": "55" * 64}, + "Tappd.RawQuote": {"report_data": "11" * 64}, + "Tappd.Info": {}, + "Tappd.Version": {}, + "DstackGuest.GetTlsKey": { + "subject": "localhost", + "alt_names": ["localhost"], + "usage_ra_tls": True, + "usage_server_auth": True, + "usage_client_auth": False, + "not_before": 0, + "not_after": 4102444800, + "with_app_info": True, + }, + "DstackGuest.GetKey": { + "path": f"{SCOPED}/a", + "purpose": SCOPED, + "algorithm": "ed25519", + }, + "DstackGuest.GetQuote": {"report_data": "22" * 64}, + "DstackGuest.Attest": {"report_data": "33" * 64}, + "DstackGuest.Info": {}, + "DstackGuest.Sign": {"algorithm": "ed25519", "data": "44" * 32}, + "DstackGuest.Version": {}, + "DstackGuest.Verify": { + "algorithm": "ed25519", + "data": "44" * 32, + "signature": "00" * 64, + "public_key": "00" * 32, + }, + "Worker.Info": {}, + "Worker.Version": {}, + "Worker.GetAttestationForAppKey": {"algorithm": "ed25519"}, + "GuestApi.Info": {}, + "GuestApi.SysInfo": {}, + "GuestApi.NetworkInfo": {}, + "GuestApi.ListContainers": {}, + "GuestApi.GpuInfo": {}, + "Gateway.AcmeInfo": {}, + "Gateway.Info": {}, + "Gateway.GetPeers": {}, + "Debug.Info": {}, + "Debug.GetSyncData": {}, + "Debug.GetProxyState": {}, + "Admin.Status": {}, + "Admin.GetMeta": {}, + "Admin.WaveKvStatus": {}, + "Admin.GetGlobalConnections": {}, + "Admin.GetNodeStatuses": {}, + "Admin.ListDnsCredentials": {}, + "Admin.GetDefaultDnsCredential": {}, + "Admin.ListZtDomains": {}, + "Admin.GetCertbotConfig": {}, + "Admin.ListRejectedInstances": {}, + "Admin.GetInfo": {"id": f"{SCOPED}-absent-instance"}, + "Admin.GetInstanceHandshakes": {"instance_id": f"{SCOPED}-absent-instance"}, + "Admin.GetDnsCredential": {"id": f"{SCOPED}-absent-credential"}, + "Admin.GetZtDomain": {"domain": f"{SCOPED}.invalid"}, + "Admin.ListCertAttestations": {"domain": f"{SCOPED}.invalid", "limit": 1}, + "Admin.GetInstancePortPolicy": {"instance_id": f"{SCOPED}-absent-instance"}, + "Admin.CreateDnsCredential": { + "name": f"{SCOPED}-credential", + "provider_type": "cloudflare", + "cf_api_token": f"{SCOPED}-token", + "cf_zone_id": f"{SCOPED}-zone", + "set_as_default": False, + "cf_api_url": "http://127.0.0.1:9/never-called", + "dns_txt_ttl": 60, + "max_dns_wait": 1, + }, + "Admin.UpdateDnsCredential": { + "id": f"{SCOPED}-absent-credential", + "name": f"{SCOPED}-credential", + "cf_api_token": f"{SCOPED}-token", + "cf_zone_id": f"{SCOPED}-zone", + "cf_api_url": "http://127.0.0.1:9/never-called", + }, + "Admin.DeleteDnsCredential": {"id": f"{SCOPED}-absent-credential"}, + "Admin.SetDefaultDnsCredential": {"id": f"{SCOPED}-absent-credential"}, + "Admin.AddZtDomain": { + "domain": f"{SCOPED}.invalid", + "dns_cred_id": f"{SCOPED}-absent-credential", + "port": 8443, + "node": 0, + "priority": 0, + "challenge": "dns-01", + }, + "Admin.UpdateZtDomain": { + "domain": f"{SCOPED}.invalid", + "dns_cred_id": f"{SCOPED}-absent-credential", + "port": 8443, + "node": 0, + "priority": 0, + "challenge": "dns-01", + }, + "Admin.DeleteZtDomain": {"domain": f"{SCOPED}.invalid"}, + "Admin.ClearInstancePortPolicy": {"instance_id": f"{SCOPED}-absent-instance"}, + "KMS.GetMeta": {}, + "KMS.GetTempCaCert": {}, + "KMS.GetAppEnvEncryptPubKey": {"app_id": "11" * 20}, + "Onboard.GetAttestationInfo": {}, +} + + +def load_policy() -> dict[str, str]: + """Read the per-method policy table.""" + document = json.loads((HERE / "api-contract-matrix-policy.json").read_text()) + return document["policies"] + + +def atomic_json(path: pathlib.Path, value: Any) -> None: + """Write JSON atomically.""" + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + "w", encoding="utf-8", dir=path.parent, delete=False + ) as output: + json.dump(value, output, ensure_ascii=False, indent=2, sort_keys=True) + output.write("\n") + temporary = pathlib.Path(output.name) + temporary.replace(path) + + +def service_policy( + inventory: dict[str, Any], component: str, service: str +) -> tuple[list[dict[str, Any]], dict[str, str]]: + """Return one service's inventory entries and the policy for each method.""" + entries = [ + entry + for entry in inventory["components"][component]["rpc_methods"] + if entry["service"] == service + ] + if not entries: + raise RuntimeError(f"no inventory entry for {component}/{service}") + policy = load_policy() + applied: dict[str, str] = {} + for entry in entries: + key = f"{service}.{entry['method']}" + applied[key] = policy.get(key, "") + if applied[key] not in ("full", "rejection-only", "skip"): + raise RuntimeError(f"no valid contract-matrix policy for {key}") + return entries, applied + + +def wrong_type_for(field_def: dict[str, Any]) -> Any: + """Return a JSON value of the wrong type for this field.""" + if field_def["type"] in ("string", "bytes"): + return 123 + return {"not": "a scalar"} + + +def wrong_wiretype(target: Target, entry: dict[str, Any]) -> list[Call]: + """Send the first declared field number with the wrong wire type.""" + fields = entry.get("request_fields", []) + if not fields: + return [] + first = fields[0] + wire = 0 if first["type"] in ("string", "bytes") else 2 + body = varint((int(first["number"]) << 3) | wire) + body += b"\x01" if wire == 0 else b"\x01A" + return [ + invoke( + target, + entry["method"], + "application/octet-stream", + body, + f"{entry['service']}.{entry['method']}|wrong-wiretype|protobuf", + "protobuf", + ) + ] + + +def sweep_rejection_only(target: Target, entry: dict[str, Any]) -> list[Call]: + """Vectors a service must refuse before it can dispatch the method.""" + service, method = entry["service"], entry["method"] + key = f"{service}.{method}" + fields = entry.get("request_fields", []) + calls: list[Call] = [] + + # A wrong JSON type for a declared field fails deserialization, so the + # handler never runs and no state can change. + for field_def in fields: + payload = {field_def["name"]: wrong_type_for(field_def)} + calls.append( + invoke( + target, + method, + "application/json", + json.dumps(payload).encode(), + f"{key}|{field_def['name']}=wrong-type|json", + "json", + ) + ) + + calls.extend(wrong_wiretype(target, entry)) + return calls + + +def sweep_framing(target: Target, method: str) -> list[Call]: + """Representation-level malformed bodies, once per service.""" + calls = [] + for label, content_type, body in malformed_bodies(): + representation = "json" if content_type == "application/json" else "protobuf" + calls.append( + invoke( + target, method, content_type, body, f"framing|{label}", representation + ) + ) + calls.append( + invoke( + target, + method, + "application/json", + b"{}", + "framing|unknown-method", + "json", + route_override=target.route(method) + "NoSuchSuffix", + ) + ) + return calls + + +def sweep_full( + target: Target, entry: dict[str, Any], markers: dict[str, str] +) -> list[Call]: + """Build the complete vector set for one method.""" + service, method = entry["service"], entry["method"] + key = f"{service}.{method}" + fields = entry.get("request_fields", []) + calls: list[Call] = [] + + # 1. Absent input: every field default, in both representations. + calls.append( + invoke(target, method, "application/json", b"{}", f"{key}|absent|json", "json") + ) + calls.append( + invoke( + target, + method, + "application/octet-stream", + b"", + f"{key}|absent|protobuf", + "protobuf", + ) + ) + + valid = VALID_PAYLOADS.get(key) + if valid is None: + return calls + + # 2. Valid input in both representations. + calls.append( + invoke( + target, + method, + "application/json", + json.dumps(valid).encode(), + f"{key}|valid|json", + "json", + ) + ) + calls.append( + invoke( + target, + method, + "application/octet-stream", + encode_request(fields, valid), + f"{key}|valid|protobuf", + "protobuf", + ) + ) + + # 3. Unknown field: protobuf must ignore it, JSON must be consistent, and + # neither may echo it back. + with_unknown = {**valid, UNKNOWN_JSON_FIELD: ECHO_MARKER} + label = f"{key}|unknown-field|json" + markers[label] = ECHO_MARKER + calls.append( + invoke( + target, + method, + "application/json", + json.dumps(with_unknown).encode(), + label, + "json", + ) + ) + highest = max((int(f["number"]) for f in fields), default=0) + body = ( + encode_request(fields, valid) + varint(((highest + 1000) << 3) | 0) + varint(7) + ) + calls.append( + invoke( + target, + method, + "application/octet-stream", + body, + f"{key}|unknown-field|protobuf", + "protobuf", + ) + ) + + # 4. Per-field adversarial vectors over the valid baseline. + for field_def in fields: + for name, value in field_vectors(field_def): + payload = copy.deepcopy(valid) + payload[field_def["name"]] = value + calls.append( + invoke( + target, + method, + "application/json", + json.dumps(payload).encode(), + f"{key}|{field_def['name']}={name}|json", + "json", + ) + ) + + # 5. The same field vectors in the protobuf representation, where encodable. + for field_def in fields: + kind = field_def["type"] + if kind == "bytes": + candidates = [("boundary-63", "11" * 63), ("boundary-65", "11" * 65)] + elif kind == "string": + candidates = [("long-1m", "A" * (1 << 20)), ("empty", "")] + elif kind.startswith(("uint", "int", "sint", "fixed", "sfixed")): + candidates = [("u64-max", (1 << 64) - 1)] + else: + continue + for name, value in candidates: + payload = copy.deepcopy(valid) + payload[field_def["name"]] = [value] if field_def.get("repeated") else value + try: + body = encode_request(fields, payload) + except (ValueError, TypeError, OverflowError): + continue + calls.append( + invoke( + target, + method, + "application/octet-stream", + body, + f"{key}|{field_def['name']}={name}|protobuf", + "protobuf", + ) + ) + + # 6. Wrong wire type for a declared field number. + calls.extend(wrong_wiretype(target, entry)) + return calls + + +def run( + target: Target, inventory: dict[str, Any], component: str, service: str +) -> dict[str, Any]: + """Run the matrix for one service and return the evidence document.""" + entries, applied = service_policy(inventory, component, service) + markers: dict[str, str] = {} + calls: list[Call] = [] + for entry in entries: + key = f"{service}.{entry['method']}" + decision = applied[key] + if decision == "skip": + continue + if decision == "rejection-only": + calls.extend(sweep_rejection_only(target, entry)) + else: + calls.extend(sweep_full(target, entry, markers)) + + # A method driven at `full` carries the representation-level vectors and + # is the liveness probe. + live = next( + ( + entry["method"] + for entry in entries + if applied.get(f"{service}.{entry['method']}") == "full" + and f"{service}.{entry['method']}" in VALID_PAYLOADS + ), + None, + ) + if live is not None: + calls.extend(sweep_framing(target, live)) + + violations = check_invariants(calls, markers) + + if live is not None: + liveness = invoke( + target, + live, + "application/json", + json.dumps(VALID_PAYLOADS[f"{service}.{live}"]).encode(), + "liveness|final", + "json", + ) + if liveness.transport_error is not None or liveness.http != 200: + violations.append( + { + "label": "liveness|final", + "method": live, + "representation": "json", + "http": liveness.http, + "seconds": round(liveness.seconds, 3), + "invariant": "L3", + "detail": "the listener did not answer a valid request after the " + f"matrix: {liveness.transport_error or liveness.text[:256]}", + } + ) + + histogram: dict[str, int] = {} + for call in calls: + bucket = "transport-error" if call.transport_error else str(call.http) + histogram[bucket] = histogram.get(bucket, 0) + 1 + + return { + "component": component, + "service": service, + "policy": applied, + "calls": len(calls), + "liveness_method": live, + "status_histogram": histogram, + "max_seconds": round(max((c.seconds for c in calls), default=0.0), 3), + "violations": violations, + "observations": [ + { + "label": call.label, + "http": call.http, + "bytes": len(call.body), + "seconds": round(call.seconds, 3), + "transport_error": call.transport_error, + "response_content_type": call.response_content_type, + } + for call in calls + ], + } + + +# case_id -> (component, service, fixture selector) +CASES: dict[str, tuple[str, str, str]] = { + "tc-gos-contract-001": ("guest-os", "Tappd", "guest-socket"), + "tc-gos-contract-002": ("guest-os", "DstackGuest", "guest-socket"), + "tc-gos-contract-003": ("guest-os", "Worker", "guest-socket"), + "tc-gos-contract-004": ("guest-os", "GuestApi", "guest-socket"), + "tc-gw-contract-001": ("gateway", "Gateway", "gateway-rpc"), + "tc-gw-contract-002": ("gateway", "Debug", "gateway-debug"), + "tc-gw-contract-003": ("gateway", "Admin", "gateway-admin"), + "tc-kms-contract-001": ("kms", "KMS", "kms-rpc"), + "tc-kms-contract-002": ("kms", "Onboard", "kms-onboard"), +} + + +def resolve_target(manifest: dict[str, Any], selector: str, service: str) -> Target: + """Build the request target from the lease-owned fixture.""" + values = manifest["values"] + if selector == "guest-socket": + fixture = values["services"][service] + socket = fixture["socket"] + if not pathlib.Path(socket).is_socket(): + raise RuntimeError(f"fixture socket is not available: {socket}") + return Target(route_template=fixture["route"], socket=socket) + + gateway = values.get("gateway") or {} + services = values.get("services") or {} + headers: dict[str, str] = {} + # The gateway fixture publishes each listener with its `/prpc` mount + # already in the url, so the method name is appended directly. The KMS + # `rpc_url` is the one exception and carries no mount. + if selector == "gateway-rpc": + base = gateway.get("rpc_url") or (services.get("rpc") or {}).get("url") + route = "/" + elif selector == "gateway-debug": + base = gateway.get("debug_url") or (services.get("debug") or {}).get("url") + route = "/" + elif selector == "gateway-admin": + base = gateway.get("admin_url") or (services.get("admin") or {}).get("url") + route = "/" + token_file = gateway.get("admin_auth_token_file") or ( + services.get("admin") or {} + ).get("auth_token_file") + if token_file: + token = pathlib.Path(token_file).read_text(encoding="utf-8").strip() + if token: + headers["Authorization"] = f"Bearer {token}" + elif selector == "kms-rpc": + kms = values["kms"] + base = kms.get("rpc_url") or kms.get("url") + route = "/prpc/" + elif selector == "kms-onboard": + # The fixture publishes the onboarding listener with its `/prpc` mount + # already in the URL, unlike the KMS rpc url. + base = values["services"]["onboard"]["url"] + route = "/" + else: + raise RuntimeError(f"unsupported fixture selector: {selector}") + if not base: + raise RuntimeError(f"the fixture manifest has no url for {selector}") + return Target( + route_template=route, + base_url=str(base).rstrip("/"), + headers=headers, + curl_extra=["--insecure"], + ) + + +def standalone() -> int: + """Development entrypoint.""" + parser = argparse.ArgumentParser() + parser.add_argument("--standalone", action="store_true") + parser.add_argument("--component", required=True) + parser.add_argument("--service", required=True) + parser.add_argument("--socket") + parser.add_argument("--base-url") + parser.add_argument("--route", required=True) + parser.add_argument("--inventory") + parser.add_argument("--output") + arguments = parser.parse_args() + inventory_path = ( + pathlib.Path(arguments.inventory) + if arguments.inventory + else HERE.parent.parent / "catalog" / "api-inventory.json" + ) + target = Target( + route_template=arguments.route, + socket=arguments.socket, + base_url=arguments.base_url, + curl_extra=[] if arguments.socket else ["--insecure"], + ) + report = run( + target, + json.loads(inventory_path.read_text()), + arguments.component, + arguments.service, + ) + if arguments.output: + pathlib.Path(arguments.output).write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n" + ) + print( + json.dumps( + { + "service": report["service"], + "calls": report["calls"], + "status_histogram": report["status_histogram"], + "violations": report["violations"], + }, + indent=2, + sort_keys=True, + ) + ) + return 1 if report["violations"] else 0 + + +def main() -> int: + """Case-harness entrypoint.""" + if "--standalone" in sys.argv: + return standalone() + case_id = os.environ["DSTACK_TEST_CASE_ID"] + result_dir = pathlib.Path(os.environ["DSTACK_TEST_RESULT_DIR"]) + plan_root = pathlib.Path(os.environ["DSTACK_TEST_PLAN_DIR"]) + manifest = json.loads( + pathlib.Path(os.environ["DSTACK_TEST_CASE_MANIFEST"]).read_text() + ) + component, service, selector = CASES[case_id] + inventory = json.loads((plan_root / "catalog" / "api-inventory.json").read_text()) + artifacts = result_dir / "artifacts" + artifacts.mkdir(parents=True, exist_ok=True) + + steps: list[dict[str, Any]] = [] + status = "PASS" + failure: str | None = None + report: dict[str, Any] = {"case_id": case_id} + + def passed(number: int, observed: str) -> None: + step_id = f"{case_id}-step-{number:02d}" + steps.append({"id": step_id, "status": "PASS", "observed": observed}) + print(f"EVIDENCE {step_id} - {observed}", flush=True) + print(f"STEP {step_id} END - PASS", flush=True) + + def require(violations: list[dict[str, Any]]) -> None: + if violations: + raise AssertionError( + f"{len(violations)} contract invariant violations: " + + json.dumps(violations[:5], sort_keys=True) + ) + + try: + print(f"STEP {case_id}-step-01 START", flush=True) + target = resolve_target(manifest, selector, service) + _, applied = service_policy(inventory, component, service) + passed( + 1, f"The listener resolved and all {len(applied)} methods have a policy." + ) + + print(f"STEP {case_id}-step-02 START", flush=True) + report.update(run(target, inventory, component, service)) + violations = report["violations"] + require([v for v in violations if v["invariant"] != "L3"]) + print(json.dumps(report["status_histogram"], sort_keys=True), flush=True) + passed( + 2, + f"{report['calls']} adversarial requests across {len(applied)} methods " + "held invariants L1, L2, L4, L5 and L6.", + ) + + print(f"STEP {case_id}-step-03 START", flush=True) + require([v for v in violations if v["invariant"] == "L3"]) + passed(3, "The listener answered a valid request after the whole matrix.") + except Exception as error: # noqa: BLE001 - the harness reports, never crashes + status = "FAIL" + failure = f"{type(error).__name__}: {error}" + for number in range(len(steps) + 1, 4): + step_id = f"{case_id}-step-{number:02d}" + steps.append({"id": step_id, "status": "FAIL", "observed": failure}) + print(failure, file=sys.stderr, flush=True) + + report["status"] = status + report["failure"] = failure + atomic_json(artifacts / "api-contract-matrix.json", report) + artifact = { + "name": "API contract matrix", + "path": "artifacts/api-contract-matrix.json", + "step_id": f"{case_id}-step-02", + "description": "Every request field of every indexed method under absent, " + "default, valid, boundary-invalid, wrong-type, unknown-field and " + "malformed-framing input, in both representations, with the policy applied " + "to each method and the observed status of every call.", + } + atomic_json(artifacts / "manifest.json", {"artifacts": [artifact]}) + atomic_json( + result_dir / "result.json", + { + "schema_version": "1.0", + "case_id": case_id, + "provisional": False, + "status": status, + "summary": "The request-contract matrix held for every indexed method." + if status == "PASS" + else failure, + "steps": steps, + "artifacts": [artifact], + "remarks": "This confirms RPC request-contract behavior, not physical TEE " + "trust properties.", + }, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test-suites/shared/automation/api-contract-matrix-policy.json b/test-suites/shared/automation/api-contract-matrix-policy.json new file mode 100644 index 000000000..293c230d7 --- /dev/null +++ b/test-suites/shared/automation/api-contract-matrix-policy.json @@ -0,0 +1,94 @@ +{ + "description": "Per-method policy for the request-contract matrix. `full` runs every vector against a lease-owned fixture; `rejection-only` runs only the vectors a service must refuse before it dispatches; `skip` is for methods whose valid invocation ends the process or consumes an external resource. Every method of a service the matrix covers must appear here, or its case fails.", + "policies": { + "Admin.AddZtDomain": "full", + "Admin.ClearInstancePortPolicy": "full", + "Admin.CreateDnsCredential": "full", + "Admin.DeleteDnsCredential": "full", + "Admin.DeleteZtDomain": "full", + "Admin.Exit": "skip", + "Admin.ForceReleaseCertLock": "rejection-only", + "Admin.GetCertbotConfig": "full", + "Admin.GetDefaultDnsCredential": "full", + "Admin.GetDnsCredential": "full", + "Admin.GetGlobalConnections": "full", + "Admin.GetInfo": "full", + "Admin.GetInstanceHandshakes": "full", + "Admin.GetInstancePortPolicy": "full", + "Admin.GetMeta": "full", + "Admin.GetNodeStatuses": "full", + "Admin.GetZtDomain": "full", + "Admin.ListCertAttestations": "full", + "Admin.ListDnsCredentials": "full", + "Admin.ListRejectedInstances": "full", + "Admin.ListZtDomains": "full", + "Admin.ReloadCert": "skip", + "Admin.RemoveCvm": "rejection-only", + "Admin.RemoveNode": "rejection-only", + "Admin.RenewCert": "skip", + "Admin.RenewZtDomainCert": "rejection-only", + "Admin.SetCaa": "skip", + "Admin.SetCertbotConfig": "rejection-only", + "Admin.SetDefaultDnsCredential": "full", + "Admin.SetInstancePortPolicy": "full", + "Admin.SetNodeStatus": "rejection-only", + "Admin.SetNodeUrl": "rejection-only", + "Admin.Status": "full", + "Admin.UpdateDnsCredential": "full", + "Admin.UpdateZtDomain": "full", + "Admin.WaveKvStatus": "full", + "Debug.GetProxyState": "full", + "Debug.GetSyncData": "full", + "Debug.Info": "full", + "Debug.RegisterCvm": "rejection-only", + "DstackGuest.Attest": "full", + "DstackGuest.GetKey": "full", + "DstackGuest.GetQuote": "full", + "DstackGuest.GetTlsKey": "full", + "DstackGuest.Info": "full", + "DstackGuest.Sign": "full", + "DstackGuest.Verify": "full", + "DstackGuest.Version": "full", + "Gateway.AcmeInfo": "full", + "Gateway.GetPeers": "full", + "Gateway.Info": "full", + "Gateway.RegisterCvm": "rejection-only", + "GuestApi.GpuInfo": "full", + "GuestApi.Info": "full", + "GuestApi.ListContainers": "full", + "GuestApi.NetworkInfo": "full", + "GuestApi.Shutdown": "rejection-only", + "GuestApi.SysInfo": "full", + "KMS.GetAppEnvEncryptPubKey": "full", + "KMS.GetAppKey": "rejection-only", + "KMS.GetKmsKey": "rejection-only", + "KMS.GetMeta": "full", + "KMS.GetTempCaCert": "full", + "KMS.SignCert": "rejection-only", + "Onboard.Bootstrap": "skip", + "Onboard.Finish": "skip", + "Onboard.GetAttestationInfo": "full", + "Onboard.Onboard": "skip", + "Tappd.DeriveK256Key": "full", + "Tappd.DeriveKey": "full", + "Tappd.Info": "full", + "Tappd.RawQuote": "full", + "Tappd.TdxQuote": "full", + "Tappd.Version": "full", + "Worker.GetAttestationForAppKey": "full", + "Worker.Info": "full", + "Worker.Version": "full" + }, + "rationale": { + "skip": { + "Admin.Exit": "a valid call stops the gateway, so the fixture could not be reused for the rest of the matrix", + "Admin.RenewCert": "a valid call places an ACME order against the configured directory", + "Admin.ReloadCert": "a valid call replaces the live certificate store for every listener", + "Admin.SetCaa": "a valid call mutates CAA records in a real DNS zone", + "Onboard.Bootstrap": "a valid call fixes the KMS root identity for the life of the deployment", + "Onboard.Onboard": "a valid call makes the KMS fetch root private keys from a caller-supplied URL", + "Onboard.Finish": "a valid call ends the onboarding listener and the bring-up that depends on it" + }, + "rejection-only": "the method mutates state that outlives one case, or requires an attested identity the matrix does not hold, so only inputs that must be refused before dispatch are sent" + } +} diff --git a/test-suites/shared/automation/api_contract_matrix.py b/test-suites/shared/automation/api_contract_matrix.py new file mode 100644 index 000000000..fb1bc680a --- /dev/null +++ b/test-suites/shared/automation/api_contract_matrix.py @@ -0,0 +1,465 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +"""Adversarial request-contract matrix for dstack pRPC services. + +Invariants asserted for every method, whatever it means: + + L1 a malformed request never produces a 5xx or a transport failure; + L2 a rejected JSON request gets a JSON ``error`` string, a rejected protobuf + request gets a decodable ``ProtoError``; + L3 the listener still answers a valid request after the whole matrix; + L4 a rejection neither echoes an unbounded amount of input nor grows with + the field it refused; + L5 no request exceeds the per-call deadline; + L6 a rejection's ``Content-Type`` matches the request's representation. +""" + +from __future__ import annotations + +import json +import subprocess +import time +from dataclasses import dataclass, field +from typing import Any + +CALL_TIMEOUT = 20 +# A rejection may quote the value it refused, but must not scale with it. +MAX_ECHO = 8192 +MAX_ECHO_SCALING = 256 + + +@dataclass +class Target: + """Where to send a request.""" + + route_template: str # contains "" + socket: str | None = None + base_url: str | None = None + headers: dict[str, str] = field(default_factory=dict) + curl_extra: list[str] = field(default_factory=list) + + def route(self, method: str) -> str: + """Resolve the route for one method.""" + return self.route_template.replace("", method) + + +@dataclass +class Call: + """One observed request/response pair.""" + + label: str + method: str + representation: str + http: int | None + body: bytes + seconds: float + transport_error: str | None = None + response_content_type: str = "" + + @property + def text(self) -> str: + """The response body as lossy text.""" + return self.body.decode("utf-8", errors="replace") + + +def invoke( + target: Target, + method: str, + content_type: str, + body: bytes, + label: str, + representation: str, + route_override: str | None = None, +) -> Call: + """Issue one request and never raise for a protocol-level answer.""" + route = route_override if route_override is not None else target.route(method) + marker = b"\nDSTACK_HTTP_TRAILER:" + argv = ["curl", "--silent", "--show-error"] + if target.socket: + argv += ["--unix-socket", target.socket] + argv += target.curl_extra + for name, value in target.headers.items(): + argv += ["--header", f"{name}: {value}"] + if content_type: + argv += ["--header", f"Content-Type: {content_type}"] + else: + argv += ["--header", "Content-Type:"] + argv += [ + "--max-time", + str(CALL_TIMEOUT), + "--request", + "POST", + "--data-binary", + "@-", + "--write-out", + marker.decode() + "%{http_code} %{content_type}", + (target.base_url or "http://localhost") + route, + ] + started = time.monotonic() + try: + process = subprocess.run( + argv, + input=body, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=CALL_TIMEOUT + 10, + check=False, + ) + except subprocess.TimeoutExpired: + return Call( + label, + method, + representation, + None, + b"", + time.monotonic() - started, + "harness timeout", + ) + elapsed = time.monotonic() - started + if process.returncode: + return Call( + label, + method, + representation, + None, + b"", + elapsed, + process.stderr.decode(errors="replace")[-400:], + ) + response, trailer = process.stdout.rsplit(marker, 1) + parts = trailer.decode(errors="replace").strip().split(" ", 1) + code = int(parts[0]) + response_content_type = parts[1].strip() if len(parts) > 1 else "" + return Call( + label, + method, + representation, + code, + response, + elapsed, + response_content_type=response_content_type, + ) + + +# -------------------------------------------------------------------------- +# protobuf wire helpers +# -------------------------------------------------------------------------- + + +def varint(value: int) -> bytes: + """Encode a non-negative integer as a protobuf varint.""" + out = bytearray() + value &= (1 << 64) - 1 + while value > 0x7F: + out.append((value & 0x7F) | 0x80) + value >>= 7 + out.append(value) + return bytes(out) + + +def encode_field(field_def: dict[str, Any], value: Any) -> bytes: + """Encode one request field onto the protobuf wire.""" + kind = field_def["type"] + number = int(field_def["number"]) + if kind == "bytes": + raw = bytes.fromhex(str(value)) + return varint((number << 3) | 2) + varint(len(raw)) + raw + if kind == "string": + raw = str(value).encode() + return varint((number << 3) | 2) + varint(len(raw)) + raw + if kind == "bool": + return varint((number << 3) | 0) + varint(1 if value else 0) + if kind.startswith(("uint", "int", "sint", "fixed", "sfixed")): + return varint((number << 3) | 0) + varint(int(value)) + # message, enum or anything else: an opaque empty length-delimited blob. + return varint((number << 3) | 2) + varint(0) + + +def encode_request(fields: list[dict[str, Any]], payload: dict[str, Any]) -> bytes: + """Encode a request body from the inventory field list.""" + out = bytearray() + for field_def in fields: + if field_def["name"] not in payload: + continue + value = payload[field_def["name"]] + values = value if field_def.get("repeated") else [value] + if not isinstance(values, list): + values = [values] + for item in values: + out.extend(encode_field(field_def, item)) + return bytes(out) + + +# -------------------------------------------------------------------------- +# adversarial vectors +# -------------------------------------------------------------------------- + +NUL = "a" + chr(0) + "b" +CONTROL = chr(0) + chr(0xFFFD) + chr(0x1F600) + + +def field_vectors(field_def: dict[str, Any]) -> list[tuple[str, Any]]: + """Return (label, JSON value) pairs a field must reject or bound.""" + kind = field_def["type"] + vectors: list[tuple[str, Any]] = [] + if kind == "string": + vectors += [ + ("empty", ""), + ("wrong-type-int", 123), + ("wrong-type-obj", {"a": 1}), + ("wrong-type-null", None), + ("nul-byte", NUL), + ("newline-injection", "a\nb\rc"), + ("path-traversal", "../../../../etc/passwd"), + ("absolute-path", "/etc/shadow"), + ("control-chars", CONTROL), + ("long-64k", "A" * 65536), + ("long-1m", "A" * (1 << 20)), + ] + elif kind == "bytes": + vectors += [ + ("empty", ""), + ("odd-hex", "abc"), + ("non-hex", "zzzz"), + ("wrong-type-int", 123), + ("wrong-type-list", [1, 2, 3]), + ("short-31", "11" * 31), + ("boundary-63", "11" * 63), + ("boundary-64", "11" * 64), + ("boundary-65", "11" * 65), + ("long-64k", "11" * 65536), + ] + elif kind == "bool": + vectors += [ + ("wrong-type-str", "true"), + ("wrong-type-int", 2), + ("wrong-type-null", None), + ] + elif kind.startswith(("uint", "int", "sint", "fixed", "sfixed")): + vectors += [ + ("zero", 0), + ("negative", -1), + ("u32-max", (1 << 32) - 1), + ("u64-max", (1 << 64) - 1), + ("over-u64", 1 << 64), + ("huge", 10**40), + ("wrong-type-str", "abc"), + ("wrong-type-null", None), + ] + else: + vectors += [ + ("wrong-type-str", "abc"), + ("wrong-type-int", 1), + ("wrong-type-null", None), + ] + if field_def.get("repeated"): + vectors += [ + ("repeated-empty", []), + ("repeated-scalar", "not-a-list"), + ("repeated-wide-4096", ["x"] * 4096), + ] + return vectors + + +def malformed_bodies() -> list[tuple[str, str, bytes]]: + """Representation-level malformed bodies every listener must survive.""" + return [ + ("json-empty-body", "application/json", b""), + ("json-not-object", "application/json", b"[]"), + ("json-scalar", "application/json", b"42"), + ("json-null", "application/json", b"null"), + ("json-truncated", "application/json", b'{"a":'), + ("json-trailing-garbage", "application/json", b"{} trailing"), + ("json-deep-nesting", "application/json", b"[" * 2000 + b"]" * 2000), + ("json-duplicate-keys", "application/json", b'{"path":"a","path":"b"}'), + ( + "json-huge-4mib", + "application/json", + b'{"path":"' + b"A" * (4 << 20) + b'"}', + ), + ("pb-truncated-varint", "application/octet-stream", b"\x08"), + ("pb-runaway-varint", "application/octet-stream", b"\x08" + b"\xff" * 12), + ( + "pb-length-overflow", + "application/octet-stream", + b"\x0a" + b"\xff\xff\xff\xff\x0f", + ), + ("pb-length-beyond-body", "application/octet-stream", b"\x0a\x7f" + b"A" * 4), + ("pb-unknown-wiretype", "application/octet-stream", b"\x0b\x00"), + ("pb-group-wiretype", "application/octet-stream", b"\x0c"), + ("pb-field-zero", "application/octet-stream", b"\x00\x00"), + ( + "pb-huge-field-number", + "application/octet-stream", + b"\xf8\xff\xff\xff\x0f\x01", + ), + ("pb-random-8k", "application/octet-stream", bytes(range(256)) * 32), + ("ct-mismatch-json-as-pb", "application/octet-stream", b'{"path":"a"}'), + ("ct-mismatch-pb-as-json", "application/json", b"\x08\x01"), + ("ct-absent", "", b"{}"), + ("ct-bogus", "application/x-not-a-thing", b"{}"), + ] + + +# -------------------------------------------------------------------------- +# invariants +# -------------------------------------------------------------------------- + + +def json_error(call: Call) -> tuple[bool, str | None]: + """Return whether a rejection carries a structured JSON ``error`` string.""" + if not call.body: + return False, "empty body" + try: + value = json.loads(call.body) + except (json.JSONDecodeError, UnicodeDecodeError): + return False, "body is not JSON" + if not isinstance(value, dict): + return False, "body is not a JSON object" + if not isinstance(value.get("error"), str): + return False, "body has no error string" + return True, value["error"] + + +def proto_error(call: Call) -> tuple[bool, str | None]: + """Return whether a rejection decodes as a one-field ``ProtoError``.""" + body = call.body + if len(body) < 2 or body[0] != 0x0A: + return False, "body is not a ProtoError message" + length = 0 + shift = 0 + offset = 1 + while offset < len(body): + byte = body[offset] + offset += 1 + length |= (byte & 0x7F) << shift + if byte < 0x80: + break + shift += 7 + if shift > 63: + return False, "ProtoError length varint is malformed" + if offset + length != len(body): + return False, "ProtoError length does not match the body" + return True, body[offset:].decode("utf-8", errors="replace") + + +def check_invariants(calls: list[Call], echoed_markers: dict[str, str]) -> list[dict]: + """Return one violation record per invariant breach.""" + violations: list[dict[str, Any]] = [] + for call in calls: + base = { + "label": call.label, + "method": call.method, + "representation": call.representation, + "http": call.http, + "seconds": round(call.seconds, 3), + } + if call.transport_error is not None: + violations.append( + { + **base, + "invariant": "L1", + "detail": f"transport failure: {call.transport_error}", + } + ) + continue + if call.http is not None and 500 <= call.http < 600: + violations.append( + { + **base, + "invariant": "L1", + "detail": f"server error {call.http}: {call.text[:MAX_ECHO]}", + } + ) + continue + sent_json = call.representation == "json" + if call.http is not None and call.http >= 400: + if sent_json: + ok, detail = json_error(call) + expected = "application/json" + else: + ok, detail = proto_error(call) + expected = "application/octet-stream" + if not ok: + violations.append( + { + **base, + "invariant": "L2", + "detail": f"unstructured rejection ({detail}): " + f"{call.text[:256]!r}", + } + ) + elif len(call.body) > MAX_ECHO: + violations.append( + { + **base, + "invariant": "L4", + "detail": f"rejection body is {len(call.body)} bytes", + } + ) + if ( + ok + and call.response_content_type + and expected not in call.response_content_type + ): + violations.append( + { + **base, + "invariant": "L6", + "detail": "a request sent as " + f"{'JSON' if sent_json else 'protobuf'} was rejected with " + f"Content-Type {call.response_content_type}", + } + ) + marker = echoed_markers.get(call.label) + if marker and marker in call.text: + violations.append( + { + **base, + "invariant": "L4", + "detail": "response echoed the input marker", + } + ) + if call.seconds > CALL_TIMEOUT: + violations.append( + {**base, "invariant": "L5", "detail": "call exceeded the deadline"} + ) + violations.extend(check_echo_scaling(calls)) + return violations + + +# Vector labels that differ only in how large one field is. A rejection that +# grows with the request is an amplifier whatever its absolute size. +SCALING_PAIRS = (("=long-64k|", "=long-1m|"), ("=short-31|", "=long-64k|")) + + +def check_echo_scaling(calls: list[Call]) -> list[dict[str, Any]]: + """Return a violation per rejection pair whose size tracks its input.""" + by_label = {call.label: call for call in calls} + violations: list[dict[str, Any]] = [] + for small_tag, large_tag in SCALING_PAIRS: + for label, small in by_label.items(): + if small_tag not in label: + continue + large = by_label.get(label.replace(small_tag, large_tag)) + if large is None or small.http is None or large.http is None: + continue + if small.http < 400 or large.http < 400: + continue + growth = len(large.body) - len(small.body) + if growth > MAX_ECHO_SCALING: + violations.append( + { + "label": large.label, + "method": large.method, + "representation": large.representation, + "http": large.http, + "seconds": round(large.seconds, 3), + "invariant": "L4", + "detail": "the rejection grew by " + f"{growth} bytes when the request field grew: " + f"{len(small.body)} -> {len(large.body)}", + } + ) + return violations diff --git a/test-suites/shared/automation/promoted-passing-cases.json b/test-suites/shared/automation/promoted-passing-cases.json index db4ea511f..d16e8140f 100644 --- a/test-suites/shared/automation/promoted-passing-cases.json +++ b/test-suites/shared/automation/promoted-passing-cases.json @@ -104,6 +104,87 @@ "environment": "MINED_REPLAY", "notes": "mined from central-fixtures-20260724T032131Z and verified by replay" }, + { + "case_id": "tc-gos-contract-001", + "source_run": "cov841v-all-5860e60a4", + "source_status": "PASS", + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "environment": "INTEGRATION", + "notes": "The full adversarial request-contract matrix over every Tappd method the inventory declares, under the policy in api-contract-matrix-policy.json. Executed on the tdxlab physical host against a lease-owned fixture; 18 of 18 scheduled cases passed in run cov841v-all-5860e60a4.", + "promoted_at": "2026-09-20T05:20:00Z" + }, + { + "case_id": "tc-gos-contract-002", + "source_run": "cov841v-all-5860e60a4", + "source_status": "PASS", + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "environment": "INTEGRATION", + "notes": "The full adversarial request-contract matrix over every DstackGuest method the inventory declares, under the policy in api-contract-matrix-policy.json. Executed on the tdxlab physical host against a lease-owned fixture; 18 of 18 scheduled cases passed in run cov841v-all-5860e60a4.", + "promoted_at": "2026-09-20T05:20:00Z" + }, + { + "case_id": "tc-gos-contract-003", + "source_run": "cov841v-all-5860e60a4", + "source_status": "PASS", + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "environment": "INTEGRATION", + "notes": "The full adversarial request-contract matrix over every Worker method the inventory declares, under the policy in api-contract-matrix-policy.json. Executed on the tdxlab physical host against a lease-owned fixture; 18 of 18 scheduled cases passed in run cov841v-all-5860e60a4.", + "promoted_at": "2026-09-20T05:20:00Z" + }, + { + "case_id": "tc-gos-contract-004", + "source_run": "cov841v-all-5860e60a4", + "source_status": "PASS", + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "environment": "INTEGRATION", + "notes": "The full adversarial request-contract matrix over every GuestApi method the inventory declares, under the policy in api-contract-matrix-policy.json. Executed on the tdxlab physical host against a lease-owned fixture; 18 of 18 scheduled cases passed in run cov841v-all-5860e60a4.", + "promoted_at": "2026-09-20T05:20:00Z" + }, + { + "case_id": "tc-gw-contract-001", + "source_run": "cov841v-all-5860e60a4", + "source_status": "PASS", + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "environment": "INTEGRATION", + "notes": "The full adversarial request-contract matrix over every Gateway method the inventory declares, under the policy in api-contract-matrix-policy.json. Executed on the tdxlab physical host against a lease-owned fixture; 18 of 18 scheduled cases passed in run cov841v-all-5860e60a4.", + "promoted_at": "2026-09-20T05:20:00Z" + }, + { + "case_id": "tc-gw-contract-002", + "source_run": "cov841v-all-5860e60a4", + "source_status": "PASS", + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "environment": "INTEGRATION", + "notes": "The full adversarial request-contract matrix over every Debug method the inventory declares, under the policy in api-contract-matrix-policy.json. Executed on the tdxlab physical host against a lease-owned fixture; 18 of 18 scheduled cases passed in run cov841v-all-5860e60a4.", + "promoted_at": "2026-09-20T05:20:00Z" + }, + { + "case_id": "tc-gw-contract-003", + "source_run": "cov841v-all-5860e60a4", + "source_status": "PASS", + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "environment": "INTEGRATION", + "notes": "The full adversarial request-contract matrix over every Admin method the inventory declares, under the policy in api-contract-matrix-policy.json. Executed on the tdxlab physical host against a lease-owned fixture; 18 of 18 scheduled cases passed in run cov841v-all-5860e60a4.", + "promoted_at": "2026-09-20T05:20:00Z" + }, + { + "case_id": "tc-kms-contract-001", + "source_run": "cov841v-all-5860e60a4", + "source_status": "PASS", + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "environment": "INTEGRATION", + "notes": "The full adversarial request-contract matrix over every KMS method the inventory declares, under the policy in api-contract-matrix-policy.json. Executed on the tdxlab physical host against a lease-owned fixture; 18 of 18 scheduled cases passed in run cov841v-all-5860e60a4.", + "promoted_at": "2026-09-20T05:20:00Z" + }, + { + "case_id": "tc-kms-contract-002", + "source_run": "cov841v-all-5860e60a4", + "source_status": "PASS", + "entrypoint": "shared/automation/api-contract-matrix-case.py", + "environment": "INTEGRATION", + "notes": "The full adversarial request-contract matrix over every Onboard method the inventory declares, under the policy in api-contract-matrix-policy.json. Executed on the tdxlab physical host against a lease-owned fixture; 18 of 18 scheduled cases passed in run cov841v-all-5860e60a4.", + "promoted_at": "2026-09-20T05:20:00Z" + }, { "case_id": "tc-gos-concurrency-001", "source_run": "cov841v-all-5860e60a4",