From c0a81c7336a0043953d93f93c4a17779e3ab9274 Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Sat, 7 Nov 2026 11:35:32 +0800 Subject: [PATCH] ci(tck): gate mandatory ACP v2 outcomes Gate MANDATORY outcomes in the v2 lane, not only the v1 lane, so a v2 FAIL or NOT_TESTED now fails CI. The v1 lane still additionally requires the TCK's own conformance verdict; the v2 lane does not gate that verdict and never gates ADVISORY results. Accept only the two conditional auth skips alpha.5 reports for an agent advertising no auth methods (ACP-AUTH-206/207). A skip counts only when the requirement is SKIPPED, the recorded capabilities report no `auth`, and each child test is SKIPPED on the pinned node id with the pinned reason parsed exactly; anything else stays gated. --- docs/tck.md | 16 ++++++-- scripts/run_tck.py | 60 +++++++++++++++++++++++++++--- tests/test_run_tck_gate.py | 75 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+), 9 deletions(-) create mode 100644 tests/test_run_tck_gate.py diff --git a/docs/tck.md b/docs/tck.md index ab0a1fb0..beb4ad28 100644 --- a/docs/tck.md +++ b/docs/tck.md @@ -36,10 +36,18 @@ environment and lockfile; it never supplies the agent's SDK package. It writes ` `build/tck/v2.json`, `build/tck/manifest.json`, and a detailed log for each suite. The v1 lane is a CI gate: the official TCK verdict and every mandatory test outcome must pass. -This catches cancellation tests that TCK's aggregate verdict would otherwise allow to skip. The -v2 lane always runs and reports its current behavior. Even with the alpha.5 TCK, the SDK -currently fails mandatory version-negotiation and batch-response checks, so v2 is not yet a CI -gate. Its failures remain visible in the report. Promote it after those SDK behaviors are fixed. +This catches cancellation tests that TCK's aggregate verdict would otherwise allow to skip. + +The v2 lane gates mandatory outcomes only: any `MANDATORY` `FAIL`, `NOT_TESTED`, or unexpected +`SKIPPED` requirement fails CI. The two conditional auth checks `ACP-AUTH-206` and `ACP-AUTH-207` +are the sole exception — alpha.5 skips them when the agent advertises no auth methods, and the +runner accepts those skips only when the fixture's recorded capabilities report no `auth` and +each child test is `SKIPPED` on the pinned node id with the pinned skip reason exactly. There is +no substring matching: a missing or unparseable reason, a missing capabilities object, a `FAIL` +row, or any third skip stays gated. The v2 lane does not gate the TCK's own verdict (which +also folds in `CAPABILITY` failures) and never gates `ADVISORY` results, which stay as advisory +reporting. + The CI job uploads all reports and logs even when its gate fails. The probes advertise only the capabilities they implement. Optional capability tests that remain diff --git a/scripts/run_tck.py b/scripts/run_tck.py index 14b334ba..e1ddac2a 100644 --- a/scripts/run_tck.py +++ b/scripts/run_tck.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import ast import json import shutil import subprocess @@ -21,6 +22,20 @@ } CANCEL_PROMPT = "wait_for_cancel" +# The only v2 MANDATORY SKIPs the gate accepts: alpha.5's two conditional auth checks, which have +# nothing to inspect unless the agent advertises auth methods. Pinned per requirement id to the test +# that must skip and its exact reason, so any other MANDATORY SKIP fails CI. +V2_PINNED_SKIPS = { + "ACP-AUTH-206": ( + "test_authentication.py::test_auth_method_type_is_a_defined_or_prefixed_value", + "Skipped: agent advertises no authMethods", + ), + "ACP-AUTH-207": ( + "test_authentication.py::test_terminal_auth_method_descriptor_shape", + "Skipped: agent advertises no type=='terminal' authMethods entry; nothing to check", + ), +} + def executable(name: str) -> str: path = shutil.which(name) @@ -80,11 +95,45 @@ def run_suite(tck_dir: Path, output_dir: Path, python: Path, version: int, timeo ).returncode -def check_mandatory(requirements: list[dict[str, Any]], version: int) -> list[str]: +def pinned_auth_skip(requirement: dict[str, Any], version: int, report: dict[str, Any]) -> bool: + """True only for the two pinned alpha.5 conditional auth SKIPs. + + Fail-closed on every axis: the v2 lane only; the requirement itself ``SKIPPED`` (a ``FAIL`` + row stays gated even when its children claim ``SKIPPED``); a fixture whose recorded + capabilities object reports no ``auth`` (missing or non-object capabilities count as + advertising it); and every child test a ``SKIPPED`` on the pinned node id whose message + parses to exactly the pinned reason. alpha.5 records a message as + ``repr((file, line, reason))``; any other shape is rejected, never substring-matched. + """ + pinned = V2_PINNED_SKIPS.get(str(requirement.get("id"))) if version == 2 else None + capabilities = report.get("agent_capabilities") + tests = requirement.get("tests") or [] + if pinned is None or requirement.get("status") != "SKIPPED" or not tests: + return False + if not isinstance(capabilities, dict) or capabilities.get("auth") is not None: + return False + nodeid, reason = pinned + for test in tests: + message = test.get("message") + if test.get("status") != "SKIPPED" or test.get("nodeid") != nodeid or not isinstance(message, str): + return False + try: + parsed = ast.literal_eval(message) + except (SyntaxError, ValueError): + return False + if not (isinstance(parsed, tuple) and parsed[-1:] == (reason,)): + return False + return True + + +def check_mandatory(requirements: list[dict[str, Any]], version: int, report: dict[str, Any]) -> list[str]: errors = [] for item in requirements: if item.get("tier") != "MANDATORY": continue + if pinned_auth_skip(item, version, report): + print(f"v{version}: {item['id']} is SKIPPED (pinned conditional auth check)", flush=True) + continue if item.get("status") != "PASS": errors.append(f"v{version}: {item['id']} is {item.get('status')}, expected PASS") for test in item.get("tests", []): @@ -135,10 +184,11 @@ def inspect_report( flush=True, ) - if strict: - if tck_exit != 0 or verdict.get("conformant") is not True: - errors.append(f"v{version}: TCK conformance verdict failed") - errors.extend(check_mandatory(requirements, version)) + if strict and (tck_exit != 0 or verdict.get("conformant") is not True): + errors.append(f"v{version}: TCK conformance verdict failed") + # Both lanes gate MANDATORY outcomes; only the v1 lane additionally requires the TCK's own + # verdict, which also folds in CAPABILITY failures. ADVISORY results are never gated. + errors.extend(check_mandatory(requirements, version, report)) return errors, report diff --git a/tests/test_run_tck_gate.py b/tests/test_run_tck_gate.py new file mode 100644 index 00000000..b81faea9 --- /dev/null +++ b/tests/test_run_tck_gate.py @@ -0,0 +1,75 @@ +"""The v2 MANDATORY gate in `scripts/run_tck.py` and its two pinned alpha.5 auth skips. + +`scripts/run_tck.py` drives an external TCK checkout, so these tests call `check_mandatory` +directly with synthetic requirement rows instead of running a suite. +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from scripts.run_tck import V2_PINNED_SKIPS, check_mandatory + +NO_AUTH = {"session": {}} +MANDATORY_SKIPS = ["ACP-AUTH-206", "ACP-AUTH-207"] + + +def row(requirement_id: str, *, status: str = "SKIPPED", reason: str | None = None, nodeid: str | None = None): + """A MANDATORY row whose single child test is SKIPPED on the pinned node id.""" + pinned_nodeid, pinned_reason = V2_PINNED_SKIPS.get(requirement_id, (nodeid, reason)) + message = repr(("/tck/test_authentication.py", 1, reason if reason is not None else pinned_reason)) + return { + "id": requirement_id, + "tier": "MANDATORY", + "status": status, + "tests": [{"nodeid": nodeid or pinned_nodeid, "status": "SKIPPED", "message": message}], + } + + +def gate(rows: list[dict[str, Any]], version: int = 2, capabilities: Any = NO_AUTH) -> list[str]: + return check_mandatory(rows, version, {"agent_capabilities": capabilities}) + + +def test_both_pinned_auth_skips_are_accepted_for_an_unauthenticated_v2_fixture() -> None: + assert gate([row("ACP-AUTH-206"), row("ACP-AUTH-207")]) == [] + + +@pytest.mark.parametrize("requirement_id", MANDATORY_SKIPS) +def test_pin_requires_the_exact_reason_and_node_id(requirement_id: str) -> None: + assert gate([row(requirement_id, reason=f"{V2_PINNED_SKIPS[requirement_id][1]} (drifted)")]) != [] + assert gate([row(requirement_id, nodeid="test_authentication.py::test_other")]) != [] + + +@pytest.mark.parametrize("capabilities", [None, [], "unexpected", {"session": {}, "auth": {"terminal": {}}}]) +def test_pin_requires_a_fixture_that_reports_no_auth(capabilities: Any) -> None: + assert gate([row("ACP-AUTH-206"), row("ACP-AUTH-207")], capabilities=capabilities) != [] + + +def test_pin_accepts_capabilities_that_omit_auth() -> None: + assert gate([row("ACP-AUTH-206")], capabilities={}) == [] + assert gate([row("ACP-AUTH-206")], capabilities={"auth": None}) == [] + + +def test_fail_row_with_pinned_id_and_children_is_still_gated() -> None: + errors = gate([row("ACP-AUTH-206", status="FAIL")]) + assert "v2: ACP-AUTH-206 is FAIL, expected PASS" in errors + + +def test_unpinned_skip_and_other_lanes_stay_gated() -> None: + assert gate([row("ACP-AUTH-202", reason="Skipped: anything", nodeid="test_authentication.py::test_other")]) != [] + assert gate([row("ACP-AUTH-206")], version=1) != [] + + +def test_mandatory_failures_are_gated_but_advisory_results_are_not() -> None: + failing = [ + {"id": "ACP-BATCH-201", "tier": "MANDATORY", "status": "FAIL", "tests": []}, + {"id": "ACP-INIT-201", "tier": "MANDATORY", "status": "NOT_TESTED", "tests": []}, + ] + assert gate(failing) != [] + advisory = [ + {"id": "ACP-SESSION-004", "tier": "ADVISORY", "status": "SKIPPED", "tests": []}, + {"id": "ACP-AUTH-203", "tier": "CAPABILITY", "status": "SKIPPED", "tests": []}, + ] + assert gate(advisory) == []