ci(tck): gate mandatory ACP v2 outcomes

docs/tck.md

@@ -36,10 +36,18 @@ environment and lockfile; it never supplies the agent's SDK package. It writes `
 `build/tck/v2.json`, `build/tck/manifest.json`, and a detailed log for each suite.
 
 The v1 lane is a CI gate: the official TCK verdict and every mandatory test outcome must pass.
-This catches cancellation tests that TCK's aggregate verdict would otherwise allow to skip. The
-v2 lane always runs and reports its current behavior. Even with the alpha.5 TCK, the SDK
-currently fails mandatory version-negotiation and batch-response checks, so v2 is not yet a CI
-gate. Its failures remain visible in the report. Promote it after those SDK behaviors are fixed.
+This catches cancellation tests that TCK's aggregate verdict would otherwise allow to skip.
+
+The v2 lane gates mandatory outcomes only: any `MANDATORY` `FAIL`, `NOT_TESTED`, or unexpected
+`SKIPPED` requirement fails CI. The two conditional auth checks `ACP-AUTH-206` and `ACP-AUTH-207`
+are the sole exception — alpha.5 skips them when the agent advertises no auth methods, and the
+runner accepts those skips only when the fixture's recorded capabilities report no `auth` and
+each child test is `SKIPPED` on the pinned node id with the pinned skip reason exactly. There is
+no substring matching: a missing or unparseable reason, a missing capabilities object, a `FAIL`
+row, or any third skip stays gated. The v2 lane does not gate the TCK's own verdict (which
+also folds in `CAPABILITY` failures) and never gates `ADVISORY` results, which stay as advisory
+reporting.
+
 The CI job uploads all reports and logs even when its gate fails.
 
 The probes advertise only the capabilities they implement. Optional capability tests that remain

scripts/run_tck.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import argparse
+import ast
 import json
 import shutil
 import subprocess
@@ -21,6 +22,20 @@
 }
 CANCEL_PROMPT = "wait_for_cancel"
 
+# The only v2 MANDATORY SKIPs the gate accepts: alpha.5's two conditional auth checks, which have
+# nothing to inspect unless the agent advertises auth methods. Pinned per requirement id to the test
+# that must skip and its exact reason, so any other MANDATORY SKIP fails CI.
+V2_PINNED_SKIPS = {
+    "ACP-AUTH-206": (
+        "test_authentication.py::test_auth_method_type_is_a_defined_or_prefixed_value",
+        "Skipped: agent advertises no authMethods",
+    ),
+    "ACP-AUTH-207": (
+        "test_authentication.py::test_terminal_auth_method_descriptor_shape",
+        "Skipped: agent advertises no type=='terminal' authMethods entry; nothing to check",
+    ),
+}
+
 
 def executable(name: str) -> str:
     path = shutil.which(name)
@@ -80,11 +95,45 @@ def run_suite(tck_dir: Path, output_dir: Path, python: Path, version: int, timeo
         ).returncode
 
 
-def check_mandatory(requirements: list[dict[str, Any]], version: int) -> list[str]:
+def pinned_auth_skip(requirement: dict[str, Any], version: int, report: dict[str, Any]) -> bool:
+    """True only for the two pinned alpha.5 conditional auth SKIPs.
+
+    Fail-closed on every axis: the v2 lane only; the requirement itself ``SKIPPED`` (a ``FAIL``
+    row stays gated even when its children claim ``SKIPPED``); a fixture whose recorded
+    capabilities object reports no ``auth`` (missing or non-object capabilities count as
+    advertising it); and every child test a ``SKIPPED`` on the pinned node id whose message
+    parses to exactly the pinned reason. alpha.5 records a message as
+    ``repr((file, line, reason))``; any other shape is rejected, never substring-matched.
+    """
+    pinned = V2_PINNED_SKIPS.get(str(requirement.get("id"))) if version == 2 else None
+    capabilities = report.get("agent_capabilities")
+    tests = requirement.get("tests") or []
+    if pinned is None or requirement.get("status") != "SKIPPED" or not tests:
+        return False
+    if not isinstance(capabilities, dict) or capabilities.get("auth") is not None:
+        return False
+    nodeid, reason = pinned
+    for test in tests:
+        message = test.get("message")
+        if test.get("status") != "SKIPPED" or test.get("nodeid") != nodeid or not isinstance(message, str):
+            return False
+        try:
+            parsed = ast.literal_eval(message)
+        except (SyntaxError, ValueError):
+            return False
+        if not (isinstance(parsed, tuple) and parsed[-1:] == (reason,)):
+            return False
+    return True
+
+
+def check_mandatory(requirements: list[dict[str, Any]], version: int, report: dict[str, Any]) -> list[str]:
     errors = []
     for item in requirements:
         if item.get("tier") != "MANDATORY":
             continue
+        if pinned_auth_skip(item, version, report):
+            print(f"v{version}: {item['id']} is SKIPPED (pinned conditional auth check)", flush=True)
+            continue
         if item.get("status") != "PASS":
             errors.append(f"v{version}: {item['id']} is {item.get('status')}, expected PASS")
         for test in item.get("tests", []):
@@ -135,10 +184,11 @@ def inspect_report(
         flush=True,
     )
 
-    if strict:
-        if tck_exit != 0 or verdict.get("conformant") is not True:
-            errors.append(f"v{version}: TCK conformance verdict failed")
-        errors.extend(check_mandatory(requirements, version))
+    if strict and (tck_exit != 0 or verdict.get("conformant") is not True):
+        errors.append(f"v{version}: TCK conformance verdict failed")
+    # Both lanes gate MANDATORY outcomes; only the v1 lane additionally requires the TCK's own
+    # verdict, which also folds in CAPABILITY failures. ADVISORY results are never gated.
+    errors.extend(check_mandatory(requirements, version, report))
     return errors, report
 
 

tests/test_run_tck_gate.py

@@ -0,0 +1,75 @@
+"""The v2 MANDATORY gate in `scripts/run_tck.py` and its two pinned alpha.5 auth skips.
+
+`scripts/run_tck.py` drives an external TCK checkout, so these tests call `check_mandatory`
+directly with synthetic requirement rows instead of running a suite.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+
+from scripts.run_tck import V2_PINNED_SKIPS, check_mandatory
+
+NO_AUTH = {"session": {}}
+MANDATORY_SKIPS = ["ACP-AUTH-206", "ACP-AUTH-207"]
+
+
+def row(requirement_id: str, *, status: str = "SKIPPED", reason: str | None = None, nodeid: str | None = None):
+    """A MANDATORY row whose single child test is SKIPPED on the pinned node id."""
+    pinned_nodeid, pinned_reason = V2_PINNED_SKIPS.get(requirement_id, (nodeid, reason))
+    message = repr(("/tck/test_authentication.py", 1, reason if reason is not None else pinned_reason))
+    return {
+        "id": requirement_id,
+        "tier": "MANDATORY",
+        "status": status,
+        "tests": [{"nodeid": nodeid or pinned_nodeid, "status": "SKIPPED", "message": message}],
+    }
+
+
+def gate(rows: list[dict[str, Any]], version: int = 2, capabilities: Any = NO_AUTH) -> list[str]:
+    return check_mandatory(rows, version, {"agent_capabilities": capabilities})
+
+
+def test_both_pinned_auth_skips_are_accepted_for_an_unauthenticated_v2_fixture() -> None:
+    assert gate([row("ACP-AUTH-206"), row("ACP-AUTH-207")]) == []
+
+
+@pytest.mark.parametrize("requirement_id", MANDATORY_SKIPS)
+def test_pin_requires_the_exact_reason_and_node_id(requirement_id: str) -> None:
+    assert gate([row(requirement_id, reason=f"{V2_PINNED_SKIPS[requirement_id][1]} (drifted)")]) != []
+    assert gate([row(requirement_id, nodeid="test_authentication.py::test_other")]) != []
+
+
+@pytest.mark.parametrize("capabilities", [None, [], "unexpected", {"session": {}, "auth": {"terminal": {}}}])
+def test_pin_requires_a_fixture_that_reports_no_auth(capabilities: Any) -> None:
+    assert gate([row("ACP-AUTH-206"), row("ACP-AUTH-207")], capabilities=capabilities) != []
+
+
+def test_pin_accepts_capabilities_that_omit_auth() -> None:
+    assert gate([row("ACP-AUTH-206")], capabilities={}) == []
+    assert gate([row("ACP-AUTH-206")], capabilities={"auth": None}) == []
+
+
+def test_fail_row_with_pinned_id_and_children_is_still_gated() -> None:
+    errors = gate([row("ACP-AUTH-206", status="FAIL")])
+    assert "v2: ACP-AUTH-206 is FAIL, expected PASS" in errors
+
+
+def test_unpinned_skip_and_other_lanes_stay_gated() -> None:
+    assert gate([row("ACP-AUTH-202", reason="Skipped: anything", nodeid="test_authentication.py::test_other")]) != []
+    assert gate([row("ACP-AUTH-206")], version=1) != []
+
+
+def test_mandatory_failures_are_gated_but_advisory_results_are_not() -> None:
+    failing = [
+        {"id": "ACP-BATCH-201", "tier": "MANDATORY", "status": "FAIL", "tests": []},
+        {"id": "ACP-INIT-201", "tier": "MANDATORY", "status": "NOT_TESTED", "tests": []},
+    ]
+    assert gate(failing) != []
+    advisory = [
+        {"id": "ACP-SESSION-004", "tier": "ADVISORY", "status": "SKIPPED", "tests": []},
+        {"id": "ACP-AUTH-203", "tier": "CAPABILITY", "status": "SKIPPED", "tests": []},
+    ]
+    assert gate(advisory) == []