From 05f831c58136e8e4c45e36f3c267706a975ec4fa Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Thu, 5 Nov 2026 02:13:12 +0800 Subject: [PATCH] ci: run ACP TCK agent-side suites against pinned revisions Add the initial Agent-side TCK integration infrastructure: probe agents for protocol v1 and experimental v2, a runner that pins TCK revisions and verifies report provenance, a CI job that uploads reports even when the v2 gate is not yet green, and docs describing local usage. The v1 lane is a CI gate (conformant, 31/31 mandatory PASS). The v2 alpha.5 lane reports known failures in version negotiation and batch responses; it is not promoted to a gate until those SDK behaviors are fixed. --- .github/workflows/tck.yml | 54 ++++++++++ README.md | 2 + docs/tck.md | 49 +++++++++ mkdocs.yml | 1 + scripts/run_tck.py | 204 ++++++++++++++++++++++++++++++++++++++ tests/tck/agent_v1.py | 70 +++++++++++++ tests/tck/agent_v2.py | 132 ++++++++++++++++++++++++ 7 files changed, 512 insertions(+) create mode 100644 .github/workflows/tck.yml create mode 100644 docs/tck.md create mode 100644 scripts/run_tck.py create mode 100644 tests/tck/agent_v1.py create mode 100644 tests/tck/agent_v2.py diff --git a/.github/workflows/tck.yml b/.github/workflows/tck.yml new file mode 100644 index 00000000..3006eac2 --- /dev/null +++ b/.github/workflows/tck.yml @@ -0,0 +1,54 @@ +name: ACP TCK + +on: + push: + branches: [main] + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +jobs: + agent-stdio: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out SDK + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 + + - name: Install uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e + with: + version: "0.11.14" + enable-cache: true + + - name: Install Python 3.14 + run: uv python install 3.14 + + - name: Install current SDK checkout + run: uv sync --frozen --python 3.14 + + - name: Check out pinned v1 ACP TCK + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 + with: + repository: agentclientprotocol/acp-tck + ref: b7f40fb3c003c45926547be5a1f7e2b374035815 + path: .tck-v1 + persist-credentials: false + + - name: Check out alpha.5 v2 ACP TCK + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 + with: + repository: agentclientprotocol/acp-tck + ref: fa00ccdc13dfaa4b288a91877c1f3e468af4aa7d + path: .tck-v2 + persist-credentials: false + + - name: Run version-matched TCK suites + run: .venv/bin/python scripts/run_tck.py --tck-v1-dir .tck-v1 --tck-v2-dir .tck-v2 + + - name: Upload TCK reports + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: acp-tck-reports + path: build/tck/* + if-no-files-found: warn diff --git a/README.md b/README.md index b8aaac8a..9b16c7c9 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ See real adopters like kimi-cli in the [Use Cases list](https://agentclientproto | Quickstart | | | Use cases | | | Contrib helpers | | +| ACP TCK checks | | | Releasing workflow | | | Examples | | | Tests | | @@ -63,6 +64,7 @@ See real adopters like kimi-cli in the [Use Cases list](https://agentclientproto - `make install` provisions the `uv` virtualenv and installs pre-commit hooks. - `make check` runs Ruff formatting/linting, type analysis, dependency hygiene, and lock verification. - `make test` executes `pytest` (with doctests) inside the managed environment. +- `scripts/run_tck.py --tck-v1-dir --tck-v2-dir ` runs the pinned external Agent-side TCK against this checkout (see [ACP TCK checks](docs/tck.md)). - `ACP_SCHEMA_VERSION= make gen-all` refreshes protocol artifacts when the schema advances. Keep docs and examples current whenever you ship public API or transport changes, and prefer Conventional Commits (`feat:`, `fix:`, etc.) when submitting patches. diff --git a/docs/tck.md b/docs/tck.md new file mode 100644 index 00000000..ab0a1fb0 --- /dev/null +++ b/docs/tck.md @@ -0,0 +1,49 @@ +# Agent-side protocol checks with ACP TCK + +The [ACP TCK](https://github.com/agentclientprotocol/acp-tck) is an independent stdio client. It +launches the probe agents in `tests/tck/` and checks their wire behavior. The probes use this +checkout's public SDK entry points and generated models; they do not implement JSON-RPC framing +or validation themselves. + +| Lane | SDK entry point | TCK suite | SDK schema source | +| --- | --- | --- | --- | +| v1 | `acp.run_agent` | `--protocol-version 1` | `schema/VERSION` | +| v2 | `acp.experimental.v2.run_agent` | `--protocol-version 2` | `schema/v2/VERSION` | + +The v1 lane uses TCK commit `b7f40fb3c003c45926547be5a1f7e2b374035815`, with a schema +snapshot from spec commit `beac4ba96190018e9f6d82e148ff275084c5aa3e`. The v2 lane uses +TCK commit `fa00ccdc13dfaa4b288a91877c1f3e468af4aa7d`, whose schema snapshot is +`2.0.0-alpha.5` from spec commit `9af0e9f748db9f4cc4c410a7b212ead4f98ae78c`. +The SDK's generated schemas come from `schema-v1.23.0` and `schema-v2.0.0-alpha.5`. +The v2 lane checks the same draft version, although TCK uses the upstream unstable superset +and the SDK uses the published schema. Reports record both TCK and SDK provenance. + +## Run locally + +Install the SDK in its own environment and check out both pinned TCK commits in sibling directories: + +```sh +uv python install 3.14 +uv sync --frozen --python 3.14 +/usr/bin/gh repo clone agentclientprotocol/acp-tck ../acp-tck +git -C ../acp-tck checkout --detach b7f40fb3c003c45926547be5a1f7e2b374035815 +git -C ../acp-tck worktree add --detach ../acp-tck-alpha5 fa00ccdc13dfaa4b288a91877c1f3e468af4aa7d +.venv/bin/python scripts/run_tck.py --tck-v1-dir ../acp-tck --tck-v2-dir ../acp-tck-alpha5 +``` + +The runner verifies that the agent Python imports `acp` from this checkout. TCK has its own uv +environment and lockfile; it never supplies the agent's SDK package. It writes `build/tck/v1.json`, +`build/tck/v2.json`, `build/tck/manifest.json`, and a detailed log for each suite. + +The v1 lane is a CI gate: the official TCK verdict and every mandatory test outcome must pass. +This catches cancellation tests that TCK's aggregate verdict would otherwise allow to skip. The +v2 lane always runs and reports its current behavior. Even with the alpha.5 TCK, the SDK +currently fails mandatory version-negotiation and batch-response checks, so v2 is not yet a CI +gate. Its failures remain visible in the report. Promote it after those SDK behaviors are fixed. +The CI job uploads all reports and logs even when its gate fails. + +The probes advertise only the capabilities they implement. Optional capability tests that remain +`SKIPPED` are outside the current probe coverage. The v1 probe intentionally leaves MCP connection +to application code, so its `ACP-SESSION-004` advisory result is not an SDK transport verdict. +Client-side behavior and HTTP/WebSocket transports are covered by the repository's own tests, +not by these TCK lanes. diff --git a/mkdocs.yml b/mkdocs.yml index 1938cf48..8b706fa9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -13,6 +13,7 @@ nav: - Use Cases: use-cases.md - Web Transport (HTTP/WS): web-transport.md - Experimental Protocol v2: experimental-v2.md + - ACP TCK: tck.md - Experimental Contrib: contrib.md - Releasing: releasing.md - 0.11 Migration Guide: migration-guide-0.11.md diff --git a/scripts/run_tck.py b/scripts/run_tck.py new file mode 100644 index 00000000..14b334ba --- /dev/null +++ b/scripts/run_tck.py @@ -0,0 +1,204 @@ +"""Run the pinned ACP TCK against the current checkout's v1 and v2 agent APIs.""" + +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +TCK_REVISIONS = { + 1: "b7f40fb3c003c45926547be5a1f7e2b374035815", + 2: "fa00ccdc13dfaa4b288a91877c1f3e468af4aa7d", +} +TCK_SCHEMA_REVISIONS = { + 1: "beac4ba96190018e9f6d82e148ff275084c5aa3e", + 2: "9af0e9f748db9f4cc4c410a7b212ead4f98ae78c", +} +CANCEL_PROMPT = "wait_for_cancel" + + +def executable(name: str) -> str: + path = shutil.which(name) + if path is None: + raise RuntimeError(f"{name} is not on PATH") + return path + + +def git_revision(path: Path) -> str: + return subprocess.check_output( # noqa: S603 + [executable("git"), "-C", str(path), "rev-parse", "HEAD"], text=True + ).strip() + + +def sdk_python() -> Path: + python = ROOT / ".venv" / "bin" / "python" + if not python.exists(): + raise RuntimeError("SDK virtual environment is missing; run uv sync --frozen first") + module = subprocess.check_output( # noqa: S603 + [str(python), "-c", "import acp; print(acp.__file__)"], text=True + ).strip() + expected = ROOT / "src" / "acp" / "__init__.py" + if Path(module).resolve() != expected.resolve(): + raise RuntimeError(f"Agent Python imports {module}, expected {expected}") + return python + + +def run_suite(tck_dir: Path, output_dir: Path, python: Path, version: int, timeout: int) -> int: + report = output_dir / f"v{version}.json" + log = output_dir / f"v{version}.log" + report.unlink(missing_ok=True) + command = [ + executable("uv"), + "--project", + str(tck_dir), + "run", + "--locked", + "acp-tck", + "--protocol-version", + str(version), + "--cancel-prompt", + CANCEL_PROMPT, + "--timeout", + str(timeout), + "--test-timeout", + str(max(45, timeout * 4)), + "--report-json", + str(report), + "--", + str(python), + str(ROOT / "tests" / "tck" / f"agent_v{version}.py"), + ] + print(f"Running ACP v{version} TCK; detailed output: {log}", flush=True) + with log.open("w") as output: + return subprocess.run( # noqa: S603 + command, cwd=ROOT, check=False, stdout=output, stderr=subprocess.STDOUT + ).returncode + + +def check_mandatory(requirements: list[dict[str, Any]], version: int) -> list[str]: + errors = [] + for item in requirements: + if item.get("tier") != "MANDATORY": + continue + if item.get("status") != "PASS": + errors.append(f"v{version}: {item['id']} is {item.get('status')}, expected PASS") + for test in item.get("tests", []): + if test.get("status") != "PASS": + errors.append(f"v{version}: {test.get('nodeid')} is {test.get('status')}, expected PASS") + return errors + + +def check_provenance(report: dict[str, Any], version: int, python: Path) -> list[str]: + errors = [] + if report.get("protocol_version") != version: + errors.append(f"v{version}: report protocol_version is {report.get('protocol_version')!r}") + if report.get("schema_revision") != TCK_SCHEMA_REVISIONS[version]: + errors.append(f"v{version}: unexpected TCK schema revision {report.get('schema_revision')!r}") + command = report.get("agent_command", []) + if not command or Path(command[0]).resolve() != python.resolve(): + errors.append(f"v{version}: report does not identify the current checkout's Python") + expected_probe = ROOT / "tests" / "tck" / f"agent_v{version}.py" + if len(command) < 2 or Path(command[1]).resolve() != expected_probe.resolve(): + errors.append(f"v{version}: report does not identify the v{version} probe") + return errors + + +def inspect_report( + path: Path, version: int, *, strict: bool, tck_exit: int, python: Path +) -> tuple[list[str], dict[str, Any]]: + if not path.exists(): + return [f"v{version}: TCK wrote no JSON report (exit {tck_exit})"], {} + try: + report = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + return [f"v{version}: cannot read TCK report: {exc}"], {} + + errors = check_provenance(report, version, python) + verdict = report.get("verdict", {}) + for flag in ("blocked_by_auth", "blocked_by_version_mismatch"): + if verdict.get(flag): + errors.append(f"v{version}: {flag}") + + requirements = report.get("requirements", []) + if not isinstance(requirements, list) or not requirements: + errors.append(f"v{version}: report contains no requirements") + requirements = [] + failures = [f"{item['id']} ({item['tier']})" for item in requirements if item.get("status") == "FAIL"] + print( + f"v{version}: TCK exit={tck_exit}, conformant={verdict.get('conformant')}, " + f"failures={', '.join(failures) or 'none'}, schema={report.get('schema_revision')}", + flush=True, + ) + + if strict: + if tck_exit != 0 or verdict.get("conformant") is not True: + errors.append(f"v{version}: TCK conformance verdict failed") + errors.extend(check_mandatory(requirements, version)) + return errors, report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tck-v1-dir", type=Path, required=True, help="checkout of the pinned v1 TCK commit") + parser.add_argument("--tck-v2-dir", type=Path, required=True, help="checkout of the alpha.5 v2 TCK commit") + parser.add_argument("--output-dir", type=Path, default=ROOT / "build" / "tck") + parser.add_argument("--timeout", type=int, default=10, help="TCK per-response timeout in seconds") + args = parser.parse_args() + tck_dirs = {1: args.tck_v1_dir.resolve(), 2: args.tck_v2_dir.resolve()} + output_dir = args.output_dir.resolve() + + for version, tck_dir in tck_dirs.items(): + if git_revision(tck_dir) != TCK_REVISIONS[version]: + parser.error(f"v{version} TCK must be checked out at {TCK_REVISIONS[version]}") + python = sdk_python() + output_dir.mkdir(parents=True, exist_ok=True) + + errors: list[str] = [] + summaries: dict[str, Any] = {} + for version in (1, 2): + tck_dir = tck_dirs[version] + exit_code = run_suite(tck_dir, output_dir, python, version, args.timeout) + problems, report = inspect_report( + output_dir / f"v{version}.json", + version, + strict=version == 1, + tck_exit=exit_code, + python=python, + ) + errors.extend(problems) + summaries[f"v{version}"] = { + "tck_revision": TCK_REVISIONS[version], + "tck_exit": exit_code, + "tck_schema_revision": report.get("schema_revision"), + "verdict": report.get("verdict"), + } + + manifest = { + "sdk_revision": git_revision(ROOT), + "sdk_schema_refs": { + "v1": (ROOT / "schema" / "VERSION").read_text().strip(), + "v2": (ROOT / "schema" / "v2" / "VERSION").read_text().strip(), + }, + "sdk_python": str(python.resolve()), + "sdk_python_version": subprocess.check_output([str(python), "--version"], text=True).strip(), # noqa: S603 + "tck_python_versions": { + f"v{version}": subprocess.check_output( # noqa: S603 + [str(tck_dir / ".venv" / "bin" / "python"), "--version"], text=True + ).strip() + for version, tck_dir in tck_dirs.items() + }, + "runs": summaries, + } + (output_dir / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + for error in errors: + print(error, file=sys.stderr) + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/tck/agent_v1.py b/tests/tck/agent_v1.py new file mode 100644 index 00000000..6ea60f13 --- /dev/null +++ b/tests/tck/agent_v1.py @@ -0,0 +1,70 @@ +"""Deterministic ACP v1 agent for exercising the SDK with the external TCK. + +This probe supplies only application state. The SDK owns stdio framing, routing, +model validation, serialization, and JSON-RPC errors. +""" + +from __future__ import annotations + +import asyncio +from typing import Any, cast +from uuid import uuid4 + +from acp import PROTOCOL_VERSION, Agent, InitializeResponse, NewSessionResponse, PromptResponse, run_agent +from acp.interfaces import Client +from acp.schema import AgentCapabilities, AgentMessageChunk, Implementation, TextContentBlock + +CANCEL_PROMPT = "wait_for_cancel" + + +class ProbeAgent: + """A plain class so unimplemented Protocol methods do not become handlers.""" + + def __init__(self) -> None: + self._client: Client | None = None + self._pending: dict[str, asyncio.Event] = {} + + def on_connect(self, client: Client) -> None: + self._client = client + + async def initialize(self, protocol_version: int, **kwargs: Any) -> InitializeResponse: + return InitializeResponse( + protocol_version=PROTOCOL_VERSION, + agent_capabilities=AgentCapabilities(), + agent_info=Implementation(name="python-sdk-tck-v1", version="1"), + ) + + async def new_session(self, cwd: str, **kwargs: Any) -> NewSessionResponse: + return NewSessionResponse(session_id=uuid4().hex) + + async def prompt(self, session_id: str, prompt: list[Any], **kwargs: Any) -> PromptResponse: + assert self._client is not None + wait_for_cancel = ( + len(prompt) == 1 and isinstance(prompt[0], TextContentBlock) and prompt[0].text == CANCEL_PROMPT + ) + event = asyncio.Event() if wait_for_cancel else None + if event is not None: + self._pending[session_id] = event + try: + await self._client.session_update( + session_id=session_id, + update=AgentMessageChunk(content=TextContentBlock(text="ready")), + ) + if event is not None: + await event.wait() + return PromptResponse(stop_reason="cancelled" if event is not None else "end_turn") + finally: + self._pending.pop(session_id, None) + + async def cancel(self, session_id: str, **kwargs: Any) -> None: + if event := self._pending.get(session_id): + event.set() + + +async def main() -> None: + # The public Protocol lists optional handlers that this probe intentionally omits. + await run_agent(cast(Agent, ProbeAgent())) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/tck/agent_v2.py b/tests/tck/agent_v2.py new file mode 100644 index 00000000..83c152d4 --- /dev/null +++ b/tests/tck/agent_v2.py @@ -0,0 +1,132 @@ +"""Deterministic ACP v2 agent for exercising the experimental SDK runtime. + +The session capability includes the v2 lifecycle baseline. Protocol framing, +validation, routing, and errors remain the SDK's responsibility. +""" + +from __future__ import annotations + +import asyncio +import contextlib +from typing import Any +from uuid import uuid4 + +from acp.exceptions import RequestError +from acp.experimental.v2 import PROTOCOL_VERSION, run_agent +from acp.experimental.v2.interfaces import Client +from acp.experimental.v2.schema import ( + AgentCapabilities, + AgentMessageChunk, + CloseSessionResponse, + IdleSessionStateUpdate, + Implementation, + InitializeResponse, + ListSessionsResponse, + NewSessionResponse, + PromptResponse, + ResumeSessionResponse, + RunningSessionStateUpdate, + SessionCapabilities, + SessionInfo, + TextContentBlock, + UserMessageUpdate, +) + +CANCEL_PROMPT = "wait_for_cancel" +_CANCEL_FALLBACK_SECONDS = 60 + + +class ProbeAgent: + def __init__(self) -> None: + self._client: Client | None = None + self._sessions: dict[str, str] = {} + self._cancel_events: dict[str, asyncio.Event] = {} + self._turns: dict[str, asyncio.Task[None]] = {} + + def on_connect(self, client: Client) -> None: + self._client = client + + async def initialize(self, protocol_version: int, info: Implementation, **kwargs: Any) -> InitializeResponse: + return InitializeResponse( + protocol_version=PROTOCOL_VERSION, + info=Implementation(name="python-sdk-tck-v2", version="1"), + capabilities=AgentCapabilities(session=SessionCapabilities()), + ) + + async def new_session(self, cwd: str, **kwargs: Any) -> NewSessionResponse: + session_id = uuid4().hex + self._sessions[session_id] = cwd + return NewSessionResponse(session_id=session_id) + + async def list_sessions(self, cwd: str | None = None, **kwargs: Any) -> ListSessionsResponse: + sessions = [ + SessionInfo(session_id=session_id, cwd=session_cwd) + for session_id, session_cwd in self._sessions.items() + if cwd is None or cwd == session_cwd + ] + return ListSessionsResponse(sessions=sessions) + + async def resume_session( + self, session_id: str, cwd: str, replay_from: Any = None, **kwargs: Any + ) -> ResumeSessionResponse: + if replay_from is not None and getattr(replay_from, "type", None) != "start": + raise RequestError.invalid_params({"details": "unsupported replay cursor"}) + self._sessions[session_id] = cwd + assert self._client is not None + update = RunningSessionStateUpdate() if session_id in self._cancel_events else IdleSessionStateUpdate() + await self._client.session_update(session_id=session_id, update=update) + return ResumeSessionResponse() + + async def close_session(self, session_id: str, **kwargs: Any) -> CloseSessionResponse: + self._sessions.pop(session_id, None) + if event := self._cancel_events.get(session_id): + event.set() + if turn := self._turns.get(session_id): + await turn + return CloseSessionResponse() + + async def prompt(self, session_id: str, prompt: list[Any], **kwargs: Any) -> PromptResponse: + message_id = uuid4().hex + self._turns[session_id] = asyncio.create_task(self._run_turn(session_id, message_id, prompt)) + return PromptResponse(message_id=message_id) + + async def cancel_session(self, session_id: str, **kwargs: Any) -> None: + if event := self._cancel_events.get(session_id): + event.set() + + async def _run_turn(self, session_id: str, message_id: str, prompt: list[Any]) -> None: + assert self._client is not None + event = asyncio.Event() + self._cancel_events[session_id] = event + try: + await self._client.session_update( + session_id=session_id, + update=UserMessageUpdate(message_id=message_id, content=prompt), + ) + await self._client.session_update(session_id=session_id, update=RunningSessionStateUpdate()) + + texts = [block.text for block in prompt if isinstance(block, TextContentBlock)] + if texts == [CANCEL_PROMPT]: + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(event.wait(), timeout=_CANCEL_FALLBACK_SECONDS) + else: + for text in texts: + await self._client.session_update( + session_id=session_id, + update=AgentMessageChunk(message_id=message_id, content=TextContentBlock(text=text)), + ) + await self._client.session_update( + session_id=session_id, + update=IdleSessionStateUpdate(stop_reason="cancelled" if event.is_set() else "end_turn"), + ) + finally: + self._cancel_events.pop(session_id, None) + self._turns.pop(session_id, None) + + +async def main() -> None: + await run_agent(ProbeAgent()) + + +if __name__ == "__main__": + asyncio.run(main())