From 63d6fc7ff4ebafa4b7688e4db0bf87ed05e6159e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 10 Nov 2026 01:21:56 -0800 Subject: [PATCH 1/4] ci(sdk): pin Python 3.13 for SDK tests The job used the runner image's system Python. On ubuntu-26.04 that is 3.14, for which the locked pydantic-core 2.33.2 has no wheel, and its source build fails because PyO3 0.24 supports at most 3.13. Signed-off-by: Kevin Wang --- .github/workflows/sdk.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/sdk.yaml b/.github/workflows/sdk.yaml index 550806ff97..27c07d6be2 100644 --- a/.github/workflows/sdk.yaml +++ b/.github/workflows/sdk.yaml @@ -84,6 +84,14 @@ jobs: # the entries that are actually shared. save-if: ${{ github.event_name == 'push' }} + # Pin the interpreter: the runner image's system Python can be newer than + # the PyO3 versions in the Python SDK's locked dependencies support. + - name: Install Python + if: steps.changes.outputs.relevant == 'true' + uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: SDK tests if: steps.changes.outputs.relevant == 'true' run: cd sdk && ./run-tests.sh From c88f60e781a62d542e0843b333f18a48d1458b88 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 10 Nov 2026 17:40:52 +0800 Subject: [PATCH 2/4] fix(vmm): write instance info atomically (#1553) prepare_mr_config_v3() rewrites .instance_info on every launch with a truncating fs::write. A failed write or power loss leaves the file empty, losing instance_id_seed and with it the instance ID that keys the VM's persistent disk. Use safe_write, as the guest-reported instance info path already does. Signed-off-by: Kevin Wang --- dstack/vmm/src/app/mr_config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack/vmm/src/app/mr_config.rs b/dstack/vmm/src/app/mr_config.rs index e9aa83539b..926bcc6756 100644 --- a/dstack/vmm/src/app/mr_config.rs +++ b/dstack/vmm/src/app/mr_config.rs @@ -173,7 +173,7 @@ impl VmWorkDir { }; instance_info.app_id = app_id.clone(); instance_info.instance_id = instance_id.clone(); - fs::write( + safe_write::safe_write( self.instance_info_path(), serde_json::to_string(&instance_info).context("failed to serialize instance info")?, ) From e3c5252ddef760692636616d7046e4df926e2978 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 11 Nov 2026 15:51:40 +0800 Subject: [PATCH 3/4] ci: run checks on PRs to feat/** and test vmm UI and vmm-cli (#1559) * ci: run checks on PRs to feat/** and test vmm UI and vmm-cli PRs targeting feature branches only got the Foundry checks. Also fail the Rust job when cryptsetup is missing so the LUKS tests cannot silently skip. Signed-off-by: Kevin Wang * ci: check auth-eth-bun on feature branch PRs and fix the vmm-cli deps cryptography 50.0.0 needs cffi 2.0.0 or later, so requirements.txt could not be installed with cffi pinned to 1.17.1. The file exists on every branch, so install it unconditionally. Signed-off-by: Kevin Wang --------- Signed-off-by: Kevin Wang --- .github/workflows/kms-auth-eth-bun.yml | 2 +- .github/workflows/prek-check.yml | 2 +- .github/workflows/rust.yml | 8 +++++- .github/workflows/sdk.yaml | 2 +- .github/workflows/spdx-check.yml | 2 +- .github/workflows/vmm-ui.yml | 35 +++++++++++++++++++++++--- dstack/vmm/requirements.txt | 2 +- 7 files changed, 44 insertions(+), 9 deletions(-) diff --git a/.github/workflows/kms-auth-eth-bun.yml b/.github/workflows/kms-auth-eth-bun.yml index 10cba37d51..687dd90d61 100644 --- a/.github/workflows/kms-auth-eth-bun.yml +++ b/.github/workflows/kms-auth-eth-bun.yml @@ -24,7 +24,7 @@ on: - 'dstack/kms/auth-eth-bun/**' - '.github/workflows/kms-auth-eth-bun.yml' pull_request: - branches: [ next, 'release/**' ] + branches: [ next, 'release/**', 'feat/**' ] paths: - 'dstack/kms/auth-eth-bun/**' - '.github/workflows/kms-auth-eth-bun.yml' diff --git a/.github/workflows/prek-check.yml b/.github/workflows/prek-check.yml index 267f21c55b..38a6f60ce5 100644 --- a/.github/workflows/prek-check.yml +++ b/.github/workflows/prek-check.yml @@ -8,7 +8,7 @@ on: push: branches: [ next, 'release/**' ] pull_request: - branches: [ next, 'release/**' ] + branches: [ next, 'release/**', 'feat/**' ] permissions: contents: read diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 0cb18deec0..c93e8a3f56 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -8,7 +8,7 @@ on: push: branches: [ next, 'release/**' ] pull_request: - branches: [ next, 'release/**' ] + branches: [ next, 'release/**', 'feat/**' ] env: CARGO_TERM_COLOR: always @@ -78,6 +78,12 @@ jobs: if: steps.changes.outputs.relevant == 'true' run: cargo fmt --check --all + # The LUKS tests in dstack-util skip and pass when cryptsetup is missing. + # Fail here instead, so a runner without it cannot hide them. + - name: Require cryptsetup + if: steps.changes.outputs.relevant == 'true' + run: cryptsetup --version + - name: Run tests if: steps.changes.outputs.relevant == 'true' run: ./run-tests.sh diff --git a/.github/workflows/sdk.yaml b/.github/workflows/sdk.yaml index 27c07d6be2..50b2f78f24 100644 --- a/.github/workflows/sdk.yaml +++ b/.github/workflows/sdk.yaml @@ -11,7 +11,7 @@ on: push: branches: [next, 'release/**'] pull_request: - branches: [next, 'release/**'] + branches: [next, 'release/**', 'feat/**'] env: CARGO_TERM_COLOR: always diff --git a/.github/workflows/spdx-check.yml b/.github/workflows/spdx-check.yml index dc24dfd081..559bb3faf6 100644 --- a/.github/workflows/spdx-check.yml +++ b/.github/workflows/spdx-check.yml @@ -8,7 +8,7 @@ on: push: branches: [ next, 'release/**' ] pull_request: - branches: [ next, 'release/**' ] + branches: [ next, 'release/**', 'feat/**' ] jobs: reuse-lint: diff --git a/.github/workflows/vmm-ui.yml b/.github/workflows/vmm-ui.yml index 2598655418..0f02647129 100644 --- a/.github/workflows/vmm-ui.yml +++ b/.github/workflows/vmm-ui.yml @@ -7,18 +7,24 @@ name: VMM UI build permissions: contents: read -# Only the VMM's web UI is built here. Not a required status check, so a -# workflow-level filter is safe. +# The VMM's web UI and vmm-cli are built and tested here. Not a required +# status check, so a workflow-level filter is safe. on: push: branches: [ next, 'release/**' ] paths: - 'dstack/vmm/ui/**' + - 'dstack/vmm/src/vmm-cli.py' + - 'dstack/vmm/src/tests/test_vmm_cli.py' + - 'dstack/vmm/requirements.txt' - '.github/workflows/vmm-ui.yml' pull_request: - branches: [ next, 'release/**' ] + branches: [ next, 'release/**', 'feat/**' ] paths: - 'dstack/vmm/ui/**' + - 'dstack/vmm/src/vmm-cli.py' + - 'dstack/vmm/src/tests/test_vmm_cli.py' + - 'dstack/vmm/requirements.txt' - '.github/workflows/vmm-ui.yml' jobs: @@ -47,3 +53,26 @@ jobs: git status --porcelain exit 1 fi + + # `--if-present`: the UI test script is not on every branch yet. + - name: Test vmm UI + run: npm run test --if-present + working-directory: dstack/vmm/ui + + vmm-cli: + runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }} + steps: + - uses: actions/checkout@v5 + + - name: Install Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install vmm-cli dependencies + run: pip install -r requirements.txt + working-directory: dstack/vmm + + - name: Test vmm-cli + run: python3 -m unittest -v tests/test_vmm_cli.py + working-directory: dstack/vmm/src diff --git a/dstack/vmm/requirements.txt b/dstack/vmm/requirements.txt index 5a4636513e..2f43172374 100644 --- a/dstack/vmm/requirements.txt +++ b/dstack/vmm/requirements.txt @@ -1,5 +1,5 @@ annotated-types==0.7.0 -cffi==1.17.1 +cffi==2.0.0 cryptography==50.0.0 cytoolz==1.0.1 eth-hash==0.7.1 From 741bfa2a52ecfe33c09fb569b2ee6624b9eb094a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 11 Nov 2026 20:09:30 +0800 Subject: [PATCH 4/4] docs(kms): require admin TLS for onboarding handover (#1558) * docs(kms): require admin TLS for onboarding handover Admin.GetKmsKey needs the caller's RA-TLS certificate, which a plain-HTTP admin listener cannot receive: without [core.admin.tls] the call fails with "No attestation provided" and the onboarding client fails to verify the source. Document the source-side admin setup (TLS, a reachable address, the token), fix the onboarding example's port (8001, not the gateway's 9202), and use https in the README admin example that already configures TLS. Signed-off-by: Kevin Wang * docs(kms): note that admin TLS needs no mutual section Signed-off-by: Kevin Wang --------- Signed-off-by: Kevin Wang --- docs/deployment.md | 21 +++++++++++++++++++-- dstack/kms/README.md | 10 ++++++---- dstack/kms/kms.toml | 4 +++- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index ec67cdedde..9d010e67b7 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -477,6 +477,23 @@ Additional KMS instances can onboard from an existing KMS to share the same root 6. New KMS verifies the source's attestation on that same connection and checks it against its own policy before using the keys 7. Both KMS instances now derive identical keys +**Configure the source KMS for key handover:** + +`Admin.GetKmsKey` only works on an admin listener with `[core.admin.tls]`: without TLS the listener speaks plain HTTP, so the new KMS can present no RA-TLS certificate and the call fails with `No attestation provided`. The admin listener binds to `127.0.0.1` by default; bind it to an address the new KMS can reach and restrict access to the KMS nodes at the network layer. + +```toml +[core.admin] +enabled = true +address = "0.0.0.0" # reachable by the new KMS; firewall it to KMS nodes only +port = 8001 +auth_token = "" + +[core.admin.tls] +key = "/etc/kms/certs/rpc.key" +certs = "/etc/kms/certs/rpc.crt" +# No [core.admin.tls.mutual]: client certificates are verified by their attestation. +``` + **Configure new KMS for onboarding:** ```toml @@ -492,7 +509,7 @@ port = 9203 # HTTP port for onboard UI ```bash curl -X POST http://:9203/prpc/Onboard.Onboard?json \ -H "Content-Type: application/json" \ - -d '{"source_url": "https://:9202/prpc", "source_token": "", "domain": "kms2.example.com"}' + -d '{"source_url": "https://:8001/prpc", "source_token": "", "domain": "kms2.example.com"}' ``` **Finish and restart:** @@ -511,7 +528,7 @@ curl http://:9203/finish > **Admin authentication.** The KMS *admin* RPCs (for example `ClearImageCache`) are served on a dedicated `[core.admin]` listener behind the shared HTTP authenticator, just like the VMM and gateway: set `[core.admin] enabled = true` with an `auth_token` (or the `DSTACK_KMS_ADMIN_TOKEN` / `ADMIN_API_TOKEN` env vars), and clients send `Authorization: Bearer ` or `X-Admin-Token`. Enabled with neither `auth_token` nor `htpasswd_file` (and `insecure_no_auth = false`) fails closed — the KMS refuses to start. See [dstack-kms admin authentication](../dstack/kms/README.md#admin-api-authentication). > -> **Key handover.** It is served as `Admin.GetKmsKey` on the `[core.admin]` listener, which needs `[core.admin.tls]` so it can verify the onboarding KMS's RA-TLS certificate. Pass the source admin URL and its token (`source_token`) in the onboarding request. `core.onboard.public_key_handover = true` keeps the legacy public `KMS.GetKmsKey`. +> **Key handover.** It is served as `Admin.GetKmsKey` on the `[core.admin]` listener, which needs `[core.admin.tls]` so it can verify the onboarding KMS's RA-TLS certificate. Pass the source admin URL (`https://`) and its token (`source_token`) in the onboarding request. `core.onboard.public_key_handover = true` keeps the legacy public `KMS.GetKmsKey`: omit `source_token` and pass the source's public RPC URL instead. --- diff --git a/dstack/kms/README.md b/dstack/kms/README.md index 49d15e6184..815424fd33 100644 --- a/dstack/kms/README.md +++ b/dstack/kms/README.md @@ -57,10 +57,11 @@ certs = "/etc/kms/certs/rpc.crt" The token can also be supplied via the `DSTACK_KMS_ADMIN_TOKEN` or `ADMIN_API_TOKEN` environment variables instead of the config file. The admin listener fails closed: enabled with neither `auth_token` nor `htpasswd_file` -(and `insecure_no_auth = false`) refuses to start. Call it with, for example: +(and `insecure_no_auth = false`) refuses to start. With `[core.admin.tls]` the +listener serves HTTPS (use `http://` without it). Call it with, for example: ```bash -curl -X POST "http://127.0.0.1:8001/prpc/Admin.ClearImageCache?json" \ +curl -k -X POST "https://127.0.0.1:8001/prpc/Admin.ClearImageCache?json" \ -H "Authorization: Bearer $TOKEN" \ -H "Content-Type: application/json" \ -d '{"image_hash": "", "config_hash": ""}' @@ -70,8 +71,9 @@ This admin authentication is separate from the on-chain / webhook authorization below, which decides whether a CVM may boot and receive keys. KMS-to-KMS onboarding calls `Admin.GetKmsKey` with a self-issued RA-TLS client -certificate plus the source's admin bearer token; other admin RPCs need no client -certificate. `core.onboard.public_key_handover = true` (the default) keeps +certificate plus the source's admin bearer token, so it requires +`[core.admin.tls]` and an admin address the new KMS can reach; other admin RPCs +need no client certificate. `core.onboard.public_key_handover = true` (the default) keeps `KMS.GetKmsKey` on the public listener for older clients. diff --git a/dstack/kms/kms.toml b/dstack/kms/kms.toml index 22ee6d7b5f..c24484d166 100644 --- a/dstack/kms/kms.toml +++ b/dstack/kms/kms.toml @@ -102,7 +102,9 @@ htpasswd_file = "" insecure_no_auth = false # Optional TLS for the admin listener. When configured, clients may present an -# RA-TLS certificate. GetKmsKey requires it; other admin RPCs do not. +# RA-TLS certificate. GetKmsKey requires it; other admin RPCs do not. To serve +# KMS onboarding, enable this and bind `address` to one the new KMS can reach. +# No `[core.admin.tls.mutual]` is needed, as for `[rpc.tls]` above. # [core.admin.tls] # key = "/etc/kms/certs/rpc.key" # certs = "/etc/kms/certs/rpc.crt"