From 7f9e569a096af1cbbc26b2c5d9df5771d047cd29 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 10 Nov 2026 19:54:38 -0800 Subject: [PATCH 1/2] docs(kms): require admin TLS for onboarding handover Admin.GetKmsKey needs the caller's RA-TLS certificate, which a plain-HTTP admin listener cannot receive: without [core.admin.tls] the call fails with "No attestation provided" and the onboarding client fails to verify the source. Document the source-side admin setup (TLS, a reachable address, the token), fix the onboarding example's port (8001, not the gateway's 9202), and use https in the README admin example that already configures TLS. Signed-off-by: Kevin Wang --- docs/deployment.md | 20 ++++++++++++++++++-- dstack/kms/README.md | 10 ++++++---- dstack/kms/kms.toml | 3 ++- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index ec67cdedde..b6814067fc 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -477,6 +477,22 @@ Additional KMS instances can onboard from an existing KMS to share the same root 6. New KMS verifies the source's attestation on that same connection and checks it against its own policy before using the keys 7. Both KMS instances now derive identical keys +**Configure the source KMS for key handover:** + +`Admin.GetKmsKey` only works on an admin listener with `[core.admin.tls]`: without TLS the listener speaks plain HTTP, so the new KMS can present no RA-TLS certificate and the call fails with `No attestation provided`. The admin listener binds to `127.0.0.1` by default; bind it to an address the new KMS can reach and restrict access to the KMS nodes at the network layer. + +```toml +[core.admin] +enabled = true +address = "0.0.0.0" # reachable by the new KMS; firewall it to KMS nodes only +port = 8001 +auth_token = "" + +[core.admin.tls] +key = "/etc/kms/certs/rpc.key" +certs = "/etc/kms/certs/rpc.crt" +``` + **Configure new KMS for onboarding:** ```toml @@ -492,7 +508,7 @@ port = 9203 # HTTP port for onboard UI ```bash curl -X POST http://:9203/prpc/Onboard.Onboard?json \ -H "Content-Type: application/json" \ - -d '{"source_url": "https://:9202/prpc", "source_token": "", "domain": "kms2.example.com"}' + -d '{"source_url": "https://:8001/prpc", "source_token": "", "domain": "kms2.example.com"}' ``` **Finish and restart:** @@ -511,7 +527,7 @@ curl http://:9203/finish > **Admin authentication.** The KMS *admin* RPCs (for example `ClearImageCache`) are served on a dedicated `[core.admin]` listener behind the shared HTTP authenticator, just like the VMM and gateway: set `[core.admin] enabled = true` with an `auth_token` (or the `DSTACK_KMS_ADMIN_TOKEN` / `ADMIN_API_TOKEN` env vars), and clients send `Authorization: Bearer ` or `X-Admin-Token`. Enabled with neither `auth_token` nor `htpasswd_file` (and `insecure_no_auth = false`) fails closed — the KMS refuses to start. See [dstack-kms admin authentication](../dstack/kms/README.md#admin-api-authentication). > -> **Key handover.** It is served as `Admin.GetKmsKey` on the `[core.admin]` listener, which needs `[core.admin.tls]` so it can verify the onboarding KMS's RA-TLS certificate. Pass the source admin URL and its token (`source_token`) in the onboarding request. `core.onboard.public_key_handover = true` keeps the legacy public `KMS.GetKmsKey`. +> **Key handover.** It is served as `Admin.GetKmsKey` on the `[core.admin]` listener, which needs `[core.admin.tls]` so it can verify the onboarding KMS's RA-TLS certificate. Pass the source admin URL (`https://`) and its token (`source_token`) in the onboarding request. `core.onboard.public_key_handover = true` keeps the legacy public `KMS.GetKmsKey`: omit `source_token` and pass the source's public RPC URL instead. --- diff --git a/dstack/kms/README.md b/dstack/kms/README.md index 49d15e6184..815424fd33 100644 --- a/dstack/kms/README.md +++ b/dstack/kms/README.md @@ -57,10 +57,11 @@ certs = "/etc/kms/certs/rpc.crt" The token can also be supplied via the `DSTACK_KMS_ADMIN_TOKEN` or `ADMIN_API_TOKEN` environment variables instead of the config file. The admin listener fails closed: enabled with neither `auth_token` nor `htpasswd_file` -(and `insecure_no_auth = false`) refuses to start. Call it with, for example: +(and `insecure_no_auth = false`) refuses to start. With `[core.admin.tls]` the +listener serves HTTPS (use `http://` without it). Call it with, for example: ```bash -curl -X POST "http://127.0.0.1:8001/prpc/Admin.ClearImageCache?json" \ +curl -k -X POST "https://127.0.0.1:8001/prpc/Admin.ClearImageCache?json" \ -H "Authorization: Bearer $TOKEN" \ -H "Content-Type: application/json" \ -d '{"image_hash": "", "config_hash": ""}' @@ -70,8 +71,9 @@ This admin authentication is separate from the on-chain / webhook authorization below, which decides whether a CVM may boot and receive keys. KMS-to-KMS onboarding calls `Admin.GetKmsKey` with a self-issued RA-TLS client -certificate plus the source's admin bearer token; other admin RPCs need no client -certificate. `core.onboard.public_key_handover = true` (the default) keeps +certificate plus the source's admin bearer token, so it requires +`[core.admin.tls]` and an admin address the new KMS can reach; other admin RPCs +need no client certificate. `core.onboard.public_key_handover = true` (the default) keeps `KMS.GetKmsKey` on the public listener for older clients. diff --git a/dstack/kms/kms.toml b/dstack/kms/kms.toml index 22ee6d7b5f..fed11f0014 100644 --- a/dstack/kms/kms.toml +++ b/dstack/kms/kms.toml @@ -102,7 +102,8 @@ htpasswd_file = "" insecure_no_auth = false # Optional TLS for the admin listener. When configured, clients may present an -# RA-TLS certificate. GetKmsKey requires it; other admin RPCs do not. +# RA-TLS certificate. GetKmsKey requires it; other admin RPCs do not. To serve +# KMS onboarding, enable this and bind `address` to one the new KMS can reach. # [core.admin.tls] # key = "/etc/kms/certs/rpc.key" # certs = "/etc/kms/certs/rpc.crt" From db71147ed61b6d54f53742df77c0fe195844ba5c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 10 Nov 2026 23:49:50 -0800 Subject: [PATCH 2/2] docs(kms): note that admin TLS needs no mutual section Signed-off-by: Kevin Wang --- docs/deployment.md | 1 + dstack/kms/kms.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/deployment.md b/docs/deployment.md index b6814067fc..9d010e67b7 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -491,6 +491,7 @@ auth_token = "" [core.admin.tls] key = "/etc/kms/certs/rpc.key" certs = "/etc/kms/certs/rpc.crt" +# No [core.admin.tls.mutual]: client certificates are verified by their attestation. ``` **Configure new KMS for onboarding:** diff --git a/dstack/kms/kms.toml b/dstack/kms/kms.toml index fed11f0014..c24484d166 100644 --- a/dstack/kms/kms.toml +++ b/dstack/kms/kms.toml @@ -104,6 +104,7 @@ insecure_no_auth = false # Optional TLS for the admin listener. When configured, clients may present an # RA-TLS certificate. GetKmsKey requires it; other admin RPCs do not. To serve # KMS onboarding, enable this and bind `address` to one the new KMS can reach. +# No `[core.admin.tls.mutual]` is needed, as for `[rpc.tls]` above. # [core.admin.tls] # key = "/etc/kms/certs/rpc.key" # certs = "/etc/kms/certs/rpc.crt"