diff --git a/.github/workflows/kms-auth-eth.yml b/.github/workflows/kms-auth-eth.yml index 2d74f7c3d..4eb49c251 100644 --- a/.github/workflows/kms-auth-eth.yml +++ b/.github/workflows/kms-auth-eth.yml @@ -26,7 +26,7 @@ on: - 'dstack/kms/auth-eth/*' - '.github/workflows/kms-auth-eth.yml' pull_request: - branches: [ next, 'release/**' ] + branches: [ next, 'release/**', 'feat/**' ] paths: - 'dstack/kms/auth-eth/*' - '.github/workflows/kms-auth-eth.yml' diff --git a/.github/workflows/prek-check.yml b/.github/workflows/prek-check.yml index 267f21c55..38a6f60ce 100644 --- a/.github/workflows/prek-check.yml +++ b/.github/workflows/prek-check.yml @@ -8,7 +8,7 @@ on: push: branches: [ next, 'release/**' ] pull_request: - branches: [ next, 'release/**' ] + branches: [ next, 'release/**', 'feat/**' ] permissions: contents: read diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 0cb18deec..c93e8a3f5 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -8,7 +8,7 @@ on: push: branches: [ next, 'release/**' ] pull_request: - branches: [ next, 'release/**' ] + branches: [ next, 'release/**', 'feat/**' ] env: CARGO_TERM_COLOR: always @@ -78,6 +78,12 @@ jobs: if: steps.changes.outputs.relevant == 'true' run: cargo fmt --check --all + # The LUKS tests in dstack-util skip and pass when cryptsetup is missing. + # Fail here instead, so a runner without it cannot hide them. + - name: Require cryptsetup + if: steps.changes.outputs.relevant == 'true' + run: cryptsetup --version + - name: Run tests if: steps.changes.outputs.relevant == 'true' run: ./run-tests.sh diff --git a/.github/workflows/sdk.yaml b/.github/workflows/sdk.yaml index c17c8269f..50b2f78f2 100644 --- a/.github/workflows/sdk.yaml +++ b/.github/workflows/sdk.yaml @@ -11,7 +11,7 @@ on: push: branches: [next, 'release/**'] pull_request: - branches: [next, 'release/**'] + branches: [next, 'release/**', 'feat/**'] env: CARGO_TERM_COLOR: always @@ -84,11 +84,13 @@ jobs: # the entries that are actually shared. save-if: ${{ github.event_name == 'push' }} - - name: Set up Python + # Pin the interpreter: the runner image's system Python can be newer than + # the PyO3 versions in the Python SDK's locked dependencies support. + - name: Install Python if: steps.changes.outputs.relevant == 'true' uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.13' - name: SDK tests if: steps.changes.outputs.relevant == 'true' diff --git a/.github/workflows/spdx-check.yml b/.github/workflows/spdx-check.yml index dc24dfd08..559bb3faf 100644 --- a/.github/workflows/spdx-check.yml +++ b/.github/workflows/spdx-check.yml @@ -8,7 +8,7 @@ on: push: branches: [ next, 'release/**' ] pull_request: - branches: [ next, 'release/**' ] + branches: [ next, 'release/**', 'feat/**' ] jobs: reuse-lint: diff --git a/.github/workflows/vmm-ui.yml b/.github/workflows/vmm-ui.yml index 259865541..0f0264712 100644 --- a/.github/workflows/vmm-ui.yml +++ b/.github/workflows/vmm-ui.yml @@ -7,18 +7,24 @@ name: VMM UI build permissions: contents: read -# Only the VMM's web UI is built here. Not a required status check, so a -# workflow-level filter is safe. +# The VMM's web UI and vmm-cli are built and tested here. Not a required +# status check, so a workflow-level filter is safe. on: push: branches: [ next, 'release/**' ] paths: - 'dstack/vmm/ui/**' + - 'dstack/vmm/src/vmm-cli.py' + - 'dstack/vmm/src/tests/test_vmm_cli.py' + - 'dstack/vmm/requirements.txt' - '.github/workflows/vmm-ui.yml' pull_request: - branches: [ next, 'release/**' ] + branches: [ next, 'release/**', 'feat/**' ] paths: - 'dstack/vmm/ui/**' + - 'dstack/vmm/src/vmm-cli.py' + - 'dstack/vmm/src/tests/test_vmm_cli.py' + - 'dstack/vmm/requirements.txt' - '.github/workflows/vmm-ui.yml' jobs: @@ -47,3 +53,26 @@ jobs: git status --porcelain exit 1 fi + + # `--if-present`: the UI test script is not on every branch yet. + - name: Test vmm UI + run: npm run test --if-present + working-directory: dstack/vmm/ui + + vmm-cli: + runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }} + steps: + - uses: actions/checkout@v5 + + - name: Install Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install vmm-cli dependencies + run: pip install -r requirements.txt + working-directory: dstack/vmm + + - name: Test vmm-cli + run: python3 -m unittest -v tests/test_vmm_cli.py + working-directory: dstack/vmm/src diff --git a/docs/deployment.md b/docs/deployment.md index a64710d1c..36f07addb 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -499,6 +499,23 @@ Additional KMS instances can onboard from an existing KMS to share the same root 6. New KMS verifies the source's attestation on that same connection and checks it against its own policy before using the keys 7. Both KMS instances now derive identical keys +**Configure the source KMS for key handover:** + +`Admin.GetKmsKey` only works on an admin listener with `[core.admin.tls]`: without TLS the listener speaks plain HTTP, so the new KMS can present no RA-TLS certificate and the call fails with `No attestation provided`. The admin listener binds to `127.0.0.1` by default; bind it to an address the new KMS can reach and restrict access to the KMS nodes at the network layer. + +```toml +[core.admin] +enabled = true +address = "0.0.0.0" # reachable by the new KMS; firewall it to KMS nodes only +port = 8001 +auth_token = "" + +[core.admin.tls] +key = "/etc/kms/certs/rpc.key" +certs = "/etc/kms/certs/rpc.crt" +# No [core.admin.tls.mutual]: client certificates are verified by their attestation. +``` + **Configure new KMS for onboarding:** ```toml @@ -514,7 +531,7 @@ port = 9203 # HTTP port for onboard UI ```bash curl -X POST http://:9203/prpc/Onboard.Onboard?json \ -H "Content-Type: application/json" \ - -d '{"source_url": "https://:9202/prpc", "source_token": "", "domain": "kms2.example.com"}' + -d '{"source_url": "https://:8001/prpc", "source_token": "", "domain": "kms2.example.com"}' ``` **Finish and restart:** @@ -533,7 +550,7 @@ curl http://:9203/finish > **Admin authentication.** The KMS *admin* RPCs (for example `ClearImageCache`) are served on a dedicated `[core.admin]` listener behind the shared HTTP authenticator, just like the VMM and gateway: set `[core.admin] enabled = true` with an `auth_token` (or the `DSTACK_KMS_ADMIN_TOKEN` / `ADMIN_API_TOKEN` env vars), and clients send `Authorization: Bearer ` or `X-Admin-Token`. Enabled with neither `auth_token` nor `htpasswd_file` (and `insecure_no_auth = false`) fails closed — the KMS refuses to start. See [dstack-kms admin authentication](../dstack/kms/README.md#admin-api-authentication). > -> **Key handover.** It is served as `Admin.GetKmsKey` on the `[core.admin]` listener, which needs `[core.admin.tls]` so it can verify the onboarding KMS's RA-TLS certificate. Pass the source admin URL and its token (`source_token`) in the onboarding request. The legacy public `KMS.GetKmsKey` is off by default; set `core.onboard.public_key_handover = true` on the source only when an onboarding KMS cannot use the admin listener, such as a 0.5.x KMS or an onboarding request without `source_token`. +> **Key handover.** It is served as `Admin.GetKmsKey` on the `[core.admin]` listener, which needs `[core.admin.tls]` so it can verify the onboarding KMS's RA-TLS certificate. Pass the source admin URL (`https://`) and its token (`source_token`) in the onboarding request. The legacy public `KMS.GetKmsKey` is off by default; set `core.onboard.public_key_handover = true` on the source only when an onboarding KMS cannot use the admin listener, such as a 0.5.x KMS or an onboarding request without `source_token`. #### Rotating KMS Root Keys diff --git a/dstack/kms/README.md b/dstack/kms/README.md index e14fd8dac..e4d0c5625 100644 --- a/dstack/kms/README.md +++ b/dstack/kms/README.md @@ -57,10 +57,11 @@ certs = "/etc/kms/certs/rpc.crt" The token can also be supplied via the `DSTACK_KMS_ADMIN_TOKEN` or `ADMIN_API_TOKEN` environment variables instead of the config file. The admin listener fails closed: enabled with neither `auth_token` nor `htpasswd_file` -(and `insecure_no_auth = false`) refuses to start. Call it with, for example: +(and `insecure_no_auth = false`) refuses to start. With `[core.admin.tls]` the +listener serves HTTPS (use `http://` without it). Call it with, for example: ```bash -curl -X POST "http://127.0.0.1:8001/prpc/Admin.ClearImageCache?json" \ +curl -k -X POST "https://127.0.0.1:8001/prpc/Admin.ClearImageCache?json" \ -H "Authorization: Bearer $TOKEN" \ -H "Content-Type: application/json" \ -d '{"image_hash": "", "config_hash": ""}' @@ -70,8 +71,9 @@ This admin authentication is separate from the on-chain / webhook authorization below, which decides whether a CVM may boot and receive keys. KMS-to-KMS onboarding calls `Admin.GetKmsKey` with a self-issued RA-TLS client -certificate plus the source's admin bearer token; other admin RPCs need no client -certificate. `core.onboard.public_key_handover` is off by default; set it to +certificate plus the source's admin bearer token, so it requires +`[core.admin.tls]` and an admin address the new KMS can reach; other admin RPCs +need no client certificate. `core.onboard.public_key_handover` is off by default; set it to `true` only to also serve `KMS.GetKmsKey` on the public listener for older clients. diff --git a/dstack/kms/kms.toml b/dstack/kms/kms.toml index 5673d189b..7c169d338 100644 --- a/dstack/kms/kms.toml +++ b/dstack/kms/kms.toml @@ -102,7 +102,9 @@ htpasswd_file = "" insecure_no_auth = false # Optional TLS for the admin listener. When configured, clients may present an -# RA-TLS certificate. GetKmsKey requires it; other admin RPCs do not. +# RA-TLS certificate. GetKmsKey requires it; other admin RPCs do not. To serve +# KMS onboarding, enable this and bind `address` to one the new KMS can reach. +# No `[core.admin.tls.mutual]` is needed, as for `[rpc.tls]` above. # [core.admin.tls] # key = "/etc/kms/certs/rpc.key" # certs = "/etc/kms/certs/rpc.crt" diff --git a/dstack/vmm/src/app/mr_config.rs b/dstack/vmm/src/app/mr_config.rs index e9aa83539..926bcc675 100644 --- a/dstack/vmm/src/app/mr_config.rs +++ b/dstack/vmm/src/app/mr_config.rs @@ -173,7 +173,7 @@ impl VmWorkDir { }; instance_info.app_id = app_id.clone(); instance_info.instance_id = instance_id.clone(); - fs::write( + safe_write::safe_write( self.instance_info_path(), serde_json::to_string(&instance_info).context("failed to serialize instance info")?, )