From 1a6dfad009d6077a80ed6b5bb9078d4cfc984bed Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 16 Nov 2026 19:49:40 -0800 Subject: [PATCH 1/3] feat(kms)!: archive each root epoch's attestation and move the keyring to keyring_dir Key handover sent the root keys of every epoch but not the attestation produced when each was generated, and the single bootstrap-info.json was overwritten by a rotation and never written by a join or auto-bootstrap (#539), so the evidence of older epochs was lost. - KmsKeys gains `attestation`, one per epoch. The source fills it from its keyring; the receiver archives it as is and does not verify it, which is left to relying parties. Older sources send it empty. - Every epoch is stored alike under /epochs// with its root-attestation; root-epoch is written last and marks a complete keyring. The newest epoch no longer lives at the top level, and bootstrap-info.json is gone: GetMeta.bootstrap_info reports the served epoch's attestation. - The keyring moves out of cert_dir into the new core.keyring_dir. It has no default, so a config cannot silently put the root keys on storage that is not persistent. Older on-disk layouts are not read: a KMS always starts from bootstrap or onboarding, never from another version's disk. Signed-off-by: Kevin Wang --- CHANGELOG.md | 17 + docs/deployment.md | 2 +- docs/kms-emergency-key-rotation-protocol.md | 6 +- docs/tutorials/kms-build-configuration.md | 18 +- docs/tutorials/kms-cvm-deployment.md | 27 +- dstack/kms/dstack-app/compose-dev.yaml | 1 + dstack/kms/dstack-app/compose-simple.yaml | 1 + dstack/kms/kms.toml | 2 + dstack/kms/rpc/proto/kms_rpc.proto | 3 + dstack/kms/src/config.rs | 74 ++-- dstack/kms/src/main.rs | 5 +- dstack/kms/src/main_service.rs | 59 ++- dstack/kms/src/onboard_service.rs | 418 ++++++++---------- dstack/kms/src/www/onboard.html | 2 +- .../full-stack-compose/scripts/app_compose.py | 1 + tools/dev-stack.sh | 2 + 16 files changed, 329 insertions(+), 309 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 576436e92..6e5a47dba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Upgrade notes + +- **KMS requires `core.keyring_dir`.** The root keys of every epoch move out of + `cert_dir` into `/epochs//`, with `/root-epoch` + marking the newest. It has no default: point it at persistent storage. The + on-disk layout is not read from older versions, so upgrade by onboarding a + new KMS, as before. + +### Changed + +- KMS: each root epoch keeps the attestation produced when its keys were + generated (`epochs//root-attestation`), and key handover carries it in + `KmsKeys.attestation`, so joined and rotated nodes keep the evidence of every + epoch. The receiver archives it without verifying it; relying parties verify + it. `bootstrap-info.json` is gone: `GetMeta.bootstrap_info` reports the served + epoch's attestation. Auto-bootstrap now records the attestation too (#539). + ## [0.6.5] - 2026-11-15 ### Upgrade notes diff --git a/docs/deployment.md b/docs/deployment.md index 3817cae0e..30383faed 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -577,7 +577,7 @@ governance made current. See the [rotation design](./kms-emergency-key-rotation- for the full procedure; the KMS steps are: 1. If epochs were never registered, publish the attestation the KMS produced at - bootstrap (kept in `bootstrap-info.json` on the bootstrapped node) and + bootstrap (served as `bootstrap_info` by `KMS.GetMeta`) and register the current root public keys as epoch 0: `DstackKms.registerRootEpoch(caPubkey, k256Pubkey, evidenceUrl)`, via `script/Manage.s.sol:RegisterRootEpoch`, or a `rootEpochs` entry in the diff --git a/docs/kms-emergency-key-rotation-protocol.md b/docs/kms-emergency-key-rotation-protocol.md index fe628d37c..670a347ee 100644 --- a/docs/kms-emergency-key-rotation-protocol.md +++ b/docs/kms-emergency-key-rotation-protocol.md @@ -75,8 +75,10 @@ latest two. Keeping old roots does not authorize issuing old derived keys. There is no local epoch configuration: a KMS serves the current epoch on chain, provided its keyring holds it and reproduces its chain hash, and waits otherwise. When the current epoch moves, the KMS restarts to follow it and -re-signs its RPC certificate under the new root CA. A `root-epoch` marker records -the newest stored epoch. +re-signs its RPC certificate under the new root CA. Each epoch is stored under +`epochs//` with the attestation produced when its keys were generated, and +onboarding hands both over; a `root-epoch` marker records the newest stored +epoch. Historical epochs are not time-limited. Their roots are already exposed, so withholding derived keys from authorized CVMs would not reduce what an attacker diff --git a/docs/tutorials/kms-build-configuration.md b/docs/tutorials/kms-build-configuration.md index b9f88be81..bb8de8084 100644 --- a/docs/tutorials/kms-build-configuration.md +++ b/docs/tutorials/kms-build-configuration.md @@ -127,8 +127,8 @@ Create the directory structure for KMS configuration and certificates. # Configuration directory sudo mkdir -p /etc/kms -# Certificate directory -sudo mkdir -p /etc/kms/certs +# Certificate and root keyring directories +sudo mkdir -p /etc/kms/certs /etc/kms/keyring # Runtime directories sudo mkdir -p /var/run/kms @@ -188,6 +188,7 @@ port = 9100 # Core KMS Configuration [core] cert_dir = "/etc/kms/certs" +keyring_dir = "/etc/kms/keyring" subject_postfix = ".dstack" # Intel PCCS URL for TDX quote verification pccs_url = "https://pccs.phala.network/sgx/certification/v4" @@ -231,6 +232,7 @@ EOF | `[listeners.admin]` | `address`, `port`, `auth_token_hash` | Admin listener (9101), which serves the key handover; the SHA-256 of the admin token. A serving KMS requires it with TLS | | `[listeners.onboard]` | `address`, `port` | Plain-HTTP onboarding page (9100, before the KMS has keys) | | `[core]` | `cert_dir` | Directory for certificates | +| `[core]` | `keyring_dir` | Directory for the root keys of every epoch (required; must be persistent) | | `[core]` | `pccs_url` | PCCS endpoint for quote verification | | `[core.auth_api]` | `url` | auth-eth webhook service URL | | `[core.onboard]` | `enabled` | Enable bootstrap/onboard mode | @@ -356,7 +358,7 @@ RUN apt-get update && \ COPY --from=bun /usr/local/bin/bun /usr/local/bin/bun # Create directories -RUN mkdir -p /etc/kms/certs /etc/kms/images /var/run/kms /var/log/kms +RUN mkdir -p /etc/kms/certs /etc/kms/keyring /etc/kms/images /var/run/kms /var/log/kms # Copy KMS binary COPY dstack-kms /usr/local/bin/dstack-kms @@ -434,6 +436,7 @@ port = 9100 # Core KMS Configuration [core] cert_dir = "/etc/kms/certs" +keyring_dir = "/etc/kms/keyring" subject_postfix = ".dstack" pccs_url = "https://pccs.phala.network/sgx/certification/v4" @@ -459,12 +462,12 @@ port = 9100 # Onboarding Configuration [core.onboard] enabled = true -# Empty domain = manual bootstrap mode (ensures bootstrap-info.json is written) +# Empty domain = manual bootstrap mode auto_bootstrap_domain = "" EOF ``` -> **Why empty `auto_bootstrap_domain`?** With an empty domain, KMS starts in "onboard mode" — a plain HTTP server that waits for you to trigger bootstrap via an RPC call. This ensures `bootstrap-info.json` is written to disk, which is required for on-chain KMS registration. You'll provide the domain during the bootstrap step in [KMS CVM Deployment](/tutorial/kms-cvm-deployment). +> **Why empty `auto_bootstrap_domain`?** With an empty domain, KMS starts in "onboard mode" — a plain HTTP server that waits for you to trigger bootstrap via an RPC call. Its response carries the attestation and public keys required for on-chain KMS registration. You'll provide the domain during the bootstrap step in [KMS CVM Deployment](/tutorial/kms-cvm-deployment). ### Copy build artifacts and configuration @@ -542,8 +545,9 @@ services: volumes: # Mount config file from local directory - ./kms.toml:/etc/kms/kms.toml:ro - # Named volume for persistent certificates + # Named volumes for persistent certificates and root keys - kms-certs:/etc/kms/certs + - kms-keyring:/etc/kms/keyring environment: - RUST_LOG=info restart: unless-stopped @@ -551,6 +555,8 @@ services: volumes: kms-certs: # Certificates persist across container restarts + kms-keyring: + # Root keys persist across container restarts EOF ``` diff --git a/docs/tutorials/kms-cvm-deployment.md b/docs/tutorials/kms-cvm-deployment.md index 2796a5244..f566350bc 100644 --- a/docs/tutorials/kms-cvm-deployment.md +++ b/docs/tutorials/kms-cvm-deployment.md @@ -65,16 +65,22 @@ When you deploy KMS as a CVM, the following happens: ### Generated Artifacts -Inside the CVM at `/etc/kms/certs/`: +Inside the CVM, the root keyring at `/etc/kms/keyring/` (`keyring_dir`): + +| File | Purpose | +|------|---------| +| `epochs//root-ca.crt` | Root Certificate Authority of epoch `n` (self-signed) | +| `epochs//root-ca.key` | Root CA signing key of epoch `n` (P256 ECDSA) | +| `epochs//root-k256.key` | Ethereum signing key of epoch `n` (secp256k1) | +| `epochs//root-attestation` | Attestation produced when the epoch `n` keys were generated; absent if there is none | +| `root-epoch` | Newest root epoch held | + +And the RPC certificates at `/etc/kms/certs/` (`cert_dir`): | File | Purpose | |------|---------| -| `root-ca.crt` | Root Certificate Authority (self-signed) | -| `root-ca.key` | Root CA signing key (P256 ECDSA) | | `rpc.crt` | TLS certificate for RPC server | | `rpc.key` | RPC server private key | -| `root-k256.key` | Ethereum signing key (secp256k1) | -| `bootstrap-info.json` | Public keys and TDX attestation quote | --- @@ -139,6 +145,7 @@ services: volumes: - /var/run/dstack.sock:/var/run/dstack.sock - kms-certs:/etc/kms/certs + - kms-keyring:/etc/kms/keyring environment: - RUST_LOG=info - KMS_DOMAIN=kms.yourdomain.com @@ -152,6 +159,7 @@ services: volumes: kms-certs: + kms-keyring: configs: kms_config: @@ -184,6 +192,7 @@ configs: [core] cert_dir = "/etc/kms/certs" + keyring_dir = "/etc/kms/keyring" pccs_url = "https://pccs.phala.network/sgx/certification/v4" [core.image] @@ -234,7 +243,7 @@ This docker-compose uses a Docker `configs` section to inject a complete `kms.to | `[core.auth_api]` | Authentication via auth-eth webhook on localhost:9200 | | `[core.onboard]` | Bootstrap settings — `auto_bootstrap_domain` is empty so KMS enters onboard mode for manual bootstrap | -> **Why manual bootstrap?** With `auto_bootstrap_domain` left empty, KMS starts in "onboard mode" — a plain HTTP server on port 9100 that waits for you to trigger bootstrap via an RPC call. This ensures `bootstrap-info.json` (containing the TDX attestation quote and public keys) is written to disk. You'll need this file later to register the KMS on-chain. +> **Why manual bootstrap?** With `auto_bootstrap_domain` left empty, KMS starts in "onboard mode" — a plain HTTP server on port 9100 that waits for you to trigger bootstrap via an RPC call. The response carries the TDX attestation quote and public keys you'll need later to register the KMS on-chain. **Environment variables explained:** @@ -356,7 +365,7 @@ Onboarding ### Step 6: Bootstrap KMS -With KMS in onboard mode, trigger key generation by calling the Bootstrap RPC endpoint. This generates root keys, a TDX attestation quote, and writes `bootstrap-info.json`: +With KMS in onboard mode, trigger key generation by calling the Bootstrap RPC endpoint. This generates root keys and a TDX attestation quote over them: ```bash # Inspect the KMS measurement before bootstrap @@ -544,9 +553,9 @@ For detailed solutions, see the [KMS Deployment Troubleshooting Guide](/tutorial ### Understanding Storage -CVM certificates are stored in a Docker named volume (`kms-certs`). This provides: +CVM certificates and root keys are stored in Docker named volumes (`kms-certs` and `kms-keyring`). This provides: -- **Container restart persistence** - Certificates survive container restarts +- **Container restart persistence** - Certificates and root keys survive container restarts - **CVM restart consideration** - Depending on VMM configuration, volumes may or may not persist ### Backup Recommendations diff --git a/dstack/kms/dstack-app/compose-dev.yaml b/dstack/kms/dstack-app/compose-dev.yaml index 955f92687..70a10e1ed 100644 --- a/dstack/kms/dstack-app/compose-dev.yaml +++ b/dstack/kms/dstack-app/compose-dev.yaml @@ -64,6 +64,7 @@ configs: [core] cert_dir = "/kms/certs" + keyring_dir = "/kms/keyring" [core.image] cache_dir = "/kms/images" diff --git a/dstack/kms/dstack-app/compose-simple.yaml b/dstack/kms/dstack-app/compose-simple.yaml index 3dbd6f2a5..ba27e2be7 100644 --- a/dstack/kms/dstack-app/compose-simple.yaml +++ b/dstack/kms/dstack-app/compose-simple.yaml @@ -37,6 +37,7 @@ configs: [core] cert_dir = "/kms/certs" + keyring_dir = "/kms/keyring" [core.image] cache_dir = "/kms/images" diff --git a/dstack/kms/kms.toml b/dstack/kms/kms.toml index cc66eb25f..1bfad44bb 100644 --- a/dstack/kms/kms.toml +++ b/dstack/kms/kms.toml @@ -69,6 +69,8 @@ port = 8000 [core] cert_dir = "/etc/kms/certs" +# Root keys of every epoch. Required, with no default: point it at persistent +# storage, e.g. keyring_dir = "/etc/kms/keyring". subject_postfix = ".dstack" site_name = "" # Whether trusted RPCs require the KMS to first attest itself to its own diff --git a/dstack/kms/rpc/proto/kms_rpc.proto b/dstack/kms/rpc/proto/kms_rpc.proto index c2088b978..e04e7e6c6 100644 --- a/dstack/kms/rpc/proto/kms_rpc.proto +++ b/dstack/kms/rpc/proto/kms_rpc.proto @@ -93,6 +93,9 @@ message GetKmsKeyRequest { message KmsKeys { string ca_key = 1; bytes k256_key = 2; + // Attestation produced when this epoch's keys were generated, as registered + // with governance; empty if none. Archived by the receiver, not verified. + bytes attestation = 3; } message KmsKeyResponse { diff --git a/dstack/kms/src/config.rs b/dstack/kms/src/config.rs index 62fcc2dc2..5a57a829e 100644 --- a/dstack/kms/src/config.rs +++ b/dstack/kms/src/config.rs @@ -70,9 +70,9 @@ const RPC_CERT: &str = "rpc.crt"; const RPC_KEY: &str = "rpc.key"; const RPC_DOMAIN: &str = "rpc-domain"; const K256_KEY: &str = "root-k256.key"; +const ROOT_ATTESTATION: &str = "root-attestation"; const ROOT_EPOCH: &str = "root-epoch"; const EPOCHS_DIR: &str = "epochs"; -const BOOTSTRAP_INFO: &str = "bootstrap-info.json"; #[derive(Debug, Clone, Deserialize)] pub(crate) struct ImageConfig { @@ -85,6 +85,9 @@ pub(crate) struct ImageConfig { #[derive(Debug, Clone, Deserialize)] pub(crate) struct KmsConfig { pub cert_dir: PathBuf, + /// Root keys of every epoch. Required: it is the only state the KMS + /// cannot regenerate, so it must be placed on persistent storage on purpose. + pub keyring_dir: PathBuf, #[serde(default)] pub attestation: AttestationVerifierConfig, pub auth_api: AuthApi, @@ -207,19 +210,15 @@ pub(crate) fn check_key_handover( } impl KmsConfig { - /// The root keys are the last files a bootstrap writes, so both existing - /// marks a completed one. Anything short of that is left over from an - /// interrupted bootstrap and may be overwritten. + /// The root epoch marker is the last file a bootstrap or onboarding writes, + /// so its presence marks a completed one. Anything short of that is left + /// over from an interrupted attempt and may be overwritten. pub fn root_keys_exist(&self) -> bool { - self.root_ca_key().exists() && self.k256_key().exists() + self.root_epoch().exists() } pub fn keys_exists(&self) -> bool { - self.root_ca_cert().exists() - && self.root_ca_key().exists() - && self.rpc_cert().exists() - && self.rpc_key().exists() - && self.k256_key().exists() + self.root_keys_exist() && self.rpc_cert().exists() && self.rpc_key().exists() } pub fn tmp_ca_cert(&self) -> PathBuf { @@ -230,14 +229,6 @@ impl KmsConfig { self.cert_dir.join(TEMP_CA_KEY) } - pub fn root_ca_cert(&self) -> PathBuf { - self.cert_dir.join(ROOT_CA_CERT) - } - - pub fn root_ca_key(&self) -> PathBuf { - self.cert_dir.join(ROOT_CA_KEY) - } - pub fn rpc_cert(&self) -> PathBuf { self.cert_dir.join(RPC_CERT) } @@ -250,35 +241,21 @@ impl KmsConfig { self.cert_dir.join(RPC_DOMAIN) } - pub fn k256_key(&self) -> PathBuf { - self.cert_dir.join(K256_KEY) - } - - /// Records the newest epoch this KMS holds, whose roots are the top-level - /// root files. Absent means 0. + /// Records the newest epoch this KMS holds. pub fn root_epoch(&self) -> PathBuf { - self.cert_dir.join(ROOT_EPOCH) + self.keyring_dir.join(ROOT_EPOCH) } - /// Root key paths of `epoch` in a keyring whose newest epoch is `latest`: - /// the newest roots are the top-level root files, so paths that predate - /// epochs are unchanged. Older epochs live in `epochs//`. - pub fn root_key_paths(&self, epoch: u32, latest: u32) -> RootKeyPaths { - let dir = if epoch == latest { - self.cert_dir.clone() - } else { - self.cert_dir.join(EPOCHS_DIR).join(epoch.to_string()) - }; + /// Root key paths of `epoch`, under `epochs//`. + pub fn root_key_paths(&self, epoch: u32) -> RootKeyPaths { + let dir = self.keyring_dir.join(EPOCHS_DIR).join(epoch.to_string()); RootKeyPaths { ca_cert: dir.join(ROOT_CA_CERT), ca_key: dir.join(ROOT_CA_KEY), k256_key: dir.join(K256_KEY), + attestation: dir.join(ROOT_ATTESTATION), } } - - pub fn bootstrap_info(&self) -> PathBuf { - self.cert_dir.join(BOOTSTRAP_INFO) - } } #[derive(Debug, Clone, Deserialize)] @@ -318,6 +295,7 @@ pub(crate) struct RootKeyPaths { pub ca_cert: PathBuf, pub ca_key: PathBuf, pub k256_key: PathBuf, + pub attestation: PathBuf, } #[derive(Debug, Clone, Deserialize)] @@ -334,9 +312,23 @@ mod tests { use super::*; use rocket::figment::providers::{Format, Toml}; + /// The default config plus the settings it leaves to the operator. + fn default_figment() -> Figment { + load_config_figment(None).merge(Toml::string("[core]\nkeyring_dir = \"/keyring\"\n")) + } + + #[test] + fn keyring_dir_has_no_default() { + let err = load_config_figment(None) + .focus("core") + .extract::() + .unwrap_err(); + assert!(err.to_string().contains("keyring_dir"), "{err}"); + } + #[test] fn default_config_serves_the_handover_on_the_admin_listener() { - let figment = load_config_figment(None); + let figment = default_figment(); let config: KmsConfig = figment .focus("core") .extract() @@ -358,7 +350,7 @@ mod tests { } fn handover_check(toml: &str) -> Result<()> { - let figment = load_config_figment(None).merge(Toml::string(toml)); + let figment = default_figment().merge(Toml::string(toml)); let config: KmsConfig = figment.focus("core").extract().unwrap(); check_key_handover(&config, &ListenersConfig::load(&figment).unwrap(), &figment) } @@ -429,7 +421,7 @@ mod tests { #[test] fn rpc_cert_is_attested_by_default() { - let figment = load_config_figment(None); + let figment = default_figment(); let config: KmsConfig = figment .focus("core") .extract() diff --git a/dstack/kms/src/main.rs b/dstack/kms/src/main.rs index be905b912..3525ebd6b 100644 --- a/dstack/kms/src/main.rs +++ b/dstack/kms/src/main.rs @@ -204,7 +204,7 @@ async fn main() -> Result<()> { info!("Issuing RPC certificate under root epoch {rpc_epoch}"); info!("Updating certs"); - if let Err(err) = onboard_service::update_certs(&config, rpc_epoch, keyring.latest()).await { + if let Err(err) = onboard_service::update_certs(&config, rpc_epoch).await { if config.attest_rpc_cert { return Err(err).context( "Failed to reissue the attested KMS RPC certificate; refusing to start with a \ @@ -222,8 +222,7 @@ async fn main() -> Result<()> { let metrics_enabled = config.metrics.enabled; // Reload certificates after renewal before exposing them through RPCs. - let rpc_ca_cert = - fs_err::read_to_string(config.root_key_paths(rpc_epoch, keyring.latest()).ca_cert)?; + let rpc_ca_cert = fs_err::read_to_string(config.root_key_paths(rpc_epoch).ca_cert)?; let keyring = main_service::Keyring::load(&config).context("Failed to reload the root keys")?; let state = main_service::KmsState::new(config, keyring, rpc_ca_cert) .context("Failed to initialize KMS state")?; diff --git a/dstack/kms/src/main_service.rs b/dstack/kms/src/main_service.rs index b38f78730..d9c643149 100644 --- a/dstack/kms/src/main_service.rs +++ b/dstack/kms/src/main_service.rs @@ -13,7 +13,7 @@ use std::{ use anyhow::{anyhow, bail, Context, Result}; use dstack_kms_rpc::{ kms_server::{KmsRpc, KmsServer}, - AppId, AppKeyResponse, GetAppKeyRequest, GetKmsKeyRequest, GetMetaResponse, + AppId, AppKeyResponse, BootstrapResponse, GetAppKeyRequest, GetKmsKeyRequest, GetMetaResponse, GetTempCaCertResponse, KmsKeyResponse, KmsKeys, PublicKeyResponse, SealedEnvPubKeyResponse, SignCertRequest, SignCertResponse, }; @@ -80,6 +80,8 @@ impl std::ops::Deref for KmsState { struct RootKeys { ca: CaCert, k256: SigningKey, + /// Evidence produced when the keys were generated; empty if there is none. + attestation: Vec, } impl RootKeys { @@ -88,7 +90,12 @@ impl RootKeys { .context("failed to load root CA certificate")?; let key_bytes = fs::read(&paths.k256_key).context("failed to read ECDSA root key")?; let k256 = SigningKey::from_slice(&key_bytes).context("failed to load ECDSA root key")?; - Ok(Self { ca, k256 }) + let attestation = read_root_attestation(paths)?; + Ok(Self { + ca, + k256, + attestation, + }) } } @@ -103,7 +110,7 @@ impl Keyring { let latest = read_root_epoch(config)?; let roots = (0..=latest) .map(|epoch| { - RootKeys::load(&config.root_key_paths(epoch, latest)) + RootKeys::load(&config.root_key_paths(epoch)) .with_context(|| format!("failed to load root keys of epoch {epoch}")) }) .collect::>>()?; @@ -479,14 +486,23 @@ fn derive_app_keys( }) } -pub(crate) fn read_root_epoch(config: &KmsConfig) -> Result { - match fs::read_to_string(config.root_epoch()) { - Ok(epoch) => epoch.trim().parse().context("invalid root epoch file"), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(0), - Err(err) => Err(err).context("failed to read root epoch file"), +/// The attestation of an epoch's root keys; empty if it has none. +pub(crate) fn read_root_attestation(paths: &RootKeyPaths) -> Result> { + match fs::read(&paths.attestation) { + Ok(attestation) => Ok(attestation), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()), + Err(err) => Err(err).context("failed to read root key attestation"), } } +pub(crate) fn read_root_epoch(config: &KmsConfig) -> Result { + fs::read_to_string(config.root_epoch()) + .context("failed to read root epoch file")? + .trim() + .parse() + .context("invalid root epoch file") +} + pub struct RpcHandler { state: KmsState, attestation: Option, @@ -606,6 +622,7 @@ impl RpcHandler { .map(|root| KmsKeys { ca_key: root.ca.key.serialize_pem(), k256_key: root.k256.to_bytes().to_vec(), + attestation: root.attestation.clone(), }) .collect(), }) @@ -1021,9 +1038,6 @@ impl KmsRpc for RpcHandler { } async fn get_meta(self) -> Result { - let bootstrap_info = fs::read_to_string(self.state.config.bootstrap_info()) - .ok() - .and_then(|s| serde_json::from_str(&s).ok()); let info = self .state .auth_api_info @@ -1032,10 +1046,16 @@ impl KmsRpc for RpcHandler { .map_err(|err| anyhow!("{err:#}"))?; let epoch = self.state.keyring.servable_epoch(info.root_epochs)?; let root = self.state.epoch_root(epoch)?; + let k256_pubkey = root.k256.verifying_key().to_sec1_bytes().to_vec(); + let bootstrap_info = (!root.attestation.is_empty()).then(|| BootstrapResponse { + ca_pubkey: root.ca.key.public_key_der(), + k256_pubkey: k256_pubkey.clone(), + attestation: root.attestation.clone(), + }); Ok(GetMetaResponse { ca_cert: root.ca.pem_cert.clone(), allow_any_upgrade: self.state.inner.config.auth_api.is_dev(), - k256_pubkey: root.k256.verifying_key().to_sec1_bytes().to_vec(), + k256_pubkey, bootstrap_info, is_dev: self.state.config.auth_api.is_dev(), kms_contract_address: info.kms_contract_address, @@ -1844,18 +1864,23 @@ mod tests { .build() .self_signed() .unwrap(); - fs::write(cert_dir.join("root-ca.key"), ca_key.serialize_pem()).unwrap(); - fs::write(cert_dir.join("root-ca.crt"), ca_cert.pem()).unwrap(); + let keyring_dir = cert_dir.join("keyring"); + let root_dir = keyring_dir.join("epochs/0"); + fs::create_dir_all(&root_dir).unwrap(); + fs::write(root_dir.join("root-ca.key"), ca_key.serialize_pem()).unwrap(); + fs::write(root_dir.join("root-ca.crt"), ca_cert.pem()).unwrap(); fs::write( - cert_dir.join("root-k256.key"), + root_dir.join("root-k256.key"), SigningKey::random(&mut rand::rngs::OsRng).to_bytes(), ) .unwrap(); + fs::write(keyring_dir.join("root-epoch"), "0").unwrap(); let overrides = format!( r#" [core] cert_dir = "{}" + keyring_dir = "{}" attest_rpc_cert = false enforce_self_authorization = false @@ -1869,6 +1894,7 @@ mod tests { url = "{webhook_url}" "#, cert_dir.display(), + keyring_dir.display(), cert_dir.display(), ); let config: KmsConfig = Figment::from(rocket::Config::default()) @@ -1930,7 +1956,7 @@ mod tests { ); } - /// The keyring in `config.cert_dir` with a second root epoch appended. + /// The keyring in `config.keyring_dir` with a second root epoch appended. fn rotated_keyring(config: &KmsConfig) -> Keyring { let mut keyring = Keyring::load(config).unwrap(); let key = KeyPair::generate_for(&ra_tls::rcgen::PKCS_ECDSA_P256_SHA256).unwrap(); @@ -1944,6 +1970,7 @@ mod tests { keyring.roots.push(RootKeys { ca: CaCert::from_parts(key, cert), k256: SigningKey::random(&mut rand::rngs::OsRng), + attestation: Vec::new(), }); keyring.chain_hashes = root_chain_hashes(keyring.roots.iter().map(|root| (&root.ca.key, &root.k256))); diff --git a/dstack/kms/src/onboard_service.rs b/dstack/kms/src/onboard_service.rs index f661d296b..ff17e9a78 100644 --- a/dstack/kms/src/onboard_service.rs +++ b/dstack/kms/src/onboard_service.rs @@ -34,9 +34,10 @@ use tokio::sync::Mutex as AsyncMutex; use tracing::info; use crate::{ - config::KmsConfig, + config::{KmsConfig, RootKeyPaths}, main_service::{ - build_boot_info_for_attestation, root_chain_hashes, servable_epoch, + build_boot_info_for_attestation, read_root_attestation, read_root_epoch, root_chain_hashes, + servable_epoch, upgrade_authority::{ app_attest, dstack_client, ensure_kms_allowed, ensure_self_kms_allowed, pad64, RootEpochRegistration, RootEpochRegistry, @@ -108,28 +109,22 @@ fn validate_onboarding_domain(domain: &str) -> Result<()> { Ok(()) } -fn stored_k256_pubkey(cfg: &KmsConfig) -> Result> { - let bytes = fs::read(cfg.k256_key()).context("failed to read k256 key")?; - let key = SigningKey::from_slice(&bytes).context("failed to parse k256 key")?; - Ok(key.verifying_key().to_sec1_bytes().to_vec()) -} - fn setup_status(cfg: &KmsConfig) -> Result { if !cfg.root_keys_exist() { return Ok(SetupStatusResponse::default()); } - let k256_pubkey = stored_k256_pubkey(cfg)?; - let info = fs::read(cfg.bootstrap_info()) - .ok() - .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) - .filter(|info| info.k256_pubkey == k256_pubkey) - .unwrap_or_default(); + let root = cfg.root_key_paths(read_root_epoch(cfg)?); + let ca_key = + KeyPair::from_pem(&fs::read_to_string(&root.ca_key)?).context("failed to parse CA key")?; + let k256_key = + SigningKey::from_slice(&fs::read(&root.k256_key)?).context("failed to parse k256 key")?; + let attestation = read_root_attestation(&root)?; Ok(SetupStatusResponse { provisioned: true, ready: cfg.keys_exists(), - k256_pubkey, - ca_pubkey: info.ca_pubkey, - attestation: info.attestation, + k256_pubkey: k256_key.verifying_key().to_sec1_bytes().to_vec(), + ca_pubkey: ca_key.public_key_der(), + attestation, }) } @@ -144,12 +139,10 @@ impl OnboardRpc for OnboardHandler { ensure_self_kms_allowed(cfg, &self.state.attestation_verifier) .await .context("KMS is not allowed to bootstrap")?; - let keys = Keys::generate(&request.domain, self.state.config.attest_rpc_cert) + let mut keys = Keys::generate(&request.domain, self.state.config.attest_rpc_cert) .await .context("Failed to generate keys")?; - let response = keys.attest().await?; - // Store the bootstrap info - safe_write(cfg.bootstrap_info(), serde_json::to_vec(&response)?)?; + let response = keys.latest.attest().await?; keys.store(cfg)?; Ok(response) } @@ -167,7 +160,7 @@ impl OnboardRpc for OnboardHandler { } else { format!("{source_url}/prpc") }; - let keys = Keys::onboard( + let mut keys = Keys::onboard( cfg, &source_url, &request.source_token, @@ -179,7 +172,7 @@ impl OnboardRpc for OnboardHandler { .await .context("Failed to onboard")?; if !request.rotate { - let k256_pubkey = keys.k256_key.verifying_key().to_sec1_bytes().to_vec(); + let k256_pubkey = keys.latest.k256_pubkey(); keys.store(cfg).context("Failed to store keys")?; return Ok(OnboardResponse { k256_pubkey, @@ -188,8 +181,7 @@ impl OnboardRpc for OnboardHandler { } // The attestation over the new epoch is what governance registers, so // it is persisted with the keys and stays readable via GetSetupStatus. - let info = keys.attest().await?; - safe_write(cfg.bootstrap_info(), serde_json::to_vec(&info)?)?; + let info = keys.latest.attest().await?; keys.store(cfg).context("Failed to store keys")?; Ok(OnboardResponse { k256_pubkey: info.k256_pubkey, @@ -479,7 +471,8 @@ mod tests { let overrides = format!( r#" [core] - cert_dir = "{}" + cert_dir = "{0}" + keyring_dir = "{0}/keyring" attest_rpc_cert = false enforce_self_authorization = false @@ -503,8 +496,9 @@ mod tests { let verifier = AttestationVerifier::load(&cfg.attestation).unwrap(); bootstrap_keys(&cfg, &verifier).await.unwrap(); - let root_ca_key = fs::read(cfg.root_ca_key()).unwrap(); - let k256_key = fs::read(cfg.k256_key()).unwrap(); + let root = cfg.root_key_paths(0); + let root_ca_key = fs::read(&root.ca_key).unwrap(); + let k256_key = fs::read(&root.k256_key).unwrap(); // A single missing certificate is enough to make `keys_exists()` false, // which is what sends the KMS back into onboarding on the next start. @@ -516,8 +510,8 @@ mod tests { format!("{err:#}").contains("already been bootstrapped"), "unexpected error: {err:#}" ); - assert_eq!(fs::read(cfg.root_ca_key()).unwrap(), root_ca_key); - assert_eq!(fs::read(cfg.k256_key()).unwrap(), k256_key); + assert_eq!(fs::read(&root.ca_key).unwrap(), root_ca_key); + assert_eq!(fs::read(&root.k256_key).unwrap(), k256_key); } #[rocket::async_test] @@ -526,13 +520,15 @@ mod tests { let cfg = auto_bootstrap_config(cert_dir.path()); let verifier = AttestationVerifier::load(&cfg.attestation).unwrap(); - // A crash between the two root key writes. - fs::write(cfg.root_ca_key(), "stale").unwrap(); + // A crash before the root epoch marker was written. + let root = cfg.root_key_paths(0); + fs::create_dir_all(root.ca_key.parent().unwrap()).unwrap(); + fs::write(&root.ca_key, "stale").unwrap(); assert!(!cfg.root_keys_exist()); bootstrap_keys(&cfg, &verifier).await.unwrap(); assert!(cfg.keys_exists()); - assert_ne!(fs::read(cfg.root_ca_key()).unwrap(), b"stale"); + assert_ne!(fs::read(&root.ca_key).unwrap(), b"stale"); } #[test] @@ -551,13 +547,14 @@ mod tests { KmsKeyResponse { temp_ca_key: String::new(), keys: (0..epochs) - .map(|_| KmsKeys { + .map(|epoch| KmsKeys { ca_key: KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256) .unwrap() .serialize_pem(), k256_key: SigningKey::random(&mut rand::rngs::OsRng) .to_bytes() .to_vec(), + attestation: vec![epoch as u8; 4], }) .collect(), } @@ -655,7 +652,7 @@ mod tests { let cert_dir = tempfile::tempdir().unwrap(); let cfg = auto_bootstrap_config(cert_dir.path()); let res = handover(3); - let sent: Vec<_> = res.keys.iter().map(|k| k.k256_key.clone()).collect(); + let sent = res.keys.clone(); let keys = onboard(&cfg, res, false, unregistered(RootEpochRegistry::Unchecked)) .await @@ -664,14 +661,12 @@ mod tests { assert!(cfg.tmp_ca_key().exists()); assert!(cfg.tmp_ca_cert().exists()); - assert_eq!(crate::main_service::read_root_epoch(&cfg).unwrap(), 2); - for (epoch, k256) in (0..).zip(&sent) { - assert_eq!( - &fs::read(cfg.root_key_paths(epoch, 2).k256_key).unwrap(), - k256 - ); + assert_eq!(read_root_epoch(&cfg).unwrap(), 2); + for (epoch, sent) in (0..).zip(&sent) { + let root = cfg.root_key_paths(epoch); + assert_eq!(fs::read(&root.k256_key).unwrap(), sent.k256_key); + assert_eq!(fs::read(&root.attestation).unwrap(), sent.attestation); } - assert_eq!(&fs::read(cfg.k256_key()).unwrap(), &sent[2]); } #[rocket::async_test] @@ -683,14 +678,11 @@ mod tests { let sent: Vec<_> = res.keys.iter().map(|k| k.k256_key.clone()).collect(); let keys = onboard(&cfg, res, true, registration).await.unwrap(); - assert_eq!(keys.epoch, 2); + assert_eq!(keys.epoch(), 2); keys.store(&cfg).unwrap(); - assert_eq!( - fs::read(cfg.root_key_paths(1, 2).k256_key).unwrap(), - sent[1] - ); - let current = fs::read(cfg.k256_key()).unwrap(); + assert_eq!(fs::read(cfg.root_key_paths(1).k256_key).unwrap(), sent[1]); + let current = fs::read(cfg.root_key_paths(2).k256_key).unwrap(); assert!(!sent.contains(¤t)); } @@ -748,7 +740,7 @@ mod tests { let keys = onboard(&cfg, res.clone(), false, registered(&res, 1)) .await .unwrap(); - assert_eq!(keys.epoch, 2); + assert_eq!(keys.epoch(), 2); // A tail nobody registered is refused. let error = onboard(&cfg, res.clone(), false, registered_up_to(&res, 1, 1)) .await @@ -816,7 +808,7 @@ mod tests { onboard(&cfg, current.clone(), true, registered(¤t, 1)) .await .unwrap() - .epoch, + .epoch(), 2 ); } @@ -896,61 +888,23 @@ mod tests { } #[rocket::async_test] - async fn setup_status_includes_bootstrap_info_when_present() { + async fn setup_status_reports_the_latest_epoch() { let cert_dir = tempfile::tempdir().unwrap(); let cfg = auto_bootstrap_config(cert_dir.path()); - let keys = Keys::generate("kms.example.com", false).await.unwrap(); + let mut keys = Keys::generate("kms.example.com", false).await.unwrap(); + keys.latest.attestation = vec![7, 8, 9]; keys.store(&cfg).unwrap(); - let expected_k256 = keys.k256_key.verifying_key().to_sec1_bytes().to_vec(); - let stored = BootstrapResponse { - ca_pubkey: vec![1, 2, 3], - k256_pubkey: expected_k256.clone(), - attestation: vec![7, 8, 9], - }; - fs::write(cfg.bootstrap_info(), serde_json::to_vec(&stored).unwrap()).unwrap(); let status = setup_status(&cfg).unwrap(); assert!(status.provisioned); assert!(status.ready); - assert_eq!(status.ca_pubkey, stored.ca_pubkey); - assert_eq!(status.attestation, stored.attestation); - assert_eq!(status.k256_pubkey, expected_k256); - } + assert_eq!(status.k256_pubkey, keys.latest.k256_pubkey()); + assert_eq!(status.ca_pubkey, keys.latest.ca_key.public_key_der()); + assert_eq!(status.attestation, [7, 8, 9]); - #[rocket::async_test] - async fn setup_status_ignores_absent_or_invalid_bootstrap_info() { - let cert_dir = tempfile::tempdir().unwrap(); - let cfg = auto_bootstrap_config(cert_dir.path()); - let keys = Keys::generate("kms.example.com", false).await.unwrap(); - let expected_k256 = keys.k256_key.verifying_key().to_sec1_bytes().to_vec(); - keys.store(&cfg).unwrap(); - let stored = BootstrapResponse { - ca_pubkey: vec![1, 2, 3], - k256_pubkey: vec![4, 5, 6], - attestation: vec![7, 8, 9], - }; - for ready in [true, false] { - if !ready { - fs::remove_file(cfg.rpc_cert()).unwrap(); - fs::remove_file(cfg.bootstrap_info()).unwrap(); - } - for info in [ - None, - Some(b"invalid json".to_vec()), - Some(serde_json::to_vec(&stored).unwrap()), - ] { - if let Some(info) = info { - fs::write(cfg.bootstrap_info(), info).unwrap(); - } - let status = setup_status(&cfg).unwrap(); - assert!(status.provisioned); - assert_eq!(status.ready, ready); - assert_eq!(status.k256_pubkey, expected_k256); - assert!(status.ca_pubkey.is_empty()); - assert!(status.attestation.is_empty()); - } - } + fs::remove_file(cfg.rpc_cert()).unwrap(); + assert!(!setup_status(&cfg).unwrap().ready); - fs::write(cfg.k256_key(), b"invalid key").unwrap(); + fs::write(cfg.root_key_paths(0).k256_key, b"invalid key").unwrap(); let err = setup_status(&cfg).unwrap_err(); assert!(format!("{err:#}").contains("failed to parse k256 key")); } @@ -966,31 +920,70 @@ mod tests { } } -/// Root keys of an epoch older than the current one. -struct HistoricalRoot { +/// The root keys of one epoch. +struct EpochRoot { ca_key: KeyPair, ca_cert: Certificate, k256_key: SigningKey, + /// Evidence produced when the keys were generated; empty if there is none. + attestation: Vec, } -impl HistoricalRoot { - fn new(ca_key: KeyPair, k256_key: SigningKey) -> Result { +impl EpochRoot { + fn new(ca_key: KeyPair, k256_key: SigningKey, attestation: Vec) -> Result { let ca_cert = self_signed_root_ca(&ca_key)?; Ok(Self { ca_key, ca_cert, k256_key, + attestation, + }) + } + + fn generate() -> Result { + Self::new( + KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256)?, + SigningKey::random(&mut rand::rngs::OsRng), + Vec::new(), + ) + } + + fn k256_pubkey(&self) -> Vec { + self.k256_key.verifying_key().to_sec1_bytes().to_vec() + } + + /// Attest the freshly generated keys and keep the attestation with them. + async fn attest(&mut self) -> Result { + let ca_pubkey = self.ca_key.public_key_der(); + let k256_pubkey = self.k256_pubkey(); + self.attestation = attest_keys(&ca_pubkey, &k256_pubkey).await?; + Ok(BootstrapResponse { + ca_pubkey, + k256_pubkey, + attestation: self.attestation.clone(), }) } + + fn store(&self, paths: &RootKeyPaths) -> Result<()> { + safe_write(&paths.ca_cert, self.ca_cert.pem())?; + if !self.attestation.is_empty() { + safe_write(&paths.attestation, &self.attestation)?; + } else if let Err(err) = fs::remove_file(&paths.attestation) { + // An interrupted attempt may have left one for other keys. + if err.kind() != std::io::ErrorKind::NotFound { + return Err(err.into()); + } + } + safe_write_with_mode(&paths.ca_key, self.ca_key.serialize_pem(), 0o600)?; + safe_write_with_mode(&paths.k256_key, self.k256_key.to_bytes(), 0o600)?; + Ok(()) + } } struct Keys { - /// Epoch of the root keys below; `history` holds epochs `0..epoch`. - epoch: u32, - history: Vec, - k256_key: SigningKey, - ca_key: KeyPair, - ca_cert: Certificate, + /// Root keys of the epochs before the latest, indexed by epoch. + history: Vec, + latest: EpochRoot, rpc_key: KeyPair, rpc_cert: Certificate, rpc_domain: String, @@ -998,73 +991,38 @@ struct Keys { impl Keys { async fn generate(domain: &str, attest_rpc_cert: bool) -> Result { - let ca_key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256)?; - let rpc_key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256)?; - let k256_key = SigningKey::random(&mut rand::rngs::OsRng); - Self::from_keys(ca_key, rpc_key, k256_key, domain, attest_rpc_cert).await - } - - /// Attest the current root public keys. - async fn attest(&self) -> Result { - let ca_pubkey = self.ca_key.public_key_der(); - let k256_pubkey = self.k256_key.verifying_key().to_sec1_bytes().to_vec(); - let attestation = attest_keys(&ca_pubkey, &k256_pubkey).await?; - Ok(BootstrapResponse { - ca_pubkey, - k256_pubkey, - attestation, - }) + Self::new(Vec::new(), EpochRoot::generate()?, domain, attest_rpc_cert).await } - async fn from_keys( - ca_key: KeyPair, - rpc_key: KeyPair, - k256_key: SigningKey, + /// Issue a fresh RPC certificate under the `latest` root. + async fn new( + history: Vec, + latest: EpochRoot, domain: &str, attest_rpc_cert: bool, ) -> Result { - let ca_cert = self_signed_root_ca(&ca_key)?; - // The only place the KMS embeds its own attestation. Skipping it lets - // the KMS run outside a TEE for development; it does not affect the - // verification of quotes presented *to* the KMS, which is a separate - // path (main_service::ensure_app_attestation_allowed) and stays on. - let attestation = if attest_rpc_cert { - let pubkey = rpc_key.public_key_der(); - let report_data = QuoteContentType::RaTlsCert.to_report_data(&pubkey); - let response = app_attest(report_data.to_vec()).await.context( - "failed to get a quote for the KMS RPC certificate. The KMS attests \ - itself through the dstack guest agent, so it must run inside a dstack \ - CVM. For local development set attest_rpc_cert = false", - )?; - Some( - VersionedAttestation::from_bytes(&response.attestation) - .context("Invalid attestation")?, - ) - } else { - None - }; - - // Sign WWW server cert with KMS cert - let rpc_cert = CertRequest::builder() - .subject(domain) - .alt_names(&[domain.to_string()]) - .special_usage("kms:rpc") - .maybe_attestation(attestation.as_ref()) - .key(&rpc_key) - .build() - .signed_by(&ca_cert, &ca_key)?; - Ok(Keys { - epoch: 0, - history: Vec::new(), - k256_key, - ca_key, - ca_cert, + let rpc_key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256)?; + let rpc_cert = issue_rpc_cert( + &latest.ca_cert, + &latest.ca_key, + &rpc_key, + domain, + attest_rpc_cert, + ) + .await?; + Ok(Self { + history, + latest, rpc_key, rpc_cert, rpc_domain: domain.to_string(), }) } + fn epoch(&self) -> u32 { + self.history.len() as u32 + } + async fn onboard( cfg: &KmsConfig, other_kms_url: &str, @@ -1169,73 +1127,79 @@ impl Keys { rotate: bool, registration: RootEpochRegistration, ) -> Result { - let latest = keys_res - .keys - .len() - .checked_sub(1) - .context("source KMS handed over no keys")? as u32; - let mut roots = keys_res + let mut history = keys_res .keys .into_iter() .map(|keys| { let ca_key = KeyPair::from_pem(&keys.ca_key).context("Failed to parse CA key")?; let k256_key = SigningKey::from_slice(&keys.k256_key).context("Failed to parse ECDSA key")?; - Ok((ca_key, k256_key)) + EpochRoot::new(ca_key, k256_key, keys.attestation) }) .collect::>>()?; - ensure_handover_registered(&roots, registration, rotate)?; - let epoch = if rotate { latest + 1 } else { latest }; - let (ca_key, k256_key) = if rotate { - ( - KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256)?, - SigningKey::random(&mut rand::rngs::OsRng), - ) + if history.is_empty() { + bail!("source KMS handed over no keys"); + } + ensure_handover_registered(&history, registration, rotate)?; + let latest = if rotate { + EpochRoot::generate()? } else { - roots.pop().context("source KMS handed over no keys")? + history.pop().context("source KMS handed over no keys")? }; - let history = roots - .into_iter() - .map(|(ca_key, k256_key)| HistoricalRoot::new(ca_key, k256_key)) - .collect::>>()?; - - let rpc_key = KeyPair::generate_for(&PKCS_ECDSA_P256_SHA256)?; - let keys = Self::from_keys(ca_key, rpc_key, k256_key, domain, cfg.attest_rpc_cert).await?; - Ok(Self { - epoch, - history, - ..keys - }) + Self::new(history, latest, domain, cfg.attest_rpc_cert).await } - /// The root keys go last: see [`KmsConfig::root_keys_exist`]. + /// The root epoch marker goes last: see [`KmsConfig::root_keys_exist`]. fn store(&self, cfg: &KmsConfig) -> Result<()> { ensure_temp_ca(cfg)?; - for (epoch, root) in (0..).zip(&self.history) { - let paths = cfg.root_key_paths(epoch, self.epoch); - safe_write(&paths.ca_cert, root.ca_cert.pem())?; - safe_write_with_mode(&paths.ca_key, root.ca_key.serialize_pem(), 0o600)?; - safe_write_with_mode(&paths.k256_key, root.k256_key.to_bytes(), 0o600)?; - } - self.store_certs(cfg)?; + safe_write(cfg.rpc_cert(), self.rpc_cert.pem())?; safe_write(cfg.rpc_domain(), self.rpc_domain.as_bytes())?; - safe_write(cfg.root_epoch(), self.epoch.to_string())?; - self.store_keys(cfg)?; - Ok(()) - } - - fn store_keys(&self, cfg: &KmsConfig) -> Result<()> { safe_write_with_mode(cfg.rpc_key(), self.rpc_key.serialize_pem(), 0o600)?; - safe_write_with_mode(cfg.root_ca_key(), self.ca_key.serialize_pem(), 0o600)?; - safe_write_with_mode(cfg.k256_key(), self.k256_key.to_bytes(), 0o600)?; + for (epoch, root) in (0..).zip(self.history.iter().chain([&self.latest])) { + root.store(&cfg.root_key_paths(epoch))?; + } + safe_write(cfg.root_epoch(), self.epoch().to_string())?; Ok(()) } +} - fn store_certs(&self, cfg: &KmsConfig) -> Result<()> { - safe_write(cfg.root_ca_cert(), self.ca_cert.pem())?; - safe_write(cfg.rpc_cert(), self.rpc_cert.pem())?; - Ok(()) - } +/// Issue the KMS RPC certificate under the root CA `ca_cert`. +async fn issue_rpc_cert( + ca_cert: &Certificate, + ca_key: &KeyPair, + rpc_key: &KeyPair, + domain: &str, + attest_rpc_cert: bool, +) -> Result { + // The only place the KMS embeds its own attestation. Skipping it lets + // the KMS run outside a TEE for development; it does not affect the + // verification of quotes presented *to* the KMS, which is a separate + // path (main_service::ensure_app_attestation_allowed) and stays on. + let attestation = if attest_rpc_cert { + let pubkey = rpc_key.public_key_der(); + let report_data = QuoteContentType::RaTlsCert.to_report_data(&pubkey); + let response = app_attest(report_data.to_vec()).await.context( + "failed to get a quote for the KMS RPC certificate. The KMS attests \ + itself through the dstack guest agent, so it must run inside a dstack \ + CVM. For local development set attest_rpc_cert = false", + )?; + Some( + VersionedAttestation::from_bytes(&response.attestation) + .context("Invalid attestation")?, + ) + } else { + None + }; + + // Sign WWW server cert with KMS cert + CertRequest::builder() + .subject(domain) + .alt_names(&[domain.to_string()]) + .special_usage("kms:rpc") + .maybe_attestation(attestation.as_ref()) + .key(rpc_key) + .build() + .signed_by(ca_cert, ca_key) } /// Keep the public compatibility CA independent of all epoch root keys. @@ -1269,17 +1233,12 @@ pub(crate) fn ensure_temp_ca(cfg: &KmsConfig) -> Result { /// Reissue the RPC certificate under the root CA of the served `epoch`, so /// guests reach this KMS through the CA of the keys it serves. -pub(crate) async fn update_certs(cfg: &KmsConfig, epoch: u32, latest: u32) -> Result<()> { +pub(crate) async fn update_certs(cfg: &KmsConfig, epoch: u32) -> Result<()> { ensure_temp_ca(cfg)?; - let roots = cfg.root_key_paths(epoch, latest); - // Read existing keys - let ca_key = KeyPair::from_pem(&fs::read_to_string(&roots.ca_key)?)?; + let root = cfg.root_key_paths(epoch); + let ca_key = KeyPair::from_pem(&fs::read_to_string(&root.ca_key)?)?; let rpc_key = KeyPair::from_pem(&fs::read_to_string(cfg.rpc_key())?)?; - // Read k256 key - let k256_key_bytes = fs::read(&roots.k256_key)?; - let k256_key = SigningKey::from_slice(&k256_key_bytes)?; - let domain = if cfg.onboard.auto_bootstrap_domain.is_empty() { fs::read_to_string(cfg.rpc_domain())? } else { @@ -1287,19 +1246,15 @@ pub(crate) async fn update_certs(cfg: &KmsConfig, epoch: u32, latest: u32) -> Re }; let domain = domain.trim(); - // Regenerate certificates using existing keys - let keys = Keys::from_keys(ca_key, rpc_key, k256_key, domain, cfg.attest_rpc_cert) + let ca_cert = self_signed_root_ca(&ca_key)?; + let rpc_cert = issue_rpc_cert(&ca_cert, &ca_key, &rpc_key, domain, cfg.attest_rpc_cert) .await .context("Failed to regenerate certificates")?; - renew_ca_cert_if_expiring( - &roots.ca_cert, - keys.ca_cert.pem(), - "KMS root CA certificate", - )?; + renew_ca_cert_if_expiring(&root.ca_cert, ca_cert.pem(), "KMS root CA certificate")?; // The RPC leaf depends on the refreshed domain and platform attestation, so // it is reissued on every startup. - safe_write(cfg.rpc_cert(), keys.rpc_cert.pem())?; + safe_write(cfg.rpc_cert(), rpc_cert.pem())?; info!("Reissued the KMS RPC certificate for {domain}"); Ok(()) @@ -1349,9 +1304,12 @@ pub(crate) async fn bootstrap_keys(cfg: &KmsConfig, verifier: &AttestationVerifi ensure_self_kms_allowed(cfg, verifier) .await .context("KMS is not allowed to auto-bootstrap")?; - let keys = Keys::generate(&cfg.onboard.auto_bootstrap_domain, cfg.attest_rpc_cert) + let mut keys = Keys::generate(&cfg.onboard.auto_bootstrap_domain, cfg.attest_rpc_cert) .await .context("Failed to generate keys")?; + if cfg.attest_rpc_cert { + keys.latest.attest().await?; + } keys.store(cfg)?; Ok(()) } @@ -1363,7 +1321,7 @@ pub(crate) async fn bootstrap_keys(cfg: &KmsConfig, verifier: &AttestationVerifi /// from the current epoch with nothing pending activation, or it would collide /// with the next registered epoch. fn ensure_handover_registered( - roots: &[(KeyPair, SigningKey)], + roots: &[EpochRoot], registration: RootEpochRegistration, rotate: bool, ) -> Result<()> { @@ -1371,7 +1329,7 @@ fn ensure_handover_registered( if rotate && registry == RootEpochRegistry::Empty { bail!("register root epoch 0 before rotating"); } - let hashes = root_chain_hashes(roots.iter().map(|(ca, k256)| (ca, k256))); + let hashes = root_chain_hashes(roots.iter().map(|root| (&root.ca_key, &root.k256_key))); let current = servable_epoch(&hashes, registry)?; let held = roots.len() as u32 - 1; if let Some(latest) = latest { diff --git a/dstack/kms/src/www/onboard.html b/dstack/kms/src/www/onboard.html index 5fe3029cc..ea233938f 100644 --- a/dstack/kms/src/www/onboard.html +++ b/dstack/kms/src/www/onboard.html @@ -365,7 +365,7 @@

Onboard from an Existing KMS Instance

return; } if (data.attestation) { - this.success = 'New root keys generated!'; + this.success = 'Root keys ready!'; this.result = JSON.stringify({ caPubkey: '0x' + data.ca_pubkey, k256Pubkey: '0x' + data.k256_pubkey, diff --git a/test-suites/full-stack-compose/scripts/app_compose.py b/test-suites/full-stack-compose/scripts/app_compose.py index 5b7b93e9e..1172f1ec8 100755 --- a/test-suites/full-stack-compose/scripts/app_compose.py +++ b/test-suites/full-stack-compose/scripts/app_compose.py @@ -86,6 +86,7 @@ def kms(args: argparse.Namespace) -> None: config = f"""{listeners} [core] cert_dir = "/kms/certs" +keyring_dir = "/kms/keyring" admin_token_hash = "" pccs_url = "" enforce_self_authorization = true diff --git a/tools/dev-stack.sh b/tools/dev-stack.sh index 7d1988e53..6278cb83b 100755 --- a/tools/dev-stack.sh +++ b/tools/dev-stack.sh @@ -13,6 +13,7 @@ ACTION=$1 YOCTO_DIR=$REPO_ROOT/os/yocto DSTACK_DIR=$REPO_ROOT/dstack CERTS_DIR=$(pwd)/certs +KEYRING_DIR=$(pwd)/keyring IMAGES_DIR=$(pwd)/images RUN_DIR=$(pwd)/run RUST_BUILD_DIR=$(pwd)/rust-target @@ -186,6 +187,7 @@ port = $KMS_RPC_LISTEN_PORT [core] cert_dir = "$CERTS_DIR" +keyring_dir = "$KEYRING_DIR" attest_rpc_cert = false enforce_self_authorization = false From f67c155fcd4ddb65e1dbc4bc3edeba51d8e3ab2a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 16 Nov 2026 20:01:20 -0800 Subject: [PATCH 2/3] docs(kms): keep KMS state in one volume at /var/lib/kms The tutorials mounted one volume per state directory under /etc/kms. Keep certificates and the root keyring together in a single kms-data volume at /var/lib/kms instead, leaving /etc/kms to configuration: a volume over /etc/kms would freeze the kms.toml baked into the image at first start. Signed-off-by: Kevin Wang --- docs/security/security-best-practices.md | 2 +- docs/tutorials/kms-build-configuration.md | 33 +++++++++++------------ docs/tutorials/kms-cvm-deployment.md | 22 +++++++-------- 3 files changed, 26 insertions(+), 31 deletions(-) diff --git a/docs/security/security-best-practices.md b/docs/security/security-best-practices.md index 48eb452d6..00af26f90 100644 --- a/docs/security/security-best-practices.md +++ b/docs/security/security-best-practices.md @@ -131,7 +131,7 @@ The VMM RPC is a management interface for trusted operators and front-end servic ## Keep private material owner-only -Secret-bearing files should be owner-only (`0600`) wherever possible, including app keys, decrypted env files, KMS root keys, gateway WireGuard/TLS keys, and ACME credentials. Preserve restrictive permissions when copying volumes, backing up `/etc/kms/certs`, or moving gateway and certbot state between hosts. Public issue [#606](https://github.com/Dstack-TEE/dstack/issues/606) tracks the remaining low-cost hardening work in dstack-managed file writes. +Secret-bearing files should be owner-only (`0600`) wherever possible, including app keys, decrypted env files, KMS root keys, gateway WireGuard/TLS keys, and ACME credentials. Preserve restrictive permissions when copying volumes, backing up the KMS `keyring_dir`, or moving gateway and certbot state between hosts. Public issue [#606](https://github.com/Dstack-TEE/dstack/issues/606) tracks the remaining low-cost hardening work in dstack-managed file writes. ## docker logs is public available by default diff --git a/docs/tutorials/kms-build-configuration.md b/docs/tutorials/kms-build-configuration.md index bb8de8084..618772c7a 100644 --- a/docs/tutorials/kms-build-configuration.md +++ b/docs/tutorials/kms-build-configuration.md @@ -127,8 +127,8 @@ Create the directory structure for KMS configuration and certificates. # Configuration directory sudo mkdir -p /etc/kms -# Certificate and root keyring directories -sudo mkdir -p /etc/kms/certs /etc/kms/keyring +# Certificate directory +sudo mkdir -p /etc/kms/certs # Runtime directories sudo mkdir -p /var/run/kms @@ -171,8 +171,8 @@ log_level = "info" # TLS certificate for the listeners (not the plain-HTTP onboard page) [listeners.default.tls] -key = "/etc/kms/certs/rpc.key" -certs = "/etc/kms/certs/rpc.crt" +key = "/var/lib/kms/certs/rpc.key" +certs = "/var/lib/kms/certs/rpc.crt" # KMS RPCs for CVMs [listeners.guest] @@ -187,8 +187,8 @@ port = 9100 # Core KMS Configuration [core] -cert_dir = "/etc/kms/certs" -keyring_dir = "/etc/kms/keyring" +cert_dir = "/var/lib/kms/certs" +keyring_dir = "/var/lib/kms/keyring" subject_postfix = ".dstack" # Intel PCCS URL for TDX quote verification pccs_url = "https://pccs.phala.network/sgx/certification/v4" @@ -358,7 +358,7 @@ RUN apt-get update && \ COPY --from=bun /usr/local/bin/bun /usr/local/bin/bun # Create directories -RUN mkdir -p /etc/kms/certs /etc/kms/keyring /etc/kms/images /var/run/kms /var/log/kms +RUN mkdir -p /etc/kms/images /var/lib/kms/certs /var/lib/kms/keyring /var/run/kms /var/log/kms # Copy KMS binary COPY dstack-kms /usr/local/bin/dstack-kms @@ -419,8 +419,8 @@ log_level = "info" # TLS certificate for the listeners (not the plain-HTTP onboard page) [listeners.default.tls] -key = "/etc/kms/certs/rpc.key" -certs = "/etc/kms/certs/rpc.crt" +key = "/var/lib/kms/certs/rpc.key" +certs = "/var/lib/kms/certs/rpc.crt" # KMS RPCs for CVMs [listeners.guest] @@ -435,8 +435,8 @@ port = 9100 # Core KMS Configuration [core] -cert_dir = "/etc/kms/certs" -keyring_dir = "/etc/kms/keyring" +cert_dir = "/var/lib/kms/certs" +keyring_dir = "/var/lib/kms/keyring" subject_postfix = ".dstack" pccs_url = "https://pccs.phala.network/sgx/certification/v4" @@ -545,18 +545,15 @@ services: volumes: # Mount config file from local directory - ./kms.toml:/etc/kms/kms.toml:ro - # Named volumes for persistent certificates and root keys - - kms-certs:/etc/kms/certs - - kms-keyring:/etc/kms/keyring + # Named volume for persistent state: certificates and root keys + - kms-data:/var/lib/kms environment: - RUST_LOG=info restart: unless-stopped volumes: - kms-certs: - # Certificates persist across container restarts - kms-keyring: - # Root keys persist across container restarts + kms-data: + # Certificates and root keys persist across container restarts EOF ``` diff --git a/docs/tutorials/kms-cvm-deployment.md b/docs/tutorials/kms-cvm-deployment.md index f566350bc..9ad991b9a 100644 --- a/docs/tutorials/kms-cvm-deployment.md +++ b/docs/tutorials/kms-cvm-deployment.md @@ -65,7 +65,7 @@ When you deploy KMS as a CVM, the following happens: ### Generated Artifacts -Inside the CVM, the root keyring at `/etc/kms/keyring/` (`keyring_dir`): +Inside the CVM, in the `kms-data` volume at `/var/lib/kms/`. The root keyring under `keyring/` (`keyring_dir`): | File | Purpose | |------|---------| @@ -75,7 +75,7 @@ Inside the CVM, the root keyring at `/etc/kms/keyring/` (`keyring_dir`): | `epochs//root-attestation` | Attestation produced when the epoch `n` keys were generated; absent if there is none | | `root-epoch` | Newest root epoch held | -And the RPC certificates at `/etc/kms/certs/` (`cert_dir`): +And the RPC certificates under `certs/` (`cert_dir`): | File | Purpose | |------|---------| @@ -144,8 +144,7 @@ services: - "9101:9101" volumes: - /var/run/dstack.sock:/var/run/dstack.sock - - kms-certs:/etc/kms/certs - - kms-keyring:/etc/kms/keyring + - kms-data:/var/lib/kms environment: - RUST_LOG=info - KMS_DOMAIN=kms.yourdomain.com @@ -158,15 +157,14 @@ services: restart: unless-stopped volumes: - kms-certs: - kms-keyring: + kms-data: configs: kms_config: content: | [listeners.default.tls] - key = "/etc/kms/certs/rpc.key" - certs = "/etc/kms/certs/rpc.crt" + key = "/var/lib/kms/certs/rpc.key" + certs = "/var/lib/kms/certs/rpc.crt" [listeners.guest] address = "0.0.0.0" @@ -191,8 +189,8 @@ configs: # certificate signing verifies CSR signature and attestation. [core] - cert_dir = "/etc/kms/certs" - keyring_dir = "/etc/kms/keyring" + cert_dir = "/var/lib/kms/certs" + keyring_dir = "/var/lib/kms/keyring" pccs_url = "https://pccs.phala.network/sgx/certification/v4" [core.image] @@ -508,7 +506,7 @@ The TDX quote can be verified by: │ │ │ │ └────┬────┘ └──────┬───────┘ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ ▼ ▼ │ │ │ │ -│ │ │ │ /etc/kms/certs Ethereum RPC │ │ │ │ +│ │ │ │ /var/lib/kms Ethereum RPC │ │ │ │ │ │ │ └──────────────────────────────────┘ │ │ │ │ │ │ │ │ │ │ │ │ guest-agent (/var/run/dstack.sock) │ │ │ @@ -553,7 +551,7 @@ For detailed solutions, see the [KMS Deployment Troubleshooting Guide](/tutorial ### Understanding Storage -CVM certificates and root keys are stored in Docker named volumes (`kms-certs` and `kms-keyring`). This provides: +CVM certificates and root keys are stored in the Docker named volume `kms-data`, while `/etc/kms` holds only configuration. This provides: - **Container restart persistence** - Certificates and root keys survive container restarts - **CVM restart consideration** - Depending on VMM configuration, volumes may or may not persist From 774723444f1573dc3199ca9632e3f6682aad57a9 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 16 Nov 2026 20:06:47 -0800 Subject: [PATCH 3/3] feat(kms): default the KMS state paths to /var/lib/kms Give keyring_dir a default in kms.toml rather than requiring it, and move cert_dir and the default listener TLS paths there too, so all mutable KMS state sits under one directory and /etc/kms holds only configuration. Signed-off-by: Kevin Wang --- CHANGELOG.md | 13 ++++++++----- docs/deployment.md | 4 ++-- docs/tutorials/kms-build-configuration.md | 2 +- dstack/kms/kms.toml | 11 ++++++----- dstack/kms/src/config.rs | 23 ++++------------------- 5 files changed, 21 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e5a47dba..3e757beb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Upgrade notes -- **KMS requires `core.keyring_dir`.** The root keys of every epoch move out of - `cert_dir` into `/epochs//`, with `/root-epoch` - marking the newest. It has no default: point it at persistent storage. The - on-disk layout is not read from older versions, so upgrade by onboarding a - new KMS, as before. +- **KMS state moves to `/var/lib/kms`.** The root keys of every epoch move out + of `cert_dir` into the new `core.keyring_dir` (default + `/var/lib/kms/keyring`), as `epochs//` with `root-epoch` marking the + newest. `cert_dir` and the default listener TLS paths move from + `/etc/kms/certs` to `/var/lib/kms/certs`. A deployment that persists + `/etc/kms/certs` and relies on the defaults must persist `/var/lib/kms` or + set both paths. The on-disk layout is not read from older versions, so + upgrade by onboarding a new KMS, as before. ### Changed diff --git a/docs/deployment.md b/docs/deployment.md index 30383faed..7953e34e6 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -522,8 +522,8 @@ port = 8001 auth_token = "" [listeners.default.tls] -key = "/etc/kms/certs/rpc.key" -certs = "/etc/kms/certs/rpc.crt" +key = "/var/lib/kms/certs/rpc.key" +certs = "/var/lib/kms/certs/rpc.crt" # No `mutual` section: client certificates are verified by their attestation. ``` diff --git a/docs/tutorials/kms-build-configuration.md b/docs/tutorials/kms-build-configuration.md index 618772c7a..f9432a1fd 100644 --- a/docs/tutorials/kms-build-configuration.md +++ b/docs/tutorials/kms-build-configuration.md @@ -232,7 +232,7 @@ EOF | `[listeners.admin]` | `address`, `port`, `auth_token_hash` | Admin listener (9101), which serves the key handover; the SHA-256 of the admin token. A serving KMS requires it with TLS | | `[listeners.onboard]` | `address`, `port` | Plain-HTTP onboarding page (9100, before the KMS has keys) | | `[core]` | `cert_dir` | Directory for certificates | -| `[core]` | `keyring_dir` | Directory for the root keys of every epoch (required; must be persistent) | +| `[core]` | `keyring_dir` | Directory for the root keys of every epoch (must be persistent) | | `[core]` | `pccs_url` | PCCS endpoint for quote verification | | `[core.auth_api]` | `url` | auth-eth webhook service URL | | `[core.onboard]` | `enabled` | Enable bootstrap/onboard mode | diff --git a/dstack/kms/kms.toml b/dstack/kms/kms.toml index 1bfad44bb..a54c9a325 100644 --- a/dstack/kms/kms.toml +++ b/dstack/kms/kms.toml @@ -18,8 +18,8 @@ proxy_protocol = false # No `mutual` section: client certificates are verified by the attestation they # carry, not by an issuer CA (see ra-rpc's ratls_client_verifier). [listeners.default.tls] -key = "/etc/kms/certs/rpc.key" -certs = "/etc/kms/certs/rpc.crt" +key = "/var/lib/kms/certs/rpc.key" +certs = "/var/lib/kms/certs/rpc.crt" # For CVMs: all KMS RPCs, including those that release keys. [listeners.guest] @@ -68,9 +68,10 @@ address = "0.0.0.0" port = 8000 [core] -cert_dir = "/etc/kms/certs" -# Root keys of every epoch. Required, with no default: point it at persistent -# storage, e.g. keyring_dir = "/etc/kms/keyring". +cert_dir = "/var/lib/kms/certs" +# Root keys of every epoch, the only state the KMS cannot regenerate. Keep it on +# persistent storage. +keyring_dir = "/var/lib/kms/keyring" subject_postfix = ".dstack" site_name = "" # Whether trusted RPCs require the KMS to first attest itself to its own diff --git a/dstack/kms/src/config.rs b/dstack/kms/src/config.rs index 5a57a829e..80f594326 100644 --- a/dstack/kms/src/config.rs +++ b/dstack/kms/src/config.rs @@ -85,8 +85,7 @@ pub(crate) struct ImageConfig { #[derive(Debug, Clone, Deserialize)] pub(crate) struct KmsConfig { pub cert_dir: PathBuf, - /// Root keys of every epoch. Required: it is the only state the KMS - /// cannot regenerate, so it must be placed on persistent storage on purpose. + /// Root keys of every epoch. pub keyring_dir: PathBuf, #[serde(default)] pub attestation: AttestationVerifierConfig, @@ -312,23 +311,9 @@ mod tests { use super::*; use rocket::figment::providers::{Format, Toml}; - /// The default config plus the settings it leaves to the operator. - fn default_figment() -> Figment { - load_config_figment(None).merge(Toml::string("[core]\nkeyring_dir = \"/keyring\"\n")) - } - - #[test] - fn keyring_dir_has_no_default() { - let err = load_config_figment(None) - .focus("core") - .extract::() - .unwrap_err(); - assert!(err.to_string().contains("keyring_dir"), "{err}"); - } - #[test] fn default_config_serves_the_handover_on_the_admin_listener() { - let figment = default_figment(); + let figment = load_config_figment(None); let config: KmsConfig = figment .focus("core") .extract() @@ -350,7 +335,7 @@ mod tests { } fn handover_check(toml: &str) -> Result<()> { - let figment = default_figment().merge(Toml::string(toml)); + let figment = load_config_figment(None).merge(Toml::string(toml)); let config: KmsConfig = figment.focus("core").extract().unwrap(); check_key_handover(&config, &ListenersConfig::load(&figment).unwrap(), &figment) } @@ -421,7 +406,7 @@ mod tests { #[test] fn rpc_cert_is_attested_by_default() { - let figment = default_figment(); + let figment = load_config_figment(None); let config: KmsConfig = figment .focus("core") .extract()