From b5ff174827f04d699f0b8e9cd5c2a7cbb8e5b9af Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 16 Nov 2026 18:57:43 -0800 Subject: [PATCH 1/2] feat(kms): merge a second KMS root as the next epoch One-off build (0.6.5-mrgk) for joining two independently bootstrapped off-chain KMS clusters, not for regular releases. Onboard takes merge_source_url/merge_source_token: the source's whole keyring is imported as usual, and the single root of the merge source is appended as the next epoch instead of a generated one. The merge source is pinned to the root CA it publishes, once its public keys extend the imported keyring to the latest registered chain hash, and is authorized by the auth API like the source. The merged keyring is then checked as an ordinary join with the new epoch pending activation. Signed-off-by: Kevin Wang --- docs/kms-root-merge.md | 47 +++++ dstack/kms/rpc/proto/kms_rpc.proto | 6 + dstack/kms/src/onboard_service.rs | 305 ++++++++++++++++++++++------- dstack/kms/src/www/onboard.html | 17 +- 4 files changed, 298 insertions(+), 77 deletions(-) create mode 100644 docs/kms-root-merge.md diff --git a/docs/kms-root-merge.md b/docs/kms-root-merge.md new file mode 100644 index 000000000..04568f601 --- /dev/null +++ b/docs/kms-root-merge.md @@ -0,0 +1,47 @@ +# Merging two KMS roots (0.6.5-mrgk) + +This build exists for one operation and is not part of any regular release. +It joins the roots of two independently bootstrapped KMS clusters into one +keyring: the existing cluster's roots stay epochs `0..N-1`, and the other +cluster's single root becomes epoch `N`. It is otherwise identical to 0.6.5. + +The result is what a rotation would produce, except that epoch `N` is imported +instead of generated. Derived keys depend on the root keys and the app epoch, +never on the root epoch number, so every app of the merged cluster keeps its +keys once epoch `N` is current. + +## Onboard + +`Onboard` takes two extra fields: + +- `merge_source_url`: a KMS holding exactly one root epoch, such as a 0.6.3 KMS. +- `merge_source_token`: optional bearer token for its admin listener. + +They cannot be combined with `rotate`. The KMS then: + +1. Imports the whole keyring from `source_url`, as an ordinary onboarding. +2. Reads the root public keys the merge source publishes (`GetMeta`) and + requires them to extend the imported keyring to the latest registered chain + hash, so epoch `N` must already be registered and pending activation. +3. Fetches the merge source's root keys, pinned to that root CA, and requires + its auth API to authorize the merge source for key export at epoch `N`. +4. Checks the merged keyring against governance as a join: the current epoch + and the pending epoch `N` must both match their chain hashes. + +The merged KMS serves the current epoch, so it serves the existing cluster's +roots until epoch `N` is activated. Ordinary 0.6.5 KMS instances then onboard +from it without `merge_source_url`; the merge build is retired afterwards. + +## Procedure + +1. Register epoch `N` with the merge source's root public keys and its + bootstrap attestation as evidence. Do not activate it. +2. Allow the merge build's measurement to import keys from both clusters, and + to export them on the merged cluster. +3. Onboard one merge build instance with both sources, then onboard 0.6.5 + instances from it, and retire the merge build. +4. Until activation, deny apps of the merge source's cluster on the merged + cluster: it would serve them the old roots. +5. Activate epoch `N` with the rotation of the existing cluster's apps, then + move the merge source's apps over. A CVM that booted on the merged cluster + records epoch `N` on its disk and cannot return to the merge source. diff --git a/dstack/kms/rpc/proto/kms_rpc.proto b/dstack/kms/rpc/proto/kms_rpc.proto index c2088b978..ea7e0eb25 100644 --- a/dstack/kms/rpc/proto/kms_rpc.proto +++ b/dstack/kms/rpc/proto/kms_rpc.proto @@ -184,6 +184,12 @@ message OnboardRequest { // This instance's launch ticket, presented to the source; empty when the // KMS policy requires none. string launch_ticket = 5; + // Merge build only: a second source holding a single root epoch, imported as + // the epoch after the source's. Governance must register it as the latest + // epoch first. + string merge_source_url = 6; + // Optional bearer token for the merge source KMS admin listener. + string merge_source_token = 7; } message OnboardResponse { diff --git a/dstack/kms/src/onboard_service.rs b/dstack/kms/src/onboard_service.rs index f661d296b..534c9cae4 100644 --- a/dstack/kms/src/onboard_service.rs +++ b/dstack/kms/src/onboard_service.rs @@ -13,7 +13,7 @@ use dstack_kms_rpc::{ kms_client::KmsClient, onboard_server::{OnboardRpc, OnboardServer}, AttestationInfoResponse, BootstrapRequest, BootstrapResponse, GetKmsKeyRequest, KmsKeyResponse, - OnboardRequest, OnboardResponse, SetupStatusResponse, + KmsKeys, OnboardRequest, OnboardResponse, SetupStatusResponse, }; use fs_err as fs; use k256::ecdsa::SigningKey; @@ -39,7 +39,7 @@ use crate::{ build_boot_info_for_attestation, root_chain_hashes, servable_epoch, upgrade_authority::{ app_attest, dstack_client, ensure_kms_allowed, ensure_self_kms_allowed, pad64, - RootEpochRegistration, RootEpochRegistry, + CurrentRootEpoch, RootEpochRegistration, RootEpochRegistry, }, }, }; @@ -89,6 +89,15 @@ impl RpcCall for OnboardHandler { } } +fn prpc_url(url: &str) -> String { + let url = url.trim_end_matches('/'); + if url.ends_with("/prpc") { + url.to_string() + } else { + format!("{url}/prpc") + } +} + fn validate_onboarding_domain(domain: &str) -> Result<()> { if domain.is_empty() || domain.len() > 253 || !domain.is_ascii() { bail!("domain must be a non-empty ASCII DNS name of at most 253 bytes"); @@ -161,17 +170,19 @@ impl OnboardRpc for OnboardHandler { if cfg.root_keys_exist() { bail!("KMS has already been onboarded"); } - let source_url = request.source_url.trim_end_matches('/').to_string(); - let source_url = if source_url.ends_with("/prpc") { - source_url - } else { - format!("{source_url}/prpc") - }; + let merge_url = + (!request.merge_source_url.is_empty()).then(|| prpc_url(&request.merge_source_url)); + if merge_url.is_some() && request.rotate { + bail!("a merge imports the next root epoch, so it cannot rotate"); + } let keys = Keys::onboard( cfg, - &source_url, + &prpc_url(&request.source_url), &request.source_token, &request.launch_ticket, + merge_url + .as_deref() + .map(|url| (url, request.merge_source_token.as_str())), &request.domain, request.rotate, self.state.attestation_verifier.clone(), @@ -296,8 +307,6 @@ mod tests { use crate::main_service::amd_attest::{ compute_expected_measurement, MeasurementInput, OvmfSectionParam, }; - use crate::main_service::upgrade_authority::CurrentRootEpoch; - use dstack_kms_rpc::KmsKeys; use sha2::Digest; fn hex_of(byte: u8, len: usize) -> String { @@ -796,6 +805,35 @@ mod tests { ); } + #[test] + fn a_merge_source_is_pinned_only_as_the_registered_next_epoch() { + let res = handover(2); + let published = |keys: &KmsKeys| { + let ca = KeyPair::from_pem(&keys.ca_key).unwrap(); + let k256 = SigningKey::from_slice(&keys.k256_key).unwrap(); + ( + ca.public_key_der(), + self_signed_root_ca(&ca).unwrap().pem(), + k256.verifying_key().to_sec1_bytes().to_vec(), + ) + }; + let (held, next) = res.keys.split_at(1); + let (ca_pubkey, ca_cert, k256) = published(&next[0]); + let latest = epoch_of(&res, 1); + assert_eq!( + merge_source_pin(held, &ca_cert, &k256, latest).unwrap(), + ca_pubkey + ); + let (_, other_cert, other_k256) = published(&handover(1).keys[0]); + assert!(merge_source_pin(held, &other_cert, &k256, latest).is_err()); + assert!(merge_source_pin(held, &ca_cert, &other_k256, latest).is_err()); + let error = merge_source_pin(&res.keys, &ca_cert, &k256, latest).unwrap_err(); + assert!( + format!("{error:#}").contains("must be the latest registered one"), + "{error:#}" + ); + } + #[rocket::async_test] async fn rotation_refuses_a_registered_epoch_pending_activation() { let cert_dir = tempfile::tempdir().unwrap(); @@ -1065,38 +1103,17 @@ impl Keys { }) } + #[allow(clippy::too_many_arguments)] async fn onboard( cfg: &KmsConfig, other_kms_url: &str, source_token: &str, launch_ticket: &str, + merge_source: Option<(&str, &str)>, domain: &str, rotate: bool, attestation_verifier: Arc, ) -> Result { - let attestation_slot = Arc::new(Mutex::new(None::)); - let attestation_slot_out = attestation_slot.clone(); - let capture_source_attestation = Box::new(move |info: Option| { - let Some(info) = info else { - bail!("Source KMS did not present a TLS certificate"); - }; - let Some(attestation) = info.attestation else { - bail!("Source KMS certificate does not contain attestation"); - }; - let mut slot = attestation_slot_out - .lock() - .map_err(|_| anyhow::anyhow!("source attestation mutex poisoned"))?; - *slot = Some(attestation); - Ok(()) - }); - let source_attestation = || { - attestation_slot - .lock() - .map_err(|_| anyhow::anyhow!("source attestation mutex poisoned"))? - .clone() - .context("Missing source KMS attestation") - }; - // Send the launch ticket and the admin token only to a KMS serving the // registered roots: at the handshake, its RPC certificate must be // signed by the current root CA registered with our own auth API. @@ -1104,47 +1121,15 @@ impl Keys { .auth_api .root_ca_pubkey() .await - .context("failed to get the registered root CA key")? - .map(|pubkey| PinnedIssuer { - pubkey, - usage: "kms:rpc".into(), - }); - let pinned_client = || { - RaClientConfig::builder() - .tls_skip_ca_verification(true) - .maybe_tls_pinned_issuer(pin.clone()) - .remote_uri(other_kms_url.to_string()) - }; - let temp_ca = if source_token.is_empty() { - let client = pinned_client().build().into_client()?; - let ca = KmsClient::new(client) - .get_temp_ca_cert() - .await - .context("failed to get the temporary CA from the source KMS")?; - Some((ca.temp_ca_cert, ca.temp_ca_key)) - } else { - None - }; - let (ra_cert, ra_key) = gen_ra_cert(temp_ca).await?; - let client = pinned_client() - .tls_built_in_root_certs(false) - .tls_client_cert(ra_cert) - .tls_client_key(ra_key) - .cert_validator(capture_source_attestation) - .attestation_verifier(attestation_verifier.clone()) - .maybe_bearer_token((!source_token.is_empty()).then(|| source_token.to_string())) - .build() - .into_client()?; - let info = dstack_client().info().await.context("Failed to get info")?; - let request = GetKmsKeyRequest { - vm_config: info.vm_config, - launch_ticket: launch_ticket.to_string(), - }; - let keys_res = if source_token.is_empty() { - KmsClient::new(client).get_kms_key(request).await - } else { - AdminClient::new(client).get_kms_key(request).await - } + .context("failed to get the registered root CA key")?; + let (mut keys_res, attestation) = fetch_keys( + other_kms_url, + source_token, + launch_ticket, + pin, + attestation_verifier.clone(), + ) + .await .context("failed to get the keys from the source KMS")?; // The attestation is captured only on the connection that returned the keys; // nothing received is used before the source passes this check. @@ -1154,9 +1139,22 @@ impl Keys { .checked_sub(1) .context("source KMS handed over no keys")?; let highest = u32::try_from(highest).context("too many root epochs")?; - let registration = ensure_kms_allowed(cfg, &source_attestation()?, highest) + let registration = ensure_kms_allowed(cfg, &attestation, highest) .await .context("source KMS is not allowed for onboarding")?; + if let Some((url, token)) = merge_source { + let next = fetch_merge_keys( + cfg, + &keys_res.keys, + url, + token, + registration, + attestation_verifier, + ) + .await + .context("failed to merge the keys of the merge source KMS")?; + keys_res.keys.push(next); + } Self::from_key_response(cfg, keys_res, domain, rotate, registration).await } @@ -1356,6 +1354,161 @@ pub(crate) async fn bootstrap_keys(cfg: &KmsConfig, verifier: &AttestationVerifi Ok(()) } +/// Fetch the root keys from one source KMS, with the attestation it presented +/// on that connection. With `pin`, nothing is sent to a source whose RPC +/// certificate the CA key `pin` did not issue. +async fn fetch_keys( + other_kms_url: &str, + source_token: &str, + launch_ticket: &str, + pin: Option>, + attestation_verifier: Arc, +) -> Result<(KmsKeyResponse, VerifiedAttestation)> { + let attestation_slot = Arc::new(Mutex::new(None::)); + let attestation_slot_out = attestation_slot.clone(); + let capture_source_attestation = Box::new(move |info: Option| { + let Some(info) = info else { + bail!("Source KMS did not present a TLS certificate"); + }; + let Some(attestation) = info.attestation else { + bail!("Source KMS certificate does not contain attestation"); + }; + let mut slot = attestation_slot_out + .lock() + .map_err(|_| anyhow::anyhow!("source attestation mutex poisoned"))?; + *slot = Some(attestation); + Ok(()) + }); + let source_attestation = || { + attestation_slot + .lock() + .map_err(|_| anyhow::anyhow!("source attestation mutex poisoned"))? + .clone() + .context("Missing source KMS attestation") + }; + + let pin = pin.map(|pubkey| PinnedIssuer { + pubkey, + usage: "kms:rpc".into(), + }); + let pinned_client = || { + RaClientConfig::builder() + .tls_skip_ca_verification(true) + .maybe_tls_pinned_issuer(pin.clone()) + .remote_uri(other_kms_url.to_string()) + }; + let temp_ca = if source_token.is_empty() { + let client = pinned_client().build().into_client()?; + let ca = KmsClient::new(client) + .get_temp_ca_cert() + .await + .context("failed to get the temporary CA from the source KMS")?; + Some((ca.temp_ca_cert, ca.temp_ca_key)) + } else { + None + }; + let (ra_cert, ra_key) = gen_ra_cert(temp_ca).await?; + let client = pinned_client() + .tls_built_in_root_certs(false) + .tls_client_cert(ra_cert) + .tls_client_key(ra_key) + .cert_validator(capture_source_attestation) + .attestation_verifier(attestation_verifier) + .maybe_bearer_token((!source_token.is_empty()).then(|| source_token.to_string())) + .build() + .into_client()?; + let info = dstack_client().info().await.context("Failed to get info")?; + let request = GetKmsKeyRequest { + vm_config: info.vm_config, + launch_ticket: launch_ticket.to_string(), + }; + let keys_res = if source_token.is_empty() { + KmsClient::new(client).get_kms_key(request).await + } else { + AdminClient::new(client).get_kms_key(request).await + }?; + Ok((keys_res, source_attestation()?)) +} + +/// Fetch the root keys of the next epoch after `held` from a second source +/// holding a single epoch, which governance must have registered as the latest +/// one. The source is pinned to the root CA key it publishes, once that key +/// and its k256 key extend `held` to the registered chain hash. +async fn fetch_merge_keys( + cfg: &KmsConfig, + held: &[KmsKeys], + url: &str, + token: &str, + registration: RootEpochRegistration, + attestation_verifier: Arc, +) -> Result { + let latest = registration + .latest + .context("merging requires a registered root epoch")?; + let client = RaClientConfig::builder() + .tls_skip_ca_verification(true) + .remote_uri(url.to_string()) + .build() + .into_client()?; + let meta = KmsClient::new(client) + .get_meta() + .await + .context("failed to get the merge source metadata")?; + let pin = merge_source_pin(held, &meta.ca_cert, &meta.k256_pubkey, latest)?; + let (res, attestation) = fetch_keys(url, token, "", Some(pin), attestation_verifier).await?; + ensure_kms_allowed(cfg, &attestation, latest.epoch) + .await + .context("merge source KMS is not allowed for onboarding")?; + let [keys] = <[KmsKeys; 1]>::try_from(res.keys) + .map_err(|_| anyhow::anyhow!("the merge source must hold exactly one root epoch"))?; + Ok(keys) +} + +/// The root CA key of a merge source publishing `ca_cert` and `k256_pubkey`, +/// if those are the latest registered epoch, directly after `held`. +fn merge_source_pin( + held: &[KmsKeys], + ca_cert: &str, + k256_pubkey: &[u8], + latest: CurrentRootEpoch, +) -> Result> { + let next = held.len(); + if latest.epoch as usize != next { + bail!( + "root epoch {next} must be the latest registered one to merge, but the latest is {}", + latest.epoch + ); + } + let (_, pem) = x509_parser::pem::parse_x509_pem(ca_cert.as_bytes()) + .context("invalid merge source CA certificate")?; + let ca_pubkey = pem + .parse_x509() + .context("invalid merge source CA certificate")? + .public_key() + .raw + .to_vec(); + let mut pubkeys = held + .iter() + .map(|keys| { + let ca = KeyPair::from_pem(&keys.ca_key).context("Failed to parse CA key")?; + let k256 = + SigningKey::from_slice(&keys.k256_key).context("Failed to parse ECDSA key")?; + Ok(( + ca.public_key_der(), + k256.verifying_key().to_sec1_bytes().to_vec(), + )) + }) + .collect::>>()?; + pubkeys.push((ca_pubkey.clone(), k256_pubkey.to_vec())); + let hashes = dstack_types::root_epoch::chain_hashes( + pubkeys.iter().map(|(ca, k256)| (&ca[..], &k256[..])), + ); + if hashes[next] != latest.chain_hash { + bail!("the merge source root keys are not registered as root epoch {next}"); + } + Ok(ca_pubkey) +} + /// Check the handed-over roots against governance: they must include the /// current epoch with a matching chain hash, and nothing beyond the latest /// registered epoch. When the source holds the latest registered epoch, its diff --git a/dstack/kms/src/www/onboard.html b/dstack/kms/src/www/onboard.html index 5fe3029cc..71ff3ee43 100644 --- a/dstack/kms/src/www/onboard.html +++ b/dstack/kms/src/www/onboard.html @@ -269,6 +269,17 @@

Onboard from an Existing KMS Instance

placeholder="Required when the KMS contract requires a launch ticket">
+ + +
+
+ + +
+