diff --git a/docs/kms-root-merge.md b/docs/kms-root-merge.md new file mode 100644 index 000000000..e188b59ef --- /dev/null +++ b/docs/kms-root-merge.md @@ -0,0 +1,50 @@ +# Merging two KMS roots (0.6.5-mrgk) + +This build exists for one operation and is not part of any regular release. +It joins the roots of two independently bootstrapped KMS clusters into one +keyring: the existing cluster's roots stay epochs `0..N-1`, and the other +cluster's single root becomes epoch `N`. It is otherwise identical to 0.6.5. + +The result is what a rotation would produce, except that epoch `N` is imported +instead of generated. Derived keys depend on the root keys and the app epoch, +never on the root epoch number, so every app of the merged cluster keeps its +keys once epoch `N` is current. + +## Onboard + +`Onboard` takes two extra fields: + +- `merge_source_url`: a KMS holding exactly one root epoch, such as a 0.6.3 KMS. +- `merge_source_token`: optional bearer token for its admin listener. + +They cannot be combined with `rotate`. The KMS then: + +1. Imports the whole keyring from `source_url`, as an ordinary onboarding. +2. Fetches the merge source's keys, requires it to hold exactly one root epoch, + and requires the auth API to authorize it for key export at epoch `N`. +3. Checks the merged keyring against governance as a join: the current epoch + and epoch `N`, registered and pending activation, must both match their + chain hashes. + +The merge source is not pinned to a root CA: only the auth API knows epoch +`N`, by its chain hash, and the keys are checked against it before use. The +merge source still releases its keys only to an attested KMS its own auth API +allows, with or without its admin token. + +The merged KMS serves the current epoch, so it serves the existing cluster's +roots until epoch `N` is activated. Ordinary 0.6.5 KMS instances then onboard +from it without `merge_source_url`; the merge build is retired afterwards. + +## Procedure + +1. Register epoch `N` with the merge source's root public keys and its + bootstrap attestation as evidence. Do not activate it. +2. Allow the merge build's measurement to import keys from both clusters, and + to export them on the merged cluster. +3. Onboard one merge build instance with both sources, then onboard 0.6.5 + instances from it, and retire the merge build. +4. Until activation, deny apps of the merge source's cluster on the merged + cluster: it would serve them the old roots. +5. Activate epoch `N` with the rotation of the existing cluster's apps, then + move the merge source's apps over. A CVM that booted on the merged cluster + records epoch `N` on its disk and cannot return to the merge source. diff --git a/dstack/kms/rpc/proto/kms_rpc.proto b/dstack/kms/rpc/proto/kms_rpc.proto index c2088b978..ea7e0eb25 100644 --- a/dstack/kms/rpc/proto/kms_rpc.proto +++ b/dstack/kms/rpc/proto/kms_rpc.proto @@ -184,6 +184,12 @@ message OnboardRequest { // This instance's launch ticket, presented to the source; empty when the // KMS policy requires none. string launch_ticket = 5; + // Merge build only: a second source holding a single root epoch, imported as + // the epoch after the source's. Governance must register it as the latest + // epoch first. + string merge_source_url = 6; + // Optional bearer token for the merge source KMS admin listener. + string merge_source_token = 7; } message OnboardResponse { diff --git a/dstack/kms/src/onboard_service.rs b/dstack/kms/src/onboard_service.rs index f661d296b..cf1946a2c 100644 --- a/dstack/kms/src/onboard_service.rs +++ b/dstack/kms/src/onboard_service.rs @@ -13,7 +13,7 @@ use dstack_kms_rpc::{ kms_client::KmsClient, onboard_server::{OnboardRpc, OnboardServer}, AttestationInfoResponse, BootstrapRequest, BootstrapResponse, GetKmsKeyRequest, KmsKeyResponse, - OnboardRequest, OnboardResponse, SetupStatusResponse, + KmsKeys, OnboardRequest, OnboardResponse, SetupStatusResponse, }; use fs_err as fs; use k256::ecdsa::SigningKey; @@ -89,6 +89,15 @@ impl RpcCall for OnboardHandler { } } +fn prpc_url(url: &str) -> String { + let url = url.trim_end_matches('/'); + if url.ends_with("/prpc") { + url.to_string() + } else { + format!("{url}/prpc") + } +} + fn validate_onboarding_domain(domain: &str) -> Result<()> { if domain.is_empty() || domain.len() > 253 || !domain.is_ascii() { bail!("domain must be a non-empty ASCII DNS name of at most 253 bytes"); @@ -161,17 +170,19 @@ impl OnboardRpc for OnboardHandler { if cfg.root_keys_exist() { bail!("KMS has already been onboarded"); } - let source_url = request.source_url.trim_end_matches('/').to_string(); - let source_url = if source_url.ends_with("/prpc") { - source_url - } else { - format!("{source_url}/prpc") - }; + let merge_url = + (!request.merge_source_url.is_empty()).then(|| prpc_url(&request.merge_source_url)); + if merge_url.is_some() && request.rotate { + bail!("a merge imports the next root epoch, so it cannot rotate"); + } let keys = Keys::onboard( cfg, - &source_url, + &prpc_url(&request.source_url), &request.source_token, &request.launch_ticket, + merge_url + .as_deref() + .map(|url| (url, request.merge_source_token.as_str())), &request.domain, request.rotate, self.state.attestation_verifier.clone(), @@ -297,7 +308,6 @@ mod tests { compute_expected_measurement, MeasurementInput, OvmfSectionParam, }; use crate::main_service::upgrade_authority::CurrentRootEpoch; - use dstack_kms_rpc::KmsKeys; use sha2::Digest; fn hex_of(byte: u8, len: usize) -> String { @@ -1065,38 +1075,17 @@ impl Keys { }) } + #[allow(clippy::too_many_arguments)] async fn onboard( cfg: &KmsConfig, other_kms_url: &str, source_token: &str, launch_ticket: &str, + merge_source: Option<(&str, &str)>, domain: &str, rotate: bool, attestation_verifier: Arc, ) -> Result { - let attestation_slot = Arc::new(Mutex::new(None::)); - let attestation_slot_out = attestation_slot.clone(); - let capture_source_attestation = Box::new(move |info: Option| { - let Some(info) = info else { - bail!("Source KMS did not present a TLS certificate"); - }; - let Some(attestation) = info.attestation else { - bail!("Source KMS certificate does not contain attestation"); - }; - let mut slot = attestation_slot_out - .lock() - .map_err(|_| anyhow::anyhow!("source attestation mutex poisoned"))?; - *slot = Some(attestation); - Ok(()) - }); - let source_attestation = || { - attestation_slot - .lock() - .map_err(|_| anyhow::anyhow!("source attestation mutex poisoned"))? - .clone() - .context("Missing source KMS attestation") - }; - // Send the launch ticket and the admin token only to a KMS serving the // registered roots: at the handshake, its RPC certificate must be // signed by the current root CA registered with our own auth API. @@ -1104,47 +1093,15 @@ impl Keys { .auth_api .root_ca_pubkey() .await - .context("failed to get the registered root CA key")? - .map(|pubkey| PinnedIssuer { - pubkey, - usage: "kms:rpc".into(), - }); - let pinned_client = || { - RaClientConfig::builder() - .tls_skip_ca_verification(true) - .maybe_tls_pinned_issuer(pin.clone()) - .remote_uri(other_kms_url.to_string()) - }; - let temp_ca = if source_token.is_empty() { - let client = pinned_client().build().into_client()?; - let ca = KmsClient::new(client) - .get_temp_ca_cert() - .await - .context("failed to get the temporary CA from the source KMS")?; - Some((ca.temp_ca_cert, ca.temp_ca_key)) - } else { - None - }; - let (ra_cert, ra_key) = gen_ra_cert(temp_ca).await?; - let client = pinned_client() - .tls_built_in_root_certs(false) - .tls_client_cert(ra_cert) - .tls_client_key(ra_key) - .cert_validator(capture_source_attestation) - .attestation_verifier(attestation_verifier.clone()) - .maybe_bearer_token((!source_token.is_empty()).then(|| source_token.to_string())) - .build() - .into_client()?; - let info = dstack_client().info().await.context("Failed to get info")?; - let request = GetKmsKeyRequest { - vm_config: info.vm_config, - launch_ticket: launch_ticket.to_string(), - }; - let keys_res = if source_token.is_empty() { - KmsClient::new(client).get_kms_key(request).await - } else { - AdminClient::new(client).get_kms_key(request).await - } + .context("failed to get the registered root CA key")?; + let (mut keys_res, attestation) = fetch_keys( + other_kms_url, + source_token, + launch_ticket, + pin, + attestation_verifier.clone(), + ) + .await .context("failed to get the keys from the source KMS")?; // The attestation is captured only on the connection that returned the keys; // nothing received is used before the source passes this check. @@ -1154,9 +1111,15 @@ impl Keys { .checked_sub(1) .context("source KMS handed over no keys")?; let highest = u32::try_from(highest).context("too many root epochs")?; - let registration = ensure_kms_allowed(cfg, &source_attestation()?, highest) + let registration = ensure_kms_allowed(cfg, &attestation, highest) .await .context("source KMS is not allowed for onboarding")?; + if let Some((url, token)) = merge_source { + let next = fetch_merge_keys(cfg, highest + 1, url, token, attestation_verifier) + .await + .context("failed to get the keys from the merge source KMS")?; + keys_res.keys.push(next); + } Self::from_key_response(cfg, keys_res, domain, rotate, registration).await } @@ -1356,6 +1319,100 @@ pub(crate) async fn bootstrap_keys(cfg: &KmsConfig, verifier: &AttestationVerifi Ok(()) } +/// Fetch the root keys from one source KMS, with the attestation it presented +/// on that connection. With `pin`, nothing is sent to a source whose RPC +/// certificate the CA key `pin` did not issue. +async fn fetch_keys( + other_kms_url: &str, + source_token: &str, + launch_ticket: &str, + pin: Option>, + attestation_verifier: Arc, +) -> Result<(KmsKeyResponse, VerifiedAttestation)> { + let attestation_slot = Arc::new(Mutex::new(None::)); + let attestation_slot_out = attestation_slot.clone(); + let capture_source_attestation = Box::new(move |info: Option| { + let Some(info) = info else { + bail!("Source KMS did not present a TLS certificate"); + }; + let Some(attestation) = info.attestation else { + bail!("Source KMS certificate does not contain attestation"); + }; + let mut slot = attestation_slot_out + .lock() + .map_err(|_| anyhow::anyhow!("source attestation mutex poisoned"))?; + *slot = Some(attestation); + Ok(()) + }); + let source_attestation = || { + attestation_slot + .lock() + .map_err(|_| anyhow::anyhow!("source attestation mutex poisoned"))? + .clone() + .context("Missing source KMS attestation") + }; + + let pin = pin.map(|pubkey| PinnedIssuer { + pubkey, + usage: "kms:rpc".into(), + }); + let pinned_client = || { + RaClientConfig::builder() + .tls_skip_ca_verification(true) + .maybe_tls_pinned_issuer(pin.clone()) + .remote_uri(other_kms_url.to_string()) + }; + let temp_ca = if source_token.is_empty() { + let client = pinned_client().build().into_client()?; + let ca = KmsClient::new(client) + .get_temp_ca_cert() + .await + .context("failed to get the temporary CA from the source KMS")?; + Some((ca.temp_ca_cert, ca.temp_ca_key)) + } else { + None + }; + let (ra_cert, ra_key) = gen_ra_cert(temp_ca).await?; + let client = pinned_client() + .tls_built_in_root_certs(false) + .tls_client_cert(ra_cert) + .tls_client_key(ra_key) + .cert_validator(capture_source_attestation) + .attestation_verifier(attestation_verifier) + .maybe_bearer_token((!source_token.is_empty()).then(|| source_token.to_string())) + .build() + .into_client()?; + let info = dstack_client().info().await.context("Failed to get info")?; + let request = GetKmsKeyRequest { + vm_config: info.vm_config, + launch_ticket: launch_ticket.to_string(), + }; + let keys_res = if source_token.is_empty() { + KmsClient::new(client).get_kms_key(request).await + } else { + AdminClient::new(client).get_kms_key(request).await + }?; + Ok((keys_res, source_attestation()?)) +} + +/// Fetch the single root epoch of a merge source, to import as `epoch`. Its +/// keys are checked against governance with the merged keyring. +async fn fetch_merge_keys( + cfg: &KmsConfig, + epoch: u32, + url: &str, + token: &str, + attestation_verifier: Arc, +) -> Result { + let (res, attestation) = fetch_keys(url, token, "", None, attestation_verifier).await?; + ensure_kms_allowed(cfg, &attestation, epoch) + .await + .context("merge source KMS is not allowed for onboarding")?; + let [keys] = <[KmsKeys; 1]>::try_from(res.keys) + .map_err(|_| anyhow::anyhow!("the merge source must hold exactly one root epoch"))?; + Ok(keys) +} + /// Check the handed-over roots against governance: they must include the /// current epoch with a matching chain hash, and nothing beyond the latest /// registered epoch. When the source holds the latest registered epoch, its diff --git a/dstack/kms/src/www/onboard.html b/dstack/kms/src/www/onboard.html index 5fe3029cc..71ff3ee43 100644 --- a/dstack/kms/src/www/onboard.html +++ b/dstack/kms/src/www/onboard.html @@ -269,6 +269,17 @@

Onboard from an Existing KMS Instance

placeholder="Required when the KMS contract requires a launch ticket">
+ + +
+
+ + +
+