From 2275694057ce464b3d941cd12b974369c5f868eb Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 22 Sep 2026 23:58:44 -0700 Subject: [PATCH 1/2] fix(verifier): bind downloaded images end to end before caching them - Check sha256(sha256sum.txt) against os_image_hash before parsing the manifest or hashing any file. - Require the bios, kernel and initrd that metadata.json names to be manifest entries. image_dir.join() of an absolute or `..` path, or of an unlisted name, would otherwise measure bytes os_image_hash never committed to. - Bound the downloaded archive and its extracted size at 100 MiB each; measurement images carry no rootfs. --- dstack/verifier/src/verification.rs | 121 ++++++++++++++++++++++------ 1 file changed, 96 insertions(+), 25 deletions(-) diff --git a/dstack/verifier/src/verification.rs b/dstack/verifier/src/verification.rs index 8337dca81..13608f8fc 100644 --- a/dstack/verifier/src/verification.rs +++ b/dstack/verifier/src/verification.rs @@ -95,6 +95,10 @@ fn decode_key_provider_info(bytes: &[u8]) -> Option PathBuf { + Path::new(&self.image_cache_dir).join("images") + } + fn image_content_digest(image_dir: &Path) -> Result>> { let sha256sum_path = image_dir.join("sha256sum.txt"); if !sha256sum_path.exists() { @@ -352,6 +360,47 @@ impl CvmVerifier { Ok(entries) } + /// Bind an extracted image to `os_image_hash` and drop everything the + /// binding does not cover. + fn verify_extracted_image(extracted_dir: &Path, hex_os_image_hash: &str) -> Result<()> { + // All image modes are addressed by sha256(sha256sum.txt); extra + // measurement CBOR files are ordinary entries and do not define + // alternate image hashes. Check it before anything else in the archive + // is parsed or hashed. + let files_doc = fs_err::read_to_string(extracted_dir.join("sha256sum.txt")) + .context("failed to read sha256sum.txt")?; + if hex::encode(Sha256::digest(&files_doc)) != hex_os_image_hash { + bail!("os_image_hash does not match sha256(sha256sum.txt)"); + } + let manifest = Self::verify_image_manifest(extracted_dir, &files_doc)?; + Self::prune_unlisted_image_files(extracted_dir, &manifest)?; + Self::check_measured_files_are_listed(extracted_dir, &manifest) + } + + /// `metadata.json` picks the firmware, kernel and initrd that get measured. + /// The manifest binds their bytes only if they are among its entries; any + /// other path -- absolute, `..`, or just unlisted -- would be measured from + /// bytes `os_image_hash` never committed to. + fn check_measured_files_are_listed( + extracted_dir: &Path, + manifest: &[sha256sum::Entry], + ) -> Result<()> { + let listed = |name: &str| manifest.iter().any(|entry| entry.name == name); + if !listed("metadata.json") { + bail!("sha256sum.txt does not list metadata.json"); + } + let image_info = fs_err::read_to_string(extracted_dir.join("metadata.json")) + .context("failed to read image metadata")?; + let image_info: dstack_types::ImageInfo = + serde_json::from_str(&image_info).context("failed to parse image metadata")?; + for name in [&image_info.bios, &image_info.kernel, &image_info.initrd] { + if !listed(name) { + bail!("metadata.json names {name}, which sha256sum.txt does not list"); + } + } + Ok(()) + } + fn extract_image_archive(tarball_path: &Path, extracted_dir: &Path) -> Result<()> { let file = fs_err::File::open(tarball_path).context("Failed to open image archive")?; // `MultiGzDecoder`, not `GzDecoder`: the latter stops at the first gzip @@ -359,6 +408,7 @@ impl CvmVerifier { // extract partially without any error. let decoder = flate2::read::MultiGzDecoder::new(file); let mut archive = tar::Archive::new(decoder); + let mut extracted = 0u64; for entry in archive.entries().context("Failed to read image archive")? { let mut entry = entry.context("Failed to read image archive entry")?; let path = entry @@ -374,6 +424,10 @@ impl CvmVerifier { path.display() ); } + extracted += entry.size(); + if extracted > MAX_IMAGE_SIZE { + bail!("image archive extracts to more than {MAX_IMAGE_SIZE} bytes"); + } if !entry .unpack_in(extracted_dir) .context("Failed to extract image archive entry")? @@ -487,9 +541,7 @@ impl CvmVerifier { let hex_os_image_hash = hex::encode(&vm_config.os_image_hash); // Get image directory - let image_dir = Path::new(&self.image_cache_dir) - .join("images") - .join(&hex_os_image_hash); + let image_dir = self.images_dir().join(&hex_os_image_hash); let metadata_path = image_dir.join("metadata.json"); if !metadata_path.exists() { @@ -975,7 +1027,7 @@ impl CvmVerifier { .replace("{OS_IMAGE_HASH}", hex_os_image_hash); // Create a temporary directory for extraction within the cache directory - let cache_dir = Path::new(&self.image_cache_dir).join("images").join("tmp"); + let cache_dir = self.images_dir().join("tmp"); fs_err::create_dir_all(&cache_dir).context("Failed to create cache directory")?; let auto_delete_temp_dir = tempfile::Builder::new() .prefix("tmp-download-") @@ -1004,7 +1056,12 @@ impl CvmVerifier { .await .context("Failed to create tarball file")?; let mut response = response; + let mut downloaded = 0u64; while let Some(chunk) = response.chunk().await? { + downloaded += chunk.len() as u64; + if downloaded > MAX_IMAGE_SIZE { + bail!("image archive exceeds {MAX_IMAGE_SIZE} bytes"); + } file.write_all(&chunk) .await .context("Failed to write chunk to file")?; @@ -1019,27 +1076,7 @@ impl CvmVerifier { drop(file); Self::extract_image_archive(&tarball_path, &extracted_dir)?; - let sha256sum_path = extracted_dir.join("sha256sum.txt"); - let files_doc = - fs_err::read_to_string(&sha256sum_path).context("Failed to read sha256sum.txt")?; - let manifest = Self::verify_image_manifest(&extracted_dir, &files_doc)?; - - // Remove the files that are not listed in sha256sum.txt - Self::prune_unlisted_image_files(&extracted_dir, &manifest)?; - - // All image modes are addressed by sha256(sha256sum.txt). Extra - // measurement CBOR files are ordinary sha256sum.txt entries and do not - // define alternate image hashes. - let legacy_os_image_hash = Sha256::new_with_prefix(files_doc.as_bytes()).finalize(); - if hex::encode(legacy_os_image_hash) != hex_os_image_hash { - bail!("os_image_hash does not match sha256(sha256sum.txt)"); - } - - // Move the extracted files to the destination directory - let metadata_path = extracted_dir.join("metadata.json"); - if !metadata_path.exists() { - bail!("metadata.json not found in the extracted archive"); - } + Self::verify_extracted_image(&extracted_dir, hex_os_image_hash)?; if dst_dir.exists() { fs_err::remove_dir_all(dst_dir).context("Failed to remove destination directory")?; @@ -1717,6 +1754,40 @@ mod tests { assert_ne!(key(&resized), key(&base)); } + /// `os_image_hash` is checked before the manifest is parsed, and the files + /// `metadata.json` measures must be manifest entries. + #[test] + fn extracted_image_binds_the_files_metadata_measures() { + let verify = |kernel: &str| { + let metadata = format!( + r#"{{"cmdline":"","kernel":"{kernel}","initrd":"initrd","bios":"ovmf.fd"}}"# + ); + let files: &[(&str, &[u8])] = &[ + ("metadata.json", metadata.as_bytes()), + ("bzImage", b"kernel"), + ("initrd", b"initrd"), + ("ovmf.fd", b"firmware"), + ]; + let files_doc: String = files + .iter() + .map(|(name, payload)| { + format!("{} {name}\n", hex::encode(Sha256::digest(payload))) + }) + .collect(); + let dir = image_dir_with_manifest(&files_doc, files); + let hash = hex::encode(Sha256::digest(&files_doc)); + CvmVerifier::verify_extracted_image(dir.path(), &hash) + }; + verify("bzImage").unwrap(); + for kernel in ["/boot/bzImage", "../bzImage", "unlisted"] { + assert!(verify(kernel).is_err(), "accepted kernel {kernel}"); + } + + let dir = image_dir_with_manifest("not a manifest", &[]); + let err = CvmVerifier::verify_extracted_image(dir.path(), &"00".repeat(32)).unwrap_err(); + assert!(err.to_string().contains("os_image_hash"), "{err:#}"); + } + #[test] fn image_cache_pruning_keeps_checksum_identity() { let files_doc = format!("{} metadata.json\n", hex::encode(Sha256::digest(b"{}"))); From e300871ce490e8fb7186cd8922483345b436a629 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 23 Sep 2026 20:14:18 -0700 Subject: [PATCH 2/2] fix(verifier): download each image once and install it off the async runtime Concurrent requests for an uncached image each downloaded it, and every finisher after the first ran remove_dir_all on the installed image while the others could be measuring from it, then raced the rename. Take a per-image lock before the cache check and hold it until the image is installed. Extraction, the manifest check and the install run in one spawn_blocking task that owns the lock guard and the temporary directory, so a caller that hits download_timeout cannot leave a still-running install to collide with the next download. --- dstack/verifier/src/verification.rs | 127 ++++++++++++++++++++++++++-- 1 file changed, 118 insertions(+), 9 deletions(-) diff --git a/dstack/verifier/src/verification.rs b/dstack/verifier/src/verification.rs index 13608f8fc..95223946c 100644 --- a/dstack/verifier/src/verification.rs +++ b/dstack/verifier/src/verification.rs @@ -3,9 +3,10 @@ // SPDX-License-Identifier: Apache-2.0 use std::{ + collections::HashMap, ffi::OsStr, path::{Component, Path, PathBuf}, - sync::Arc, + sync::{Arc, Mutex, PoisonError, Weak}, time::Duration, }; @@ -26,7 +27,7 @@ use ra_tls::attestation::{ }; use serde::{Deserialize, Serialize}; use sha2::{Digest as _, Sha256}; -use tokio::io::AsyncWriteExt; +use tokio::{io::AsyncWriteExt, sync::Mutex as AsyncMutex}; use tpm_qvl::verify::VerifiedReport as TpmVerifiedReport; use tracing::{debug, info, warn}; @@ -121,6 +122,7 @@ pub struct CvmVerifier { pub download_url: String, pub download_timeout: Duration, pub attestation_verifier: Arc, + download_locks: Mutex>>>, } impl CvmVerifier { @@ -135,9 +137,26 @@ impl CvmVerifier { download_url, download_timeout, attestation_verifier, + download_locks: Mutex::default(), } } + /// One lock per image, held from the cache check until the image is + /// installed, so concurrent requests for an image download it once. + fn download_lock(&self, hex_os_image_hash: &str) -> Arc> { + let mut locks = self + .download_locks + .lock() + .unwrap_or_else(PoisonError::into_inner); + locks.retain(|_, lock| lock.strong_count() > 0); + if let Some(lock) = locks.get(hex_os_image_hash).and_then(Weak::upgrade) { + return lock; + } + let lock = Arc::default(); + locks.insert(hex_os_image_hash.to_owned(), Arc::downgrade(&lock)); + lock + } + fn measurement_cache_dir(&self) -> PathBuf { Path::new(&self.image_cache_dir).join("measurements") } @@ -1022,6 +1041,10 @@ impl CvmVerifier { } pub async fn download_image(&self, hex_os_image_hash: &str, dst_dir: &Path) -> Result<()> { + let guard = self.download_lock(hex_os_image_hash).lock_owned().await; + if dst_dir.join("metadata.json").exists() { + return Ok(()); + } let url = self .download_url .replace("{OS_IMAGE_HASH}", hex_os_image_hash); @@ -1067,15 +1090,29 @@ impl CvmVerifier { .context("Failed to write chunk to file")?; } - let extracted_dir = tmp_dir.join("extracted"); - fs_err::create_dir_all(&extracted_dir).context("Failed to create extraction directory")?; - file.flush() .await .context("Failed to flush image archive")?; drop(file); - Self::extract_image_archive(&tarball_path, &extracted_dir)?; + let hex_os_image_hash = hex_os_image_hash.to_owned(); + let dst_dir = dst_dir.to_owned(); + tokio::task::spawn_blocking(move || { + // A blocking task outlives its caller: if the caller is cancelled + // (e.g. by `download_timeout`), this still runs to the end. Keep + // the lock until the install finishes, or the next download for + // this image would race it on `dst_dir`. + let _guard = guard; + Self::install_image(auto_delete_temp_dir.path(), &hex_os_image_hash, &dst_dir) + }) + .await + .context("image install task failed")? + } + + fn install_image(tmp_dir: &Path, hex_os_image_hash: &str, dst_dir: &Path) -> Result<()> { + let extracted_dir = tmp_dir.join("extracted"); + fs_err::create_dir_all(&extracted_dir).context("Failed to create extraction directory")?; + Self::extract_image_archive(&tmp_dir.join("image.tar.gz"), &extracted_dir)?; Self::verify_extracted_image(&extracted_dir, hex_os_image_hash)?; if dst_dir.exists() { @@ -1083,10 +1120,8 @@ impl CvmVerifier { } let dst_dir_parent = dst_dir.parent().context("Failed to get parent directory")?; fs_err::create_dir_all(dst_dir_parent).context("Failed to create parent directory")?; - // Move the extracted files to the destination directory fs_err::rename(extracted_dir, dst_dir) - .context("Failed to move extracted files to destination directory")?; - Ok(()) + .context("Failed to move extracted files to destination directory") } } @@ -1853,6 +1888,80 @@ mod tests { } } + #[tokio::test(flavor = "multi_thread")] + async fn concurrent_requests_download_an_image_once() { + let metadata = br#"{"cmdline":"","kernel":"bzImage","initrd":"initrd","bios":"ovmf.fd"}"#; + let files: &[(&str, &[u8])] = &[ + ("metadata.json", metadata), + ("bzImage", b"kernel"), + ("initrd", b"initrd"), + ("ovmf.fd", b"firmware"), + ]; + let files_doc: String = files + .iter() + .map(|(name, payload)| format!("{} {name}\n", hex::encode(Sha256::digest(payload)))) + .collect(); + let hash = hex::encode(Sha256::digest(&files_doc)); + let mut archive = tar::Builder::new(flate2::write::GzEncoder::new( + Vec::new(), + flate2::Compression::default(), + )); + for (name, payload) in files + .iter() + .chain([&("sha256sum.txt", files_doc.as_bytes())]) + { + let mut header = tar::Header::new_gnu(); + header.set_size(payload.len() as u64); + header.set_mode(0o644); + archive.append_data(&mut header, name, *payload).unwrap(); + } + let tarball = archive.into_inner().unwrap().finish().unwrap(); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!( + "http://{}/{{OS_IMAGE_HASH}}", + listener.local_addr().unwrap() + ); + let served = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + tokio::spawn({ + let served = served.clone(); + async move { + use tokio::io::AsyncReadExt; + while let Ok((mut stream, _)) = listener.accept().await { + served.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let _ = stream.read(&mut [0; 4096]).await; + tokio::time::sleep(Duration::from_millis(100)).await; + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + tarball.len() + ); + let _ = stream.write_all(head.as_bytes()).await; + let _ = stream.write_all(&tarball).await; + } + } + }); + + let cache = tempfile::tempdir().unwrap(); + let verifier = Arc::new(CvmVerifier::new( + cache.path().to_str().unwrap().into(), + url, + Duration::from_secs(10), + test_attestation_verifier(), + )); + let dst_dir = verifier.images_dir().join(&hash); + let downloads: Vec<_> = (0..8) + .map(|_| { + let (verifier, hash, dst_dir) = (verifier.clone(), hash.clone(), dst_dir.clone()); + tokio::spawn(async move { verifier.download_image(&hash, &dst_dir).await }) + }) + .collect(); + for download in downloads { + download.await.unwrap().unwrap(); + } + assert_eq!(served.load(std::sync::atomic::Ordering::SeqCst), 1); + assert_eq!(fs_err::read(dst_dir.join("bzImage")).unwrap(), b"kernel"); + } + #[test] fn image_archive_rejects_links_and_accepts_regular_files() { let directory = tempfile::tempdir().unwrap();