From 77aa2c25b41c26b7223bd0c183740bcdde97e7a2 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Mon, 25 May 2026 21:38:01 -0700 Subject: [PATCH 01/67] feat: add amd sev-snp attestation support --- Cargo.lock | 1 + dstack-attest/Cargo.toml | 1 + dstack-attest/src/attestation.rs | 63 ++++++++++++- dstack-attest/src/lib.rs | 2 + dstack-attest/src/sev_snp.rs | 147 +++++++++++++++++++++++++++++++ dstack-attest/src/v1.rs | 45 ++++++++++ kms/src/onboard_service.rs | 1 + verifier/src/verification.rs | 7 ++ vmm/src/app/qemu.rs | 43 ++++++++- vmm/src/config.rs | 60 +++++++++++++ vmm/vmm.toml | 2 + 11 files changed, 366 insertions(+), 6 deletions(-) create mode 100644 dstack-attest/src/sev_snp.rs diff --git a/Cargo.lock b/Cargo.lock index 3c8510b4e..cb7f87263 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2301,6 +2301,7 @@ dependencies = [ "hex", "hex_fmt", "insta", + "libc", "nsm-attest", "nsm-qvl", "or-panic", diff --git a/dstack-attest/Cargo.toml b/dstack-attest/Cargo.toml index 0dfd0e8a9..430c8cc00 100644 --- a/dstack-attest/Cargo.toml +++ b/dstack-attest/Cargo.toml @@ -19,6 +19,7 @@ ez-hash.workspace = true fs-err.workspace = true hex.workspace = true hex_fmt.workspace = true +libc.workspace = true or-panic.workspace = true scale = { workspace = true, features = ["derive"] } serde.workspace = true diff --git a/dstack-attest/src/attestation.rs b/dstack-attest/src/attestation.rs index 355477292..f9b581c20 100644 --- a/dstack-attest/src/attestation.rs +++ b/dstack-attest/src/attestation.rs @@ -34,6 +34,7 @@ pub use tpm_types::TpmQuote; pub use crate::v1::{Attestation as AttestationV1, PlatformEvidence, StackEvidence}; const DSTACK_TDX: &str = "dstack-tdx"; +const DSTACK_AMD_SEV_SNP: &str = "dstack-amd-sev-snp"; const DSTACK_GCP_TDX: &str = "dstack-gcp-tdx"; const DSTACK_NITRO_ENCLAVE: &str = "dstack-nitro-enclave"; @@ -87,6 +88,9 @@ fn platform_from_legacy_quote(quote: AttestationQuote) -> PlatformEvidence { AttestationQuote::DstackTdx(TdxQuote { quote, event_log }) => { PlatformEvidence::Tdx { quote, event_log } } + AttestationQuote::DstackAmdSevSnp(SnpQuote { report, cert_chain }) => { + PlatformEvidence::SevSnp { report, cert_chain } + } AttestationQuote::DstackGcpTdx(DstackGcpTdxQuote { tdx_quote: TdxQuote { quote, event_log }, tpm_quote, @@ -106,6 +110,9 @@ fn platform_into_legacy_quote(platform: PlatformEvidence) -> AttestationQuote { PlatformEvidence::Tdx { quote, event_log } => { AttestationQuote::DstackTdx(TdxQuote { quote, event_log }) } + PlatformEvidence::SevSnp { report, cert_chain } => { + AttestationQuote::DstackAmdSevSnp(SnpQuote { report, cert_chain }) + } PlatformEvidence::GcpTdx { quote, event_log, @@ -123,6 +130,7 @@ fn platform_into_legacy_quote(platform: PlatformEvidence) -> AttestationQuote { fn platform_attestation_mode(platform: &PlatformEvidence) -> AttestationMode { match platform { PlatformEvidence::Tdx { .. } => AttestationMode::DstackTdx, + PlatformEvidence::SevSnp { .. } => AttestationMode::DstackAmdSevSnp, PlatformEvidence::GcpTdx { .. } => AttestationMode::DstackGcpTdx, PlatformEvidence::NitroEnclave { .. } => AttestationMode::DstackNitroEnclave, } @@ -168,6 +176,9 @@ pub enum AttestationMode { #[default] #[serde(rename = "dstack-tdx")] DstackTdx, + /// AMD SEV-SNP report generated by the dstack attestation SDK + #[serde(rename = "dstack-amd-sev-snp")] + DstackAmdSevSnp, /// GCP TDX with DCAP quote only #[serde(rename = "dstack-gcp-tdx")] DstackGcpTdx, @@ -180,15 +191,20 @@ impl AttestationMode { /// Detect attestation mode from system pub fn detect() -> Result { let has_tdx = std::path::Path::new("/dev/tdx_guest").exists(); + let has_sev_snp = std::path::Path::new("/dev/sev-guest").exists() + || std::path::Path::new("/sys/kernel/config/tsm/report").exists(); // First, try to detect platform from DMI product name let platform = Platform::detect_or_dstack(); match platform { Platform::Dstack => { + if has_sev_snp { + return Ok(Self::DstackAmdSevSnp); + } if has_tdx { return Ok(Self::DstackTdx); } - bail!("Unsupported platform: Dstack(-tdx)"); + bail!("Unsupported platform: Dstack(-tdx/-amd-sev-snp)"); } Platform::Gcp => { // GCP platform: TDX + TPM dual mode @@ -205,6 +221,7 @@ impl AttestationMode { pub fn has_tdx(&self) -> bool { match self { Self::DstackTdx => true, + Self::DstackAmdSevSnp => false, Self::DstackGcpTdx => true, Self::DstackNitroEnclave => false, } @@ -215,6 +232,7 @@ impl AttestationMode { match self { Self::DstackGcpTdx => Some(14), Self::DstackTdx => None, + Self::DstackAmdSevSnp => None, Self::DstackNitroEnclave => None, } } @@ -223,6 +241,7 @@ impl AttestationMode { pub fn as_str(&self) -> &'static str { match self { Self::DstackTdx => DSTACK_TDX, + Self::DstackAmdSevSnp => DSTACK_AMD_SEV_SNP, Self::DstackGcpTdx => DSTACK_GCP_TDX, Self::DstackNitroEnclave => DSTACK_NITRO_ENCLAVE, } @@ -232,6 +251,10 @@ impl AttestationMode { pub fn is_composable(&self) -> bool { match self { Self::DstackTdx => true, + // SEV-SNP launch measurement does not provide a TDX RTMR3-equivalent + // runtime event extension path yet, so runtime events are + // informational until an SNP-specific app binding is added. + Self::DstackAmdSevSnp => false, Self::DstackGcpTdx => true, Self::DstackNitroEnclave => false, } @@ -357,6 +380,15 @@ pub struct TdxQuote { pub event_log: Vec, } +/// Represents an AMD SEV-SNP attestation report. +#[derive(Clone, Encode, Decode)] +pub struct SnpQuote { + /// Raw SNP report bytes. + pub report: Vec, + /// Optional certificate chain blobs, when exposed by the kernel/firmware path. + pub cert_chain: Vec>, +} + /// Represents an NSM (Nitro Security Module) attestation document #[derive(Clone, Encode, Decode)] pub struct NsmQuote { @@ -586,6 +618,9 @@ impl AttestationV1 { nsm_quote: nsm_quote.clone(), })? } + PlatformEvidence::SevSnp { .. } => { + bail!("Unsupported attestation quote"); + } }; let compose_hash = if platform_attestation_mode(&self.platform).is_composable() { find_event_payload(runtime_events, "compose-hash").unwrap_or_default() @@ -708,6 +743,12 @@ impl AttestationV1 { timestamp: verified_report.timestamp, }) } + PlatformEvidence::SevSnp { .. } => { + bail!( + "Unsupported attestation mode: {:?}", + platform_attestation_mode(&platform) + ); + } }; Ok(VerifiedAttestation { @@ -821,12 +862,14 @@ pub enum AttestationQuote { DstackTdx(TdxQuote), DstackGcpTdx(DstackGcpTdxQuote), DstackNitroEnclave(DstackNitroQuote), + DstackAmdSevSnp(SnpQuote), } impl AttestationQuote { pub fn mode(&self) -> AttestationMode { match self { AttestationQuote::DstackTdx { .. } => AttestationMode::DstackTdx, + AttestationQuote::DstackAmdSevSnp { .. } => AttestationMode::DstackAmdSevSnp, AttestationQuote::DstackGcpTdx { .. } => AttestationMode::DstackGcpTdx, AttestationQuote::DstackNitroEnclave { .. } => AttestationMode::DstackNitroEnclave, } @@ -860,6 +903,7 @@ impl Attestation { pub fn tdx_quote_mut(&mut self) -> Option<&mut TdxQuote> { match &mut self.quote { AttestationQuote::DstackTdx(quote) => Some(quote), + AttestationQuote::DstackAmdSevSnp(_) => None, AttestationQuote::DstackGcpTdx(q) => Some(&mut q.tdx_quote), AttestationQuote::DstackNitroEnclave(_) => None, } @@ -868,6 +912,7 @@ impl Attestation { pub fn tdx_quote(&self) -> Option<&TdxQuote> { match &self.quote { AttestationQuote::DstackTdx(quote) => Some(quote), + AttestationQuote::DstackAmdSevSnp(_) => None, AttestationQuote::DstackGcpTdx(q) => Some(&q.tdx_quote), AttestationQuote::DstackNitroEnclave(_) => None, } @@ -876,6 +921,7 @@ impl Attestation { pub fn tpm_quote(&self) -> Option<&TpmQuote> { match &self.quote { AttestationQuote::DstackTdx(_) => None, + AttestationQuote::DstackAmdSevSnp(_) => None, AttestationQuote::DstackGcpTdx(q) => Some(&q.tpm_quote), AttestationQuote::DstackNitroEnclave(_) => None, } @@ -1200,6 +1246,9 @@ impl Attestation { AttestationQuote::DstackTdx(q) => { self.decode_mr_tdx(boottime_mr, &mr_key_provider, q)? } + AttestationQuote::DstackAmdSevSnp(_) => { + bail!("unsupported attestation quote for app info decoding"); + } AttestationQuote::DstackGcpTdx(q) => { self.decode_mr_gcp_tpm(boottime_mr, &mr_key_provider, &os_image_hash, &q.tpm_quote)? } @@ -1340,6 +1389,11 @@ impl Attestation { cc_eventlog::tdx::read_event_log().context("Failed to read event log")?; AttestationQuote::DstackTdx(TdxQuote { quote, event_log }) } + AttestationMode::DstackAmdSevSnp => { + let quote = crate::sev_snp::get_report(*report_data) + .context("Failed to get SEV-SNP report")?; + AttestationQuote::DstackAmdSevSnp(quote) + } AttestationMode::DstackGcpTdx => { let quote = tdx_attest::get_quote(report_data).context("Failed to get quote")?; let event_log = @@ -1363,7 +1417,9 @@ impl Attestation { } }; let config = match "e { - AttestationQuote::DstackTdx(_) | AttestationQuote::DstackGcpTdx(_) => { + AttestationQuote::DstackAmdSevSnp(_) + | AttestationQuote::DstackTdx(_) + | AttestationQuote::DstackGcpTdx(_) => { read_vm_config().context("Failed to read vm config")? } AttestationQuote::DstackNitroEnclave(quote) => { @@ -1403,6 +1459,9 @@ impl Attestation { let report = self.verify_tdx(pccs_url, &q.quote).await?; DstackVerifiedReport::DstackTdx(report) } + AttestationQuote::DstackAmdSevSnp(_) => { + bail!("Unsupported attestation mode: {:?}", self.quote.mode()); + } AttestationQuote::DstackGcpTdx(q) => { let tdx_report = self.verify_tdx(pccs_url, &q.tdx_quote.quote).await?; let tpm_report = self diff --git a/dstack-attest/src/lib.rs b/dstack-attest/src/lib.rs index c0395112a..6452b7c36 100644 --- a/dstack-attest/src/lib.rs +++ b/dstack-attest/src/lib.rs @@ -14,6 +14,8 @@ pub use tpm_attest as tpm; use crate::attestation::AttestationMode; pub mod attestation; +#[cfg(feature = "quote")] +mod sev_snp; mod v1; /// Serializes runtime event emission within this process. diff --git a/dstack-attest/src/sev_snp.rs b/dstack-attest/src/sev_snp.rs new file mode 100644 index 000000000..441367adf --- /dev/null +++ b/dstack-attest/src/sev_snp.rs @@ -0,0 +1,147 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Minimal AMD SEV-SNP guest report support. + +use std::{fs::OpenOptions, io, path::Path}; + +use anyhow::{bail, Context, Result}; + +use crate::attestation::SnpQuote; + +const TSM_REPORT_ROOT: &str = "/sys/kernel/config/tsm/report"; +const SEV_GUEST_DEVICE: &str = "/dev/sev-guest"; +const SNP_REPORT_SIZE: usize = 1184; +const SNP_REPORT_RESP_SIZE: usize = 4000; +const SNP_GET_REPORT: libc::c_ulong = 0xc020_5300; + +#[repr(C)] +#[derive(Clone, Copy)] +struct SnpReportReq { + report_data: [u8; 64], + vmpl: u32, + rsvd: [u8; 28], +} + +#[repr(C)] +struct SnpReportResp { + data: [u8; SNP_REPORT_RESP_SIZE], +} + +#[repr(C)] +struct SnpGuestRequestIoctl { + msg_version: u8, + req_data: u64, + resp_data: u64, + fw_err: u64, +} + +pub fn get_report(report_data: [u8; 64]) -> Result { + if Path::new(TSM_REPORT_ROOT).exists() { + match get_report_configfs(report_data) { + Ok(quote) => return Ok(quote), + Err(err) => tracing::debug!("failed to get sev-snp report from configfs tsm: {err:#}"), + } + } + if Path::new(SEV_GUEST_DEVICE).exists() { + return get_report_ioctl(report_data); + } + bail!("sev-snp report is unavailable: neither {TSM_REPORT_ROOT} nor {SEV_GUEST_DEVICE} exists") +} + +fn get_report_configfs(report_data: [u8; 64]) -> Result { + let root = Path::new(TSM_REPORT_ROOT); + let dir = root.join(format!("dstack-{}", std::process::id())); + if !dir.exists() { + fs_err::create_dir(&dir).with_context(|| format!("failed to create {}", dir.display()))?; + } + + let hex_report_data = hex::encode(report_data); + write_first_existing( + &[ + dir.join("inblob"), + dir.join("reportdata"), + dir.join("report_data"), + ], + &report_data, + hex_report_data.as_bytes(), + )?; + + let report = read_first_existing(&[dir.join("outblob"), dir.join("report")])?; + if report.is_empty() { + bail!("sev-snp configfs tsm returned an empty report"); + } + Ok(SnpQuote { + report, + cert_chain: read_cert_chain_configfs(&dir), + }) +} + +fn write_first_existing(paths: &[std::path::PathBuf], binary: &[u8], hex: &[u8]) -> Result<()> { + let mut last_err = None; + for path in paths { + if !path.exists() { + continue; + } + match fs_err::write(path, binary).or_else(|_| fs_err::write(path, hex)) { + Ok(()) => return Ok(()), + Err(err) => last_err = Some(err), + } + } + match last_err { + Some(err) => Err(err).context("failed to write sev-snp tsm report data"), + None => bail!("failed to find sev-snp tsm report input file"), + } +} + +fn read_first_existing(paths: &[std::path::PathBuf]) -> Result> { + for path in paths { + if path.exists() { + return fs_err::read(path) + .with_context(|| format!("failed to read {}", path.display())); + } + } + bail!("failed to find sev-snp tsm report output file") +} + +fn read_cert_chain_configfs(dir: &Path) -> Vec> { + ["certs", "cert_chain", "auxblob"] + .iter() + .filter_map(|name| fs_err::read(dir.join(name)).ok()) + .filter(|bytes| !bytes.is_empty()) + .collect() +} + +fn get_report_ioctl(report_data: [u8; 64]) -> Result { + let file = OpenOptions::new() + .read(true) + .write(true) + .open(SEV_GUEST_DEVICE) + .with_context(|| format!("failed to open {SEV_GUEST_DEVICE}"))?; + let mut req = SnpReportReq { + report_data, + vmpl: 0, + rsvd: [0; 28], + }; + let mut resp = SnpReportResp { + data: [0; SNP_REPORT_RESP_SIZE], + }; + let mut ioctl_req = SnpGuestRequestIoctl { + msg_version: 1, + req_data: (&mut req as *mut SnpReportReq) as u64, + resp_data: (&mut resp as *mut SnpReportResp) as u64, + fw_err: 0, + }; + + let rc = unsafe { libc::ioctl(file.as_raw_fd(), SNP_GET_REPORT, &mut ioctl_req) }; + if rc < 0 { + return Err(io::Error::last_os_error()).context("sev-snp get report ioctl failed"); + } + Ok(SnpQuote { + report: resp.data[..SNP_REPORT_SIZE].to_vec(), + cert_chain: Vec::new(), + }) +} + +use std::os::fd::AsRawFd; diff --git a/dstack-attest/src/v1.rs b/dstack-attest/src/v1.rs index 7c9aa3932..62d54cc81 100644 --- a/dstack-attest/src/v1.rs +++ b/dstack-attest/src/v1.rs @@ -17,6 +17,11 @@ pub enum PlatformEvidence { quote: Vec, event_log: Vec, }, + #[serde(rename = "sev-snp")] + SevSnp { + report: Vec, + cert_chain: Vec>, + }, #[serde(rename = "gcp-tdx")] GcpTdx { quote: Vec, @@ -58,6 +63,20 @@ impl PlatformEvidence { } } + pub fn sev_snp_report(&self) -> Option<&[u8]> { + match self { + Self::SevSnp { report, .. } => Some(report.as_slice()), + _ => None, + } + } + + pub fn sev_snp_cert_chain(&self) -> Option<&[Vec]> { + match self { + Self::SevSnp { cert_chain, .. } => Some(cert_chain.as_slice()), + _ => None, + } + } + pub fn into_stripped(self) -> Self { match self { Self::Tdx { quote, event_log } => Self::Tdx { @@ -328,4 +347,30 @@ mod tests { _ => panic!("expected dstack-pod stack evidence"), } } + + #[test] + fn sev_snp_msgpack_roundtrip_preserves_evidence() { + let attestation = Attestation::new( + PlatformEvidence::SevSnp { + report: vec![0x11; 1184], + cert_chain: vec![vec![0x22, 0x33]], + }, + StackEvidence::Dstack { + report_data: vec![9u8; 64], + runtime_events: vec![], + config: "{}".into(), + }, + ); + + let encoded = attestation.to_msgpack().expect("encode msgpack"); + let decoded = Attestation::from_msgpack(&encoded).expect("decode msgpack"); + assert_eq!( + decoded.platform.sev_snp_report(), + Some(vec![0x11; 1184].as_slice()) + ); + assert_eq!( + decoded.platform.sev_snp_cert_chain(), + Some(vec![vec![0x22, 0x33]].as_slice()) + ); + } } diff --git a/kms/src/onboard_service.rs b/kms/src/onboard_service.rs index bb4086894..e1c5c277b 100644 --- a/kms/src/onboard_service.rs +++ b/kms/src/onboard_service.rs @@ -116,6 +116,7 @@ impl OnboardRpc for OnboardHandler { .context("Failed to decode attestation")?; let attestation_mode = match &attestation.clone().into_v1().platform { PlatformEvidence::Tdx { .. } => "dstack-tdx", + PlatformEvidence::SevSnp { .. } => "dstack-amd-sev-snp", PlatformEvidence::GcpTdx { .. } => "dstack-gcp-tdx", PlatformEvidence::NitroEnclave { .. } => "dstack-nitro-enclave", } diff --git a/verifier/src/verification.rs b/verifier/src/verification.rs index 00c0e1991..bb5638c2a 100644 --- a/verifier/src/verification.rs +++ b/verifier/src/verification.rs @@ -32,6 +32,7 @@ fn tee_platform_name(quote: &AttestationQuote) -> &'static str { AttestationQuote::DstackTdx(_) => "tdx", AttestationQuote::DstackGcpTdx(_) => "gcp-tdx", AttestationQuote::DstackNitroEnclave(_) => "nitro", + AttestationQuote::DstackAmdSevSnp(_) => "sev-snp", } } @@ -526,6 +527,12 @@ impl CvmVerifier { }; self.verify_os_image_hash_for_nitro_enclave(&vm_config, &report.pcrs)?; } + AttestationQuote::DstackAmdSevSnp(_) => { + bail!( + "Unsupported attestation quote: {:?}", + attestation.quote.mode() + ); + } } Ok(vm_config) } diff --git a/vmm/src/app/qemu.rs b/vmm/src/app/qemu.rs index 456068510..0b4c2b77f 100644 --- a/vmm/src/app/qemu.rs +++ b/vmm/src/app/qemu.rs @@ -5,7 +5,9 @@ //! QEMU related code use crate::{ app::Manifest, - config::{CvmConfig, GatewayConfig, Networking, NetworkingMode, ProcessAnnotation}, + config::{ + CvmConfig, GatewayConfig, Networking, NetworkingMode, ProcessAnnotation, TeePlatform, + }, }; use std::{collections::HashMap, os::unix::fs::PermissionsExt}; use std::{ @@ -796,10 +798,27 @@ impl VmConfig { return Ok(()); } - command - .arg("-machine") - .arg("q35,kernel-irqchip=split,confidential-guest-support=tdx,hpet=off"); + match cfg.platform.resolve() { + TeePlatform::Tdx | TeePlatform::Auto => { + command + .arg("-machine") + .arg("q35,kernel-irqchip=split,confidential-guest-support=tdx,hpet=off"); + self.configure_tdx_guest(command, workdir, cfg, app_compose)?; + } + TeePlatform::AmdSevSnp => { + self.configure_amd_sev_snp_guest(command, cfg); + } + } + Ok(()) + } + fn configure_tdx_guest( + &self, + command: &mut Command, + workdir: &VmWorkDir, + cfg: &CvmConfig, + app_compose: &AppCompose, + ) -> Result<()> { let img_ver = self.image.info.version_tuple().unwrap_or_default(); let support_mr_config_id = img_ver >= (0, 5, 2); @@ -876,6 +895,22 @@ impl VmConfig { Ok(()) } + fn configure_amd_sev_snp_guest(&self, command: &mut Command, cfg: &CvmConfig) { + command.arg("-object").arg(format!( + "memory-backend-memfd,id=ram1,size={}M,share=true,prealloc=false", + self.manifest.memory + )); + command + .arg("-object") + .arg("sev-snp-guest,id=sev0,policy=0x30000,sev-device=/dev/sev,kernel-hashes=on,cbitpos=51,reduced-phys-bits=1"); + command.arg("-machine").arg( + "q35,kernel-irqchip=split,confidential-guest-support=sev0,memory-backend=ram1,hpet=off", + ); + if cfg.qgs_port.is_some() { + tracing::warn!("qgs_port is ignored for amd sev-snp guests"); + } + } + fn configure_smbios(&self, command: &mut Command, cfg: &CvmConfig) { let p = &cfg.product; diff --git a/vmm/src/config.rs b/vmm/src/config.rs index cdb587456..5e43e4479 100644 --- a/vmm/src/config.rs +++ b/vmm/src/config.rs @@ -106,6 +106,42 @@ impl Protocol { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum TeePlatform { + Auto, + Tdx, + AmdSevSnp, +} + +impl Default for TeePlatform { + fn default() -> Self { + Self::Auto + } +} + +impl TeePlatform { + pub fn resolve(self) -> Self { + match self { + Self::Auto => Self::resolve_from_cpuinfo( + &fs_err::read_to_string("/proc/cpuinfo").unwrap_or_default(), + ), + platform => platform, + } + } + + pub fn resolve_from_cpuinfo(cpuinfo: &str) -> Self { + if cpuinfo + .split_whitespace() + .any(|flag| flag.eq_ignore_ascii_case("sev_snp")) + { + Self::AmdSevSnp + } else { + Self::Tdx + } + } +} + #[derive(Debug, Clone, Deserialize, Serialize)] pub struct PortRange { pub protocol: Protocol, @@ -143,6 +179,9 @@ impl PortMappingConfig { #[derive(Debug, Clone, Deserialize)] pub struct CvmConfig { + /// TEE platform to use when launching CVMs. + #[serde(default)] + pub platform: TeePlatform, pub qemu_path: PathBuf, /// The URL of the KMS server pub kms_urls: Vec, @@ -605,4 +644,25 @@ mod tests { let result = parse_qemu_version_from_output(output); assert!(result.is_err()); } + + #[test] + fn tee_platform_deserializes_amd_sev_snp() { + let platform: TeePlatform = serde_json::from_str("\"amd-sev-snp\"").unwrap(); + assert_eq!(platform, TeePlatform::AmdSevSnp); + } + + #[test] + fn tee_platform_auto_resolves_amd_from_cpu_flags() { + let cpuinfo = "flags : fpu svm sev sev_es sev_snp debug_swap"; + assert_eq!( + TeePlatform::resolve_from_cpuinfo(cpuinfo), + TeePlatform::AmdSevSnp + ); + } + + #[test] + fn tee_platform_auto_falls_back_to_tdx_without_amd_snp_flag() { + let cpuinfo = "flags : fpu vmx tdx_guest"; + assert_eq!(TeePlatform::resolve_from_cpuinfo(cpuinfo), TeePlatform::Tdx); + } } diff --git a/vmm/vmm.toml b/vmm/vmm.toml index ba8e2a88a..73d8c124a 100644 --- a/vmm/vmm.toml +++ b/vmm/vmm.toml @@ -21,6 +21,8 @@ node_name = "" registry = "" [cvm] +# TEE platform: "auto", "tdx", or "amd-sev-snp". Auto selects AMD SEV-SNP when host CPU flags include sev_snp, otherwise TDX. +platform = "auto" qemu_path = "" kms_urls = ["http://127.0.0.1:8081"] gateway_urls = ["http://127.0.0.1:8082"] From 87c78d797990a7be0f319e4bef6929b1dcbc3ba9 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Mon, 25 May 2026 21:51:30 -0700 Subject: [PATCH 02/67] fix: harden sev-snp report data binding --- dstack-attest/src/attestation.rs | 2 ++ dstack-attest/src/sev_snp.rs | 42 ++++++++++++++++++++++++++++++-- dstack-attest/src/v1.rs | 36 ++++++++++++++++++++++++++- 3 files changed, 77 insertions(+), 3 deletions(-) diff --git a/dstack-attest/src/attestation.rs b/dstack-attest/src/attestation.rs index f9b581c20..93e5913bb 100644 --- a/dstack-attest/src/attestation.rs +++ b/dstack-attest/src/attestation.rs @@ -33,6 +33,8 @@ pub use tpm_types::TpmQuote; pub use crate::v1::{Attestation as AttestationV1, PlatformEvidence, StackEvidence}; +pub const SNP_REPORT_DATA_RANGE: std::ops::Range = 0x50..0x90; + const DSTACK_TDX: &str = "dstack-tdx"; const DSTACK_AMD_SEV_SNP: &str = "dstack-amd-sev-snp"; const DSTACK_GCP_TDX: &str = "dstack-gcp-tdx"; diff --git a/dstack-attest/src/sev_snp.rs b/dstack-attest/src/sev_snp.rs index 441367adf..e485503ee 100644 --- a/dstack-attest/src/sev_snp.rs +++ b/dstack-attest/src/sev_snp.rs @@ -8,7 +8,7 @@ use std::{fs::OpenOptions, io, path::Path}; use anyhow::{bail, Context, Result}; -use crate::attestation::SnpQuote; +use crate::attestation::{SnpQuote, SNP_REPORT_DATA_RANGE}; const TSM_REPORT_ROOT: &str = "/sys/kernel/config/tsm/report"; const SEV_GUEST_DEVICE: &str = "/dev/sev-guest"; @@ -72,6 +72,7 @@ fn get_report_configfs(report_data: [u8; 64]) -> Result { if report.is_empty() { bail!("sev-snp configfs tsm returned an empty report"); } + ensure_report_data_matches(&report, &report_data)?; Ok(SnpQuote { report, cert_chain: read_cert_chain_configfs(&dir), @@ -138,10 +139,47 @@ fn get_report_ioctl(report_data: [u8; 64]) -> Result { if rc < 0 { return Err(io::Error::last_os_error()).context("sev-snp get report ioctl failed"); } + let report = resp.data[..SNP_REPORT_SIZE].to_vec(); + ensure_report_data_matches(&report, &report_data)?; Ok(SnpQuote { - report: resp.data[..SNP_REPORT_SIZE].to_vec(), + report, cert_chain: Vec::new(), }) } +fn ensure_report_data_matches(report: &[u8], report_data: &[u8; 64]) -> Result<()> { + if report.len() < SNP_REPORT_DATA_RANGE.end { + bail!( + "sev-snp report too short: expected at least {} bytes, got {}", + SNP_REPORT_DATA_RANGE.end, + report.len() + ); + } + if &report[SNP_REPORT_DATA_RANGE] != report_data { + bail!("sev-snp report_data mismatch"); + } + Ok(()) +} + use std::os::fd::AsRawFd; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_report_with_wrong_report_data() { + let expected = [0x42; 64]; + let mut report = vec![0u8; SNP_REPORT_SIZE]; + report[SNP_REPORT_DATA_RANGE].copy_from_slice(&[0x24; 64]); + assert!(ensure_report_data_matches(&report, &expected).is_err()); + } + + #[test] + fn accepts_report_with_matching_report_data() { + let expected = [0x42; 64]; + let mut report = vec![0u8; SNP_REPORT_SIZE]; + report[SNP_REPORT_DATA_RANGE].copy_from_slice(&expected); + ensure_report_data_matches(&report, &expected).unwrap(); + } +} diff --git a/dstack-attest/src/v1.rs b/dstack-attest/src/v1.rs index 62d54cc81..0d2561852 100644 --- a/dstack-attest/src/v1.rs +++ b/dstack-attest/src/v1.rs @@ -245,7 +245,7 @@ impl Attestation { /// Return a new attestation with the report_data patched in both platform quote and stack. pub fn with_report_data(self, report_data: [u8; 64]) -> Self { - use crate::attestation::TDX_QUOTE_REPORT_DATA_RANGE; + use crate::attestation::{SNP_REPORT_DATA_RANGE, TDX_QUOTE_REPORT_DATA_RANGE}; let platform = match self.platform { PlatformEvidence::Tdx { @@ -257,6 +257,15 @@ impl Attestation { } PlatformEvidence::Tdx { quote, event_log } } + PlatformEvidence::SevSnp { + mut report, + cert_chain, + } => { + if report.len() >= SNP_REPORT_DATA_RANGE.end { + report[SNP_REPORT_DATA_RANGE].copy_from_slice(&report_data); + } + PlatformEvidence::SevSnp { report, cert_chain } + } other => other, }; let stack = match self.stack { @@ -373,4 +382,29 @@ mod tests { Some(vec![vec![0x22, 0x33]].as_slice()) ); } + + #[test] + fn sev_snp_with_report_data_patches_report_and_stack() { + let mut report = vec![0x11; 1184]; + report[crate::attestation::SNP_REPORT_DATA_RANGE].copy_from_slice(&[0x22; 64]); + let attestation = Attestation::new( + PlatformEvidence::SevSnp { + report, + cert_chain: vec![], + }, + StackEvidence::Dstack { + report_data: vec![0x22; 64], + runtime_events: vec![], + config: "{}".into(), + }, + ); + + let patched = attestation.with_report_data([0x33; 64]); + assert_eq!(patched.report_data().unwrap(), [0x33; 64]); + let report = patched.platform.sev_snp_report().unwrap(); + assert_eq!( + &report[crate::attestation::SNP_REPORT_DATA_RANGE], + &[0x33; 64] + ); + } } From 0701e6332ef4ea2996c4c75cef34a90bce2bb732 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Tue, 26 May 2026 09:11:50 -0700 Subject: [PATCH 03/67] fix: address sev-snp draft review findings --- dstack-attest/src/attestation.rs | 100 ++++++++++++++++++++++++++----- dstack-attest/src/sev_snp.rs | 82 ++++++++++++++++++++++++- vmm/src/app/qemu.rs | 29 ++++++--- vmm/src/config.rs | 21 +++---- 4 files changed, 195 insertions(+), 37 deletions(-) diff --git a/dstack-attest/src/attestation.rs b/dstack-attest/src/attestation.rs index 93e5913bb..e61a094a6 100644 --- a/dstack-attest/src/attestation.rs +++ b/dstack-attest/src/attestation.rs @@ -178,36 +178,49 @@ pub enum AttestationMode { #[default] #[serde(rename = "dstack-tdx")] DstackTdx, - /// AMD SEV-SNP report generated by the dstack attestation SDK - #[serde(rename = "dstack-amd-sev-snp")] - DstackAmdSevSnp, /// GCP TDX with DCAP quote only #[serde(rename = "dstack-gcp-tdx")] DstackGcpTdx, /// Dstack attestation SDK in AWS Nitro Enclave #[serde(rename = "dstack-nitro-enclave")] DstackNitroEnclave, + /// AMD SEV-SNP report generated by the dstack attestation SDK. + /// Keep this last to preserve SCALE discriminants for existing variants. + #[serde(rename = "dstack-amd-sev-snp")] + DstackAmdSevSnp, +} + +#[cfg(feature = "quote")] +fn has_sev_snp_tsm_provider() -> bool { + crate::sev_snp::has_sev_snp_tsm_provider(std::path::Path::new("/sys/kernel/config/tsm/report")) +} + +#[cfg(not(feature = "quote"))] +fn has_sev_snp_tsm_provider() -> bool { + false +} + +fn choose_dstack_attestation_mode(has_tdx: bool, has_sev_snp: bool) -> Result { + if has_tdx { + return Ok(AttestationMode::DstackTdx); + } + if has_sev_snp { + return Ok(AttestationMode::DstackAmdSevSnp); + } + bail!("Unsupported platform: Dstack(-tdx/-amd-sev-snp)"); } impl AttestationMode { /// Detect attestation mode from system pub fn detect() -> Result { let has_tdx = std::path::Path::new("/dev/tdx_guest").exists(); - let has_sev_snp = std::path::Path::new("/dev/sev-guest").exists() - || std::path::Path::new("/sys/kernel/config/tsm/report").exists(); + let has_sev_snp = + std::path::Path::new("/dev/sev-guest").exists() || has_sev_snp_tsm_provider(); // First, try to detect platform from DMI product name let platform = Platform::detect_or_dstack(); match platform { - Platform::Dstack => { - if has_sev_snp { - return Ok(Self::DstackAmdSevSnp); - } - if has_tdx { - return Ok(Self::DstackTdx); - } - bail!("Unsupported platform: Dstack(-tdx/-amd-sev-snp)"); - } + Platform::Dstack => return choose_dstack_attestation_mode(has_tdx, has_sev_snp), Platform::Gcp => { // GCP platform: TDX + TPM dual mode if has_tdx { @@ -864,6 +877,7 @@ pub enum AttestationQuote { DstackTdx(TdxQuote), DstackGcpTdx(DstackGcpTdxQuote), DstackNitroEnclave(DstackNitroQuote), + /// Keep this last to preserve SCALE discriminants for existing variants. DstackAmdSevSnp(SnpQuote), } @@ -878,6 +892,64 @@ impl AttestationQuote { } } +#[cfg(test)] +mod compatibility_tests { + use super::*; + use scale::Encode; + + #[test] + fn attestation_mode_scale_discriminants_preserve_existing_wire_values() { + assert_eq!(AttestationMode::DstackTdx.encode(), vec![0]); + assert_eq!(AttestationMode::DstackGcpTdx.encode(), vec![1]); + assert_eq!(AttestationMode::DstackNitroEnclave.encode(), vec![2]); + assert_eq!(AttestationMode::DstackAmdSevSnp.encode(), vec![3]); + } + + #[test] + fn attestation_quote_scale_discriminants_preserve_existing_wire_values() { + let gcp = AttestationQuote::DstackGcpTdx(DstackGcpTdxQuote { + tdx_quote: TdxQuote { + quote: Vec::new(), + event_log: Vec::new(), + }, + tpm_quote: TpmQuote { + message: Vec::new(), + signature: Vec::new(), + pcr_values: Vec::new(), + ak_cert: Vec::new(), + platform: dstack_types::Platform::Gcp, + event_log: Vec::new(), + }, + }); + assert_eq!(gcp.encode()[0], 1); + let nitro = AttestationQuote::DstackNitroEnclave(DstackNitroQuote { + nsm_quote: Vec::new(), + }); + assert_eq!(nitro.encode()[0], 2); + let quote = AttestationQuote::DstackAmdSevSnp(SnpQuote { + report: Vec::new(), + cert_chain: Vec::new(), + }); + assert_eq!(quote.encode()[0], 3); + } + + #[test] + fn dstack_attestation_mode_prefers_tdx_when_both_tdx_and_tsm_exist() { + assert_eq!( + choose_dstack_attestation_mode(true, true).unwrap(), + AttestationMode::DstackTdx + ); + } + + #[test] + fn dstack_attestation_mode_uses_snp_when_only_snp_exists() { + assert_eq!( + choose_dstack_attestation_mode(false, true).unwrap(), + AttestationMode::DstackAmdSevSnp + ); + } +} + /// Attestation data #[derive(Clone, Encode, Decode)] pub struct Attestation { diff --git a/dstack-attest/src/sev_snp.rs b/dstack-attest/src/sev_snp.rs index e485503ee..c9b8cefa3 100644 --- a/dstack-attest/src/sev_snp.rs +++ b/dstack-attest/src/sev_snp.rs @@ -38,7 +38,7 @@ struct SnpGuestRequestIoctl { } pub fn get_report(report_data: [u8; 64]) -> Result { - if Path::new(TSM_REPORT_ROOT).exists() { + if has_sev_snp_tsm_provider(Path::new(TSM_REPORT_ROOT)) { match get_report_configfs(report_data) { Ok(quote) => return Ok(quote), Err(err) => tracing::debug!("failed to get sev-snp report from configfs tsm: {err:#}"), @@ -50,6 +50,41 @@ pub fn get_report(report_data: [u8; 64]) -> Result { bail!("sev-snp report is unavailable: neither {TSM_REPORT_ROOT} nor {SEV_GUEST_DEVICE} exists") } +pub(crate) fn has_sev_snp_tsm_provider(root: &Path) -> bool { + if !root.exists() { + return false; + } + + if provider_file_is_sev_guest(&root.join("provider")) { + return true; + } + + let probe = root.join(format!("dstack-probe-{}", std::process::id())); + if fs_err::create_dir(&probe).is_ok() { + let is_sev_snp = provider_file_is_sev_guest(&probe.join("provider")); + let _ = fs_err::remove_dir(&probe); + if is_sev_snp { + return true; + } + } + + let Ok(entries) = fs_err::read_dir(root) else { + return false; + }; + entries.flatten().any(|entry| { + let Ok(file_type) = entry.file_type() else { + return false; + }; + file_type.is_dir() && provider_file_is_sev_guest(&entry.path().join("provider")) + }) +} + +fn provider_file_is_sev_guest(path: &Path) -> bool { + fs_err::read_to_string(path) + .map(|provider| matches!(provider.trim(), "sev_guest" | "sev-guest")) + .unwrap_or(false) +} + fn get_report_configfs(report_data: [u8; 64]) -> Result { let root = Path::new(TSM_REPORT_ROOT); let dir = root.join(format!("dstack-{}", std::process::id())); @@ -166,6 +201,7 @@ use std::os::fd::AsRawFd; #[cfg(test)] mod tests { use super::*; + use std::time::{SystemTime, UNIX_EPOCH}; #[test] fn rejects_report_with_wrong_report_data() { @@ -182,4 +218,48 @@ mod tests { report[SNP_REPORT_DATA_RANGE].copy_from_slice(&expected); ensure_report_data_matches(&report, &expected).unwrap(); } + + #[test] + fn tsm_provider_detection_accepts_only_sev_guest_provider() { + let root = test_dir("sev-guest"); + fs_err::create_dir_all(root.join("entry")).unwrap(); + fs_err::write(root.join("entry/provider"), "sev_guest\n").unwrap(); + + assert!(has_sev_snp_tsm_provider(&root)); + + let _ = fs_err::remove_dir_all(root); + } + + #[test] + fn tsm_provider_detection_accepts_legacy_hyphenated_sev_guest_provider() { + let root = test_dir("sev-guest-hyphen"); + fs_err::create_dir_all(root.join("entry")).unwrap(); + fs_err::write(root.join("entry/provider"), "sev-guest\n").unwrap(); + + assert!(has_sev_snp_tsm_provider(&root)); + + let _ = fs_err::remove_dir_all(root); + } + + #[test] + fn tsm_provider_detection_rejects_tdx_guest_provider() { + let root = test_dir("tdx-guest"); + fs_err::create_dir_all(root.join("entry")).unwrap(); + fs_err::write(root.join("entry/provider"), "tdx-guest\n").unwrap(); + + assert!(!has_sev_snp_tsm_provider(&root)); + + let _ = fs_err::remove_dir_all(root); + } + + fn test_dir(name: &str) -> std::path::PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + std::env::temp_dir().join(format!( + "dstack-sev-snp-test-{name}-{}-{nanos}", + std::process::id() + )) + } } diff --git a/vmm/src/app/qemu.rs b/vmm/src/app/qemu.rs index 0b4c2b77f..876e14ab2 100644 --- a/vmm/src/app/qemu.rs +++ b/vmm/src/app/qemu.rs @@ -369,7 +369,7 @@ impl VmState { #[cfg(test)] mod tests { - use super::sanitize_optional; + use super::{amd_sev_snp_memory_backend_arg, sanitize_optional}; #[test] fn sanitize_optional_filters_empty_owned_values() { @@ -390,6 +390,14 @@ mod tests { Some("instance-123") ); } + + #[test] + fn amd_sev_snp_memory_backend_arg_uses_passed_final_memory_size() { + assert_eq!( + amd_sev_snp_memory_backend_arg(4096), + "memory-backend-memfd,id=ram1,size=4096M,share=true,prealloc=false" + ); + } } impl VmConfig { @@ -542,7 +550,6 @@ impl VmConfig { command.arg("-netdev").arg(netdev); command.arg("-device").arg(net_device); - self.configure_machine(&mut command, &workdir, cfg, &app_compose)?; self.configure_smbios(&mut command, cfg); if matches!(app_compose.key_provider(), KeyProviderKind::Tpm) { @@ -665,6 +672,8 @@ impl VmConfig { } } + self.configure_machine(&mut command, &workdir, cfg, &app_compose, mem)?; + // Configure GPU devices if !gpus.gpus.is_empty() { // Add iommufd object @@ -790,6 +799,7 @@ impl VmConfig { workdir: &VmWorkDir, cfg: &CvmConfig, app_compose: &AppCompose, + mem: u32, ) -> Result<()> { if self.manifest.no_tee { command @@ -806,7 +816,7 @@ impl VmConfig { self.configure_tdx_guest(command, workdir, cfg, app_compose)?; } TeePlatform::AmdSevSnp => { - self.configure_amd_sev_snp_guest(command, cfg); + self.configure_amd_sev_snp_guest(command, cfg, mem); } } Ok(()) @@ -895,11 +905,10 @@ impl VmConfig { Ok(()) } - fn configure_amd_sev_snp_guest(&self, command: &mut Command, cfg: &CvmConfig) { - command.arg("-object").arg(format!( - "memory-backend-memfd,id=ram1,size={}M,share=true,prealloc=false", - self.manifest.memory - )); + fn configure_amd_sev_snp_guest(&self, command: &mut Command, cfg: &CvmConfig, mem: u32) { + command + .arg("-object") + .arg(amd_sev_snp_memory_backend_arg(mem)); command .arg("-object") .arg("sev-snp-guest,id=sev0,policy=0x30000,sev-device=/dev/sev,kernel-hashes=on,cbitpos=51,reduced-phys-bits=1"); @@ -956,6 +965,10 @@ impl VmConfig { } } +fn amd_sev_snp_memory_backend_arg(mem: u32) -> String { + format!("memory-backend-memfd,id=ram1,size={mem}M,share=true,prealloc=false") +} + /// Round up a value to the nearest multiple of another value. /// If the value is already a multiple, it remains unchanged. fn round_up(value: u32, multiple: u32) -> u32 { diff --git a/vmm/src/config.rs b/vmm/src/config.rs index 5e43e4479..56d899cd0 100644 --- a/vmm/src/config.rs +++ b/vmm/src/config.rs @@ -130,15 +130,11 @@ impl TeePlatform { } } - pub fn resolve_from_cpuinfo(cpuinfo: &str) -> Self { - if cpuinfo - .split_whitespace() - .any(|flag| flag.eq_ignore_ascii_case("sev_snp")) - { - Self::AmdSevSnp - } else { - Self::Tdx - } + pub fn resolve_from_cpuinfo(_cpuinfo: &str) -> Self { + // Keep `auto` conservative while AMD SEV-SNP support is experimental and + // verifier/KMS/app binding are not production-ready. Operators must opt + // into SNP explicitly with `platform = "amd-sev-snp"`. + Self::Tdx } } @@ -652,12 +648,9 @@ mod tests { } #[test] - fn tee_platform_auto_resolves_amd_from_cpu_flags() { + fn tee_platform_auto_stays_tdx_even_with_amd_snp_flag_while_experimental() { let cpuinfo = "flags : fpu svm sev sev_es sev_snp debug_swap"; - assert_eq!( - TeePlatform::resolve_from_cpuinfo(cpuinfo), - TeePlatform::AmdSevSnp - ); + assert_eq!(TeePlatform::resolve_from_cpuinfo(cpuinfo), TeePlatform::Tdx); } #[test] From ce4aa6de3f491bd73dfaec319ece122b724294ee Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Tue, 26 May 2026 12:28:48 -0700 Subject: [PATCH 04/67] feat: add sev-snp verifier core --- Cargo.lock | 129 +++++++++++++++++++++++++- Cargo.toml | 1 + dstack-attest/Cargo.toml | 2 + dstack-attest/src/amd_sev_snp.rs | 151 +++++++++++++++++++++++++++++++ dstack-attest/src/attestation.rs | 23 ++++- dstack-attest/src/lib.rs | 1 + 6 files changed, 298 insertions(+), 9 deletions(-) create mode 100644 dstack-attest/src/amd_sev_snp.rs diff --git a/Cargo.lock b/Cargo.lock index cb7f87263..8d98bc018 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1070,6 +1070,12 @@ dependencies = [ "hex-conservative", ] +[[package]] +name = "bitfield" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c821a6e124197eb56d907ccc2188eab1038fb919c914f47976e64dd8dbc855d1" + [[package]] name = "bitflags" version = "1.3.2" @@ -1572,6 +1578,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "codicon" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12170080f3533d6f09a19f81596f836854d0fa4867dc32c8172b8474b4e9de61" + [[package]] name = "colorchoice" version = "1.0.5" @@ -2211,13 +2223,34 @@ dependencies = [ "crypto-common 0.2.2", ] +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys 0.4.1", +] + [[package]] name = "dirs" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" dependencies = [ - "dirs-sys", + "dirs-sys 0.5.0", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users 0.4.6", + "windows-sys 0.48.0", ] [[package]] @@ -2228,7 +2261,7 @@ checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" dependencies = [ "libc", "option-ext", - "redox_users", + "redox_users 0.5.2", "windows-sys 0.61.2", ] @@ -2291,6 +2324,7 @@ name = "dstack-attest" version = "0.5.11" dependencies = [ "anyhow", + "base64 0.22.1", "cc-eventlog", "dcap-qvl", "dstack-types", @@ -2310,6 +2344,7 @@ dependencies = [ "serde", "serde-human-bytes", "serde_json", + "sev", "sha2 0.10.9", "sha3 0.10.9", "tdx-attest", @@ -2721,7 +2756,7 @@ dependencies = [ "base64 0.22.1", "bon", "clap", - "dirs", + "dirs 6.0.0", "dstack-kms-rpc", "dstack-port-forward", "dstack-types", @@ -4204,6 +4239,12 @@ dependencies = [ "memoffset 0.9.1", ] +[[package]] +name = "iocuddle" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8972d5be69940353d5347a1344cb375d9b457d6809b428b05bb1ca2fb9ce007" + [[package]] name = "iohash" version = "0.5.11" @@ -4569,7 +4610,7 @@ dependencies = [ "rust-argon2", "secrecy", "serde", - "serde-big-array", + "serde-big-array 0.3.3", "serde_json", "sha2 0.9.9", "thiserror 1.0.69", @@ -6148,6 +6189,17 @@ dependencies = [ "bitflags 2.11.1", ] +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 1.0.69", +] + [[package]] name = "redox_users" version = "0.5.2" @@ -7082,6 +7134,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + [[package]] name = "serde-duration" version = "0.5.11" @@ -7250,6 +7311,34 @@ dependencies = [ "serde", ] +[[package]] +name = "sev" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20ac277517d8fffdf3c41096323ed705b3a7c75e397129c072fb448339839d0f" +dependencies = [ + "base64 0.22.1", + "bincode 1.3.3", + "bitfield", + "bitflags 1.3.2", + "byteorder", + "codicon", + "dirs 5.0.1", + "hex", + "iocuddle", + "lazy_static", + "libc", + "p384", + "rsa", + "serde", + "serde-big-array 0.5.1", + "serde_bytes", + "sha2 0.10.9", + "static_assertions", + "uuid", + "x509-cert", +] + [[package]] name = "sha1" version = "0.10.6" @@ -7913,6 +8002,27 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tls_codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de2e01245e2bb89d6f05801c564fa27624dbd7b1846859876c7dad82e90bf6b" +dependencies = [ + "tls_codec_derive", + "zeroize", +] + +[[package]] +name = "tls_codec_derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "tokio" version = "1.52.3" @@ -8387,6 +8497,7 @@ checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" dependencies = [ "getrandom 0.4.2", "js-sys", + "serde_core", "wasm-bindgen", ] @@ -8852,6 +8963,15 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -9237,6 +9357,7 @@ dependencies = [ "const-oid", "der", "spki", + "tls_codec", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1677b9a71..e738356e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -135,6 +135,7 @@ hex_fmt = "0.3.0" hex-literal = "1.0.0" prost = "0.13.5" prost-types = "0.13.5" +sev = { version = "=6.0.0", default-features = false, features = ["snp", "crypto_nossl"] } scale = { version = "3.7.4", package = "parity-scale-codec", features = [ "derive", ] } diff --git a/dstack-attest/Cargo.toml b/dstack-attest/Cargo.toml index 430c8cc00..e3d3fcfad 100644 --- a/dstack-attest/Cargo.toml +++ b/dstack-attest/Cargo.toml @@ -11,6 +11,7 @@ license.workspace = true [dependencies] anyhow.workspace = true +base64.workspace = true cc-eventlog.workspace = true rmp-serde.workspace = true dcap-qvl.workspace = true @@ -25,6 +26,7 @@ scale = { workspace = true, features = ["derive"] } serde.workspace = true serde-human-bytes.workspace = true serde_json.workspace = true +sev.workspace = true sha2.workspace = true sha3.workspace = true tdx-attest.workspace = true diff --git a/dstack-attest/src/amd_sev_snp.rs b/dstack-attest/src/amd_sev_snp.rs new file mode 100644 index 000000000..fa84ecf11 --- /dev/null +++ b/dstack-attest/src/amd_sev_snp.rs @@ -0,0 +1,151 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! AMD SEV-SNP attestation verification helpers. +//! +//! This module intentionally implements only the hardware report signature +//! verification slice. KMS/app authorization must still bind the verified +//! measurement to app/config identity before production key release. + +use anyhow::{bail, Context, Result}; +use base64::engine::general_purpose::STANDARD; +use base64::Engine as _; +use sev::certs::snp::{ca, Certificate, Chain, Verifiable}; +use sev::firmware::guest::AttestationReport; + +/// AMD Genoa ARK certificate (DER, base64-encoded). +/// Source: https://kdsintf.amd.com/vcek/v1/Genoa/cert_chain +const GENOA_ARK_DER_B64: &str = "MIIGYzCCBBKgAwIBAgIDAgAAMEYGCSqGSIb3DQEBCjA5oA8wDQYJYIZIAWUDBAICBQChHDAaBgkqhkiG9w0BAQgwDQYJYIZIAWUDBAICBQCiAwIBMKMDAgEBMHsxFDASBgNVBAsMC0VuZ2luZWVyaW5nMQswCQYDVQQGEwJVUzEUMBIGA1UEBwwLU2FudGEgQ2xhcmExCzAJBgNVBAgMAkNBMR8wHQYDVQQKDBZBZHZhbmNlZCBNaWNybyBEZXZpY2VzMRIwEAYDVQQDDAlBUkstR2Vub2EwHhcNMjIwMTI2MTUzNDM3WhcNNDcwMTI2MTUzNDM3WjB7MRQwEgYDVQQLDAtFbmdpbmVlcmluZzELMAkGA1UEBhMCVVMxFDASBgNVBAcMC1NhbnRhIENsYXJhMQswCQYDVQQIDAJDQTEfMB0GA1UECgwWQWR2YW5jZWQgTWljcm8gRGV2aWNlczESMBAGA1UEAwwJQVJLLUdlbm9hMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA3Cd95S/uFOuRIskW9vz9VDBF69NDQF79oRhL/L2PVQGhK3YdfEBgpF/JiwWFBsT/fXDhzA01p3LkcT/7LdjcRfKXjHl+0Qq/M4dZkh6QDoUeKzNBLDcBKDDGWo3v35NyrxbA1DnkYwUKU5AAk4P94tKXLp80oxt84ahyHoLmc/LqsGsp+oq1Bz4PPsYLwTG4iMKVaaT90/oZ4I8oibSru92vJhlqWO27d/Rxc3iUMyhNeGToOvgx/iUo4gGpG61NDpkEUvIzuKcaMx8IdTpWg2DF6SwF0IgVMffnvtJmA68BwJNWo1E4PLJdaPfBifcJpuBFwNVQIPQEVX3aP89HJSp8YbY9lySS6PlVEqTBBtaQmi4ATGmMR+n2K/e+JAhU2Gj7jIpJhOkdH9firQDnmlA2SFfJ/Cc0mGNzW9RmIhyOUnNFoclmkRhl3/AQU5Ys9Qsan1jT/EiyT+pCpmnA+y9edvhDCbOG8F2oxHGRdTBkylungrkXJGYiwGrR8kaiqv7NN8QhOBMqYjcbrkEr0f8QMKklIS5ruOfqlLMCBw8JLB3LkjpWgtD7OpxkzSsohN47Uom86RY6lp72g8eXHP1qYrnvhzaG1S70vw6OkbaaC9EjiH/uHgAJQGxon7u0Q7xgoREWA/e7JcBQwLg80Hq/sbRuqesxz7wBWSY254cCAwEAAaN+MHwwDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBSfXfn+DdjzWtAzGiXvgSlPvjGoWzAPBgNVHRMBAf8EBTADAQH/MDoGA1UdHwQzMDEwL6AtoCuGKWh0dHBzOi8va2RzaW50Zi5hbWQuY29tL3ZjZWsvdjEvR2Vub2EvY3JsMEYGCSqGSIb3DQEBCjA5oA8wDQYJYIZIAWUDBAICBQChHDAaBgkqhkiG9w0BAQgwDQYJYIZIAWUDBAICBQCiAwIBMKMDAgEBA4ICAQAdIlPBC7DQmvH7kjlOznFx3i21SzOPDs5L7SgFjMC9rR07292GQCA7Z7Ulq97JQaWeD2ofGGse5swj4OQfKfVv/zaJUFjvosZOnfZ63epu8MjWgBSXJg5QE/Al0zRsZsp53DBTdA+Uv/s33fexdenT1mpKYzhIg/cKtz4oMxq8JKWJ8Po1CXLzKcfrTphjlbkh8AVKMXeBd2SpM33B1YP4g1BOdk013kqb7bRHZ1iB2JHG5cMKKbwRCSAAGHLTzASgDcXr9Fp7Z3liDhGu/ci1opGmkp12QNiJuBbkTU+xDZHm5X8Jm99BX7NEpzlOwIVR8ClgBDyuBkBC2ljtr3ZSaUIYj2xuyWN95KFY49nWxcz90CFa3Hzmy4zMQmBe9dVyls5eL5p9bkXcgRMDTbgmVZiAf4afe8DLdmQcYcMFQbHhgVzMiyZHGJgcCrQmA7MkTwEIds1wx/HzMcwU4qqNBAoZV7oeIIPxdqFXfPqHqiRlEbRDfX1TG5NFVaeByX0GyH6jzYVuezETzruaky6fp2bl2bczxPE8HdS38ijiJmm9vl50RGUeOAXjSuInGR4bsRufeGPB9peTa9BcBOeTWzstqTUB/F/qaZCIZKr4X6TyfUuSDz/1JDAGl+lxdM0P9+lLaP9NahQjHCVf0zf1c1salVuGFk2w/wMz1R1BHg=="; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VerifiedAmdSnpReport { + pub measurement: [u8; 48], + pub report_data: [u8; 64], + pub chip_id: [u8; 64], +} + +pub struct AmdSnpAttestationInput<'a> { + pub report: &'a [u8], + pub ask_pem: &'a [u8], + pub vcek_pem: &'a [u8], +} + +pub fn verify_amd_snp_attestation( + input: &AmdSnpAttestationInput<'_>, +) -> Result { + if input.report.len() != 1184 { + bail!( + "invalid amd sev-snp report length: expected 1184 bytes, got {}", + input.report.len() + ); + } + let report = AttestationReport::from_bytes(input.report) + .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; + + let ark_der = STANDARD + .decode(GENOA_ARK_DER_B64) + .context("failed to decode amd genoa ark")?; + let ark = Certificate::from_der(&ark_der) + .map_err(|err| anyhow::anyhow!("failed to parse amd genoa ark: {err:?}"))?; + let ask = Certificate::from_pem(input.ask_pem) + .map_err(|err| anyhow::anyhow!("failed to parse amd ask certificate: {err:?}"))?; + let vcek = Certificate::from_pem(input.vcek_pem) + .map_err(|err| anyhow::anyhow!("failed to parse amd vcek certificate: {err:?}"))?; + + let chain = Chain { + ca: ca::Chain { ark, ask }, + vek: vcek.clone(), + }; + chain + .verify() + .map_err(|err| anyhow::anyhow!("amd cert chain verification failed: {err:?}"))?; + (&vcek, &report).verify().map_err(|err| { + anyhow::anyhow!("amd sev-snp report signature verification failed: {err:?}") + })?; + + let mut measurement = [0u8; 48]; + measurement.copy_from_slice( + report + .measurement + .as_ref() + .get(..48) + .context("amd sev-snp measurement too short")?, + ); + let mut report_data = [0u8; 64]; + report_data.copy_from_slice( + report + .report_data + .as_ref() + .get(..64) + .context("amd sev-snp report_data too short")?, + ); + let mut chip_id = [0u8; 64]; + chip_id.copy_from_slice( + report + .chip_id + .as_ref() + .get(..64) + .context("amd sev-snp chip_id too short")?, + ); + + Ok(VerifiedAmdSnpReport { + measurement, + report_data, + chip_id, + }) +} + +pub fn verify_amd_snp_evidence( + report: &[u8], + cert_chain: &[Vec], + expected_report_data: &[u8; 64], +) -> Result { + let (ask_pem, vcek_pem) = split_ask_vcek_pem_chain(cert_chain)?; + let verified = verify_amd_snp_attestation(&AmdSnpAttestationInput { + report, + ask_pem, + vcek_pem, + })?; + if &verified.report_data != expected_report_data { + bail!("amd sev-snp report_data mismatch"); + } + Ok(verified) +} + +fn split_ask_vcek_pem_chain(cert_chain: &[Vec]) -> Result<(&[u8], &[u8])> { + match cert_chain { + [ask_pem, vcek_pem] => Ok((ask_pem.as_slice(), vcek_pem.as_slice())), + _ => bail!("amd sev-snp cert_chain must contain exactly ASK and VCEK PEM certificates"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn missing_cert_chain_fails_closed() { + let report = vec![0u8; 1184]; + let expected_report_data = [0u8; 64]; + let err = verify_amd_snp_evidence(&report, &[], &expected_report_data).unwrap_err(); + assert!( + err.to_string() + .contains("cert_chain must contain exactly ASK and VCEK"), + "unexpected error: {err:#}" + ); + } + + #[test] + fn malformed_report_fails_closed_before_success() { + let cert_chain = vec![b"not ask".to_vec(), b"not vcek".to_vec()]; + let expected_report_data = [0u8; 64]; + let err = + verify_amd_snp_evidence(b"too short", &cert_chain, &expected_report_data).unwrap_err(); + assert!( + err.to_string() + .contains("invalid amd sev-snp report length"), + "unexpected error: {err:#}" + ); + } +} diff --git a/dstack-attest/src/attestation.rs b/dstack-attest/src/attestation.rs index e61a094a6..603be1813 100644 --- a/dstack-attest/src/attestation.rs +++ b/dstack-attest/src/attestation.rs @@ -366,6 +366,7 @@ pub struct NitroVerifiedReport { #[derive(Clone)] pub enum DstackVerifiedReport { DstackTdx(TdxVerifiedReport), + DstackAmdSevSnp(crate::amd_sev_snp::VerifiedAmdSnpReport), DstackGcpTdx { tdx_report: TdxVerifiedReport, tpm_report: TpmVerifiedReport, @@ -377,10 +378,20 @@ impl DstackVerifiedReport { pub fn tdx_report(&self) -> Option<&TdxVerifiedReport> { match self { DstackVerifiedReport::DstackTdx(report) => Some(report), + DstackVerifiedReport::DstackAmdSevSnp(_) => None, DstackVerifiedReport::DstackGcpTdx { tdx_report, .. } => Some(tdx_report), DstackVerifiedReport::DstackNitroEnclave(_) => None, } } + + pub fn amd_snp_report(&self) -> Option<&crate::amd_sev_snp::VerifiedAmdSnpReport> { + match self { + DstackVerifiedReport::DstackAmdSevSnp(report) => Some(report), + DstackVerifiedReport::DstackTdx(_) + | DstackVerifiedReport::DstackGcpTdx { .. } + | DstackVerifiedReport::DstackNitroEnclave(_) => None, + } + } } /// Represents a verified attestation @@ -758,11 +769,12 @@ impl AttestationV1 { timestamp: verified_report.timestamp, }) } - PlatformEvidence::SevSnp { .. } => { - bail!( - "Unsupported attestation mode: {:?}", - platform_attestation_mode(&platform) - ); + PlatformEvidence::SevSnp { report, cert_chain } => { + DstackVerifiedReport::DstackAmdSevSnp(crate::amd_sev_snp::verify_amd_snp_evidence( + report, + cert_chain, + &report_data, + )?) } }; @@ -1049,6 +1061,7 @@ impl GetDeviceId for DstackVerifiedReport { fn get_devide_id(&self) -> Vec { match self { DstackVerifiedReport::DstackTdx(tdx_report) => tdx_report.ppid.to_vec(), + DstackVerifiedReport::DstackAmdSevSnp(report) => report.chip_id.to_vec(), DstackVerifiedReport::DstackGcpTdx { tdx_report, .. } => tdx_report.ppid.to_vec(), DstackVerifiedReport::DstackNitroEnclave(report) => { // i-1234567890abcdef0-enc9876543210abcde -> i-1234567890abcdef0 diff --git a/dstack-attest/src/lib.rs b/dstack-attest/src/lib.rs index 6452b7c36..f89d7ece1 100644 --- a/dstack-attest/src/lib.rs +++ b/dstack-attest/src/lib.rs @@ -13,6 +13,7 @@ pub use tpm_attest as tpm; use crate::attestation::AttestationMode; +pub mod amd_sev_snp; pub mod attestation; #[cfg(feature = "quote")] mod sev_snp; From c0b442b38efd9c4a09dcc656595365d2ed7b0346 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Tue, 26 May 2026 12:53:11 -0700 Subject: [PATCH 05/67] fix: normalize sev-snp cert collateral --- dstack-attest/src/amd_sev_snp.rs | 280 ++++++++++++++++++++++++++++--- dstack-attest/src/sev_snp.rs | 103 +++++------- 2 files changed, 300 insertions(+), 83 deletions(-) diff --git a/dstack-attest/src/amd_sev_snp.rs b/dstack-attest/src/amd_sev_snp.rs index fa84ecf11..8c3d038a0 100644 --- a/dstack-attest/src/amd_sev_snp.rs +++ b/dstack-attest/src/amd_sev_snp.rs @@ -4,9 +4,11 @@ //! AMD SEV-SNP attestation verification helpers. //! -//! This module intentionally implements only the hardware report signature -//! verification slice. KMS/app authorization must still bind the verified -//! measurement to app/config identity before production key release. +//! This module implements the hardware report verification slice: certificate +//! normalization, AMD ARK/ASK/VCEK chain verification, report signature checks, +//! report_data binding, and invariant SNP policy checks. KMS/app authorization +//! must still bind the verified measurement to app/config identity before +//! production key release. use anyhow::{bail, Context, Result}; use base64::engine::general_purpose::STANDARD; @@ -18,6 +20,17 @@ use sev::firmware::guest::AttestationReport; /// Source: https://kdsintf.amd.com/vcek/v1/Genoa/cert_chain const GENOA_ARK_DER_B64: &str = "MIIGYzCCBBKgAwIBAgIDAgAAMEYGCSqGSIb3DQEBCjA5oA8wDQYJYIZIAWUDBAICBQChHDAaBgkqhkiG9w0BAQgwDQYJYIZIAWUDBAICBQCiAwIBMKMDAgEBMHsxFDASBgNVBAsMC0VuZ2luZWVyaW5nMQswCQYDVQQGEwJVUzEUMBIGA1UEBwwLU2FudGEgQ2xhcmExCzAJBgNVBAgMAkNBMR8wHQYDVQQKDBZBZHZhbmNlZCBNaWNybyBEZXZpY2VzMRIwEAYDVQQDDAlBUkstR2Vub2EwHhcNMjIwMTI2MTUzNDM3WhcNNDcwMTI2MTUzNDM3WjB7MRQwEgYDVQQLDAtFbmdpbmVlcmluZzELMAkGA1UEBhMCVVMxFDASBgNVBAcMC1NhbnRhIENsYXJhMQswCQYDVQQIDAJDQTEfMB0GA1UECgwWQWR2YW5jZWQgTWljcm8gRGV2aWNlczESMBAGA1UEAwwJQVJLLUdlbm9hMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA3Cd95S/uFOuRIskW9vz9VDBF69NDQF79oRhL/L2PVQGhK3YdfEBgpF/JiwWFBsT/fXDhzA01p3LkcT/7LdjcRfKXjHl+0Qq/M4dZkh6QDoUeKzNBLDcBKDDGWo3v35NyrxbA1DnkYwUKU5AAk4P94tKXLp80oxt84ahyHoLmc/LqsGsp+oq1Bz4PPsYLwTG4iMKVaaT90/oZ4I8oibSru92vJhlqWO27d/Rxc3iUMyhNeGToOvgx/iUo4gGpG61NDpkEUvIzuKcaMx8IdTpWg2DF6SwF0IgVMffnvtJmA68BwJNWo1E4PLJdaPfBifcJpuBFwNVQIPQEVX3aP89HJSp8YbY9lySS6PlVEqTBBtaQmi4ATGmMR+n2K/e+JAhU2Gj7jIpJhOkdH9firQDnmlA2SFfJ/Cc0mGNzW9RmIhyOUnNFoclmkRhl3/AQU5Ys9Qsan1jT/EiyT+pCpmnA+y9edvhDCbOG8F2oxHGRdTBkylungrkXJGYiwGrR8kaiqv7NN8QhOBMqYjcbrkEr0f8QMKklIS5ruOfqlLMCBw8JLB3LkjpWgtD7OpxkzSsohN47Uom86RY6lp72g8eXHP1qYrnvhzaG1S70vw6OkbaaC9EjiH/uHgAJQGxon7u0Q7xgoREWA/e7JcBQwLg80Hq/sbRuqesxz7wBWSY254cCAwEAAaN+MHwwDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBSfXfn+DdjzWtAzGiXvgSlPvjGoWzAPBgNVHRMBAf8EBTADAQH/MDoGA1UdHwQzMDEwL6AtoCuGKWh0dHBzOi8va2RzaW50Zi5hbWQuY29tL3ZjZWsvdjEvR2Vub2EvY3JsMEYGCSqGSIb3DQEBCjA5oA8wDQYJYIZIAWUDBAICBQChHDAaBgkqhkiG9w0BAQgwDQYJYIZIAWUDBAICBQCiAwIBMKMDAgEBA4ICAQAdIlPBC7DQmvH7kjlOznFx3i21SzOPDs5L7SgFjMC9rR07292GQCA7Z7Ulq97JQaWeD2ofGGse5swj4OQfKfVv/zaJUFjvosZOnfZ63epu8MjWgBSXJg5QE/Al0zRsZsp53DBTdA+Uv/s33fexdenT1mpKYzhIg/cKtz4oMxq8JKWJ8Po1CXLzKcfrTphjlbkh8AVKMXeBd2SpM33B1YP4g1BOdk013kqb7bRHZ1iB2JHG5cMKKbwRCSAAGHLTzASgDcXr9Fp7Z3liDhGu/ci1opGmkp12QNiJuBbkTU+xDZHm5X8Jm99BX7NEpzlOwIVR8ClgBDyuBkBC2ljtr3ZSaUIYj2xuyWN95KFY49nWxcz90CFa3Hzmy4zMQmBe9dVyls5eL5p9bkXcgRMDTbgmVZiAf4afe8DLdmQcYcMFQbHhgVzMiyZHGJgcCrQmA7MkTwEIds1wx/HzMcwU4qqNBAoZV7oeIIPxdqFXfPqHqiRlEbRDfX1TG5NFVaeByX0GyH6jzYVuezETzruaky6fp2bl2bczxPE8HdS38ijiJmm9vl50RGUeOAXjSuInGR4bsRufeGPB9peTa9BcBOeTWzstqTUB/F/qaZCIZKr4X6TyfUuSDz/1JDAGl+lxdM0P9+lLaP9NahQjHCVf0zf1c1salVuGFk2w/wMz1R1BHg=="; +const ASK_CERT_GUID: [u8; 16] = [ + 0x4a, 0xb7, 0xb3, 0x79, 0xbb, 0xac, 0x4f, 0xe4, 0xa0, 0x2f, 0x05, 0xae, 0xf3, 0x27, 0xc7, 0x82, +]; +const VCEK_CERT_GUID: [u8; 16] = [ + 0x63, 0xda, 0x75, 0x8d, 0xe6, 0x64, 0x45, 0x64, 0xad, 0xc5, 0xf4, 0xb9, 0x3b, 0xe8, 0xac, 0xcd, +]; +const VLEK_CERT_GUID: [u8; 16] = [ + 0xa8, 0x07, 0x4b, 0xc2, 0xa2, 0x5a, 0x48, 0x3e, 0xaa, 0xe6, 0x39, 0xc0, 0x45, 0xa0, 0xb8, 0xa1, +]; +const CERT_TABLE_ENTRY_SIZE: usize = 24; + #[derive(Debug, Clone, PartialEq, Eq)] pub struct VerifiedAmdSnpReport { pub measurement: [u8; 48], @@ -25,6 +38,18 @@ pub struct VerifiedAmdSnpReport { pub chip_id: [u8; 64], } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CertEncoding { + Pem, + Der, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct CertBytes { + bytes: Vec, + encoding: CertEncoding, +} + pub struct AmdSnpAttestationInput<'a> { pub report: &'a [u8], pub ask_pem: &'a [u8], @@ -34,13 +59,31 @@ pub struct AmdSnpAttestationInput<'a> { pub fn verify_amd_snp_attestation( input: &AmdSnpAttestationInput<'_>, ) -> Result { - if input.report.len() != 1184 { + verify_amd_snp_attestation_with_certs( + input.report, + CertBytes { + bytes: input.ask_pem.to_vec(), + encoding: CertEncoding::Pem, + }, + CertBytes { + bytes: input.vcek_pem.to_vec(), + encoding: CertEncoding::Pem, + }, + ) +} + +fn verify_amd_snp_attestation_with_certs( + report_bytes: &[u8], + ask_bytes: CertBytes, + vcek_bytes: CertBytes, +) -> Result { + if report_bytes.len() != 1184 { bail!( "invalid amd sev-snp report length: expected 1184 bytes, got {}", - input.report.len() + report_bytes.len() ); } - let report = AttestationReport::from_bytes(input.report) + let report = AttestationReport::from_bytes(report_bytes) .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; let ark_der = STANDARD @@ -48,10 +91,8 @@ pub fn verify_amd_snp_attestation( .context("failed to decode amd genoa ark")?; let ark = Certificate::from_der(&ark_der) .map_err(|err| anyhow::anyhow!("failed to parse amd genoa ark: {err:?}"))?; - let ask = Certificate::from_pem(input.ask_pem) - .map_err(|err| anyhow::anyhow!("failed to parse amd ask certificate: {err:?}"))?; - let vcek = Certificate::from_pem(input.vcek_pem) - .map_err(|err| anyhow::anyhow!("failed to parse amd vcek certificate: {err:?}"))?; + let ask = parse_certificate(&ask_bytes, "ask")?; + let vcek = parse_certificate(&vcek_bytes, "vcek")?; let chain = Chain { ca: ca::Chain { ark, ask }, @@ -63,6 +104,7 @@ pub fn verify_amd_snp_attestation( (&vcek, &report).verify().map_err(|err| { anyhow::anyhow!("amd sev-snp report signature verification failed: {err:?}") })?; + validate_amd_snp_report_policy(&report)?; let mut measurement = [0u8; 48]; measurement.copy_from_slice( @@ -101,23 +143,139 @@ pub fn verify_amd_snp_evidence( cert_chain: &[Vec], expected_report_data: &[u8; 64], ) -> Result { - let (ask_pem, vcek_pem) = split_ask_vcek_pem_chain(cert_chain)?; - let verified = verify_amd_snp_attestation(&AmdSnpAttestationInput { - report, - ask_pem, - vcek_pem, - })?; + let (ask, vcek) = normalize_ask_vcek_certs(cert_chain)?; + let verified = verify_amd_snp_attestation_with_certs(report, ask, vcek)?; if &verified.report_data != expected_report_data { bail!("amd sev-snp report_data mismatch"); } Ok(verified) } -fn split_ask_vcek_pem_chain(cert_chain: &[Vec]) -> Result<(&[u8], &[u8])> { +fn parse_certificate(cert: &CertBytes, name: &str) -> Result { + match cert.encoding { + CertEncoding::Pem => Certificate::from_pem(&cert.bytes) + .map_err(|err| anyhow::anyhow!("failed to parse amd {name} certificate: {err:?}")), + CertEncoding::Der => Certificate::from_der(&cert.bytes) + .map_err(|err| anyhow::anyhow!("failed to parse amd {name} certificate: {err:?}")), + } +} + +fn validate_amd_snp_report_policy(report: &AttestationReport) -> Result<()> { + if !matches!(report.version, 2 | 3) { + bail!("unsupported amd sev-snp report version: {}", report.version); + } + if report.vmpl != 0 { + bail!("amd sev-snp report must be generated at vmpl0"); + } + if report.policy.debug_allowed() { + bail!("amd sev-snp guest policy allows debug"); + } + if report.policy.migrate_ma_allowed() { + bail!("amd sev-snp guest policy allows migration agent"); + } + if report.key_info.mask_chip_key() { + bail!("amd sev-snp report masks the chip signing key"); + } + if report.key_info.signing_key() != 0 { + bail!( + "unsupported amd sev-snp signing key: expected vcek, got {}", + report.key_info.signing_key() + ); + } + if !report.policy.smt_allowed() && report.plat_info.smt_enabled() { + bail!("amd sev-snp platform has smt enabled but guest policy does not allow smt"); + } + if report.policy.rapl_dis() && !report.plat_info.rapl_disabled() { + bail!("amd sev-snp guest policy requires rapl disabled, but platform reports rapl enabled"); + } + if report.policy.ciphertext_hiding() && !report.plat_info.ciphertext_hiding_enabled() { + bail!( + "amd sev-snp guest policy requires ciphertext hiding, but platform does not report it" + ); + } + Ok(()) +} + +fn normalize_ask_vcek_certs(cert_chain: &[Vec]) -> Result<(CertBytes, CertBytes)> { match cert_chain { - [ask_pem, vcek_pem] => Ok((ask_pem.as_slice(), vcek_pem.as_slice())), - _ => bail!("amd sev-snp cert_chain must contain exactly ASK and VCEK PEM certificates"), + [ask, vcek] => Ok((cert_bytes_from_blob(ask), cert_bytes_from_blob(vcek))), + [auxblob] => normalize_kernel_cert_table(auxblob), + _ => bail!( + "amd sev-snp cert_chain must contain either ASK and VCEK certificates or one kernel certificate table auxblob" + ), + } +} + +fn cert_bytes_from_blob(blob: &[u8]) -> CertBytes { + let encoding = if blob.starts_with(b"-----BEGIN CERTIFICATE-----") { + CertEncoding::Pem + } else { + CertEncoding::Der + }; + CertBytes { + bytes: blob.to_vec(), + encoding, + } +} + +fn normalize_kernel_cert_table(auxblob: &[u8]) -> Result<(CertBytes, CertBytes)> { + let mut ask = None; + let mut vcek = None; + for (guid, data) in parse_kernel_cert_table(auxblob)? { + match guid { + ASK_CERT_GUID => ask = Some(data), + VCEK_CERT_GUID => vcek = Some(data), + VLEK_CERT_GUID => bail!("amd sev-snp vlek certificates are not supported yet"), + _ => {} + } } + let ask = ask.context("amd sev-snp certificate table missing ASK certificate")?; + let vcek = vcek.context("amd sev-snp certificate table missing VCEK certificate")?; + Ok(( + CertBytes { + bytes: ask, + encoding: CertEncoding::Der, + }, + CertBytes { + bytes: vcek, + encoding: CertEncoding::Der, + }, + )) +} + +fn parse_kernel_cert_table(auxblob: &[u8]) -> Result)>> { + if auxblob.len() < CERT_TABLE_ENTRY_SIZE { + bail!("amd sev-snp certificate table is too short"); + } + let mut entries = Vec::new(); + let mut pos = 0usize; + loop { + let entry = auxblob + .get(pos..pos + CERT_TABLE_ENTRY_SIZE) + .context("amd sev-snp certificate table is missing terminator")?; + let guid: [u8; 16] = entry[..16] + .try_into() + .context("amd sev-snp certificate table entry guid has invalid length")?; + let offset = u32::from_le_bytes(entry[16..20].try_into().unwrap()) as usize; + let length = u32::from_le_bytes(entry[20..24].try_into().unwrap()) as usize; + if guid == [0u8; 16] && offset == 0 && length == 0 { + break; + } + let end = offset + .checked_add(length) + .context("amd sev-snp certificate table entry length overflows")?; + if offset < CERT_TABLE_ENTRY_SIZE || end > auxblob.len() || length == 0 { + bail!("amd sev-snp certificate table entry has invalid bounds"); + } + entries.push((guid, auxblob[offset..end].to_vec())); + pos = pos + .checked_add(CERT_TABLE_ENTRY_SIZE) + .context("amd sev-snp certificate table entry count overflows")?; + if pos >= auxblob.len() { + bail!("amd sev-snp certificate table is missing terminator"); + } + } + Ok(entries) } #[cfg(test)] @@ -131,7 +289,7 @@ mod tests { let err = verify_amd_snp_evidence(&report, &[], &expected_report_data).unwrap_err(); assert!( err.to_string() - .contains("cert_chain must contain exactly ASK and VCEK"), + .contains("cert_chain must contain either ASK and VCEK certificates or one kernel certificate table auxblob"), "unexpected error: {err:#}" ); } @@ -148,4 +306,86 @@ mod tests { "unexpected error: {err:#}" ); } + + #[test] + fn normalizes_kernel_cert_table_auxblob_to_ask_and_vcek_der() { + use sev::firmware::host::{CertTableEntry, CertType}; + + let auxblob = CertTableEntry::cert_table_to_vec_bytes(&[ + CertTableEntry::new(CertType::VCEK, b"vcek-der".to_vec()), + CertTableEntry::new(CertType::ASK, b"ask-der".to_vec()), + ]) + .unwrap(); + + let (ask, vcek) = normalize_ask_vcek_certs(&[auxblob]).unwrap(); + + assert_eq!(ask.bytes, b"ask-der"); + assert_eq!(ask.encoding, CertEncoding::Der); + assert_eq!(vcek.bytes, b"vcek-der"); + assert_eq!(vcek.encoding, CertEncoding::Der); + } + + #[test] + fn malformed_single_auxblob_fails_closed_without_panic() { + let err = normalize_ask_vcek_certs(&[vec![0xff; 23]]).unwrap_err(); + + assert!( + err.to_string().contains("certificate table"), + "unexpected error: {err:#}" + ); + } + + #[test] + fn normalizes_existing_two_item_pem_chain_without_reordering() { + let ask = b"-----BEGIN CERTIFICATE-----\nask\n-----END CERTIFICATE-----\n".to_vec(); + let vcek = b"-----BEGIN CERTIFICATE-----\nvcek\n-----END CERTIFICATE-----\n".to_vec(); + + let (normalized_ask, normalized_vcek) = + normalize_ask_vcek_certs(&[ask.clone(), vcek.clone()]).unwrap(); + + assert_eq!(normalized_ask.bytes, ask); + assert_eq!(normalized_ask.encoding, CertEncoding::Pem); + assert_eq!(normalized_vcek.bytes, vcek); + assert_eq!(normalized_vcek.encoding, CertEncoding::Pem); + } + + #[test] + fn report_policy_rejects_debug_allowed() { + let mut report = base_report(); + report.policy.set_debug_allowed(true); + + let err = validate_amd_snp_report_policy(&report).unwrap_err(); + + assert!( + err.to_string().contains("debug"), + "unexpected error: {err:#}" + ); + } + + #[test] + fn report_policy_rejects_non_vmpl0() { + let mut report = base_report(); + report.vmpl = 1; + + let err = validate_amd_snp_report_policy(&report).unwrap_err(); + + assert!( + err.to_string().contains("vmpl0"), + "unexpected error: {err:#}" + ); + } + + #[test] + fn report_policy_accepts_strict_vcek_vmpl0_report() { + let report = base_report(); + + validate_amd_snp_report_policy(&report).unwrap(); + } + + fn base_report() -> AttestationReport { + AttestationReport { + version: 2, + ..Default::default() + } + } } diff --git a/dstack-attest/src/sev_snp.rs b/dstack-attest/src/sev_snp.rs index c9b8cefa3..4844585fc 100644 --- a/dstack-attest/src/sev_snp.rs +++ b/dstack-attest/src/sev_snp.rs @@ -4,38 +4,16 @@ //! Minimal AMD SEV-SNP guest report support. -use std::{fs::OpenOptions, io, path::Path}; +use std::path::Path; use anyhow::{bail, Context, Result}; +use sev::firmware::{guest::Firmware, host::CertTableEntry}; use crate::attestation::{SnpQuote, SNP_REPORT_DATA_RANGE}; const TSM_REPORT_ROOT: &str = "/sys/kernel/config/tsm/report"; const SEV_GUEST_DEVICE: &str = "/dev/sev-guest"; const SNP_REPORT_SIZE: usize = 1184; -const SNP_REPORT_RESP_SIZE: usize = 4000; -const SNP_GET_REPORT: libc::c_ulong = 0xc020_5300; - -#[repr(C)] -#[derive(Clone, Copy)] -struct SnpReportReq { - report_data: [u8; 64], - vmpl: u32, - rsvd: [u8; 28], -} - -#[repr(C)] -struct SnpReportResp { - data: [u8; SNP_REPORT_RESP_SIZE], -} - -#[repr(C)] -struct SnpGuestRequestIoctl { - msg_version: u8, - req_data: u64, - resp_data: u64, - fw_err: u64, -} pub fn get_report(report_data: [u8; 64]) -> Result { if has_sev_snp_tsm_provider(Path::new(TSM_REPORT_ROOT)) { @@ -142,51 +120,39 @@ fn read_first_existing(paths: &[std::path::PathBuf]) -> Result> { } fn read_cert_chain_configfs(dir: &Path) -> Vec> { - ["certs", "cert_chain", "auxblob"] - .iter() - .filter_map(|name| fs_err::read(dir.join(name)).ok()) - .filter(|bytes| !bytes.is_empty()) - .collect() + for name in ["certs", "cert_chain", "auxblob"] { + let Ok(bytes) = fs_err::read(dir.join(name)) else { + continue; + }; + if !bytes.is_empty() { + return vec![bytes]; + } + } + Vec::new() } fn get_report_ioctl(report_data: [u8; 64]) -> Result { - let file = OpenOptions::new() - .read(true) - .write(true) - .open(SEV_GUEST_DEVICE) - .with_context(|| format!("failed to open {SEV_GUEST_DEVICE}"))?; - let mut req = SnpReportReq { - report_data, - vmpl: 0, - rsvd: [0; 28], - }; - let mut resp = SnpReportResp { - data: [0; SNP_REPORT_RESP_SIZE], - }; - let mut ioctl_req = SnpGuestRequestIoctl { - msg_version: 1, - req_data: (&mut req as *mut SnpReportReq) as u64, - resp_data: (&mut resp as *mut SnpReportResp) as u64, - fw_err: 0, - }; - - let rc = unsafe { libc::ioctl(file.as_raw_fd(), SNP_GET_REPORT, &mut ioctl_req) }; - if rc < 0 { - return Err(io::Error::last_os_error()).context("sev-snp get report ioctl failed"); - } - let report = resp.data[..SNP_REPORT_SIZE].to_vec(); + let mut firmware = + Firmware::open().with_context(|| format!("failed to open {SEV_GUEST_DEVICE}"))?; + let (report, cert_entries) = firmware + .get_ext_report(Some(1), Some(report_data), Some(0)) + .map_err(|err| anyhow::anyhow!("sev-snp get extended report ioctl failed: {err}"))?; ensure_report_data_matches(&report, &report_data)?; - Ok(SnpQuote { - report, - cert_chain: Vec::new(), - }) + let cert_chain = match cert_entries { + Some(entries) if !entries.is_empty() => { + vec![CertTableEntry::cert_table_to_vec_bytes(&entries) + .context("failed to encode sev-snp certificate table")?] + } + _ => Vec::new(), + }; + Ok(SnpQuote { report, cert_chain }) } fn ensure_report_data_matches(report: &[u8], report_data: &[u8; 64]) -> Result<()> { - if report.len() < SNP_REPORT_DATA_RANGE.end { + if report.len() != SNP_REPORT_SIZE { bail!( - "sev-snp report too short: expected at least {} bytes, got {}", - SNP_REPORT_DATA_RANGE.end, + "sev-snp report has invalid length: expected {} bytes, got {}", + SNP_REPORT_SIZE, report.len() ); } @@ -196,8 +162,6 @@ fn ensure_report_data_matches(report: &[u8], report_data: &[u8; 64]) -> Result<( Ok(()) } -use std::os::fd::AsRawFd; - #[cfg(test)] mod tests { use super::*; @@ -252,6 +216,19 @@ mod tests { let _ = fs_err::remove_dir_all(root); } + #[test] + fn configfs_cert_chain_uses_first_supported_nonempty_blob() { + let root = test_dir("cert-chain"); + fs_err::create_dir_all(&root).unwrap(); + fs_err::write(root.join("certs"), []).unwrap(); + fs_err::write(root.join("cert_chain"), b"chain").unwrap(); + fs_err::write(root.join("auxblob"), b"auxblob").unwrap(); + + assert_eq!(read_cert_chain_configfs(&root), vec![b"chain".to_vec()]); + + let _ = fs_err::remove_dir_all(root); + } + fn test_dir(name: &str) -> std::path::PathBuf { let nanos = SystemTime::now() .duration_since(UNIX_EPOCH) From 1adfa2beb4784f5713c249e3958cdfcdfaf054da Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Tue, 26 May 2026 14:11:26 -0700 Subject: [PATCH 06/67] fix: add fail-closed sev-snp measurement binding --- kms/src/config.rs | 22 +++ kms/src/main_service.rs | 1 + kms/src/main_service/amd_attest.rs | 281 +++++++++++++++++++++++++++++ 3 files changed, 304 insertions(+) create mode 100644 kms/src/main_service/amd_attest.rs diff --git a/kms/src/config.rs b/kms/src/config.rs index ecdfb9aeb..096cfd369 100644 --- a/kms/src/config.rs +++ b/kms/src/config.rs @@ -31,6 +31,23 @@ pub(crate) struct ImageConfig { pub download_timeout: Duration, } +/// Configuration for AMD SEV-SNP measurement/app binding validation. +#[derive(Debug, Clone, Deserialize)] +pub(crate) struct SevSnpMeasureConfig { + /// Path to the AMD SEV-SNP OVMF binary used for this VM image. + /// + /// Optional when callers provide OVMF section metadata with the request. + pub ovmf_path: Option, + /// SNP guest features bitmask used at launch. Defaults to SNP with kernel + /// hashes enabled. + #[serde(default = "default_guest_features")] + pub guest_features: u64, +} + +fn default_guest_features() -> u64 { + 0x1 +} + #[derive(Debug, Clone, Deserialize)] pub(crate) struct KmsConfig { pub cert_dir: PathBuf, @@ -38,6 +55,11 @@ pub(crate) struct KmsConfig { pub auth_api: AuthApi, pub onboard: OnboardConfig, pub image: ImageConfig, + /// AMD SEV-SNP measurement verification configuration. Optional at config + /// load time for non-SNP/dev deployments; SNP binding helpers require it. + #[serde(default)] + #[allow(dead_code)] + pub sev_snp: Option, #[serde(with = "serde_human_bytes")] pub admin_token_hash: Vec, #[serde(default)] diff --git a/kms/src/main_service.rs b/kms/src/main_service.rs index 00723566b..acf25e33d 100644 --- a/kms/src/main_service.rs +++ b/kms/src/main_service.rs @@ -37,6 +37,7 @@ use crate::{ crypto::{derive_k256_key, sign_message, sign_message_with_timestamp}, }; +pub(crate) mod amd_attest; pub(crate) mod upgrade_authority; #[derive(Clone)] diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs new file mode 100644 index 000000000..659f84d4f --- /dev/null +++ b/kms/src/main_service/amd_attest.rs @@ -0,0 +1,281 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Fail-closed AMD SEV-SNP measurement/app binding validation. +//! +//! This module intentionally does not release keys and does not enable any AMD +//! KMS key-release endpoint. Until full in-KMS SNP measurement recomputation is +//! wired in, callers must provide the expected measurement from a trusted +//! recomputation path; this helper validates every required binding input and +//! compares that expected value to the hardware-verified report measurement. + +#![allow(dead_code)] + +use anyhow::{bail, Result}; + +use crate::config::SevSnpMeasureConfig; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct OvmfSectionParam { + pub gpa: u64, + pub size: u64, + /// Raw OVMF SEV metadata section type: + /// 1=SNP_SEC_MEMORY, 2=SNP_SECRETS, 3=CPUID, 4=SVSM_CAA, + /// 0x10=SNP_KERNEL_HASHES. + pub section_type: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct MeasurementInput { + pub app_id: String, + pub compose_hash: String, + pub rootfs_hash: String, + pub ovmf_hash: String, + pub kernel_hash: String, + pub initrd_hash: String, + pub vcpus: u32, + pub vcpu_type: Option, + pub ovmf_sections: Vec, + /// Trusted expected SNP MEASUREMENT from an out-of-band recomputation or + /// allowlist. This must never be copied from untrusted client input. + pub trusted_expected_measurement: Option, +} + +pub(crate) fn validate_amd_snp_measurement_binding( + config: Option<&SevSnpMeasureConfig>, + verified_measurement: &[u8; 48], + input: &MeasurementInput, +) -> Result<()> { + let config = config.ok_or_else(|| anyhow::anyhow!("sev-snp measurement config is required"))?; + if config.guest_features == 0 { + bail!("guest_features must be non-zero"); + } + + decode_required_hex("app_id", &input.app_id, 20)?; + decode_required_hex("compose_hash", &input.compose_hash, 32)?; + decode_required_hex("rootfs_hash", &input.rootfs_hash, 32)?; + decode_required_hex("ovmf_hash", &input.ovmf_hash, 48)?; + decode_required_hex("kernel_hash", &input.kernel_hash, 32)?; + decode_required_hex("initrd_hash", &input.initrd_hash, 32)?; + + if input.vcpus == 0 { + bail!("vcpus must be greater than zero"); + } + match input.vcpu_type.as_deref() { + Some(vcpu_type) if !vcpu_type.trim().is_empty() => {} + _ => bail!("vcpu_type is required"), + } + if config + .ovmf_path + .as_deref() + .unwrap_or_default() + .trim() + .is_empty() + && input.ovmf_sections.is_empty() + { + bail!("ovmf_sections are required when ovmf_path is not configured"); + } + for section in &input.ovmf_sections { + if section.size == 0 { + bail!("ovmf section size must be greater than zero"); + } + if !matches!(section.section_type, 1 | 2 | 3 | 4 | 0x10) { + bail!("unknown ovmf section_type {:#x}", section.section_type); + } + } + + let expected_measurement = input + .trusted_expected_measurement + .as_deref() + .ok_or_else(|| anyhow::anyhow!("trusted_expected_measurement is required"))?; + let expected_measurement = + decode_required_hex("trusted_expected_measurement", expected_measurement, 48)?; + if expected_measurement.as_slice() != verified_measurement { + bail!("amd sev-snp measurement mismatch"); + } + + Ok(()) +} + +fn decode_required_hex(name: &str, value: &str, expected_len: usize) -> Result> { + if value.is_empty() { + bail!("{name} must not be empty"); + } + let bytes = hex::decode(value).map_err(|_| anyhow::anyhow!("{name} must be valid hex"))?; + if bytes.len() != expected_len { + bail!("{name} must be {expected_len} bytes"); + } + Ok(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn config() -> SevSnpMeasureConfig { + SevSnpMeasureConfig { + ovmf_path: None, + guest_features: 1, + } + } + + fn hex_of(byte: u8, len: usize) -> String { + hex::encode(vec![byte; len]) + } + + fn valid_input() -> MeasurementInput { + MeasurementInput { + app_id: hex_of(0x11, 20), + compose_hash: hex_of(0x22, 32), + rootfs_hash: hex_of(0x33, 32), + ovmf_hash: hex_of(0x44, 48), + kernel_hash: hex_of(0x55, 32), + initrd_hash: hex_of(0x66, 32), + vcpus: 2, + vcpu_type: Some("epyc-v4".to_string()), + ovmf_sections: vec![OvmfSectionParam { + gpa: 0x100000, + size: 0x200000, + section_type: 1, + }], + trusted_expected_measurement: Some(hex_of(0xaa, 48)), + } + } + + fn config_with_path(path: &str) -> SevSnpMeasureConfig { + SevSnpMeasureConfig { + ovmf_path: Some(path.to_string()), + guest_features: 1, + } + } + + fn assert_rejects(input: MeasurementInput, msg: &str) { + let verified = [0xaa; 48]; + let err = validate_amd_snp_measurement_binding(Some(&config()), &verified, &input) + .expect_err("binding should reject invalid input"); + assert!( + err.to_string().contains(msg), + "expected error containing {msg:?}, got {err:?}" + ); + } + + #[test] + fn accepts_fully_bound_matching_measurement() { + let verified = [0xaa; 48]; + validate_amd_snp_measurement_binding(Some(&config()), &verified, &valid_input()) + .expect("valid binding should be accepted"); + } + + #[test] + fn rejects_missing_config() { + let verified = [0xaa; 48]; + let err = validate_amd_snp_measurement_binding(None, &verified, &valid_input()) + .expect_err("missing config must fail closed"); + assert!(err + .to_string() + .contains("sev-snp measurement config is required")); + } + + #[test] + fn rejects_empty_or_malformed_binding_hashes() { + let mut input = valid_input(); + input.app_id.clear(); + assert_rejects(input, "app_id must not be empty"); + + let mut input = valid_input(); + input.compose_hash = "not hex".to_string(); + assert_rejects(input, "compose_hash must be valid hex"); + + let mut input = valid_input(); + input.rootfs_hash = hex_of(0x33, 31); + assert_rejects(input, "rootfs_hash must be 32 bytes"); + + let mut input = valid_input(); + input.ovmf_hash = hex_of(0x44, 47); + assert_rejects(input, "ovmf_hash must be 48 bytes"); + + let mut input = valid_input(); + input.kernel_hash = hex_of(0x55, 31); + assert_rejects(input, "kernel_hash must be 32 bytes"); + + let mut input = valid_input(); + input.initrd_hash = hex_of(0x66, 31); + assert_rejects(input, "initrd_hash must be 32 bytes"); + } + + #[test] + fn rejects_missing_machine_binding_inputs() { + let mut input = valid_input(); + input.vcpus = 0; + assert_rejects(input, "vcpus must be greater than zero"); + + let mut input = valid_input(); + input.vcpu_type = None; + assert_rejects(input, "vcpu_type is required"); + + let mut input = valid_input(); + input.ovmf_sections.clear(); + assert_rejects( + input, + "ovmf_sections are required when ovmf_path is not configured", + ); + + let mut input = valid_input(); + input.ovmf_sections.clear(); + let verified = [0xaa; 48]; + validate_amd_snp_measurement_binding( + Some(&config_with_path("/opt/amd/ovmf.fd")), + &verified, + &input, + ) + .expect("configured ovmf_path should allow sections to be loaded later"); + } + + #[test] + fn rejects_unsafe_machine_config() { + let verified = [0xaa; 48]; + let err = validate_amd_snp_measurement_binding( + Some(&SevSnpMeasureConfig { + ovmf_path: None, + guest_features: 0, + }), + &verified, + &valid_input(), + ) + .expect_err("zero guest_features must fail closed"); + assert!(err.to_string().contains("guest_features must be non-zero")); + + let mut input = valid_input(); + input.ovmf_sections[0].size = 0; + assert_rejects(input, "ovmf section size must be greater than zero"); + + let mut input = valid_input(); + input.ovmf_sections[0].section_type = 0xff; + assert_rejects(input, "unknown ovmf section_type 0xff"); + + let mut input = valid_input(); + input.ovmf_sections.clear(); + let err = + validate_amd_snp_measurement_binding(Some(&config_with_path(" ")), &verified, &input) + .expect_err("blank ovmf_path must not bypass section metadata requirement"); + assert!(err + .to_string() + .contains("ovmf_sections are required when ovmf_path is not configured")); + } + + #[test] + fn rejects_missing_or_mismatched_measurement() { + let mut input = valid_input(); + input.trusted_expected_measurement = None; + assert_rejects(input, "trusted_expected_measurement is required"); + + let mut input = valid_input(); + input.trusted_expected_measurement = Some(hex_of(0xaa, 47)); + assert_rejects(input, "trusted_expected_measurement must be 48 bytes"); + + let mut input = valid_input(); + input.trusted_expected_measurement = Some(hex_of(0xbb, 48)); + assert_rejects(input, "amd sev-snp measurement mismatch"); + } +} From 0e9f5865804e4b4c2347ac8f737755c7241686a1 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Tue, 26 May 2026 14:31:34 -0700 Subject: [PATCH 07/67] fix: recompute sev-snp launch measurement --- kms/src/main_service/amd_attest.rs | 824 ++++++++++++++++++++++++++--- 1 file changed, 758 insertions(+), 66 deletions(-) diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs index 659f84d4f..2c7b5b4eb 100644 --- a/kms/src/main_service/amd_attest.rs +++ b/kms/src/main_service/amd_attest.rs @@ -5,17 +5,32 @@ //! Fail-closed AMD SEV-SNP measurement/app binding validation. //! //! This module intentionally does not release keys and does not enable any AMD -//! KMS key-release endpoint. Until full in-KMS SNP measurement recomputation is -//! wired in, callers must provide the expected measurement from a trusted -//! recomputation path; this helper validates every required binding input and -//! compares that expected value to the hardware-verified report measurement. +//! KMS key-release endpoint. It recomputes the expected SNP MEASUREMENT from +//! validated KMS configuration and launch inputs, then compares the recomputed +//! value to the hardware-verified report measurement. +//! +//! Important: this is launch measurement binding, not a complete authorization +//! decision. `app_id` is carried and validated so future SNP `BootInfo` +//! construction can consume the same input, but the current QEMU SNP launch +//! measurement does not independently bind `app_id`. Do not use this helper by +//! itself to release app keys. #![allow(dead_code)] -use anyhow::{bail, Result}; +use anyhow::{bail, Context, Result}; +use sha2::{Digest, Sha256, Sha384}; +use std::fs; use crate::config::SevSnpMeasureConfig; +const LD_BYTES: usize = 48; +const ZEROS_LD: [u8; LD_BYTES] = [0u8; LD_BYTES]; +const MAX_VCPUS: u32 = 512; +const MAX_OVMF_SECTIONS: usize = 64; +const MAX_OVMF_METADATA_PAGES: u64 = 16_777_216; // 64 GiB worth of 4 KiB pages. + // VMSA page GPA: (u64)(-1) page-aligned, bits >51 cleared. +const VMSA_GPA: u64 = 0x0000_FFFF_FFFF_F000; + #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct OvmfSectionParam { pub gpa: u64, @@ -28,18 +43,32 @@ pub(crate) struct OvmfSectionParam { #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct MeasurementInput { + /// 20-byte app identity. It is validated now even though the current SNP + /// launch measurement does not include it directly; callers must bind this + /// through the future SNP `BootInfo`/authorization path before key release. pub app_id: String, + /// 32-byte docker compose hash included in the measured kernel cmdline. pub compose_hash: String, + /// 32-byte rootfs hash included in the measured kernel cmdline. pub rootfs_hash: String, + /// Optional 32-byte additional docker files hash included in the measured + /// kernel cmdline when present. + pub docker_files_hash: Option, + /// 48-byte OVMF GCTX launch digest seed. Required when OVMF sections are + /// supplied by the request; optional only when KMS can load ovmf_path. pub ovmf_hash: String, + /// 32-byte kernel SHA-256 hash. pub kernel_hash: String, + /// 32-byte initrd SHA-256 hash. An empty string is treated as the SHA-256 of + /// an empty initrd, matching QEMU/sev-snp-measure behavior. pub initrd_hash: String, + /// GPA of the SevHashTable, from OVMF footer metadata. + pub sev_hashes_table_gpa: u64, + /// AP reset EIP, from OVMF footer metadata. + pub sev_es_reset_eip: u32, pub vcpus: u32, pub vcpu_type: Option, pub ovmf_sections: Vec, - /// Trusted expected SNP MEASUREMENT from an out-of-band recomputation or - /// allowlist. This must never be copied from untrusted client input. - pub trusted_expected_measurement: Option, } pub(crate) fn validate_amd_snp_measurement_binding( @@ -48,51 +77,96 @@ pub(crate) fn validate_amd_snp_measurement_binding( input: &MeasurementInput, ) -> Result<()> { let config = config.ok_or_else(|| anyhow::anyhow!("sev-snp measurement config is required"))?; + validate_measurement_input(config, input)?; + + let expected_measurement = compute_expected_measurement(config, input)?; + if expected_measurement.as_slice() != verified_measurement { + bail!("amd sev-snp measurement mismatch"); + } + + Ok(()) +} + +fn validate_measurement_input( + config: &SevSnpMeasureConfig, + input: &MeasurementInput, +) -> Result<()> { if config.guest_features == 0 { bail!("guest_features must be non-zero"); } - decode_required_hex("app_id", &input.app_id, 20)?; + let app_id = decode_required_hex("app_id", &input.app_id, 20)?; + if app_id.iter().all(|&b| b == 0) { + bail!("app_id must not be all-zeros"); + } decode_required_hex("compose_hash", &input.compose_hash, 32)?; decode_required_hex("rootfs_hash", &input.rootfs_hash, 32)?; - decode_required_hex("ovmf_hash", &input.ovmf_hash, 48)?; decode_required_hex("kernel_hash", &input.kernel_hash, 32)?; - decode_required_hex("initrd_hash", &input.initrd_hash, 32)?; + decode_optional_hex("initrd_hash", &input.initrd_hash, 32)?; + if let Some(docker_files_hash) = &input.docker_files_hash { + decode_required_hex("docker_files_hash", docker_files_hash, 32)?; + } if input.vcpus == 0 { bail!("vcpus must be greater than zero"); } + if input.vcpus > MAX_VCPUS { + bail!("vcpus must not exceed {MAX_VCPUS}"); + } match input.vcpu_type.as_deref() { - Some(vcpu_type) if !vcpu_type.trim().is_empty() => {} + Some(vcpu_type) if !vcpu_type.trim().is_empty() => { + vcpu_sig_from_type(vcpu_type)?; + } _ => bail!("vcpu_type is required"), } - if config - .ovmf_path - .as_deref() - .unwrap_or_default() - .trim() - .is_empty() - && input.ovmf_sections.is_empty() - { - bail!("ovmf_sections are required when ovmf_path is not configured"); + + if input.ovmf_sections.is_empty() { + if config + .ovmf_path + .as_deref() + .unwrap_or_default() + .trim() + .is_empty() + { + bail!("ovmf_sections are required when ovmf_path is not configured"); + } + if !input.ovmf_hash.is_empty() { + bail!("ovmf_hash must be empty when ovmf_path is used"); + } + return Ok(()); + } + + decode_required_hex("ovmf_hash", &input.ovmf_hash, 48)?; + if input.ovmf_sections.len() > MAX_OVMF_SECTIONS { + bail!("ovmf section count must not exceed {MAX_OVMF_SECTIONS}"); + } + if input.sev_hashes_table_gpa == 0 { + bail!("sev_hashes_table_gpa must be non-zero"); } + if input.sev_es_reset_eip == 0 { + bail!("sev_es_reset_eip must be non-zero"); + } + + let mut has_kernel_hashes_section = false; + let mut measured_pages = 0u64; for section in &input.ovmf_sections { if section.size == 0 { bail!("ovmf section size must be greater than zero"); } - if !matches!(section.section_type, 1 | 2 | 3 | 4 | 0x10) { - bail!("unknown ovmf section_type {:#x}", section.section_type); + let pages = section.size.div_ceil(4096); + measured_pages = measured_pages + .checked_add(pages) + .ok_or_else(|| anyhow::anyhow!("ovmf metadata page count overflow"))?; + if measured_pages > MAX_OVMF_METADATA_PAGES { + bail!("ovmf metadata page count must not exceed {MAX_OVMF_METADATA_PAGES}"); } + let section_type = SectionType::from_u32(section.section_type).ok_or_else(|| { + anyhow::anyhow!("unknown ovmf section_type {:#x}", section.section_type) + })?; + has_kernel_hashes_section |= section_type == SectionType::SnpKernelHashes; } - - let expected_measurement = input - .trusted_expected_measurement - .as_deref() - .ok_or_else(|| anyhow::anyhow!("trusted_expected_measurement is required"))?; - let expected_measurement = - decode_required_hex("trusted_expected_measurement", expected_measurement, 48)?; - if expected_measurement.as_slice() != verified_measurement { - bail!("amd sev-snp measurement mismatch"); + if !has_kernel_hashes_section { + bail!("ovmf metadata does not include a snp_kernel_hashes section"); } Ok(()) @@ -102,6 +176,13 @@ fn decode_required_hex(name: &str, value: &str, expected_len: usize) -> Result Result> { + if value.is_empty() { + return Ok(Vec::new()); + } let bytes = hex::decode(value).map_err(|_| anyhow::anyhow!("{name} must be valid hex"))?; if bytes.len() != expected_len { bail!("{name} must be {expected_len} bytes"); @@ -109,6 +190,490 @@ fn decode_required_hex(name: &str, value: &str, expected_len: usize) -> Result Self { + Self { ld: ZEROS_LD } + } + + fn from_ovmf_hash(hex_value: &str) -> Result { + let raw = hex::decode(hex_value).context("ovmf_hash must be valid hex")?; + let ld: [u8; LD_BYTES] = raw + .try_into() + .map_err(|_| anyhow::anyhow!("ovmf_hash must be 48 bytes"))?; + Ok(Self { ld }) + } + + /// SNP spec §8.17.2 PAGE_INFO layout (112 bytes): current digest, + /// contents digest, length, page type, permissions/reserved, and GPA. + fn update(&mut self, page_type: u8, gpa: u64, contents: &[u8; LD_BYTES]) { + let mut buf = [0u8; 0x70]; + buf[..LD_BYTES].copy_from_slice(&self.ld); + buf[48..96].copy_from_slice(contents); + buf[96..98].copy_from_slice(&0x70u16.to_le_bytes()); + buf[98] = page_type; + buf[104..112].copy_from_slice(&gpa.to_le_bytes()); + let mut digest = [0u8; LD_BYTES]; + digest.copy_from_slice(&Sha384::digest(buf)); + self.ld = digest; + } + + fn sha384(data: &[u8]) -> [u8; LD_BYTES] { + let mut out = [0u8; LD_BYTES]; + out.copy_from_slice(&Sha384::digest(data)); + out + } + + fn update_normal_pages(&mut self, start_gpa: u64, data: &[u8]) { + for (i, chunk) in data.chunks(4096).enumerate() { + self.update(0x01, start_gpa + (i * 4096) as u64, &Self::sha384(chunk)); + } + } + + fn update_zero_pages(&mut self, gpa: u64, len: usize) { + for i in (0..len).step_by(4096) { + self.update(0x03, gpa + i as u64, &ZEROS_LD); + } + } + + fn update_secrets_page(&mut self, gpa: u64) { + self.update(0x05, gpa, &ZEROS_LD); + } + + fn update_cpuid_page(&mut self, gpa: u64) { + self.update(0x06, gpa, &ZEROS_LD); + } + + fn update_vmsa_page(&mut self, page: &[u8]) { + self.update(0x02, VMSA_GPA, &Self::sha384(page)); + } +} + +const GUID_LE_HASH_TABLE_HEADER: [u8; 16] = [ + 0x06, 0xd6, 0x38, 0x94, 0x22, 0x4f, 0xc9, 0x4c, 0xb4, 0x79, 0xa7, 0x93, 0xd4, 0x11, 0xfd, 0x21, +]; +const GUID_LE_KERNEL_ENTRY: [u8; 16] = [ + 0x37, 0x94, 0xe7, 0x4d, 0xd2, 0xab, 0x7f, 0x42, 0xb8, 0x35, 0xd5, 0xb1, 0x72, 0xd2, 0x04, 0x5b, +]; +const GUID_LE_INITRD_ENTRY: [u8; 16] = [ + 0x31, 0xf7, 0xba, 0x44, 0x2f, 0x3a, 0xd7, 0x4b, 0x9a, 0xf1, 0x41, 0xe2, 0x91, 0x69, 0x78, 0x1d, +]; +const GUID_LE_CMDLINE_ENTRY: [u8; 16] = [ + 0xd8, 0x2d, 0xd0, 0x97, 0x20, 0xbd, 0x94, 0x4c, 0xaa, 0x78, 0xe7, 0x71, 0x4d, 0x36, 0xab, 0x2a, +]; + +fn sev_entry(guid: &[u8; 16], hash: &[u8; 32]) -> [u8; 50] { + let mut entry = [0u8; 50]; + entry[..16].copy_from_slice(guid); + entry[16..18].copy_from_slice(&50u16.to_le_bytes()); + entry[18..].copy_from_slice(hash); + entry +} + +fn build_sev_hashes_page( + kernel_hash_hex: &str, + initrd_hash_hex: &str, + append: &str, + page_offset: usize, +) -> Result<[u8; 4096]> { + let kernel_hash: [u8; 32] = hex::decode(kernel_hash_hex) + .context("kernel_hash must be valid hex")? + .try_into() + .map_err(|_| anyhow::anyhow!("kernel_hash must be 32 bytes"))?; + + let initrd_hash: [u8; 32] = if initrd_hash_hex.is_empty() { + let mut h = [0u8; 32]; + h.copy_from_slice(&Sha256::digest(b"")); + h + } else { + hex::decode(initrd_hash_hex) + .context("initrd_hash must be valid hex")? + .try_into() + .map_err(|_| anyhow::anyhow!("initrd_hash must be 32 bytes"))? + }; + + let mut cmdline_bytes = append.as_bytes().to_vec(); + cmdline_bytes.push(0); + let mut cmdline_hash = [0u8; 32]; + cmdline_hash.copy_from_slice(&Sha256::digest(&cmdline_bytes)); + + let cmdline_entry = sev_entry(&GUID_LE_CMDLINE_ENTRY, &cmdline_hash); + let initrd_entry = sev_entry(&GUID_LE_INITRD_ENTRY, &initrd_hash); + let kernel_entry = sev_entry(&GUID_LE_KERNEL_ENTRY, &kernel_hash); + + const TABLE_SIZE: usize = 16 + 2 + 50 + 50 + 50; + let mut table = [0u8; TABLE_SIZE]; + table[..16].copy_from_slice(&GUID_LE_HASH_TABLE_HEADER); + table[16..18].copy_from_slice(&(TABLE_SIZE as u16).to_le_bytes()); + table[18..68].copy_from_slice(&cmdline_entry); + table[68..118].copy_from_slice(&initrd_entry); + table[118..168].copy_from_slice(&kernel_entry); + + const PADDED: usize = (TABLE_SIZE + 15) & !(15usize); + if page_offset + PADDED > 4096 { + bail!("sev hash table overflows 4096-byte page"); + } + let mut page = [0u8; 4096]; + page[page_offset..page_offset + TABLE_SIZE].copy_from_slice(&table); + Ok(page) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SectionType { + SnpSecMemory = 1, + SnpSecrets = 2, + Cpuid = 3, + SvsmCaa = 4, + SnpKernelHashes = 0x10, +} + +impl SectionType { + fn from_u32(value: u32) -> Option { + match value { + 1 => Some(Self::SnpSecMemory), + 2 => Some(Self::SnpSecrets), + 3 => Some(Self::Cpuid), + 4 => Some(Self::SvsmCaa), + 0x10 => Some(Self::SnpKernelHashes), + _ => None, + } + } +} + +struct MetadataSection { + gpa: u64, + size: u64, + section_type: SectionType, +} + +struct OvmfInfo { + data: Vec, + gpa: u64, + sections: Vec, + sev_hashes_table_gpa: u64, + sev_es_reset_eip: u32, +} + +const GUID_FOOTER_TABLE: [u8; 16] = [ + 0xde, 0x82, 0xb5, 0x96, 0xb2, 0x1f, 0xf7, 0x45, 0xba, 0xea, 0xa3, 0x66, 0xc5, 0x5a, 0x08, 0x2d, +]; +const GUID_SEV_HASH_TABLE_RV: [u8; 16] = [ + 0x1f, 0x37, 0x55, 0x72, 0x3b, 0x3a, 0x04, 0x4b, 0x92, 0x7b, 0x1d, 0xa6, 0xef, 0xa8, 0xd4, 0x54, +]; +const GUID_SEV_ES_RESET_BLK: [u8; 16] = [ + 0xde, 0x71, 0xf7, 0x00, 0x7e, 0x1a, 0xcb, 0x4f, 0x89, 0x0e, 0x68, 0xc7, 0x7e, 0x2f, 0xb4, 0x4e, +]; +const GUID_SEV_META_DATA: [u8; 16] = [ + 0x66, 0x65, 0x88, 0xdc, 0x4a, 0x98, 0x98, 0x47, 0xa7, 0x5e, 0x55, 0x85, 0xa7, 0xbf, 0x67, 0xcc, +]; + +fn read_u16_le(buf: &[u8], off: usize) -> u16 { + u16::from_le_bytes([buf[off], buf[off + 1]]) +} + +fn read_u32_le(buf: &[u8], off: usize) -> u32 { + u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]]) +} + +impl OvmfInfo { + fn load(path: &str) -> Result { + let data = fs::read(path).with_context(|| format!("cannot read ovmf binary '{path}'"))?; + let size = data.len(); + let gpa = (0x1_0000_0000u64) + .checked_sub(size as u64) + .context("ovmf binary is larger than 4 gib")?; + + const ENTRY_HDR: usize = 18; + let footer_off = size.saturating_sub(32 + ENTRY_HDR); + if footer_off + ENTRY_HDR > size { + bail!("ovmf binary too small to contain footer table"); + } + if data[footer_off + 2..footer_off + 18] != GUID_FOOTER_TABLE { + bail!("ovmf footer guid not found"); + } + let footer_total_size = read_u16_le(&data, footer_off) as usize; + if footer_total_size < ENTRY_HDR { + bail!("ovmf footer table has invalid total size"); + } + let table_size = footer_total_size - ENTRY_HDR; + if table_size > footer_off { + bail!("ovmf footer table is out of bounds"); + } + let table_start = footer_off - table_size; + let table_bytes = &data[table_start..footer_off]; + + let mut sev_hashes_table_gpa = 0u64; + let mut sev_es_reset_eip = 0u32; + let mut meta_offset_from_end = None; + + let mut pos = table_bytes.len(); + while pos >= ENTRY_HDR { + let entry_off = pos - ENTRY_HDR; + let entry_size = read_u16_le(table_bytes, entry_off) as usize; + if entry_size < ENTRY_HDR || entry_size > pos { + bail!("ovmf footer table has invalid entry size"); + } + let guid = &table_bytes[entry_off + 2..entry_off + 18]; + let data_start = pos - entry_size; + let data_end = pos - ENTRY_HDR; + let entry_data = &table_bytes[data_start..data_end]; + + if guid == GUID_SEV_HASH_TABLE_RV && entry_data.len() >= 4 { + sev_hashes_table_gpa = read_u32_le(entry_data, 0) as u64; + } else if guid == GUID_SEV_ES_RESET_BLK && entry_data.len() >= 4 { + sev_es_reset_eip = read_u32_le(entry_data, 0); + } else if guid == GUID_SEV_META_DATA && entry_data.len() >= 4 { + meta_offset_from_end = Some(read_u32_le(entry_data, 0) as usize); + } + pos -= entry_size; + } + + if sev_hashes_table_gpa == 0 { + bail!("ovmf sev hash table entry not found in footer table"); + } + if sev_es_reset_eip == 0 { + bail!("ovmf sev_es_reset_block entry not found in footer table"); + } + + let mut sections = Vec::new(); + let off_from_end = meta_offset_from_end + .ok_or_else(|| anyhow::anyhow!("ovmf sev metadata entry not found in footer table"))?; + if off_from_end > size { + bail!("ovmf sev metadata offset exceeds file size"); + } + let meta_start = size - off_from_end; + if meta_start + 16 > size { + bail!("ovmf sev metadata header out of bounds"); + } + if &data[meta_start..meta_start + 4] != b"ASEV" { + bail!("ovmf sev metadata has bad signature"); + } + let meta_version = read_u32_le(&data, meta_start + 8); + if meta_version != 1 { + bail!("ovmf sev metadata has unsupported version {meta_version}"); + } + let num_items = read_u32_le(&data, meta_start + 12) as usize; + let items_start = meta_start + 16; + if items_start + num_items * 12 > size { + bail!("ovmf sev metadata sections out of bounds"); + } + for i in 0..num_items { + let off = items_start + i * 12; + let section_type_value = read_u32_le(&data, off + 8); + let section_type = SectionType::from_u32(section_type_value).ok_or_else(|| { + anyhow::anyhow!("unknown ovmf section_type {section_type_value:#x}") + })?; + sections.push(MetadataSection { + gpa: read_u32_le(&data, off) as u64, + size: read_u32_le(&data, off + 4) as u64, + section_type, + }); + } + + Ok(Self { + data, + gpa, + sections, + sev_hashes_table_gpa, + sev_es_reset_eip, + }) + } +} + +fn write_u16_le_at(buf: &mut [u8], off: usize, value: u16) { + buf[off..off + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn write_u32_le_at(buf: &mut [u8], off: usize, value: u32) { + buf[off..off + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn write_u64_le_at(buf: &mut [u8], off: usize, value: u64) { + buf[off..off + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn write_vmcb_seg(buf: &mut [u8], off: usize, selector: u16, attrib: u16, limit: u32, base: u64) { + write_u16_le_at(buf, off, selector); + write_u16_le_at(buf, off + 2, attrib); + write_u32_le_at(buf, off + 4, limit); + write_u64_le_at(buf, off + 8, base); +} + +fn amd_cpu_sig(family: u32, model: u32, stepping: u32) -> u32 { + let (family_low, family_high) = if family > 0xf { + (0xf, (family - 0xf) & 0xff) + } else { + (family, 0) + }; + let model_low = model & 0xf; + let model_high = (model >> 4) & 0xf; + (family_high << 20) + | (model_high << 16) + | (family_low << 8) + | (model_low << 4) + | (stepping & 0xf) +} + +fn vcpu_sig_from_type(vcpu_type: &str) -> Result { + match vcpu_type.trim().to_lowercase().as_str() { + "epyc" | "epyc-v1" | "epyc-v2" | "epyc-ibpb" | "epyc-v3" | "epyc-v4" => { + Ok(amd_cpu_sig(23, 1, 2)) + } + "epyc-rome" | "epyc-rome-v1" | "epyc-rome-v2" | "epyc-rome-v3" => { + Ok(amd_cpu_sig(23, 49, 0)) + } + "epyc-milan" | "epyc-milan-v1" | "epyc-milan-v2" => Ok(amd_cpu_sig(25, 1, 1)), + "epyc-genoa" | "epyc-genoa-v1" => Ok(amd_cpu_sig(25, 17, 0)), + other => bail!("unknown vcpu_type {other:?}"), + } +} + +fn build_vmsa_page(eip: u32, vcpu_sig: u32, sev_features: u64) -> Box<[u8; 4096]> { + let mut page = Box::new([0u8; 4096]); + let p = page.as_mut_slice(); + + let cs_base = (eip as u64) & 0xffff_0000; + let rip = (eip as u64) & 0x0000_ffff; + + write_vmcb_seg(p, 0x000, 0, 0x0093, 0xffff, 0); + write_vmcb_seg(p, 0x010, 0xf000, 0x009b, 0xffff, cs_base); + write_vmcb_seg(p, 0x020, 0, 0x0093, 0xffff, 0); + write_vmcb_seg(p, 0x030, 0, 0x0093, 0xffff, 0); + write_vmcb_seg(p, 0x040, 0, 0x0093, 0xffff, 0); + write_vmcb_seg(p, 0x050, 0, 0x0093, 0xffff, 0); + write_vmcb_seg(p, 0x060, 0, 0x0000, 0xffff, 0); + write_vmcb_seg(p, 0x070, 0, 0x0082, 0xffff, 0); + write_vmcb_seg(p, 0x080, 0, 0x0000, 0xffff, 0); + write_vmcb_seg(p, 0x090, 0, 0x008b, 0xffff, 0); + + write_u64_le_at(p, 0x0D0, 0x1000); + write_u64_le_at(p, 0x148, 0x40); + write_u64_le_at(p, 0x158, 0x10); + write_u64_le_at(p, 0x160, 0x400); + write_u64_le_at(p, 0x168, 0xffff_0ff0); + write_u64_le_at(p, 0x170, 0x2); + write_u64_le_at(p, 0x178, rip); + write_u64_le_at(p, 0x268, 0x0007_0406_0007_0406); + write_u64_le_at(p, 0x310, vcpu_sig as u64); + write_u64_le_at(p, 0x3B0, sev_features); + write_u64_le_at(p, 0x3E8, 0x1); + write_u32_le_at(p, 0x408, 0x1f80); + write_u16_le_at(p, 0x410, 0x037f); + + page +} + +pub(crate) fn compute_expected_measurement( + config: &SevSnpMeasureConfig, + input: &MeasurementInput, +) -> Result<[u8; 48]> { + let vcpu_type = input + .vcpu_type + .as_deref() + .ok_or_else(|| anyhow::anyhow!("vcpu_type is required"))?; + + let mut cmdline = format!( + "console=ttyS0 loglevel=7 docker_compose_hash={} rootfs_hash={}", + input.compose_hash, input.rootfs_hash + ); + if let Some(docker_files_hash) = input.docker_files_hash.as_deref() { + cmdline.push_str(&format!( + " docker_additional_files_hash={docker_files_hash}" + )); + } + + let (mut gctx, effective_hashes_gpa, effective_reset_eip, resolved_sections) = + if !input.ovmf_sections.is_empty() { + let sections = input + .ovmf_sections + .iter() + .map(|section| { + let section_type = + SectionType::from_u32(section.section_type).ok_or_else(|| { + anyhow::anyhow!("unknown ovmf section_type {:#x}", section.section_type) + })?; + Ok(MetadataSection { + gpa: section.gpa, + size: section.size, + section_type, + }) + }) + .collect::>>()?; + ( + Gctx::from_ovmf_hash(&input.ovmf_hash)?, + input.sev_hashes_table_gpa, + input.sev_es_reset_eip, + sections, + ) + } else { + let path = config.ovmf_path.as_deref().ok_or_else(|| { + anyhow::anyhow!("ovmf_sections are required when ovmf_path is not configured") + })?; + let ovmf = OvmfInfo::load(path)?; + let gctx = if input.ovmf_hash.is_empty() { + let mut g = Gctx::new(); + g.update_normal_pages(ovmf.gpa, &ovmf.data); + g + } else { + Gctx::from_ovmf_hash(&input.ovmf_hash)? + }; + ( + gctx, + ovmf.sev_hashes_table_gpa, + ovmf.sev_es_reset_eip, + ovmf.sections, + ) + }; + + let mut has_kernel_hashes_section = false; + for section in &resolved_sections { + let gpa = section.gpa; + let size = usize::try_from(section.size) + .map_err(|_| anyhow::anyhow!("ovmf section size is too large"))?; + match section.section_type { + SectionType::SnpSecMemory => gctx.update_zero_pages(gpa, size), + SectionType::SnpSecrets => gctx.update_secrets_page(gpa), + SectionType::Cpuid => gctx.update_cpuid_page(gpa), + SectionType::SvsmCaa => gctx.update_zero_pages(gpa, size), + SectionType::SnpKernelHashes => { + has_kernel_hashes_section = true; + if effective_hashes_gpa == 0 { + bail!("snp_kernel_hashes section present but sev_hashes_table_gpa is 0"); + } + let page_offset = (effective_hashes_gpa & 0xfff) as usize; + let page = build_sev_hashes_page( + &input.kernel_hash, + &input.initrd_hash, + &cmdline, + page_offset, + )?; + gctx.update_normal_pages(gpa, &page); + } + } + } + if !has_kernel_hashes_section { + bail!("ovmf metadata does not include a snp_kernel_hashes section"); + } + + let vcpu_sig = vcpu_sig_from_type(vcpu_type)?; + let bsp_vmsa = build_vmsa_page(0xffff_fff0, vcpu_sig, config.guest_features); + let ap_vmsa = build_vmsa_page(effective_reset_eip, vcpu_sig, config.guest_features); + + for i in 0..input.vcpus as usize { + let vmsa_page = if i == 0 { + bsp_vmsa.as_ref() + } else { + ap_vmsa.as_ref() + }; + gctx.update_vmsa_page(vmsa_page); + } + + Ok(gctx.ld) +} + #[cfg(test)] mod tests { use super::*; @@ -120,6 +685,13 @@ mod tests { } } + fn config_with_path(path: &str) -> SevSnpMeasureConfig { + SevSnpMeasureConfig { + ovmf_path: Some(path.to_string()), + guest_features: 1, + } + } + fn hex_of(byte: u8, len: usize) -> String { hex::encode(vec![byte; len]) } @@ -129,24 +701,36 @@ mod tests { app_id: hex_of(0x11, 20), compose_hash: hex_of(0x22, 32), rootfs_hash: hex_of(0x33, 32), + docker_files_hash: Some(hex_of(0x77, 32)), ovmf_hash: hex_of(0x44, 48), kernel_hash: hex_of(0x55, 32), initrd_hash: hex_of(0x66, 32), + sev_hashes_table_gpa: 0x80_1000, + sev_es_reset_eip: 0xffff_fff0, vcpus: 2, vcpu_type: Some("epyc-v4".to_string()), - ovmf_sections: vec![OvmfSectionParam { - gpa: 0x100000, - size: 0x200000, - section_type: 1, - }], - trusted_expected_measurement: Some(hex_of(0xaa, 48)), - } - } - - fn config_with_path(path: &str) -> SevSnpMeasureConfig { - SevSnpMeasureConfig { - ovmf_path: Some(path.to_string()), - guest_features: 1, + ovmf_sections: vec![ + OvmfSectionParam { + gpa: 0x100000, + size: 0x2000, + section_type: 1, + }, + OvmfSectionParam { + gpa: 0x80_0000, + size: 0x1000, + section_type: 0x10, + }, + OvmfSectionParam { + gpa: 0x81_0000, + size: 0x1000, + section_type: 2, + }, + OvmfSectionParam { + gpa: 0x82_0000, + size: 0x1000, + section_type: 3, + }, + ], } } @@ -161,10 +745,67 @@ mod tests { } #[test] - fn accepts_fully_bound_matching_measurement() { - let verified = [0xaa; 48]; - validate_amd_snp_measurement_binding(Some(&config()), &verified, &valid_input()) - .expect("valid binding should be accepted"); + fn gctx_update_is_deterministic_and_order_sensitive() { + let contents = Gctx::sha384(b"page"); + let mut first = Gctx::new(); + first.update(0x01, 0x1000, &contents); + assert_eq!( + hex::encode(first.ld), + "3ebc1a70acc0bae5ae2788fae29a0371f983b19a68faf9843064f36040f58571ce5bb6bcdc9c361087073f8cffd92635" + ); + + let mut second = Gctx::new(); + second.update(0x01, 0x2000, &contents); + assert_ne!(first.ld, second.ld); + } + + #[test] + fn builds_sev_hashes_page_at_requested_offset() { + let page = build_sev_hashes_page(&hex_of(0x55, 32), "", "console=ttyS0", 0x80) + .expect("sev hashes page should build"); + assert_eq!(&page[..0x80], &[0u8; 0x80]); + assert_eq!(&page[0x80..0x90], &GUID_LE_HASH_TABLE_HEADER); + assert_eq!(u16::from_le_bytes([page[0x90], page[0x91]]), 168); + assert_eq!( + &page[0x92..0xa2], + &GUID_LE_CMDLINE_ENTRY, + "cmdline entry must be first" + ); + let empty_hash = Sha256::digest(b""); + assert_eq!(&page[0x80 + 68 + 18..0x80 + 68 + 50], empty_hash.as_slice()); + } + + #[test] + fn vcpu_type_mapping_is_strict() { + assert_eq!( + vcpu_sig_from_type("EPYC-v4").unwrap(), + amd_cpu_sig(23, 1, 2) + ); + assert_eq!( + vcpu_sig_from_type("epyc-genoa-v1").unwrap(), + amd_cpu_sig(25, 17, 0) + ); + let err = vcpu_sig_from_type("not-a-cpu").expect_err("unknown vcpu should reject"); + assert!(err.to_string().contains("unknown vcpu_type")); + } + + #[test] + fn accepts_recomputed_matching_measurement_and_rejects_mismatch() { + let input = valid_input(); + let expected = compute_expected_measurement(&config(), &input).unwrap(); + assert_eq!( + hex::encode(expected), + "2c598d7885c7a61e44c048ac5594600ce906339a8a6a3c0fa0d81c67275d55273d38d6ec5afcf5ff1d74621f0f0354d9", + "synthetic measurement vector should not drift silently" + ); + validate_amd_snp_measurement_binding(Some(&config()), &expected, &input) + .expect("matching recomputed binding should be accepted"); + + let mut mismatched = expected; + mismatched[0] ^= 0xff; + let err = validate_amd_snp_measurement_binding(Some(&config()), &mismatched, &input) + .expect_err("mismatched measurement must reject"); + assert!(err.to_string().contains("amd sev-snp measurement mismatch")); } #[test] @@ -183,6 +824,10 @@ mod tests { input.app_id.clear(); assert_rejects(input, "app_id must not be empty"); + let mut input = valid_input(); + input.app_id = hex_of(0x00, 20); + assert_rejects(input, "app_id must not be all-zeros"); + let mut input = valid_input(); input.compose_hash = "not hex".to_string(); assert_rejects(input, "compose_hash must be valid hex"); @@ -202,6 +847,16 @@ mod tests { let mut input = valid_input(); input.initrd_hash = hex_of(0x66, 31); assert_rejects(input, "initrd_hash must be 32 bytes"); + + let mut input = valid_input(); + input.initrd_hash.clear(); + let expected = compute_expected_measurement(&config(), &input).unwrap(); + validate_amd_snp_measurement_binding(Some(&config()), &expected, &input) + .expect("empty initrd hash should mean empty initrd"); + + let mut input = valid_input(); + input.docker_files_hash = Some(String::new()); + assert_rejects(input, "docker_files_hash must not be empty"); } #[test] @@ -210,10 +865,18 @@ mod tests { input.vcpus = 0; assert_rejects(input, "vcpus must be greater than zero"); + let mut input = valid_input(); + input.vcpus = MAX_VCPUS + 1; + assert_rejects(input, "vcpus must not exceed"); + let mut input = valid_input(); input.vcpu_type = None; assert_rejects(input, "vcpu_type is required"); + let mut input = valid_input(); + input.vcpu_type = Some("mystery".to_string()); + assert_rejects(input, "unknown vcpu_type"); + let mut input = valid_input(); input.ovmf_sections.clear(); assert_rejects( @@ -223,13 +886,27 @@ mod tests { let mut input = valid_input(); input.ovmf_sections.clear(); + input.ovmf_hash.clear(); let verified = [0xaa; 48]; - validate_amd_snp_measurement_binding( + let err = validate_amd_snp_measurement_binding( + Some(&config_with_path("/path/that/does/not/exist/ovmf.fd")), + &verified, + &input, + ) + .expect_err("configured ovmf_path should be used for recomputation"); + assert!(err.to_string().contains("cannot read ovmf binary")); + + let mut input = valid_input(); + input.ovmf_sections.clear(); + let err = validate_amd_snp_measurement_binding( Some(&config_with_path("/opt/amd/ovmf.fd")), &verified, &input, ) - .expect("configured ovmf_path should allow sections to be loaded later"); + .expect_err("request ovmf_hash must not override configured ovmf_path"); + assert!(err + .to_string() + .contains("ovmf_hash must be empty when ovmf_path is used")); } #[test] @@ -250,10 +927,40 @@ mod tests { input.ovmf_sections[0].size = 0; assert_rejects(input, "ovmf section size must be greater than zero"); + let mut input = valid_input(); + input.ovmf_sections = vec![ + OvmfSectionParam { + gpa: 0x1000, + size: 0x1000, + section_type: 1, + }; + MAX_OVMF_SECTIONS + 1 + ]; + assert_rejects(input, "ovmf section count must not exceed"); + + let mut input = valid_input(); + input.ovmf_sections[0].size = (MAX_OVMF_METADATA_PAGES + 1) * 4096; + assert_rejects(input, "ovmf metadata page count must not exceed"); + let mut input = valid_input(); input.ovmf_sections[0].section_type = 0xff; assert_rejects(input, "unknown ovmf section_type 0xff"); + let mut input = valid_input(); + input.ovmf_sections.retain(|s| s.section_type != 0x10); + assert_rejects( + input, + "ovmf metadata does not include a snp_kernel_hashes section", + ); + + let mut input = valid_input(); + input.sev_hashes_table_gpa = 0; + assert_rejects(input, "sev_hashes_table_gpa must be non-zero"); + + let mut input = valid_input(); + input.sev_es_reset_eip = 0; + assert_rejects(input, "sev_es_reset_eip must be non-zero"); + let mut input = valid_input(); input.ovmf_sections.clear(); let err = @@ -263,19 +970,4 @@ mod tests { .to_string() .contains("ovmf_sections are required when ovmf_path is not configured")); } - - #[test] - fn rejects_missing_or_mismatched_measurement() { - let mut input = valid_input(); - input.trusted_expected_measurement = None; - assert_rejects(input, "trusted_expected_measurement is required"); - - let mut input = valid_input(); - input.trusted_expected_measurement = Some(hex_of(0xaa, 47)); - assert_rejects(input, "trusted_expected_measurement must be 48 bytes"); - - let mut input = valid_input(); - input.trusted_expected_measurement = Some(hex_of(0xbb, 48)); - assert_rejects(input, "amd sev-snp measurement mismatch"); - } } From 5755441313619c5601228a4edf64ba6278ec44bc Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Tue, 26 May 2026 14:41:47 -0700 Subject: [PATCH 08/67] fix: add sev-snp boot info helper --- kms/src/main_service/amd_attest.rs | 257 ++++++++++++++++++++++++++++- 1 file changed, 255 insertions(+), 2 deletions(-) diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs index 2c7b5b4eb..29fa9463e 100644 --- a/kms/src/main_service/amd_attest.rs +++ b/kms/src/main_service/amd_attest.rs @@ -18,17 +18,21 @@ #![allow(dead_code)] use anyhow::{bail, Context, Result}; +use ra_tls::attestation::AttestationMode; use sha2::{Digest, Sha256, Sha384}; use std::fs; use crate::config::SevSnpMeasureConfig; +use super::upgrade_authority::BootInfo; + const LD_BYTES: usize = 48; const ZEROS_LD: [u8; LD_BYTES] = [0u8; LD_BYTES]; const MAX_VCPUS: u32 = 512; const MAX_OVMF_SECTIONS: usize = 64; -const MAX_OVMF_METADATA_PAGES: u64 = 16_777_216; // 64 GiB worth of 4 KiB pages. - // VMSA page GPA: (u64)(-1) page-aligned, bits >51 cleared. +/// 64 GiB worth of 4 KiB pages. +const MAX_OVMF_METADATA_PAGES: u64 = 16_777_216; +// VMSA page GPA: (u64)(-1) page-aligned, bits >51 cleared. const VMSA_GPA: u64 = 0x0000_FFFF_FFFF_F000; #[derive(Debug, Clone, PartialEq, Eq)] @@ -87,6 +91,153 @@ pub(crate) fn validate_amd_snp_measurement_binding( Ok(()) } +/// Builds a deterministic authorization `BootInfo` for an already-verified AMD +/// SEV-SNP report without wiring it into KMS key release. +/// +/// This helper first recomputes and validates the QEMU SNP launch measurement. +/// `mr_aggregated` is the hardware-verified 48-byte SNP `MEASUREMENT`, and +/// `device_id` is the hardware-verified 64-byte SNP `chip_id`. `app_id` is +/// decoded from the request input for authorization policy, but it is not +/// directly hardware-measured by the current QEMU SNP launch measurement model. +/// `compose_hash` and `rootfs_hash` are bound through the measured kernel +/// command line; `os_image_hash` is therefore represented by `rootfs_hash`. +/// +/// Authorization-specific digests are domain separated and deterministic: +/// * `mr_system = sha256("dstack-amd-sev-snp:mr-system:v1" || launch/system inputs)` +/// * `key_provider_info = sha256("dstack-amd-sev-snp:app-binding:v1" || mr_system || app_id || compose_hash || chip_id)` +/// * `instance_id = sha256("dstack-amd-sev-snp:instance-id:v1" || chip_id || measurement || app_id || compose_hash)` +/// +/// Keeping these as helper-only values lets future authorization policy inspect +/// exactly which SNP-specific inputs were bound while SNP key release remains +/// fail-closed unless an explicit release path is added separately. +pub(crate) fn build_amd_snp_boot_info( + config: &SevSnpMeasureConfig, + verified_measurement: &[u8; 48], + verified_chip_id: &[u8; 64], + input: &MeasurementInput, +) -> Result { + validate_amd_snp_measurement_binding(Some(config), verified_measurement, input)?; + + let app_id = decode_required_hex("app_id", &input.app_id, 20)?; + let compose_hash = decode_required_hex("compose_hash", &input.compose_hash, 32)?; + let rootfs_hash = decode_required_hex("rootfs_hash", &input.rootfs_hash, 32)?; + let mr_system = snp_mr_system_digest(config, verified_measurement, input)?; + let key_provider_info = snp_app_binding_digest( + &mr_system, + &app_id, + &compose_hash, + verified_chip_id.as_slice(), + ); + let instance_id = snp_instance_id_digest( + verified_chip_id.as_slice(), + verified_measurement, + &app_id, + &compose_hash, + ); + + Ok(BootInfo { + attestation_mode: AttestationMode::DstackAmdSevSnp, + mr_aggregated: verified_measurement.to_vec(), + os_image_hash: rootfs_hash, + mr_system, + app_id, + compose_hash, + instance_id, + device_id: verified_chip_id.to_vec(), + key_provider_info, + tcb_status: "snp-verified-basic-policy".to_string(), + advisory_ids: Vec::new(), + }) +} + +fn snp_mr_system_digest( + config: &SevSnpMeasureConfig, + verified_measurement: &[u8; 48], + input: &MeasurementInput, +) -> Result> { + let ovmf_hash = decode_optional_hex("ovmf_hash", &input.ovmf_hash, 48)?; + let kernel_hash = decode_required_hex("kernel_hash", &input.kernel_hash, 32)?; + let initrd_hash = if input.initrd_hash.is_empty() { + Sha256::digest(b"").to_vec() + } else { + decode_required_hex("initrd_hash", &input.initrd_hash, 32)? + }; + let rootfs_hash = decode_required_hex("rootfs_hash", &input.rootfs_hash, 32)?; + let docker_files_hash = input + .docker_files_hash + .as_deref() + .map(|value| decode_required_hex("docker_files_hash", value, 32)) + .transpose()?; + let vcpu_type = input + .vcpu_type + .as_deref() + .ok_or_else(|| anyhow::anyhow!("vcpu_type is required"))?; + + let mut h = Sha256::new(); + h.update(b"dstack-amd-sev-snp:mr-system:v1"); + h.update(verified_measurement); + h.update(len_prefixed(&ovmf_hash)); + h.update(kernel_hash); + h.update(initrd_hash); + h.update(rootfs_hash); + h.update(input.vcpus.to_le_bytes()); + h.update(len_prefixed(vcpu_type.as_bytes())); + h.update(config.guest_features.to_le_bytes()); + match docker_files_hash { + Some(value) => { + h.update([1]); + h.update(value); + } + None => h.update([0]), + } + h.update(input.sev_hashes_table_gpa.to_le_bytes()); + h.update(input.sev_es_reset_eip.to_le_bytes()); + h.update((input.ovmf_sections.len() as u64).to_le_bytes()); + for section in &input.ovmf_sections { + h.update(section.gpa.to_le_bytes()); + h.update(section.size.to_le_bytes()); + h.update(section.section_type.to_le_bytes()); + } + Ok(h.finalize().to_vec()) +} + +fn snp_app_binding_digest( + mr_system: &[u8], + app_id: &[u8], + compose_hash: &[u8], + chip_id: &[u8], +) -> Vec { + let mut h = Sha256::new(); + h.update(b"dstack-amd-sev-snp:app-binding:v1"); + h.update(mr_system); + h.update(app_id); + h.update(compose_hash); + h.update(chip_id); + h.finalize().to_vec() +} + +fn snp_instance_id_digest( + chip_id: &[u8], + measurement: &[u8], + app_id: &[u8], + compose_hash: &[u8], +) -> Vec { + let mut h = Sha256::new(); + h.update(b"dstack-amd-sev-snp:instance-id:v1"); + h.update(chip_id); + h.update(measurement); + h.update(app_id); + h.update(compose_hash); + h.finalize().to_vec() +} + +fn len_prefixed(bytes: &[u8]) -> Vec { + let mut out = Vec::with_capacity(8 + bytes.len()); + out.extend_from_slice(&(bytes.len() as u64).to_le_bytes()); + out.extend_from_slice(bytes); + out +} + fn validate_measurement_input( config: &SevSnpMeasureConfig, input: &MeasurementInput, @@ -808,6 +959,108 @@ mod tests { assert!(err.to_string().contains("amd sev-snp measurement mismatch")); } + #[test] + fn builds_snp_boot_info_for_matching_measurement_only() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let chip_id = [0xab; 64]; + + let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input) + .expect("matching measurement should build snp boot info"); + assert_eq!(boot_info.attestation_mode, AttestationMode::DstackAmdSevSnp); + assert_eq!(boot_info.mr_aggregated, verified.to_vec()); + assert_eq!(boot_info.device_id, chip_id.to_vec()); + assert_eq!(boot_info.app_id, vec![0x11; 20]); + assert_eq!(boot_info.compose_hash, vec![0x22; 32]); + assert_eq!(boot_info.os_image_hash, vec![0x33; 32]); + assert_eq!(boot_info.mr_system.len(), 32); + assert_eq!(boot_info.key_provider_info.len(), 32); + assert_eq!(boot_info.instance_id.len(), 32); + assert_eq!(boot_info.tcb_status, "snp-verified-basic-policy"); + assert!(boot_info.advisory_ids.is_empty()); + + let mut mismatched = verified; + mismatched[0] ^= 0xff; + let err = build_amd_snp_boot_info(&config(), &mismatched, &chip_id, &input) + .expect_err("mismatched measurement must not build boot info"); + assert!(err.to_string().contains("amd sev-snp measurement mismatch")); + } + + #[test] + fn app_id_changes_authorization_binding_not_launch_measurement() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let chip_id = [0xcd; 64]; + let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input).unwrap(); + + let mut changed = input.clone(); + changed.app_id = hex_of(0x12, 20); + let changed_measurement = compute_expected_measurement(&config(), &changed).unwrap(); + assert_eq!( + changed_measurement, verified, + "app_id is authorization input, not qemu snp launch-measured input" + ); + let changed_boot_info = + build_amd_snp_boot_info(&config(), &verified, &chip_id, &changed).unwrap(); + + assert_ne!(boot_info.app_id, changed_boot_info.app_id); + assert_ne!(boot_info.instance_id, changed_boot_info.instance_id); + assert_ne!( + boot_info.key_provider_info, + changed_boot_info.key_provider_info + ); + assert_eq!(boot_info.mr_aggregated, changed_boot_info.mr_aggregated); + assert_eq!(boot_info.mr_system, changed_boot_info.mr_system); + } + + #[test] + fn measured_input_changes_reject_until_measurement_is_recomputed() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let chip_id = [0xef; 64]; + let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input).unwrap(); + + for mutate in [ + |i: &mut MeasurementInput| i.compose_hash = hex_of(0x23, 32), + |i: &mut MeasurementInput| i.rootfs_hash = hex_of(0x34, 32), + |i: &mut MeasurementInput| i.kernel_hash = hex_of(0x56, 32), + |i: &mut MeasurementInput| i.vcpus = 3, + ] { + let mut changed = input.clone(); + mutate(&mut changed); + let err = build_amd_snp_boot_info(&config(), &verified, &chip_id, &changed) + .expect_err("stale verified measurement must reject changed measured input"); + assert!(err.to_string().contains("amd sev-snp measurement mismatch")); + + let changed_verified = compute_expected_measurement(&config(), &changed).unwrap(); + let changed_boot_info = + build_amd_snp_boot_info(&config(), &changed_verified, &chip_id, &changed) + .expect("recomputed measurement should build boot info"); + assert_ne!(boot_info.mr_aggregated, changed_boot_info.mr_aggregated); + assert_ne!(boot_info.mr_system, changed_boot_info.mr_system); + } + } + + #[test] + fn chip_id_maps_to_device_id_and_changes_chip_bound_digests() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let boot_info = build_amd_snp_boot_info(&config(), &verified, &[0x01; 64], &input).unwrap(); + let changed_boot_info = + build_amd_snp_boot_info(&config(), &verified, &[0x02; 64], &input).unwrap(); + + assert_eq!(boot_info.device_id, vec![0x01; 64]); + assert_eq!(changed_boot_info.device_id, vec![0x02; 64]); + assert_ne!(boot_info.device_id, changed_boot_info.device_id); + assert_ne!(boot_info.instance_id, changed_boot_info.instance_id); + assert_ne!( + boot_info.key_provider_info, + changed_boot_info.key_provider_info + ); + assert_eq!(boot_info.mr_aggregated, changed_boot_info.mr_aggregated); + assert_eq!(boot_info.mr_system, changed_boot_info.mr_system); + } + #[test] fn rejects_missing_config() { let verified = [0xaa; 48]; From 4b31c97fa6b5a43cd691c40c63fe6ac5c7398e03 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Tue, 26 May 2026 14:51:53 -0700 Subject: [PATCH 09/67] test: add sev-snp measurement golden vector --- kms/src/main_service/amd_attest.rs | 78 ++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs index 29fa9463e..92e5ae06d 100644 --- a/kms/src/main_service/amd_attest.rs +++ b/kms/src/main_service/amd_attest.rs @@ -1061,6 +1061,84 @@ mod tests { assert_eq!(boot_info.mr_system, changed_boot_info.mr_system); } + #[test] + #[ignore = "requires sev-snp-measure and an SNP-capable OVMF binary"] + fn recomputation_matches_sev_snp_measure_live_golden_vector() { + let ovmf_path = std::env::var("DSTACK_SEV_SNP_GOLDEN_OVMF") + .unwrap_or_else(|_| "/opt/AMDSEV/usr/local/share/qemu/OVMF.fd".to_string()); + assert!( + std::path::Path::new(&ovmf_path).exists(), + "set DSTACK_SEV_SNP_GOLDEN_OVMF to an SNP-capable OVMF binary" + ); + + let dir = tempfile::tempdir().expect("tempdir should be available"); + let kernel_path = dir.path().join("kernel.bin"); + let initrd_path = dir.path().join("initrd.bin"); + let kernel_bytes = b"golden-kernel-for-dstack-sev-snp-measure\n"; + let initrd_bytes = b"golden-initrd-for-dstack-sev-snp-measure\n"; + std::fs::write(&kernel_path, kernel_bytes).expect("kernel fixture should be written"); + std::fs::write(&initrd_path, initrd_bytes).expect("initrd fixture should be written"); + + let kernel_hash = hex::encode(Sha256::digest(kernel_bytes)); + let initrd_hash = hex::encode(Sha256::digest(initrd_bytes)); + let mut input = valid_input(); + input.docker_files_hash = None; + input.ovmf_hash.clear(); + input.ovmf_sections.clear(); + input.kernel_hash = kernel_hash; + input.initrd_hash = initrd_hash; + input.vcpus = 2; + input.vcpu_type = Some("EPYC-v4".to_string()); + + let config = config_with_path(&ovmf_path); + let recomputed = compute_expected_measurement(&config, &input) + .expect("dstack recomputation should succeed"); + + let append = format!( + "console=ttyS0 loglevel=7 docker_compose_hash={} rootfs_hash={}", + input.compose_hash, input.rootfs_hash + ); + let output = std::process::Command::new("sev-snp-measure") + .args([ + "--mode", + "snp", + "--vcpus", + "2", + "--vcpu-type", + "EPYC-v4", + "--ovmf", + &ovmf_path, + "--kernel", + kernel_path.to_str().unwrap(), + "--initrd", + initrd_path.to_str().unwrap(), + "--append", + &append, + "--guest-features", + "0x1", + "--output-format", + "hex", + ]) + .output() + .expect("sev-snp-measure should be installed"); + assert!( + output.status.success(), + "sev-snp-measure failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let tool_measurement = String::from_utf8(output.stdout) + .expect("sev-snp-measure output should be utf8") + .trim() + .to_string(); + + assert_eq!(hex::encode(recomputed), tool_measurement); + assert_eq!( + tool_measurement, + "859c646870cffdb4620077c20ea81702c1bd0bde9c967887ddbd430ebe31a89d2832a442b8d8d83e4bdd70b52bb3f009", + "live sev-snp-measure golden vector should not drift silently" + ); + } + #[test] fn rejects_missing_config() { let verified = [0xaa; 48]; From 7dda8ff03d0db61c44ef9dfa56d8d4adcc11f0e7 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Tue, 26 May 2026 14:56:01 -0700 Subject: [PATCH 10/67] fix: add sev-snp auth policy helper --- kms/src/main_service/amd_attest.rs | 189 ++++++++++++++++++++++ kms/src/main_service/upgrade_authority.rs | 2 +- 2 files changed, 190 insertions(+), 1 deletion(-) diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs index 92e5ae06d..2cf521a3a 100644 --- a/kms/src/main_service/amd_attest.rs +++ b/kms/src/main_service/amd_attest.rs @@ -150,6 +150,121 @@ pub(crate) fn build_amd_snp_boot_info( }) } +/// Explicit helper-only AMD SEV-SNP authorization policy. +/// +/// The current SNP work is intentionally not wired into key release. This type +/// makes the future release semantics auditable before any RPC/proto path is +/// added: an SNP `BootInfo` must match allowlisted hardware measurement, +/// app/config identity, device identity, and TCB/advisory policy. Empty +/// allowlists fail closed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct AmdSnpAuthPolicy { + pub allowed_measurements: Vec>, + pub allowed_app_ids: Vec>, + pub allowed_compose_hashes: Vec>, + pub allowed_os_image_hashes: Vec>, + pub allowed_device_ids: Vec>, + pub allowed_tcb_statuses: Vec, + pub allowed_advisory_ids: Vec, +} + +impl AmdSnpAuthPolicy { + /// Build a narrow exact-match policy from an already verified SNP boot + /// identity. This is useful for tests and for future allowlist materializing + /// logic, but still does not release keys by itself. + pub(crate) fn from_boot_info(boot_info: &BootInfo) -> Result { + ensure_snp_boot_info_shape(boot_info)?; + Ok(Self { + allowed_measurements: vec![boot_info.mr_aggregated.clone()], + allowed_app_ids: vec![boot_info.app_id.clone()], + allowed_compose_hashes: vec![boot_info.compose_hash.clone()], + allowed_os_image_hashes: vec![boot_info.os_image_hash.clone()], + allowed_device_ids: vec![boot_info.device_id.clone()], + allowed_tcb_statuses: vec![boot_info.tcb_status.clone()], + allowed_advisory_ids: boot_info.advisory_ids.clone(), + }) + } +} + +pub(crate) fn validate_amd_snp_auth_policy( + boot_info: &BootInfo, + policy: &AmdSnpAuthPolicy, +) -> Result<()> { + ensure_snp_boot_info_shape(boot_info)?; + ensure_allowed_bytes( + "measurement", + &boot_info.mr_aggregated, + &policy.allowed_measurements, + )?; + ensure_allowed_bytes("app_id", &boot_info.app_id, &policy.allowed_app_ids)?; + ensure_allowed_bytes( + "compose_hash", + &boot_info.compose_hash, + &policy.allowed_compose_hashes, + )?; + ensure_allowed_bytes( + "os_image_hash", + &boot_info.os_image_hash, + &policy.allowed_os_image_hashes, + )?; + ensure_allowed_bytes( + "device_id", + &boot_info.device_id, + &policy.allowed_device_ids, + )?; + ensure_allowed_string( + "tcb_status", + &boot_info.tcb_status, + &policy.allowed_tcb_statuses, + )?; + for advisory_id in &boot_info.advisory_ids { + ensure_allowed_string("advisory_id", advisory_id, &policy.allowed_advisory_ids)?; + } + Ok(()) +} + +fn ensure_snp_boot_info_shape(boot_info: &BootInfo) -> Result<()> { + if boot_info.attestation_mode != AttestationMode::DstackAmdSevSnp { + bail!("attestation mode is not amd sev-snp"); + } + ensure_len("measurement", &boot_info.mr_aggregated, 48)?; + ensure_len("app_id", &boot_info.app_id, 20)?; + ensure_len("compose_hash", &boot_info.compose_hash, 32)?; + ensure_len("os_image_hash", &boot_info.os_image_hash, 32)?; + ensure_len("device_id", &boot_info.device_id, 64)?; + ensure_len("mr_system", &boot_info.mr_system, 32)?; + ensure_len("key_provider_info", &boot_info.key_provider_info, 32)?; + ensure_len("instance_id", &boot_info.instance_id, 32)?; + if boot_info.tcb_status.trim().is_empty() { + bail!("tcb_status is not allowed"); + } + Ok(()) +} + +fn ensure_len(name: &str, value: &[u8], expected_len: usize) -> Result<()> { + if value.len() != expected_len { + bail!("{name} must be {expected_len} bytes"); + } + Ok(()) +} + +fn ensure_allowed_bytes(name: &str, value: &[u8], allowed: &[Vec]) -> Result<()> { + if allowed + .iter() + .any(|candidate| candidate.as_slice() == value) + { + return Ok(()); + } + bail!("{name} is not allowed") +} + +fn ensure_allowed_string(name: &str, value: &str, allowed: &[String]) -> Result<()> { + if allowed.iter().any(|candidate| candidate == value) { + return Ok(()); + } + bail!("{name} is not allowed") +} + fn snp_mr_system_digest( config: &SevSnpMeasureConfig, verified_measurement: &[u8; 48], @@ -1139,6 +1254,80 @@ mod tests { ); } + #[test] + fn explicit_snp_auth_policy_accepts_only_exact_verified_identity() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let chip_id = [0x42; 64]; + let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input).unwrap(); + let policy = AmdSnpAuthPolicy::from_boot_info(&boot_info) + .expect("boot info should produce an exact SNP auth policy"); + + validate_amd_snp_auth_policy(&boot_info, &policy) + .expect("exact verified SNP identity should satisfy policy"); + + let mut changed = boot_info; + changed.compose_hash[0] ^= 0xff; + let err = validate_amd_snp_auth_policy(&changed, &policy) + .expect_err("compose hash mismatch must reject"); + assert!(err.to_string().contains("compose_hash is not allowed")); + } + + #[test] + fn explicit_snp_auth_policy_rejects_incomplete_or_unsafe_tcb() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let chip_id = [0x24; 64]; + let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input).unwrap(); + let policy = AmdSnpAuthPolicy::from_boot_info(&boot_info).unwrap(); + + let mut wrong_mode = boot_info.clone(); + wrong_mode.attestation_mode = AttestationMode::DstackTdx; + let err = validate_amd_snp_auth_policy(&wrong_mode, &policy) + .expect_err("non-SNP mode must reject"); + assert!(err + .to_string() + .contains("attestation mode is not amd sev-snp")); + + let mut wrong_status = boot_info.clone(); + wrong_status.tcb_status = "UpToDate".to_string(); + let err = validate_amd_snp_auth_policy(&wrong_status, &policy) + .expect_err("unexpected tcb status must reject"); + assert!(err.to_string().contains("tcb_status is not allowed")); + + let mut advisory = boot_info.clone(); + advisory.advisory_ids.push("SNP-TEST-ADVISORY".to_string()); + let err = validate_amd_snp_auth_policy(&advisory, &policy) + .expect_err("unexpected advisory must reject by default"); + assert!(err.to_string().contains("advisory_id is not allowed")); + } + + #[test] + fn explicit_snp_auth_policy_rejects_partial_allowlists() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let chip_id = [0x35; 64]; + let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input).unwrap(); + + for mutate in [ + |p: &mut AmdSnpAuthPolicy| p.allowed_measurements.clear(), + |p: &mut AmdSnpAuthPolicy| p.allowed_app_ids.clear(), + |p: &mut AmdSnpAuthPolicy| p.allowed_compose_hashes.clear(), + |p: &mut AmdSnpAuthPolicy| p.allowed_os_image_hashes.clear(), + |p: &mut AmdSnpAuthPolicy| p.allowed_device_ids.clear(), + |p: &mut AmdSnpAuthPolicy| p.allowed_tcb_statuses.clear(), + ] { + let mut policy = AmdSnpAuthPolicy::from_boot_info(&boot_info).unwrap(); + mutate(&mut policy); + let err = validate_amd_snp_auth_policy(&boot_info, &policy) + .expect_err("partial SNP policy allowlist must reject"); + assert!( + err.to_string().contains("is not allowed"), + "unexpected error: {err:?}" + ); + } + } + #[test] fn rejects_missing_config() { let verified = [0xaa; 48]; diff --git a/kms/src/main_service/upgrade_authority.rs b/kms/src/main_service/upgrade_authority.rs index 9e1649980..622507768 100644 --- a/kms/src/main_service/upgrade_authority.rs +++ b/kms/src/main_service/upgrade_authority.rs @@ -15,7 +15,7 @@ use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use serde_human_bytes as hex_bytes; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct BootInfo { pub attestation_mode: AttestationMode, From 6c2d8178fef770ad62be60d419908b5962b8e634 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Thu, 28 May 2026 17:34:54 -0700 Subject: [PATCH 11/67] fix: bind sev-snp app id into measurement --- kms/src/main_service/amd_attest.rs | 28 ++++++++------- vmm/src/app/qemu.rs | 58 ++++++++++++++++++++++++++++-- 2 files changed, 71 insertions(+), 15 deletions(-) diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs index 2cf521a3a..0c748f8c4 100644 --- a/kms/src/main_service/amd_attest.rs +++ b/kms/src/main_service/amd_attest.rs @@ -842,8 +842,8 @@ pub(crate) fn compute_expected_measurement( .ok_or_else(|| anyhow::anyhow!("vcpu_type is required"))?; let mut cmdline = format!( - "console=ttyS0 loglevel=7 docker_compose_hash={} rootfs_hash={}", - input.compose_hash, input.rootfs_hash + "console=ttyS0 loglevel=7 docker_compose_hash={} rootfs_hash={} app_id={}", + input.compose_hash, input.rootfs_hash, input.app_id ); if let Some(docker_files_hash) = input.docker_files_hash.as_deref() { cmdline.push_str(&format!( @@ -1061,7 +1061,7 @@ mod tests { let expected = compute_expected_measurement(&config(), &input).unwrap(); assert_eq!( hex::encode(expected), - "2c598d7885c7a61e44c048ac5594600ce906339a8a6a3c0fa0d81c67275d55273d38d6ec5afcf5ff1d74621f0f0354d9", + "4753950048f296ea9cc36be3ba3e26f9cb014411188134d2ea40580a76edf277268cc46b67dfd213d1a7dfc9a9006e0f", "synthetic measurement vector should not drift silently" ); validate_amd_snp_measurement_binding(Some(&config()), &expected, &input) @@ -1102,7 +1102,7 @@ mod tests { } #[test] - fn app_id_changes_authorization_binding_not_launch_measurement() { + fn app_id_changes_launch_measurement_and_authorization_binding() { let input = valid_input(); let verified = compute_expected_measurement(&config(), &input).unwrap(); let chip_id = [0xcd; 64]; @@ -1111,12 +1111,16 @@ mod tests { let mut changed = input.clone(); changed.app_id = hex_of(0x12, 20); let changed_measurement = compute_expected_measurement(&config(), &changed).unwrap(); - assert_eq!( + assert_ne!( changed_measurement, verified, - "app_id is authorization input, not qemu snp launch-measured input" + "app_id must be launch-measured for SNP to match TDX app identity semantics" ); + let err = build_amd_snp_boot_info(&config(), &verified, &chip_id, &changed) + .expect_err("stale measurement must reject changed app_id"); + assert!(err.to_string().contains("amd sev-snp measurement mismatch")); + let changed_boot_info = - build_amd_snp_boot_info(&config(), &verified, &chip_id, &changed).unwrap(); + build_amd_snp_boot_info(&config(), &changed_measurement, &chip_id, &changed).unwrap(); assert_ne!(boot_info.app_id, changed_boot_info.app_id); assert_ne!(boot_info.instance_id, changed_boot_info.instance_id); @@ -1124,8 +1128,8 @@ mod tests { boot_info.key_provider_info, changed_boot_info.key_provider_info ); - assert_eq!(boot_info.mr_aggregated, changed_boot_info.mr_aggregated); - assert_eq!(boot_info.mr_system, changed_boot_info.mr_system); + assert_ne!(boot_info.mr_aggregated, changed_boot_info.mr_aggregated); + assert_ne!(boot_info.mr_system, changed_boot_info.mr_system); } #[test] @@ -1210,8 +1214,8 @@ mod tests { .expect("dstack recomputation should succeed"); let append = format!( - "console=ttyS0 loglevel=7 docker_compose_hash={} rootfs_hash={}", - input.compose_hash, input.rootfs_hash + "console=ttyS0 loglevel=7 docker_compose_hash={} rootfs_hash={} app_id={}", + input.compose_hash, input.rootfs_hash, input.app_id ); let output = std::process::Command::new("sev-snp-measure") .args([ @@ -1249,7 +1253,7 @@ mod tests { assert_eq!(hex::encode(recomputed), tool_measurement); assert_eq!( tool_measurement, - "859c646870cffdb4620077c20ea81702c1bd0bde9c967887ddbd430ebe31a89d2832a442b8d8d83e4bdd70b52bb3f009", + "6497fb9f90dc4a322228a8a5eb14742e09067bc44c184c2068d583ef628b5bae8c6cf15d91fe1bc0b7a8cbcc575be370", "live sev-snp-measure golden vector should not drift silently" ); } diff --git a/vmm/src/app/qemu.rs b/vmm/src/app/qemu.rs index 876e14ab2..388de20bd 100644 --- a/vmm/src/app/qemu.rs +++ b/vmm/src/app/qemu.rs @@ -369,7 +369,7 @@ impl VmState { #[cfg(test)] mod tests { - use super::{amd_sev_snp_memory_backend_arg, sanitize_optional}; + use super::{amd_sev_snp_measured_cmdline, amd_sev_snp_memory_backend_arg, sanitize_optional}; #[test] fn sanitize_optional_filters_empty_owned_values() { @@ -398,6 +398,19 @@ mod tests { "memory-backend-memfd,id=ram1,size=4096M,share=true,prealloc=false" ); } + + #[test] + fn amd_sev_snp_measured_cmdline_binds_app_identity() { + assert_eq!( + amd_sev_snp_measured_cmdline( + "console=ttyS0 loglevel=7", + "22", + "33", + "1111111111111111111111111111111111111111" + ), + "console=ttyS0 loglevel=7 docker_compose_hash=22 rootfs_hash=33 app_id=1111111111111111111111111111111111111111" + ); + } } impl VmConfig { @@ -737,8 +750,32 @@ impl VmConfig { } } - // Add kernel command line - if let Some(cmdline) = &self.image.info.cmdline { + // Add kernel command line. SNP launch measurement includes app identity + // through the measured QEMU kernel command line, matching TDX's + // app-id-in-measured-identity semantics without relying on post-launch + // RTMRs (which SNP does not have). + let cmdline = match (&self.image.info.cmdline, cfg.platform.resolve()) { + (Some(cmdline), TeePlatform::AmdSevSnp) if !self.manifest.no_tee => { + let compose_hash = hex::encode( + workdir + .app_compose_hash() + .context("Failed to get compose hash")?, + ); + let rootfs_hash = + self.image.info.rootfs_hash.as_deref().ok_or_else(|| { + anyhow::anyhow!("rootfs_hash is required for amd sev-snp") + })?; + Some(amd_sev_snp_measured_cmdline( + cmdline, + &compose_hash, + rootfs_hash, + &self.manifest.app_id, + )) + } + (Some(cmdline), _) => Some(cmdline.clone()), + (None, _) => None, + }; + if let Some(cmdline) = cmdline { command.arg("-append").arg(cmdline); } @@ -969,6 +1006,21 @@ fn amd_sev_snp_memory_backend_arg(mem: u32) -> String { format!("memory-backend-memfd,id=ram1,size={mem}M,share=true,prealloc=false") } +fn amd_sev_snp_measured_cmdline( + base_cmdline: &str, + compose_hash: &str, + rootfs_hash: &str, + app_id: &str, +) -> String { + format!( + "{} docker_compose_hash={} rootfs_hash={} app_id={}", + base_cmdline.trim(), + compose_hash, + rootfs_hash, + app_id + ) +} + /// Round up a value to the nearest multiple of another value. /// If the value is already a multiple, it remains unchanged. fn round_up(value: u32, multiple: u32) -> u32 { From be054d8b4eb61759c955f39ccb191199dba92e7d Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Thu, 28 May 2026 18:04:40 -0700 Subject: [PATCH 12/67] fix: connect sev-snp verified attestation to boot info --- Cargo.lock | 1 + kms/Cargo.toml | 3 + kms/src/main_service/amd_attest.rs | 109 +++++++++++++++++++++++++---- 3 files changed, 101 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8d98bc018..413779e42 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2518,6 +2518,7 @@ dependencies = [ "anyhow", "chrono", "clap", + "dstack-attest", "dstack-guest-agent-rpc", "dstack-kms-rpc", "dstack-mr", diff --git a/kms/Cargo.toml b/kms/Cargo.toml index bc33bc6a7..b59d31118 100644 --- a/kms/Cargo.toml +++ b/kms/Cargo.toml @@ -48,5 +48,8 @@ serde-duration.workspace = true dstack-verifier = { workspace = true, default-features = false } dstack-mr.workspace = true +[dev-dependencies] +dstack-attest.workspace = true + [features] default = [] diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs index 0c748f8c4..b2273e335 100644 --- a/kms/src/main_service/amd_attest.rs +++ b/kms/src/main_service/amd_attest.rs @@ -10,15 +10,15 @@ //! value to the hardware-verified report measurement. //! //! Important: this is launch measurement binding, not a complete authorization -//! decision. `app_id` is carried and validated so future SNP `BootInfo` -//! construction can consume the same input, but the current QEMU SNP launch -//! measurement does not independently bind `app_id`. Do not use this helper by -//! itself to release app keys. +//! decision. `app_id`, compose hash, and rootfs hash are included in the SNP +//! measured kernel command line so the recomputed launch `MEASUREMENT` changes +//! with app identity, matching dstack's TDX measured-identity semantics. Do not +//! use this helper by itself to release app keys. #![allow(dead_code)] use anyhow::{bail, Context, Result}; -use ra_tls::attestation::AttestationMode; +use ra_tls::attestation::{AttestationMode, VerifiedAttestation}; use sha2::{Digest, Sha256, Sha384}; use std::fs; @@ -47,9 +47,8 @@ pub(crate) struct OvmfSectionParam { #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct MeasurementInput { - /// 20-byte app identity. It is validated now even though the current SNP - /// launch measurement does not include it directly; callers must bind this - /// through the future SNP `BootInfo`/authorization path before key release. + /// 20-byte app identity included in the measured kernel cmdline for SNP, + /// matching TDX's app-id-in-measured-identity semantics. pub app_id: String, /// 32-byte docker compose hash included in the measured kernel cmdline. pub compose_hash: String, @@ -96,10 +95,8 @@ pub(crate) fn validate_amd_snp_measurement_binding( /// /// This helper first recomputes and validates the QEMU SNP launch measurement. /// `mr_aggregated` is the hardware-verified 48-byte SNP `MEASUREMENT`, and -/// `device_id` is the hardware-verified 64-byte SNP `chip_id`. `app_id` is -/// decoded from the request input for authorization policy, but it is not -/// directly hardware-measured by the current QEMU SNP launch measurement model. -/// `compose_hash` and `rootfs_hash` are bound through the measured kernel +/// `device_id` is the hardware-verified 64-byte SNP `chip_id`. `app_id`, +/// `compose_hash`, and `rootfs_hash` are bound through the measured kernel /// command line; `os_image_hash` is therefore represented by `rootfs_hash`. /// /// Authorization-specific digests are domain separated and deterministic: @@ -150,6 +147,25 @@ pub(crate) fn build_amd_snp_boot_info( }) } +/// Extracts the verified AMD SEV-SNP report from a verified attestation and +/// materializes the helper-only SNP `BootInfo` used by future authorization. +/// +/// This is the safe integration seam: the attestation verifier has already +/// checked the report signature/collateral/report_data, while this KMS helper +/// recomputes the launch measurement from trusted config and request inputs. +/// It still does not release keys or expose an AMD key-release RPC. +pub(crate) fn build_amd_snp_boot_info_from_verified_attestation( + config: &SevSnpMeasureConfig, + attestation: &VerifiedAttestation, + input: &MeasurementInput, +) -> Result { + let verified = attestation + .report + .amd_snp_report() + .ok_or_else(|| anyhow::anyhow!("verified attestation is not amd sev-snp"))?; + build_amd_snp_boot_info(config, &verified.measurement, &verified.chip_id, input) +} + /// Explicit helper-only AMD SEV-SNP authorization policy. /// /// The current SNP work is intentionally not wired into key release. This type @@ -1101,6 +1117,75 @@ mod tests { assert!(err.to_string().contains("amd sev-snp measurement mismatch")); } + #[test] + fn builds_snp_boot_info_from_verified_attestation_report() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let chip_id = [0xab; 64]; + let attestation = ra_tls::attestation::VerifiedAttestation { + quote: ra_tls::attestation::AttestationQuote::DstackAmdSevSnp( + ra_tls::attestation::SnpQuote { + report: Vec::new(), + cert_chain: Vec::new(), + }, + ), + runtime_events: Vec::new(), + report_data: [0x42; 64], + config: String::new(), + report: ra_tls::attestation::DstackVerifiedReport::DstackAmdSevSnp( + dstack_attest::amd_sev_snp::VerifiedAmdSnpReport { + measurement: verified, + report_data: [0x42; 64], + chip_id, + }, + ), + }; + + let boot_info = + build_amd_snp_boot_info_from_verified_attestation(&config(), &attestation, &input) + .expect("verified snp attestation should feed boot info helper"); + + assert_eq!(boot_info.mr_aggregated, verified.to_vec()); + assert_eq!(boot_info.device_id, chip_id.to_vec()); + assert_eq!(boot_info.app_id, vec![0x11; 20]); + } + + #[test] + fn verified_attestation_helper_rejects_non_snp_reports() { + let input = valid_input(); + let attestation = ra_tls::attestation::VerifiedAttestation { + quote: ra_tls::attestation::AttestationQuote::DstackNitroEnclave( + ra_tls::attestation::DstackNitroQuote { + nsm_quote: Vec::new(), + }, + ), + runtime_events: Vec::new(), + report_data: [0x42; 64], + config: String::new(), + report: ra_tls::attestation::DstackVerifiedReport::DstackNitroEnclave( + ra_tls::attestation::NitroVerifiedReport { + module_id: String::new(), + pcrs: ra_tls::attestation::NitroPcrs { + pcr0: Vec::new(), + pcr1: Vec::new(), + pcr2: Vec::new(), + }, + user_data: Vec::new(), + timestamp: 0, + }, + ), + }; + + let err = + build_amd_snp_boot_info_from_verified_attestation(&config(), &attestation, &input) + .expect_err("non-snp verified attestation must reject"); + assert!( + err.to_string() + .contains("verified attestation is not amd sev-snp"), + "unexpected error: {err:?}" + ); + } + #[test] fn app_id_changes_launch_measurement_and_authorization_binding() { let input = valid_input(); From ace87537882d611cbe4d8c6c1e35e6c6939363b7 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Thu, 28 May 2026 18:14:28 -0700 Subject: [PATCH 13/67] fix: parse sev-snp measurement inputs from vm config --- kms/src/main_service/amd_attest.rs | 194 ++++++++++++++++++++++++++++- 1 file changed, 192 insertions(+), 2 deletions(-) diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs index b2273e335..d46ce14ae 100644 --- a/kms/src/main_service/amd_attest.rs +++ b/kms/src/main_service/amd_attest.rs @@ -35,7 +35,9 @@ const MAX_OVMF_METADATA_PAGES: u64 = 16_777_216; // VMSA page GPA: (u64)(-1) page-aligned, bits >51 cleared. const VMSA_GPA: u64 = 0x0000_FFFF_FFFF_F000; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)] +#[cfg_attr(test, derive(serde::Serialize))] +#[serde(deny_unknown_fields)] pub(crate) struct OvmfSectionParam { pub gpa: u64, pub size: u64, @@ -45,7 +47,9 @@ pub(crate) struct OvmfSectionParam { pub section_type: u32, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)] +#[cfg_attr(test, derive(serde::Serialize))] +#[serde(deny_unknown_fields)] pub(crate) struct MeasurementInput { /// 20-byte app identity included in the measured kernel cmdline for SNP, /// matching TDX's app-id-in-measured-identity semantics. @@ -71,9 +75,49 @@ pub(crate) struct MeasurementInput { pub sev_es_reset_eip: u32, pub vcpus: u32, pub vcpu_type: Option, + #[serde(deserialize_with = "deserialize_ovmf_sections_bounded")] pub ovmf_sections: Vec, } +fn deserialize_ovmf_sections_bounded<'de, D>( + deserializer: D, +) -> std::result::Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + struct BoundedOvmfSections; + + impl<'de> serde::de::Visitor<'de> for BoundedOvmfSections { + type Value = Vec; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "at most {MAX_OVMF_SECTIONS} OVMF metadata sections" + ) + } + + fn visit_seq(self, mut seq: A) -> std::result::Result, A::Error> + where + A: serde::de::SeqAccess<'de>, + { + let mut sections = + Vec::with_capacity(seq.size_hint().unwrap_or(0).min(MAX_OVMF_SECTIONS)); + while let Some(section) = seq.next_element()? { + if sections.len() >= MAX_OVMF_SECTIONS { + return Err(serde::de::Error::custom(format!( + "ovmf section count must not exceed {MAX_OVMF_SECTIONS}" + ))); + } + sections.push(section); + } + Ok(sections) + } + } + + deserializer.deserialize_seq(BoundedOvmfSections) +} + pub(crate) fn validate_amd_snp_measurement_binding( config: Option<&SevSnpMeasureConfig>, verified_measurement: &[u8; 48], @@ -166,6 +210,33 @@ pub(crate) fn build_amd_snp_boot_info_from_verified_attestation( build_amd_snp_boot_info(config, &verified.measurement, &verified.chip_id, input) } +#[derive(Debug, serde::Deserialize)] +struct SevSnpMeasurementVmConfig { + sev_snp_measurement: Option, +} + +/// Parses SNP launch-measurement inputs from the KMS request `vm_config` and +/// builds helper-only SNP `BootInfo` from an already verified attestation. +/// +/// The field is intentionally explicit (`sev_snp_measurement`) so missing SNP +/// launch inputs fail closed instead of falling back to TDX event-log decoding. +pub(crate) fn build_amd_snp_boot_info_from_verified_attestation_and_vm_config( + config: &SevSnpMeasureConfig, + attestation: &VerifiedAttestation, + vm_config: &str, +) -> Result { + let input = parse_measurement_input_from_vm_config(vm_config)?; + build_amd_snp_boot_info_from_verified_attestation(config, attestation, &input) +} + +fn parse_measurement_input_from_vm_config(vm_config: &str) -> Result { + let parsed: SevSnpMeasurementVmConfig = serde_json::from_str(vm_config) + .context("failed to parse vm_config for amd sev-snp measurement")?; + parsed + .sev_snp_measurement + .ok_or_else(|| anyhow::anyhow!("sev_snp_measurement is required for amd sev-snp")) +} + /// Explicit helper-only AMD SEV-SNP authorization policy. /// /// The current SNP work is intentionally not wired into key release. This type @@ -1150,6 +1221,125 @@ mod tests { assert_eq!(boot_info.app_id, vec![0x11; 20]); } + #[test] + fn builds_snp_boot_info_from_verified_attestation_and_vm_config_json() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let chip_id = [0xab; 64]; + let attestation = ra_tls::attestation::VerifiedAttestation { + quote: ra_tls::attestation::AttestationQuote::DstackAmdSevSnp( + ra_tls::attestation::SnpQuote { + report: Vec::new(), + cert_chain: Vec::new(), + }, + ), + runtime_events: Vec::new(), + report_data: [0x42; 64], + config: String::new(), + report: ra_tls::attestation::DstackVerifiedReport::DstackAmdSevSnp( + dstack_attest::amd_sev_snp::VerifiedAmdSnpReport { + measurement: verified, + report_data: [0x42; 64], + chip_id, + }, + ), + }; + let vm_config = serde_json::json!({ + "sev_snp_measurement": input, + }) + .to_string(); + + let boot_info = build_amd_snp_boot_info_from_verified_attestation_and_vm_config( + &config(), + &attestation, + &vm_config, + ) + .expect("vm_config-carried snp measurement inputs should build boot info"); + + assert_eq!(boot_info.mr_aggregated, verified.to_vec()); + assert_eq!(boot_info.device_id, chip_id.to_vec()); + assert_eq!(boot_info.app_id, vec![0x11; 20]); + } + + #[test] + fn verified_attestation_vm_config_helper_requires_snp_measurement_input() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let attestation = ra_tls::attestation::VerifiedAttestation { + quote: ra_tls::attestation::AttestationQuote::DstackAmdSevSnp( + ra_tls::attestation::SnpQuote { + report: Vec::new(), + cert_chain: Vec::new(), + }, + ), + runtime_events: Vec::new(), + report_data: [0x42; 64], + config: String::new(), + report: ra_tls::attestation::DstackVerifiedReport::DstackAmdSevSnp( + dstack_attest::amd_sev_snp::VerifiedAmdSnpReport { + measurement: verified, + report_data: [0x42; 64], + chip_id: [0xab; 64], + }, + ), + }; + + let err = build_amd_snp_boot_info_from_verified_attestation_and_vm_config( + &config(), + &attestation, + r#"{"os_image_hash":"0x00"}"#, + ) + .expect_err("missing sev_snp_measurement must fail closed"); + assert!( + err.to_string().contains("sev_snp_measurement is required"), + "unexpected error: {err:?}" + ); + } + + #[test] + fn vm_config_measurement_parser_rejects_unknown_measurement_fields() { + let mut measurement = serde_json::to_value(valid_input()).unwrap(); + measurement["unexpected"] = serde_json::json!(true); + let vm_config = serde_json::json!({ + "sev_snp_measurement": measurement, + }) + .to_string(); + + let err = parse_measurement_input_from_vm_config(&vm_config) + .expect_err("unknown measurement fields must reject"); + assert!( + format!("{err:?}").contains("unknown field"), + "unexpected error: {err:?}" + ); + } + + #[test] + fn vm_config_measurement_parser_bounds_ovmf_sections_during_deserialization() { + let mut measurement = serde_json::to_value(valid_input()).unwrap(); + measurement["ovmf_sections"] = serde_json::Value::Array( + (0..=MAX_OVMF_SECTIONS) + .map(|_| { + serde_json::json!({ + "gpa": 0x100000u64, + "size": 0x1000u64, + "section_type": 1u32, + }) + }) + .collect(), + ); + let vm_config = serde_json::json!({ + "sev_snp_measurement": measurement, + }) + .to_string(); + + let err = parse_measurement_input_from_vm_config(&vm_config) + .expect_err("oversized ovmf_sections must reject during parse"); + assert!( + format!("{err:?}").contains("ovmf section count must not exceed"), + "unexpected error: {err:?}" + ); + } + #[test] fn verified_attestation_helper_rejects_non_snp_reports() { let input = valid_input(); From 2b730954c4e198758e46cb6c9a4fc5e819809bbf Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Thu, 28 May 2026 18:26:48 -0700 Subject: [PATCH 14/67] fix: route kms snp attestation through dry-run auth --- kms/src/main_service.rs | 201 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 193 insertions(+), 8 deletions(-) diff --git a/kms/src/main_service.rs b/kms/src/main_service.rs index acf25e33d..176b122fd 100644 --- a/kms/src/main_service.rs +++ b/kms/src/main_service.rs @@ -22,7 +22,7 @@ use fs_err as fs; use k256::ecdsa::SigningKey; use ra_rpc::{CallContext, RpcCall}; use ra_tls::{ - attestation::VerifiedAttestation, + attestation::{AttestationMode, VerifiedAttestation}, cert::{CaCert, CertRequest, CertSigningRequestV1, CertSigningRequestV2, Csr}, kdf, }; @@ -33,7 +33,7 @@ use tracing::{info, warn}; use upgrade_authority::{build_boot_info, local_kms_boot_info, BootInfo}; use crate::{ - config::KmsConfig, + config::{KmsConfig, SevSnpMeasureConfig}, crypto::{derive_k256_key, sign_message, sign_message_with_timestamp}, }; @@ -146,6 +146,31 @@ struct BootConfig { gateway_app_id: String, } +fn build_boot_info_for_attestation( + sev_snp_config: Option<&SevSnpMeasureConfig>, + att: &VerifiedAttestation, + use_boottime_mr: bool, + vm_config_str: &str, +) -> Result { + if att.report.amd_snp_report().is_some() { + let config = sev_snp_config + .ok_or_else(|| anyhow::anyhow!("sev_snp config is required for amd sev-snp"))?; + return amd_attest::build_amd_snp_boot_info_from_verified_attestation_and_vm_config( + config, + att, + vm_config_str, + ); + } + build_boot_info(att, use_boottime_mr, vm_config_str) +} + +fn ensure_snp_key_release_not_enabled(boot_info: &BootInfo) -> Result<()> { + if boot_info.attestation_mode == AttestationMode::DstackAmdSevSnp { + bail!("amd sev-snp key release is not enabled"); + } + Ok(()) +} + impl RpcHandler { async fn ensure_self_allowed(&self) -> Result<()> { if !self.state.config.enforce_self_authorization { @@ -252,7 +277,12 @@ impl RpcHandler { use_boottime_mr: bool, vm_config_str: &str, ) -> Result { - let boot_info = build_boot_info(att, use_boottime_mr, vm_config_str)?; + let boot_info = build_boot_info_for_attestation( + self.state.config.sev_snp.as_ref(), + att, + use_boottime_mr, + vm_config_str, + )?; let response = self .state .config @@ -262,9 +292,14 @@ impl RpcHandler { if !response.is_allowed { bail!("Boot denied: {}", response.reason); } - self.verify_os_image_hash(vm_config_str.into(), att) - .await - .context("Failed to verify os image hash")?; + // SNP rootfs/app/config binding is handled by the SNP launch-measurement + // helper above. The legacy OS-image verifier is TDX-oriented and still + // rejects SNP quotes; keep SNP on the explicit fail-closed helper path. + if boot_info.attestation_mode != AttestationMode::DstackAmdSevSnp { + self.verify_os_image_hash(vm_config_str.into(), att) + .await + .context("Failed to verify os image hash")?; + } Ok(BootConfig { boot_info, gateway_app_id: response.gateway_app_id, @@ -307,8 +342,10 @@ impl KmsRpc for RpcHandler { .ensure_app_boot_allowed(&request.vm_config) .await .context("App not allowed")?; + ensure_snp_key_release_not_enabled(&boot_info)?; let app_id = boot_info.app_id; let instance_id = boot_info.instance_id; + let os_image_hash = boot_info.os_image_hash; let context_data = vec![&app_id[..], &instance_id[..], b"app-disk-crypt-key"]; let app_disk_key = kdf::derive_dh_secret(&self.state.root_ca.key, &context_data) @@ -335,7 +372,7 @@ impl KmsRpc for RpcHandler { k256_signature, tproxy_app_id: gateway_app_id.clone(), gateway_app_id, - os_image_hash: boot_info.os_image_hash, + os_image_hash, }) } @@ -412,7 +449,8 @@ impl KmsRpc for RpcHandler { self.ensure_self_allowed() .await .context("KMS self authorization failed")?; - let _info = self.ensure_kms_allowed(&request.vm_config).await?; + let info = self.ensure_kms_allowed(&request.vm_config).await?; + ensure_snp_key_release_not_enabled(&info)?; Ok(KmsKeyResponse { temp_ca_key: self.state.inner.temp_ca_key.clone(), keys: vec![KmsKeys { @@ -464,6 +502,7 @@ impl KmsRpc for RpcHandler { let app_info = self .ensure_app_attestation_allowed(&attestation, false, true, &request.vm_config) .await?; + ensure_snp_key_release_not_enabled(&app_info.boot_info)?; let app_ca = self.derive_app_ca(&app_info.boot_info.app_id)?; let cert = app_ca .sign_csr(&csr, Some(&app_info.boot_info.app_id), "app:custom") @@ -503,3 +542,149 @@ impl RpcCall for RpcHandler { pub fn rpc_methods() -> &'static [&'static str] { >::supported_methods() } + +#[cfg(test)] +mod tests { + use super::*; + use crate::main_service::amd_attest::{ + compute_expected_measurement, MeasurementInput, OvmfSectionParam, + }; + + fn sev_snp_config() -> SevSnpMeasureConfig { + SevSnpMeasureConfig { + ovmf_path: None, + guest_features: 1, + } + } + + fn hex_of(byte: u8, len: usize) -> String { + hex::encode(vec![byte; len]) + } + + fn valid_snp_measurement_input() -> MeasurementInput { + MeasurementInput { + app_id: hex_of(0x11, 20), + compose_hash: hex_of(0x22, 32), + rootfs_hash: hex_of(0x33, 32), + docker_files_hash: Some(hex_of(0x77, 32)), + ovmf_hash: hex_of(0x44, 48), + kernel_hash: hex_of(0x55, 32), + initrd_hash: hex_of(0x66, 32), + sev_hashes_table_gpa: 0x80_1000, + sev_es_reset_eip: 0xffff_fff0, + vcpus: 2, + vcpu_type: Some("epyc-v4".to_string()), + ovmf_sections: vec![ + OvmfSectionParam { + gpa: 0x100000, + size: 0x2000, + section_type: 1, + }, + OvmfSectionParam { + gpa: 0x80_0000, + size: 0x1000, + section_type: 0x10, + }, + OvmfSectionParam { + gpa: 0x81_0000, + size: 0x1000, + section_type: 2, + }, + OvmfSectionParam { + gpa: 0x82_0000, + size: 0x1000, + section_type: 3, + }, + ], + } + } + + fn verified_snp_attestation(measurement: [u8; 48], chip_id: [u8; 64]) -> VerifiedAttestation { + VerifiedAttestation { + quote: ra_tls::attestation::AttestationQuote::DstackAmdSevSnp( + ra_tls::attestation::SnpQuote { + report: Vec::new(), + cert_chain: Vec::new(), + }, + ), + runtime_events: Vec::new(), + report_data: [0x42; 64], + config: String::new(), + report: ra_tls::attestation::DstackVerifiedReport::DstackAmdSevSnp( + dstack_attest::amd_sev_snp::VerifiedAmdSnpReport { + measurement, + report_data: [0x42; 64], + chip_id, + }, + ), + } + } + + #[test] + fn build_boot_info_for_attestation_accepts_snp_vm_config_path() { + let input = valid_snp_measurement_input(); + let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); + let attestation = verified_snp_attestation(measurement, [0xab; 64]); + let vm_config = serde_json::json!({ + "sev_snp_measurement": input, + }) + .to_string(); + + let boot_info = build_boot_info_for_attestation( + Some(&sev_snp_config()), + &attestation, + false, + &vm_config, + ) + .expect("snp attestation should build boot info through vm_config path"); + + assert_eq!(boot_info.attestation_mode, AttestationMode::DstackAmdSevSnp); + assert_eq!(boot_info.mr_aggregated, measurement.to_vec()); + assert_eq!(boot_info.device_id, vec![0xab; 64]); + assert_eq!(boot_info.app_id, vec![0x11; 20]); + } + + #[test] + fn build_boot_info_for_attestation_requires_snp_config_for_snp() { + let input = valid_snp_measurement_input(); + let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); + let attestation = verified_snp_attestation(measurement, [0xab; 64]); + let vm_config = serde_json::json!({ + "sev_snp_measurement": input, + }) + .to_string(); + + let err = build_boot_info_for_attestation(None, &attestation, false, &vm_config) + .expect_err("snp attestation must require sev_snp config"); + assert!( + err.to_string().contains("sev_snp config is required"), + "unexpected error: {err:?}" + ); + } + + #[test] + fn snp_boot_info_is_still_blocked_from_key_release() { + let input = valid_snp_measurement_input(); + let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); + let attestation = verified_snp_attestation(measurement, [0xab; 64]); + let vm_config = serde_json::json!({ + "sev_snp_measurement": input, + }) + .to_string(); + let boot_info = build_boot_info_for_attestation( + Some(&sev_snp_config()), + &attestation, + false, + &vm_config, + ) + .unwrap(); + + let err = ensure_snp_key_release_not_enabled(&boot_info) + .expect_err("snp boot info must not be key-release enabled yet"); + assert!( + err.to_string() + .contains("amd sev-snp key release is not enabled"), + "unexpected error: {err:?}" + ); + } +} From 922afd744dafde5af9f03613c7b4333b88fa1917 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Thu, 28 May 2026 18:50:04 -0700 Subject: [PATCH 15/67] fix: report sev-snp onboarding attestation info --- kms/src/main_service.rs | 2 +- kms/src/onboard_service.rs | 177 ++++++++++++++++++++++++++++++++----- 2 files changed, 157 insertions(+), 22 deletions(-) diff --git a/kms/src/main_service.rs b/kms/src/main_service.rs index 176b122fd..8e514e2ac 100644 --- a/kms/src/main_service.rs +++ b/kms/src/main_service.rs @@ -146,7 +146,7 @@ struct BootConfig { gateway_app_id: String, } -fn build_boot_info_for_attestation( +pub(crate) fn build_boot_info_for_attestation( sev_snp_config: Option<&SevSnpMeasureConfig>, att: &VerifiedAttestation, use_boottime_mr: bool, diff --git a/kms/src/onboard_service.rs b/kms/src/onboard_service.rs index e1c5c277b..943471933 100644 --- a/kms/src/onboard_service.rs +++ b/kms/src/onboard_service.rs @@ -18,16 +18,22 @@ use ra_rpc::{ CallContext, RpcCall, }; use ra_tls::{ - attestation::{PlatformEvidence, QuoteContentType, VerifiedAttestation, VersionedAttestation}, + attestation::{ + GetDeviceId, PlatformEvidence, QuoteContentType, VerifiedAttestation, VersionedAttestation, + }, cert::{CaCert, CertRequest}, rcgen::{Certificate, KeyPair, PKCS_ECDSA_P256_SHA256}, }; use safe_write::safe_write; +use sha2::Digest; use crate::{ - config::KmsConfig, - main_service::upgrade_authority::{ - app_attest, dstack_client, ensure_kms_allowed, ensure_self_kms_allowed, pad64, + config::{KmsConfig, SevSnpMeasureConfig}, + main_service::{ + build_boot_info_for_attestation, + upgrade_authority::{ + app_attest, dstack_client, ensure_kms_allowed, ensure_self_kms_allowed, pad64, + }, }, }; @@ -133,16 +139,6 @@ impl OnboardRpc for OnboardHandler { .await .context("Failed to get VM info")?; - // Decode app info to get device_id, mr_aggregated, os_image_hash, mr_system - let app_info = verified - .decode_app_info_ex(false, &info.vm_config) - .context("Failed to decode app info")?; - let ppid = verified - .report - .tdx_report() - .map(|report| report.ppid.to_vec()) - .unwrap_or_default(); - let (eth_rpc_url, kms_contract_address) = match self.state.config.auth_api.get_info().await { Ok(info) => ( @@ -155,16 +151,15 @@ impl OnboardRpc for OnboardHandler { } }; - Ok(AttestationInfoResponse { - device_id: app_info.device_id, - mr_aggregated: app_info.mr_aggregated.to_vec(), - os_image_hash: app_info.os_image_hash, + Ok(build_attestation_info_response( + self.state.config.sev_snp.as_ref(), + &verified, attestation_mode, - site_name: self.state.config.site_name.clone(), + &info.vm_config, + self.state.config.site_name.clone(), eth_rpc_url, kms_contract_address, - ppid, - }) + )?) } async fn finish(self) -> anyhow::Result<()> { @@ -172,6 +167,146 @@ impl OnboardRpc for OnboardHandler { } } +fn build_attestation_info_response( + sev_snp_config: Option<&SevSnpMeasureConfig>, + verified: &VerifiedAttestation, + attestation_mode: String, + vm_config: &str, + site_name: String, + eth_rpc_url: String, + kms_contract_address: String, +) -> Result { + let boot_info = build_boot_info_for_attestation(sev_snp_config, verified, false, vm_config) + .context("Failed to decode app info")?; + let raw_device_id = verified.report.get_devide_id(); + Ok(AttestationInfoResponse { + device_id: sha2::Sha256::digest(&raw_device_id).to_vec(), + mr_aggregated: boot_info.mr_aggregated, + os_image_hash: boot_info.os_image_hash, + attestation_mode, + site_name, + eth_rpc_url, + kms_contract_address, + ppid: raw_device_id, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + config::SevSnpMeasureConfig, + main_service::amd_attest::{ + compute_expected_measurement, MeasurementInput, OvmfSectionParam, + }, + }; + use sha2::Digest; + + fn sev_snp_config() -> SevSnpMeasureConfig { + SevSnpMeasureConfig { + ovmf_path: None, + guest_features: 1, + } + } + + fn hex_of(byte: u8, len: usize) -> String { + hex::encode(vec![byte; len]) + } + + fn valid_snp_measurement_input() -> MeasurementInput { + MeasurementInput { + app_id: hex_of(0x11, 20), + compose_hash: hex_of(0x22, 32), + rootfs_hash: hex_of(0x33, 32), + docker_files_hash: Some(hex_of(0x77, 32)), + ovmf_hash: hex_of(0x44, 48), + kernel_hash: hex_of(0x55, 32), + initrd_hash: hex_of(0x66, 32), + sev_hashes_table_gpa: 0x80_1000, + sev_es_reset_eip: 0xffff_fff0, + vcpus: 2, + vcpu_type: Some("epyc-v4".to_string()), + ovmf_sections: vec![ + OvmfSectionParam { + gpa: 0x100000, + size: 0x2000, + section_type: 1, + }, + OvmfSectionParam { + gpa: 0x80_0000, + size: 0x1000, + section_type: 0x10, + }, + OvmfSectionParam { + gpa: 0x81_0000, + size: 0x1000, + section_type: 2, + }, + OvmfSectionParam { + gpa: 0x82_0000, + size: 0x1000, + section_type: 3, + }, + ], + } + } + + fn verified_snp_attestation(measurement: [u8; 48], chip_id: [u8; 64]) -> VerifiedAttestation { + VerifiedAttestation { + quote: ra_tls::attestation::AttestationQuote::DstackAmdSevSnp( + ra_tls::attestation::SnpQuote { + report: Vec::new(), + cert_chain: Vec::new(), + }, + ), + runtime_events: Vec::new(), + report_data: [0x42; 64], + config: String::new(), + report: ra_tls::attestation::DstackVerifiedReport::DstackAmdSevSnp( + dstack_attest::amd_sev_snp::VerifiedAmdSnpReport { + measurement, + report_data: [0x42; 64], + chip_id, + }, + ), + } + } + + #[test] + fn attestation_info_response_uses_snp_boot_info_and_chip_id() { + let input = valid_snp_measurement_input(); + let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); + let attestation = verified_snp_attestation(measurement, [0xab; 64]); + let vm_config = serde_json::json!({ + "sev_snp_measurement": input, + }) + .to_string(); + + let response = build_attestation_info_response( + Some(&sev_snp_config()), + &attestation, + "dstack-amd-sev-snp".to_string(), + &vm_config, + "test-site".to_string(), + "https://rpc.example".to_string(), + "0x1234".to_string(), + ) + .expect("snp attestation info should be derived from snp boot info"); + + assert_eq!( + response.device_id, + sha2::Sha256::digest([0xab; 64]).to_vec() + ); + assert_eq!(response.ppid, vec![0xab; 64]); + assert_eq!(response.mr_aggregated, measurement.to_vec()); + assert_eq!(response.os_image_hash, vec![0x33; 32]); + assert_eq!(response.attestation_mode, "dstack-amd-sev-snp"); + assert_eq!(response.site_name, "test-site"); + assert_eq!(response.eth_rpc_url, "https://rpc.example"); + assert_eq!(response.kms_contract_address, "0x1234"); + } +} + struct Keys { k256_key: SigningKey, tmp_ca_key: KeyPair, From 5b36b4c7eb10409490bc4f61745aa0cbf9e581ba Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Thu, 28 May 2026 19:02:42 -0700 Subject: [PATCH 16/67] fix: use sev-snp boot info for kms self auth --- kms/src/main_service.rs | 42 +++++++++++++++++++++-- kms/src/main_service/upgrade_authority.rs | 19 ++++++---- 2 files changed, 52 insertions(+), 9 deletions(-) diff --git a/kms/src/main_service.rs b/kms/src/main_service.rs index 8e514e2ac..6638998e0 100644 --- a/kms/src/main_service.rs +++ b/kms/src/main_service.rs @@ -155,6 +155,11 @@ pub(crate) fn build_boot_info_for_attestation( if att.report.amd_snp_report().is_some() { let config = sev_snp_config .ok_or_else(|| anyhow::anyhow!("sev_snp config is required for amd sev-snp"))?; + let vm_config_str = if vm_config_str.is_empty() { + att.config.as_str() + } else { + vm_config_str + }; return amd_attest::build_amd_snp_boot_info_from_verified_attestation_and_vm_config( config, att, @@ -179,7 +184,12 @@ impl RpcHandler { let boot_info = self .state .self_boot_info - .get_or_try_init(|| local_kms_boot_info(self.state.config.pccs_url.as_deref())) + .get_or_try_init(|| { + local_kms_boot_info( + self.state.config.pccs_url.as_deref(), + self.state.config.sev_snp.as_ref(), + ) + }) .await .context("Failed to load cached self boot info")?; let response = self @@ -600,6 +610,14 @@ mod tests { } fn verified_snp_attestation(measurement: [u8; 48], chip_id: [u8; 64]) -> VerifiedAttestation { + verified_snp_attestation_with_config(measurement, chip_id, String::new()) + } + + fn verified_snp_attestation_with_config( + measurement: [u8; 48], + chip_id: [u8; 64], + config: String, + ) -> VerifiedAttestation { VerifiedAttestation { quote: ra_tls::attestation::AttestationQuote::DstackAmdSevSnp( ra_tls::attestation::SnpQuote { @@ -609,7 +627,7 @@ mod tests { ), runtime_events: Vec::new(), report_data: [0x42; 64], - config: String::new(), + config, report: ra_tls::attestation::DstackVerifiedReport::DstackAmdSevSnp( dstack_attest::amd_sev_snp::VerifiedAmdSnpReport { measurement, @@ -644,6 +662,26 @@ mod tests { assert_eq!(boot_info.app_id, vec![0x11; 20]); } + #[test] + fn build_boot_info_for_attestation_uses_embedded_snp_vm_config_when_external_is_empty() { + let input = valid_snp_measurement_input(); + let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); + let embedded_config = serde_json::json!({ + "sev_snp_measurement": input, + }) + .to_string(); + let attestation = + verified_snp_attestation_with_config(measurement, [0xab; 64], embedded_config); + + let boot_info = + build_boot_info_for_attestation(Some(&sev_snp_config()), &attestation, false, "") + .expect("snp local KMS attestation should use embedded vm_config"); + + assert_eq!(boot_info.attestation_mode, AttestationMode::DstackAmdSevSnp); + assert_eq!(boot_info.mr_aggregated, measurement.to_vec()); + assert_eq!(boot_info.app_id, vec![0x11; 20]); + } + #[test] fn build_boot_info_for_attestation_requires_snp_config_for_snp() { let input = valid_snp_measurement_input(); diff --git a/kms/src/main_service/upgrade_authority.rs b/kms/src/main_service/upgrade_authority.rs index 622507768..9f164cb60 100644 --- a/kms/src/main_service/upgrade_authority.rs +++ b/kms/src/main_service/upgrade_authority.rs @@ -2,7 +2,8 @@ // // SPDX-License-Identifier: Apache-2.0 -use crate::config::{AuthApi, KmsConfig}; +use super::build_boot_info_for_attestation; +use crate::config::{AuthApi, KmsConfig, SevSnpMeasureConfig}; use anyhow::{bail, Context, Result}; use dstack_guest_agent_rpc::{ dstack_guest_client::DstackGuestClient, AttestResponse, RawQuoteArgs, @@ -72,7 +73,10 @@ pub(crate) fn build_boot_info( }) } -pub(crate) async fn local_kms_boot_info(pccs_url: Option<&str>) -> Result { +pub(crate) async fn local_kms_boot_info( + pccs_url: Option<&str>, + sev_snp_config: Option<&SevSnpMeasureConfig>, +) -> Result { let response = app_attest(pad64([0u8; 32])) .await .context("Failed to get local KMS attestation")?; @@ -83,7 +87,7 @@ pub(crate) async fn local_kms_boot_info(pccs_url: Option<&str>) -> Result Result<()> { if !cfg.enforce_self_authorization { return Ok(()); } - let boot_info = local_kms_boot_info(cfg.pccs_url.as_deref()) + let boot_info = local_kms_boot_info(cfg.pccs_url.as_deref(), cfg.sev_snp.as_ref()) .await .context("failed to build local KMS boot info")?; let response = cfg @@ -237,15 +241,16 @@ pub(crate) async fn ensure_kms_allowed( cfg: &KmsConfig, attestation: &VerifiedAttestation, ) -> Result<()> { - let mut boot_info = build_boot_info(attestation, false, "") - .context("failed to build KMS boot info from attestation")?; + let mut boot_info = + build_boot_info_for_attestation(cfg.sev_snp.as_ref(), attestation, false, "") + .context("failed to build KMS boot info from attestation")?; // Workaround: old source KMS instances use the legacy cert format (separate TDX_QUOTE + // EVENT_LOG OIDs) which lacks vm_config, resulting in an empty os_image_hash. // Fill it from the local KMS's own value. This is safe because mrAggregated already // validates OS image integrity transitively through the RTMR measurement chain. // TODO: remove once all source KMS instances use the unified PHALA_RATLS_ATTESTATION format. if boot_info.os_image_hash.is_empty() { - let local_info = local_kms_boot_info(cfg.pccs_url.as_deref()) + let local_info = local_kms_boot_info(cfg.pccs_url.as_deref(), cfg.sev_snp.as_ref()) .await .context("failed to get local KMS boot info for os_image_hash fallback")?; boot_info.os_image_hash = local_info.os_image_hash; From a8f0e8870adbf857ee14fccec0c90fbb441d43f3 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Thu, 28 May 2026 19:19:18 -0700 Subject: [PATCH 17/67] fix: make auth-simple tcb policy explicit --- kms/auth-simple/README.md | 30 ++++++++----- kms/auth-simple/index.test.ts | 85 +++++++++++++++++++++++++++++++++++ kms/auth-simple/index.ts | 21 ++++++++- 3 files changed, 124 insertions(+), 12 deletions(-) diff --git a/kms/auth-simple/README.md b/kms/auth-simple/README.md index dbb425aa5..2bb5c180e 100644 --- a/kms/auth-simple/README.md +++ b/kms/auth-simple/README.md @@ -38,6 +38,8 @@ Add more fields as you deploy Gateway and apps: ```json { "osImages": ["0x..."], + "allowedTcbStatuses": ["UpToDate"], + "allowedAdvisoryIds": [], "gatewayAppId": "0x...", "kms": { "mrAggregated": ["0x..."], @@ -60,6 +62,8 @@ Add more fields as you deploy Gateway and apps: |-------|----------|-------------| | `osImages` | Yes | Allowed OS image hashes (from `digest.txt`) | | `gatewayAppId` | No | Gateway app ID (add after Gateway deployment) | +| `allowedTcbStatuses` | No | Allowed TCB status strings. Defaults to `["UpToDate"]`; experimental SEV-SNP dry-run authorization must explicitly opt into `"snp-verified-basic-policy"`. | +| `allowedAdvisoryIds` | No | Advisory IDs permitted in `advisoryIds`. Defaults to `[]`, which rejects any advisory. | | `kms.mrAggregated` | Yes for KMS authorization | Allowed KMS aggregated MR values. An empty array denies all KMS boots. | | `kms.devices` | No | Allowed KMS device IDs | | `kms.allowAnyDevice` | No | If true, skip device ID check for KMS | @@ -67,6 +71,8 @@ Add more fields as you deploy Gateway and apps: | `apps..devices` | No | Allowed device IDs for this app | | `apps..allowAnyDevice` | No | If true, skip device ID check for this app | +For experimental AMD SEV-SNP dry-run authorization, keep the default fail-closed TCB policy unless you intentionally want the auth webhook to accept the staged SNP `BootInfo`. To exercise the dry-run path without enabling key release, set `allowedTcbStatuses` to include `"snp-verified-basic-policy"` and allowlist the recomputed SNP `mrAggregated`, `osImageHash`, app/compose identity, and device/chip identity as usual. KMS still rejects SNP before returning app keys, KMS keys, or app certificates. + ### Getting Hash Values **OS Image Hash:** @@ -128,13 +134,15 @@ App boot authorization. **Request:** ```json { + "attestationMode": "DstackTdx", "mrAggregated": "0x...", "osImageHash": "0x...", "appId": "0x...", "composeHash": "0x...", "instanceId": "0x...", "deviceId": "0x...", - "tcbStatus": "UpToDate" + "tcbStatus": "UpToDate", + "advisoryIds": [] } ``` @@ -159,18 +167,20 @@ KMS boot authorization. ### KMS Boot Validation -1. `tcbStatus` must be "UpToDate" -2. `osImageHash` must be in `osImages` array -3. `mrAggregated` must be in `kms.mrAggregated` -4. `deviceId` must be in `kms.devices` (unless `allowAnyDevice` is true) +1. `tcbStatus` must be listed in `allowedTcbStatuses` (default: only `"UpToDate"`) +2. Every `advisoryIds` entry must be listed in `allowedAdvisoryIds` (default: none allowed) +3. `osImageHash` must be in `osImages` array +4. `mrAggregated` must be in `kms.mrAggregated` +5. `deviceId` must be in `kms.devices` (unless `allowAnyDevice` is true) ### App Boot Validation -1. `tcbStatus` must be "UpToDate" -2. `osImageHash` must be in `osImages` array -3. `appId` must exist in `apps` object -4. `composeHash` must be in app's `composeHashes` array -5. `deviceId` must be in app's `devices` (unless `allowAnyDevice` is true) +1. `tcbStatus` must be listed in `allowedTcbStatuses` (default: only `"UpToDate"`) +2. Every `advisoryIds` entry must be listed in `allowedAdvisoryIds` (default: none allowed) +3. `osImageHash` must be in `osImages` array +4. `appId` must exist in `apps` object +5. `composeHash` must be in app's `composeHashes` array +6. `deviceId` must be in app's `devices` (unless `allowAnyDevice` is true) ## Hot Reload diff --git a/kms/auth-simple/index.test.ts b/kms/auth-simple/index.test.ts index 856a0deda..f052af07a 100644 --- a/kms/auth-simple/index.test.ts +++ b/kms/auth-simple/index.test.ts @@ -92,6 +92,91 @@ describe('auth-simple', () => { expect(json.reason).toContain('TCB status'); }); + it('requires explicit opt-in for SEV-SNP placeholder TCB status', async () => { + writeTestConfig({ + gatewayAppId: '0xgateway', + osImages: ['0x1fbb0cf9cc6cfbf23d6b779776fabad2c5403d643badb9e5e238615e4960a78a'], + kms: { + mrAggregated: ['0xabc123'], + devices: ['0xdevice999'], + allowAnyDevice: false + } + }); + + const sevSnpBootInfo = { + ...baseBootInfo, + attestationMode: 'DstackAmdSevSnp', + tcbStatus: 'snp-verified-basic-policy' + }; + const denied = await app.fetch(new Request('http://localhost/bootAuth/kms', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(sevSnpBootInfo) + })); + expect((await denied.json()).isAllowed).toBe(false); + + writeTestConfig({ + gatewayAppId: '0xgateway', + osImages: ['0x1fbb0cf9cc6cfbf23d6b779776fabad2c5403d643badb9e5e238615e4960a78a'], + allowedTcbStatuses: ['snp-verified-basic-policy'], + kms: { + mrAggregated: ['0xabc123'], + devices: ['0xdevice999'], + allowAnyDevice: false + } + }); + + const allowed = await app.fetch(new Request('http://localhost/bootAuth/kms', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(sevSnpBootInfo) + })); + const allowedJson = await allowed.json(); + + expect(allowedJson.isAllowed).toBe(true); + expect(allowedJson.reason).toBe(''); + }); + + it('rejects unallowlisted advisory IDs by default', async () => { + writeTestConfig({ + gatewayAppId: '0xgateway', + osImages: ['0x1fbb0cf9cc6cfbf23d6b779776fabad2c5403d643badb9e5e238615e4960a78a'], + kms: { + mrAggregated: ['0xabc123'], + allowAnyDevice: true + } + }); + + const denied = await app.fetch(new Request('http://localhost/bootAuth/kms', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ ...baseBootInfo, advisoryIds: ['INTEL-SA-TEST'] }) + })); + const deniedJson = await denied.json(); + + expect(deniedJson.isAllowed).toBe(false); + expect(deniedJson.reason).toContain('advisory'); + + writeTestConfig({ + gatewayAppId: '0xgateway', + osImages: ['0x1fbb0cf9cc6cfbf23d6b779776fabad2c5403d643badb9e5e238615e4960a78a'], + allowedAdvisoryIds: ['INTEL-SA-TEST'], + kms: { + mrAggregated: ['0xabc123'], + allowAnyDevice: true + } + }); + + const allowed = await app.fetch(new Request('http://localhost/bootAuth/kms', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ ...baseBootInfo, advisoryIds: ['INTEL-SA-TEST'] }) + })); + const allowedJson = await allowed.json(); + + expect(allowedJson.isAllowed).toBe(true); + }); + it('rejects KMS boot with invalid OS image', async () => { writeTestConfig({ gatewayAppId: '0xgateway', diff --git a/kms/auth-simple/index.ts b/kms/auth-simple/index.ts index 7307f49cb..a24ea8808 100644 --- a/kms/auth-simple/index.ts +++ b/kms/auth-simple/index.ts @@ -9,6 +9,7 @@ import { readFileSync, existsSync } from 'fs'; // zod schemas for validation - compatible with auth-eth implementation const BootInfoSchema = z.object({ + attestationMode: z.string().optional().default(''), mrAggregated: z.string().describe('aggregated MR measurement'), osImageHash: z.string().describe('OS Image hash'), appId: z.string().describe('application ID'), @@ -46,6 +47,11 @@ const AuthConfigSchema = z.object({ chainId: z.number().default(0), appImplementation: z.string().default('0x0000000000000000000000000000000000000000'), osImages: z.array(z.string()).default([]), + // TDX production defaults remain strict. Experimental SEV-SNP dry-run + // authorization must explicitly opt into its placeholder TCB status until + // AMD TCB/revocation policy is finalized. + allowedTcbStatuses: z.array(z.string()).default(['UpToDate']), + allowedAdvisoryIds: z.array(z.string()).default([]), kms: KmsConfigSchema.default({}), apps: z.record(z.string(), AppConfigSchema).default({}) }); @@ -92,14 +98,25 @@ class ConfigBackend { const deviceId = normalizeHex(bootInfo.deviceId); // check TCB status - if (bootInfo.tcbStatus !== 'UpToDate') { + const allowedTcbStatuses = config.allowedTcbStatuses; + if (!allowedTcbStatuses.includes(bootInfo.tcbStatus)) { return { isAllowed: false, - reason: 'TCB status is not up to date', + reason: 'TCB status is not allowed', gatewayAppId: config.gatewayAppId }; } + for (const advisoryId of bootInfo.advisoryIds) { + if (!config.allowedAdvisoryIds.includes(advisoryId)) { + return { + isAllowed: false, + reason: 'advisory ID is not allowed', + gatewayAppId: config.gatewayAppId + }; + } + } + // check OS image const allowedOsImages = config.osImages.map(normalizeHex); if (!allowedOsImages.includes(osImageHash)) { From 058fd29252339471252e5c7914aff29ed2b15338 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Mon, 1 Jun 2026 15:49:52 -0700 Subject: [PATCH 18/67] fix: block sev-snp temp ca release --- kms/src/main_service.rs | 47 +++++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/kms/src/main_service.rs b/kms/src/main_service.rs index 6638998e0..da1f3b494 100644 --- a/kms/src/main_service.rs +++ b/kms/src/main_service.rs @@ -176,10 +176,17 @@ fn ensure_snp_key_release_not_enabled(boot_info: &BootInfo) -> Result<()> { Ok(()) } +fn ensure_self_key_release_not_enabled(self_boot_info: Option<&BootInfo>) -> Result<()> { + if let Some(boot_info) = self_boot_info { + ensure_snp_key_release_not_enabled(boot_info)?; + } + Ok(()) +} + impl RpcHandler { - async fn ensure_self_allowed(&self) -> Result<()> { + async fn ensure_self_allowed(&self) -> Result> { if !self.state.config.enforce_self_authorization { - return Ok(()); + return Ok(None); } let boot_info = self .state @@ -202,7 +209,7 @@ impl RpcHandler { if !response.is_allowed { bail!("KMS is not allowed: {}", response.reason); } - Ok(()) + Ok(Some(boot_info)) } fn ensure_attested(&self) -> Result<&VerifiedAttestation> { @@ -471,9 +478,11 @@ impl KmsRpc for RpcHandler { } async fn get_temp_ca_cert(self) -> Result { - self.ensure_self_allowed() + let self_boot_info = self + .ensure_self_allowed() .await .context("KMS self authorization failed")?; + ensure_self_key_release_not_enabled(self_boot_info)?; Ok(GetTempCaCertResponse { temp_ca_cert: self.state.inner.temp_ca_cert.clone(), temp_ca_key: self.state.inner.temp_ca_key.clone(), @@ -700,8 +709,7 @@ mod tests { ); } - #[test] - fn snp_boot_info_is_still_blocked_from_key_release() { + fn snp_boot_info() -> BootInfo { let input = valid_snp_measurement_input(); let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); let attestation = verified_snp_attestation(measurement, [0xab; 64]); @@ -709,13 +717,13 @@ mod tests { "sev_snp_measurement": input, }) .to_string(); - let boot_info = build_boot_info_for_attestation( - Some(&sev_snp_config()), - &attestation, - false, - &vm_config, - ) - .unwrap(); + build_boot_info_for_attestation(Some(&sev_snp_config()), &attestation, false, &vm_config) + .unwrap() + } + + #[test] + fn snp_boot_info_is_still_blocked_from_key_release() { + let boot_info = snp_boot_info(); let err = ensure_snp_key_release_not_enabled(&boot_info) .expect_err("snp boot info must not be key-release enabled yet"); @@ -725,4 +733,17 @@ mod tests { "unexpected error: {err:?}" ); } + + #[test] + fn snp_self_boot_info_is_blocked_from_temp_ca_release() { + let boot_info = snp_boot_info(); + + let err = ensure_self_key_release_not_enabled(Some(&boot_info)) + .expect_err("snp self boot info must not receive temp CA key material"); + assert!( + err.to_string() + .contains("amd sev-snp key release is not enabled"), + "unexpected error: {err:?}" + ); + } } From 73d857b3e7c8b74d57ca2160be46b36826608847 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Mon, 1 Jun 2026 16:40:53 -0700 Subject: [PATCH 19/67] fix: derive sev-snp tcb policy from report --- dstack-attest/src/amd_sev_snp.rs | 91 ++++++++++++++++++++++++- kms/auth-simple/README.md | 4 +- kms/auth-simple/index.test.ts | 6 +- kms/auth-simple/index.ts | 5 +- kms/src/main_service.rs | 2 + kms/src/main_service/amd_attest.rs | 103 +++++++++++++++++++++++++++-- kms/src/onboard_service.rs | 2 + 7 files changed, 199 insertions(+), 14 deletions(-) diff --git a/dstack-attest/src/amd_sev_snp.rs b/dstack-attest/src/amd_sev_snp.rs index 8c3d038a0..3a2f1fbc5 100644 --- a/dstack-attest/src/amd_sev_snp.rs +++ b/dstack-attest/src/amd_sev_snp.rs @@ -14,7 +14,7 @@ use anyhow::{bail, Context, Result}; use base64::engine::general_purpose::STANDARD; use base64::Engine as _; use sev::certs::snp::{ca, Certificate, Chain, Verifiable}; -use sev::firmware::guest::AttestationReport; +use sev::firmware::{guest::AttestationReport, host::TcbVersion}; /// AMD Genoa ARK certificate (DER, base64-encoded). /// Source: https://kdsintf.amd.com/vcek/v1/Genoa/cert_chain @@ -31,11 +31,62 @@ const VLEK_CERT_GUID: [u8; 16] = [ ]; const CERT_TABLE_ENTRY_SIZE: usize = 24; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct AmdSnpTcbVersion { + pub bootloader: u8, + pub tee: u8, + pub snp: u8, + pub microcode: u8, +} + +impl From for AmdSnpTcbVersion { + fn from(value: TcbVersion) -> Self { + Self { + bootloader: value.bootloader, + tee: value.tee, + snp: value.snp, + microcode: value.microcode, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct AmdSnpTcbInfo { + pub current: AmdSnpTcbVersion, + pub reported: AmdSnpTcbVersion, + pub committed: AmdSnpTcbVersion, + pub launch: AmdSnpTcbVersion, +} + +impl AmdSnpTcbInfo { + pub fn from_report(report: &AttestationReport) -> Self { + Self { + current: report.current_tcb.into(), + reported: report.reported_tcb.into(), + committed: report.committed_tcb.into(), + launch: report.launch_tcb.into(), + } + } + + pub fn tcb_status(&self) -> &'static str { + if self.current == self.reported + && self.committed == self.reported + && self.launch == self.reported + { + "UpToDate" + } else { + "OutOfDate" + } + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct VerifiedAmdSnpReport { pub measurement: [u8; 48], pub report_data: [u8; 64], pub chip_id: [u8; 64], + pub tcb_info: AmdSnpTcbInfo, + pub advisory_ids: Vec, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -135,6 +186,12 @@ fn verify_amd_snp_attestation_with_certs( measurement, report_data, chip_id, + tcb_info: AmdSnpTcbInfo::from_report(&report), + // AMD SEV-SNP attestation reports and VCEKs do not carry a direct + // advisory list. Keep this explicit and empty so downstream auth stays + // fail-closed if a future verifier adds advisories from revocation or + // external policy collateral. + advisory_ids: Vec::new(), }) } @@ -282,6 +339,38 @@ fn parse_kernel_cert_table(auxblob: &[u8]) -> Result)>> { mod tests { use super::*; + fn tcb(bootloader: u8, tee: u8, snp: u8, microcode: u8) -> AmdSnpTcbVersion { + AmdSnpTcbVersion { + bootloader, + tee, + snp, + microcode, + } + } + + #[test] + fn tcb_status_is_up_to_date_only_when_all_reported_versions_match() { + let up_to_date = AmdSnpTcbInfo { + current: tcb(1, 2, 3, 4), + reported: tcb(1, 2, 3, 4), + committed: tcb(1, 2, 3, 4), + launch: tcb(1, 2, 3, 4), + }; + assert_eq!(up_to_date.tcb_status(), "UpToDate"); + + let stale_launch = AmdSnpTcbInfo { + launch: tcb(1, 2, 3, 3), + ..up_to_date + }; + assert_eq!(stale_launch.tcb_status(), "OutOfDate"); + + let stale_vcek_reported = AmdSnpTcbInfo { + reported: tcb(1, 2, 3, 3), + ..up_to_date + }; + assert_eq!(stale_vcek_reported.tcb_status(), "OutOfDate"); + } + #[test] fn missing_cert_chain_fails_closed() { let report = vec![0u8; 1184]; diff --git a/kms/auth-simple/README.md b/kms/auth-simple/README.md index 2bb5c180e..58360895f 100644 --- a/kms/auth-simple/README.md +++ b/kms/auth-simple/README.md @@ -62,7 +62,7 @@ Add more fields as you deploy Gateway and apps: |-------|----------|-------------| | `osImages` | Yes | Allowed OS image hashes (from `digest.txt`) | | `gatewayAppId` | No | Gateway app ID (add after Gateway deployment) | -| `allowedTcbStatuses` | No | Allowed TCB status strings. Defaults to `["UpToDate"]`; experimental SEV-SNP dry-run authorization must explicitly opt into `"snp-verified-basic-policy"`. | +| `allowedTcbStatuses` | No | Allowed verifier-derived TCB status strings. Defaults to `["UpToDate"]`; non-up-to-date SNP/TDX statuses remain fail-closed unless explicitly allowlisted for testing. | | `allowedAdvisoryIds` | No | Advisory IDs permitted in `advisoryIds`. Defaults to `[]`, which rejects any advisory. | | `kms.mrAggregated` | Yes for KMS authorization | Allowed KMS aggregated MR values. An empty array denies all KMS boots. | | `kms.devices` | No | Allowed KMS device IDs | @@ -71,7 +71,7 @@ Add more fields as you deploy Gateway and apps: | `apps..devices` | No | Allowed device IDs for this app | | `apps..allowAnyDevice` | No | If true, skip device ID check for this app | -For experimental AMD SEV-SNP dry-run authorization, keep the default fail-closed TCB policy unless you intentionally want the auth webhook to accept the staged SNP `BootInfo`. To exercise the dry-run path without enabling key release, set `allowedTcbStatuses` to include `"snp-verified-basic-policy"` and allowlist the recomputed SNP `mrAggregated`, `osImageHash`, app/compose identity, and device/chip identity as usual. KMS still rejects SNP before returning app keys, KMS keys, or app certificates. +For experimental AMD SEV-SNP dry-run authorization, keep the default fail-closed TCB policy unless you intentionally want the auth webhook to accept non-up-to-date verifier-derived SNP `BootInfo`. To exercise the dry-run path without enabling key release, allowlist the recomputed SNP `mrAggregated`, `osImageHash`, app/compose identity, device/chip identity, and any non-default `allowedTcbStatuses`/`allowedAdvisoryIds` values explicitly. KMS still rejects SNP before returning app keys, KMS keys, or app certificates. ### Getting Hash Values diff --git a/kms/auth-simple/index.test.ts b/kms/auth-simple/index.test.ts index f052af07a..584d0f9e3 100644 --- a/kms/auth-simple/index.test.ts +++ b/kms/auth-simple/index.test.ts @@ -92,7 +92,7 @@ describe('auth-simple', () => { expect(json.reason).toContain('TCB status'); }); - it('requires explicit opt-in for SEV-SNP placeholder TCB status', async () => { + it('requires explicit opt-in for non-UpToDate SEV-SNP TCB status', async () => { writeTestConfig({ gatewayAppId: '0xgateway', osImages: ['0x1fbb0cf9cc6cfbf23d6b779776fabad2c5403d643badb9e5e238615e4960a78a'], @@ -106,7 +106,7 @@ describe('auth-simple', () => { const sevSnpBootInfo = { ...baseBootInfo, attestationMode: 'DstackAmdSevSnp', - tcbStatus: 'snp-verified-basic-policy' + tcbStatus: 'OutOfDate' }; const denied = await app.fetch(new Request('http://localhost/bootAuth/kms', { method: 'POST', @@ -118,7 +118,7 @@ describe('auth-simple', () => { writeTestConfig({ gatewayAppId: '0xgateway', osImages: ['0x1fbb0cf9cc6cfbf23d6b779776fabad2c5403d643badb9e5e238615e4960a78a'], - allowedTcbStatuses: ['snp-verified-basic-policy'], + allowedTcbStatuses: ['OutOfDate'], kms: { mrAggregated: ['0xabc123'], devices: ['0xdevice999'], diff --git a/kms/auth-simple/index.ts b/kms/auth-simple/index.ts index a24ea8808..b8d8c86c2 100644 --- a/kms/auth-simple/index.ts +++ b/kms/auth-simple/index.ts @@ -47,9 +47,8 @@ const AuthConfigSchema = z.object({ chainId: z.number().default(0), appImplementation: z.string().default('0x0000000000000000000000000000000000000000'), osImages: z.array(z.string()).default([]), - // TDX production defaults remain strict. Experimental SEV-SNP dry-run - // authorization must explicitly opt into its placeholder TCB status until - // AMD TCB/revocation policy is finalized. + // TDX and SEV-SNP production defaults remain strict: only UpToDate is + // accepted unless operators explicitly allow another verifier-derived status. allowedTcbStatuses: z.array(z.string()).default(['UpToDate']), allowedAdvisoryIds: z.array(z.string()).default([]), kms: KmsConfigSchema.default({}), diff --git a/kms/src/main_service.rs b/kms/src/main_service.rs index da1f3b494..23bc05887 100644 --- a/kms/src/main_service.rs +++ b/kms/src/main_service.rs @@ -642,6 +642,8 @@ mod tests { measurement, report_data: [0x42; 64], chip_id, + tcb_info: dstack_attest::amd_sev_snp::AmdSnpTcbInfo::default(), + advisory_ids: Vec::new(), }, ), } diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs index d46ce14ae..5faf94fd3 100644 --- a/kms/src/main_service/amd_attest.rs +++ b/kms/src/main_service/amd_attest.rs @@ -151,11 +151,30 @@ pub(crate) fn validate_amd_snp_measurement_binding( /// Keeping these as helper-only values lets future authorization policy inspect /// exactly which SNP-specific inputs were bound while SNP key release remains /// fail-closed unless an explicit release path is added separately. +#[cfg(test)] pub(crate) fn build_amd_snp_boot_info( config: &SevSnpMeasureConfig, verified_measurement: &[u8; 48], verified_chip_id: &[u8; 64], input: &MeasurementInput, +) -> Result { + build_amd_snp_boot_info_with_tcb_status( + config, + verified_measurement, + verified_chip_id, + "UpToDate", + &[], + input, + ) +} + +fn build_amd_snp_boot_info_with_tcb_status( + config: &SevSnpMeasureConfig, + verified_measurement: &[u8; 48], + verified_chip_id: &[u8; 64], + tcb_status: &str, + advisory_ids: &[String], + input: &MeasurementInput, ) -> Result { validate_amd_snp_measurement_binding(Some(config), verified_measurement, input)?; @@ -186,8 +205,8 @@ pub(crate) fn build_amd_snp_boot_info( instance_id, device_id: verified_chip_id.to_vec(), key_provider_info, - tcb_status: "snp-verified-basic-policy".to_string(), - advisory_ids: Vec::new(), + tcb_status: tcb_status.to_string(), + advisory_ids: advisory_ids.to_vec(), }) } @@ -207,7 +226,14 @@ pub(crate) fn build_amd_snp_boot_info_from_verified_attestation( .report .amd_snp_report() .ok_or_else(|| anyhow::anyhow!("verified attestation is not amd sev-snp"))?; - build_amd_snp_boot_info(config, &verified.measurement, &verified.chip_id, input) + build_amd_snp_boot_info_with_tcb_status( + config, + &verified.measurement, + &verified.chip_id, + verified.tcb_info.tcb_status(), + &verified.advisory_ids, + input, + ) } #[derive(Debug, serde::Deserialize)] @@ -1178,7 +1204,8 @@ mod tests { assert_eq!(boot_info.mr_system.len(), 32); assert_eq!(boot_info.key_provider_info.len(), 32); assert_eq!(boot_info.instance_id.len(), 32); - assert_eq!(boot_info.tcb_status, "snp-verified-basic-policy"); + assert_eq!(boot_info.tcb_status, "UpToDate"); + assert_ne!(boot_info.tcb_status, "snp-verified-basic-policy"); assert!(boot_info.advisory_ids.is_empty()); let mut mismatched = verified; @@ -1208,6 +1235,8 @@ mod tests { measurement: verified, report_data: [0x42; 64], chip_id, + tcb_info: dstack_attest::amd_sev_snp::AmdSnpTcbInfo::default(), + advisory_ids: Vec::new(), }, ), }; @@ -1219,6 +1248,66 @@ mod tests { assert_eq!(boot_info.mr_aggregated, verified.to_vec()); assert_eq!(boot_info.device_id, chip_id.to_vec()); assert_eq!(boot_info.app_id, vec![0x11; 20]); + assert_eq!(boot_info.tcb_status, "UpToDate"); + } + + #[test] + fn verified_attestation_tcb_status_replaces_snp_placeholder() { + let input = valid_input(); + let verified = compute_expected_measurement(&config(), &input).unwrap(); + let chip_id = [0xbc; 64]; + let tcb = dstack_attest::amd_sev_snp::AmdSnpTcbVersion { + bootloader: 1, + tee: 2, + snp: 3, + microcode: 4, + }; + let stale_tcb = dstack_attest::amd_sev_snp::AmdSnpTcbVersion { + microcode: 3, + ..tcb + }; + let attestation = ra_tls::attestation::VerifiedAttestation { + quote: ra_tls::attestation::AttestationQuote::DstackAmdSevSnp( + ra_tls::attestation::SnpQuote { + report: Vec::new(), + cert_chain: Vec::new(), + }, + ), + runtime_events: Vec::new(), + report_data: [0x42; 64], + config: String::new(), + report: ra_tls::attestation::DstackVerifiedReport::DstackAmdSevSnp( + dstack_attest::amd_sev_snp::VerifiedAmdSnpReport { + measurement: verified, + report_data: [0x42; 64], + chip_id, + tcb_info: dstack_attest::amd_sev_snp::AmdSnpTcbInfo { + current: tcb, + reported: tcb, + committed: tcb, + launch: stale_tcb, + }, + advisory_ids: vec!["SNP-TEST-ADVISORY".to_string()], + }, + ), + }; + + let boot_info = + build_amd_snp_boot_info_from_verified_attestation(&config(), &attestation, &input) + .expect("verified snp attestation should feed boot info helper"); + + assert_eq!(boot_info.tcb_status, "OutOfDate"); + assert_eq!(boot_info.advisory_ids, vec!["SNP-TEST-ADVISORY"]); + assert_ne!(boot_info.tcb_status, "snp-verified-basic-policy"); + let policy = AmdSnpAuthPolicy::from_boot_info(&boot_info).unwrap(); + let mut up_to_date_only = policy.clone(); + up_to_date_only.allowed_tcb_statuses = vec!["UpToDate".to_string()]; + let err = validate_amd_snp_auth_policy(&boot_info, &up_to_date_only) + .expect_err("out-of-date snp tcb must not satisfy up-to-date policy"); + assert!( + err.to_string().contains("tcb_status is not allowed"), + "unexpected error: {err:?}" + ); } #[test] @@ -1241,6 +1330,8 @@ mod tests { measurement: verified, report_data: [0x42; 64], chip_id, + tcb_info: dstack_attest::amd_sev_snp::AmdSnpTcbInfo::default(), + advisory_ids: Vec::new(), }, ), }; @@ -1280,6 +1371,8 @@ mod tests { measurement: verified, report_data: [0x42; 64], chip_id: [0xab; 64], + tcb_info: dstack_attest::amd_sev_snp::AmdSnpTcbInfo::default(), + advisory_ids: Vec::new(), }, ), }; @@ -1569,7 +1662,7 @@ mod tests { .contains("attestation mode is not amd sev-snp")); let mut wrong_status = boot_info.clone(); - wrong_status.tcb_status = "UpToDate".to_string(); + wrong_status.tcb_status = "OutOfDate".to_string(); let err = validate_amd_snp_auth_policy(&wrong_status, &policy) .expect_err("unexpected tcb status must reject"); assert!(err.to_string().contains("tcb_status is not allowed")); diff --git a/kms/src/onboard_service.rs b/kms/src/onboard_service.rs index 943471933..3230588ca 100644 --- a/kms/src/onboard_service.rs +++ b/kms/src/onboard_service.rs @@ -267,6 +267,8 @@ mod tests { measurement, report_data: [0x42; 64], chip_id, + tcb_info: dstack_attest::amd_sev_snp::AmdSnpTcbInfo::default(), + advisory_ids: Vec::new(), }, ), } From 52d3fac3cd50f63b74c2fbe5f7b60df1bd88286f Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Mon, 1 Jun 2026 16:59:11 -0700 Subject: [PATCH 20/67] chore: satisfy sev-snp workspace clippy --- dstack-attest/src/attestation.rs | 2 +- kms/src/onboard_service.rs | 4 ++-- vmm/src/config.rs | 9 ++------- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/dstack-attest/src/attestation.rs b/dstack-attest/src/attestation.rs index 603be1813..3030cf5ff 100644 --- a/dstack-attest/src/attestation.rs +++ b/dstack-attest/src/attestation.rs @@ -220,7 +220,7 @@ impl AttestationMode { // First, try to detect platform from DMI product name let platform = Platform::detect_or_dstack(); match platform { - Platform::Dstack => return choose_dstack_attestation_mode(has_tdx, has_sev_snp), + Platform::Dstack => choose_dstack_attestation_mode(has_tdx, has_sev_snp), Platform::Gcp => { // GCP platform: TDX + TPM dual mode if has_tdx { diff --git a/kms/src/onboard_service.rs b/kms/src/onboard_service.rs index 3230588ca..137063c86 100644 --- a/kms/src/onboard_service.rs +++ b/kms/src/onboard_service.rs @@ -151,7 +151,7 @@ impl OnboardRpc for OnboardHandler { } }; - Ok(build_attestation_info_response( + build_attestation_info_response( self.state.config.sev_snp.as_ref(), &verified, attestation_mode, @@ -159,7 +159,7 @@ impl OnboardRpc for OnboardHandler { self.state.config.site_name.clone(), eth_rpc_url, kms_contract_address, - )?) + ) } async fn finish(self) -> anyhow::Result<()> { diff --git a/vmm/src/config.rs b/vmm/src/config.rs index 56d899cd0..ce2194d07 100644 --- a/vmm/src/config.rs +++ b/vmm/src/config.rs @@ -106,20 +106,15 @@ impl Protocol { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] pub enum TeePlatform { + #[default] Auto, Tdx, AmdSevSnp, } -impl Default for TeePlatform { - fn default() -> Self { - Self::Auto - } -} - impl TeePlatform { pub fn resolve(self) -> Self { match self { From 0077ec96be1db2e3fffcb830fbd9f5e46c8fde15 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Tue, 2 Jun 2026 12:56:34 -0700 Subject: [PATCH 21/67] docs: add sev-snp review readiness note --- docs/amd-sev-snp-review-readiness.md | 103 +++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 docs/amd-sev-snp-review-readiness.md diff --git a/docs/amd-sev-snp-review-readiness.md b/docs/amd-sev-snp-review-readiness.md new file mode 100644 index 000000000..949aa00c6 --- /dev/null +++ b/docs/amd-sev-snp-review-readiness.md @@ -0,0 +1,103 @@ +# AMD SEV-SNP Review Readiness + +This branch stages AMD SEV-SNP support for review while keeping SNP key/cert release fail-closed. + +## Current review boundary + +Implemented and intended for review: + +- AMD SEV-SNP evidence plumbing in the v1 attestation format. +- SNP report verification with AMD Genoa ARK/ASK/VCEK chain verification. +- Report-data challenge binding and fail-closed report policy checks. +- SNP launch-measurement recomputation from OVMF/kernel/initrd/cmdline inputs. +- KMS SNP `BootInfo` construction from verified report measurement, chip id, launch inputs, TCB status, and advisory ids. +- Dry-run auth-policy evaluation through existing KMS auth flow. +- Onboarding attestation-info reporting for SNP identity fields. +- VMM explicit `platform = "amd-sev-snp"` launch path. + +Intentionally not enabled yet: + +- SNP app key release. +- SNP KMS/root/temp CA key release. +- SNP app certificate/signing certificate release. +- Production SNP key release policy. + +The KMS still rejects SNP `BootInfo` before returning secrets or certificates. Treat this branch as review-ready staging, not production SNP key release. + +## Fail-closed policy summary + +- `platform = "auto"` remains conservative while SNP is experimental; operators must explicitly set `platform = "amd-sev-snp"` to launch an SNP guest. +- SNP launch measurement is recomputed from trusted KMS config/input and compared to the hardware-verified report measurement. +- SNP `BootInfo.tcb_status` is verifier-derived from signed AMD SNP report TCB fields: + - `UpToDate` only when current/reported/committed/launch TCB versions all match. + - `OutOfDate` otherwise. +- SNP advisory ids are propagated from verifier output into `BootInfo`; currently this list is explicit and empty because the AMD report/VCEK evidence used here does not carry a direct advisory-list field. +- `auth-simple` defaults remain strict: only `UpToDate` is accepted and any advisory id is denied unless explicitly allowlisted. + +## Live golden-vector proof + +The ignored live regression test cross-checks dstack's pure Rust SNP measurement recomputation against `sev-snp-measure` on the SNP-capable host. + +Command: + +```bash +cargo test -p dstack-kms --all-features recomputation_matches_sev_snp_measure_live_golden_vector -- --ignored --nocapture +``` + +Latest local proof: + +```text +DSTACK_SEV_SNP_MEASURE_GOLDEN_VECTOR_BEGIN +utc=2026-06-02T19:49:14Z +host=dedicated-m24-fork +uname=Linux dedicated-m24-fork 6.11.0-rc3-snp-host-85ef1ac03941 #2 SMP Sat May 3 11:42:34 EDT 2025 x86_64 GNU/Linux +sev_snp_measure=/usr/local/bin/sev-snp-measure +sev_snp_measure_version=sev-snp-measure 0.0.10 +ovmf_path=/opt/AMDSEV/usr/local/share/qemu/OVMF.fd +ovmf_sha256=67e7a7027437823e9c166a60d00666d5d5391e13050488cad5cc2acd913fab4a +kernel_fixture_sha256=3f73f96a321b35a4c5561b05cfa6e9b5c573159380d37abe76f9a8ebe113a72e +initrd_fixture_sha256=e8790816224329cd76675c2aba4e62e885b5a4e0ec056227da70e775191d6d56 +vcpus=2 +vcpu_type=EPYC-v4 +guest_features=0x1 +append=console=ttyS0 loglevel=7 docker_compose_hash=2222222222222222222222222222222222222222222222222222222222222222 rootfs_hash=3333333333333333333333333333333333333333333333333333333333333333 app_id=1111111111111111111111111111111111111111 +sev_snp_measurement=6497fb9f90dc4a322228a8a5eb14742e09067bc44c184c2068d583ef628b5bae8c6cf15d91fe1bc0b7a8cbcc575be370 +cargo_live_test=cargo test -p dstack-kms --all-features recomputation_matches_sev_snp_measure_live_golden_vector -- --ignored --nocapture +cargo_live_test_result=passed locally on this host at 2026-06-02T19:49:14Z +DSTACK_SEV_SNP_MEASURE_GOLDEN_VECTOR_END +``` + +## Guest attestation proof + +A prior SNP guest smoke proof confirmed the guest kernel exposed SEV-SNP report support and could produce a report containing the expected challenge bytes. + +```text +Memory Encryption Features active: AMD SEV SEV-ES SEV-SNP +SEV: SNP running at VMPL0. +sev-guest sev-guest: Initialized SEV guest driver (using vmpck_id 0) +DSTACK_SEV_SNP_ATTESTATION_PROOF_BEGIN +source=configfs-tsm +report_size=1184 +report_data_offset=80 +report_contains_expected_report_data=true +DSTACK_SEV_SNP_ATTESTATION_PROOF_END +``` + +## Validation commands + +Run locally for this review-ready staging branch: + +```bash +cargo fmt --all +cargo test -p dstack-kms --all-features +cargo test -p dstack-attest --all-features +cargo test -p dstack-vmm --all-features +cargo check --workspace --all-features +cargo clippy --workspace --all-features -- -D warnings --allow unused_variables +git diff --check +cd kms/auth-simple && npx oxlint . && npx vitest run +``` + +## Next milestone after review + +Production SNP key release should be a separate milestone. Before removing the SNP release guards, define and test the final key-release policy contract, including revocation/advisory collateral handling, chip identity treatment, app/compose/rootfs binding, and every sensitive KMS output path. From 3792eb1b4227628b3170748395e2573eadfcd70f Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Tue, 2 Jun 2026 13:13:26 -0700 Subject: [PATCH 22/67] feat: enable guarded sev-snp key release --- docs/amd-sev-snp-review-readiness.md | 41 ++++++--- kms/kms.toml | 9 ++ kms/src/config.rs | 35 +++++++ kms/src/main_service.rs | 132 +++++++++++++++++++++++---- kms/src/main_service/amd_attest.rs | 26 +++--- 5 files changed, 199 insertions(+), 44 deletions(-) diff --git a/docs/amd-sev-snp-review-readiness.md b/docs/amd-sev-snp-review-readiness.md index 949aa00c6..7cbec206d 100644 --- a/docs/amd-sev-snp-review-readiness.md +++ b/docs/amd-sev-snp-review-readiness.md @@ -1,6 +1,6 @@ # AMD SEV-SNP Review Readiness -This branch stages AMD SEV-SNP support for review while keeping SNP key/cert release fail-closed. +This branch adds AMD SEV-SNP support and now includes a controlled, explicitly opt-in KMS key/cert release gate for SNP. ## Current review boundary @@ -11,18 +11,17 @@ Implemented and intended for review: - Report-data challenge binding and fail-closed report policy checks. - SNP launch-measurement recomputation from OVMF/kernel/initrd/cmdline inputs. - KMS SNP `BootInfo` construction from verified report measurement, chip id, launch inputs, TCB status, and advisory ids. -- Dry-run auth-policy evaluation through existing KMS auth flow. +- Auth-policy evaluation through the existing KMS auth flow. +- Controlled SNP key/cert release guarded by both external auth policy and local KMS config. - Onboarding attestation-info reporting for SNP identity fields. - VMM explicit `platform = "amd-sev-snp"` launch path. -Intentionally not enabled yet: +Default posture: -- SNP app key release. -- SNP KMS/root/temp CA key release. -- SNP app certificate/signing certificate release. -- Production SNP key release policy. - -The KMS still rejects SNP `BootInfo` before returning secrets or certificates. Treat this branch as review-ready staging, not production SNP key release. +- SNP app key release, KMS/root/temp CA key release, and app certificate release are still disabled by default. +- Operators must explicitly set `[core.sev_snp_key_release].enabled = true` before any SNP `BootInfo` can release sensitive material. +- KMS startup rejects `enabled = true` unless `enforce_self_authorization = true`, so the self-authorized `GetTempCaCert` path cannot silently bypass the SNP release gate in production config. +- Even with the local KMS gate enabled, the existing auth API must first allow the verified SNP `BootInfo` for the app/KMS identity. ## Fail-closed policy summary @@ -33,6 +32,26 @@ The KMS still rejects SNP `BootInfo` before returning secrets or certificates. T - `OutOfDate` otherwise. - SNP advisory ids are propagated from verifier output into `BootInfo`; currently this list is explicit and empty because the AMD report/VCEK evidence used here does not carry a direct advisory-list field. - `auth-simple` defaults remain strict: only `UpToDate` is accepted and any advisory id is denied unless explicitly allowlisted. +- The local KMS release gate mirrors that strict default: + - `[core.sev_snp_key_release].enabled = false` by default. + - `allowed_tcb_statuses = ["UpToDate"]` by default. + - `allowed_advisory_ids = []` by default, so any advisory remains fail-closed unless explicitly allowlisted. + +Example opt-in gate: + +```toml +[core.sev_snp_key_release] +enabled = true +allowed_tcb_statuses = ["UpToDate"] +allowed_advisory_ids = [] +``` + +Sensitive release surfaces using this gate: + +- `GetAppKey`: app disk/env/k256 key material. +- `GetKmsKey`: temp CA key plus root CA/k256 key material for authorized KMS transfer. +- `SignCert`: app certificate chain signing. +- `GetTempCaCert`: temp CA material for self-authorized KMS instances. ## Live golden-vector proof @@ -98,6 +117,6 @@ git diff --check cd kms/auth-simple && npx oxlint . && npx vitest run ``` -## Next milestone after review +## Remaining production follow-up -Production SNP key release should be a separate milestone. Before removing the SNP release guards, define and test the final key-release policy contract, including revocation/advisory collateral handling, chip identity treatment, app/compose/rootfs binding, and every sensitive KMS output path. +The release gate is controlled and production-oriented, but AMD advisory/revocation collateral is still limited by the evidence source available here: SNP reports/VCEKs do not directly carry an advisory list, so `advisory_ids` currently propagates as an explicit empty list. Future collateral fetchers can populate this field and will be denied by both auth-simple and the local KMS release gate unless each advisory is explicitly allowlisted. diff --git a/kms/kms.toml b/kms/kms.toml index d3d171b1f..83bccde45 100644 --- a/kms/kms.toml +++ b/kms/kms.toml @@ -33,6 +33,15 @@ site_name = "" # is unavailable. enforce_self_authorization = true +# AMD SEV-SNP key/cert release remains disabled unless this local KMS gate is +# explicitly enabled. External auth policy must still allow the verified +# BootInfo before any sensitive material is returned. Enabling this also +# requires enforce_self_authorization = true. +[core.sev_snp_key_release] +enabled = false +allowed_tcb_statuses = ["UpToDate"] +allowed_advisory_ids = [] + [core.image] verify = true cache_dir = "/usr/share/dstack/images" diff --git a/kms/src/config.rs b/kms/src/config.rs index 096cfd369..f4915c73c 100644 --- a/kms/src/config.rs +++ b/kms/src/config.rs @@ -48,6 +48,36 @@ fn default_guest_features() -> u64 { 0x1 } +#[derive(Debug, Clone, Deserialize)] +pub(crate) struct SevSnpKeyReleaseConfig { + /// Enable AMD SEV-SNP key/cert release after attestation, measurement + /// binding, and external auth-policy checks have all succeeded. + #[serde(default)] + pub enabled: bool, + /// Verifier-derived TCB statuses that are acceptable for releasing + /// sensitive key/cert material. Defaults to the strict production value. + #[serde(default = "default_allowed_tcb_statuses")] + pub allowed_tcb_statuses: Vec, + /// Advisory IDs that are acceptable for releasing sensitive key/cert + /// material. Defaults to empty, which rejects any advisory. + #[serde(default)] + pub allowed_advisory_ids: Vec, +} + +impl Default for SevSnpKeyReleaseConfig { + fn default() -> Self { + Self { + enabled: false, + allowed_tcb_statuses: default_allowed_tcb_statuses(), + allowed_advisory_ids: Vec::new(), + } + } +} + +fn default_allowed_tcb_statuses() -> Vec { + vec!["UpToDate".to_string()] +} + #[derive(Debug, Clone, Deserialize)] pub(crate) struct KmsConfig { pub cert_dir: PathBuf, @@ -60,6 +90,11 @@ pub(crate) struct KmsConfig { #[serde(default)] #[allow(dead_code)] pub sev_snp: Option, + /// Additional local release gate for AMD SEV-SNP key/cert material. This is + /// separate from the auth API so production deployments need an explicit KMS + /// opt-in as well as a successful external policy decision. + #[serde(default)] + pub sev_snp_key_release: SevSnpKeyReleaseConfig, #[serde(with = "serde_human_bytes")] pub admin_token_hash: Vec, #[serde(default)] diff --git a/kms/src/main_service.rs b/kms/src/main_service.rs index 23bc05887..f948b7ac1 100644 --- a/kms/src/main_service.rs +++ b/kms/src/main_service.rs @@ -33,7 +33,7 @@ use tracing::{info, warn}; use upgrade_authority::{build_boot_info, local_kms_boot_info, BootInfo}; use crate::{ - config::{KmsConfig, SevSnpMeasureConfig}, + config::{KmsConfig, SevSnpKeyReleaseConfig, SevSnpMeasureConfig}, crypto::{derive_k256_key, sign_message, sign_message_with_timestamp}, }; @@ -117,6 +117,10 @@ impl KmsState { "self-authorization is disabled; trusted RPCs will not be gated by KMS self-attestation - do not use in production TEE deployments" ); } + ensure_snp_key_release_config_safe( + config.enforce_self_authorization, + &config.sev_snp_key_release, + )?; Ok(Self { inner: Arc::new(KmsStateInner { config, @@ -169,16 +173,51 @@ pub(crate) fn build_boot_info_for_attestation( build_boot_info(att, use_boottime_mr, vm_config_str) } -fn ensure_snp_key_release_not_enabled(boot_info: &BootInfo) -> Result<()> { - if boot_info.attestation_mode == AttestationMode::DstackAmdSevSnp { +fn ensure_snp_key_release_allowed( + boot_info: &BootInfo, + policy: &SevSnpKeyReleaseConfig, +) -> Result<()> { + if boot_info.attestation_mode != AttestationMode::DstackAmdSevSnp { + return Ok(()); + } + if !policy.enabled { bail!("amd sev-snp key release is not enabled"); } + if !policy + .allowed_tcb_statuses + .iter() + .any(|allowed| allowed == &boot_info.tcb_status) + { + bail!("tcb_status is not allowed"); + } + for advisory_id in &boot_info.advisory_ids { + if !policy + .allowed_advisory_ids + .iter() + .any(|allowed| allowed == advisory_id) + { + bail!("advisory_id is not allowed"); + } + } + Ok(()) +} + +fn ensure_snp_key_release_config_safe( + enforce_self_authorization: bool, + policy: &SevSnpKeyReleaseConfig, +) -> Result<()> { + if policy.enabled && !enforce_self_authorization { + bail!("self-authorization is required for amd sev-snp key release"); + } Ok(()) } -fn ensure_self_key_release_not_enabled(self_boot_info: Option<&BootInfo>) -> Result<()> { +fn ensure_self_key_release_allowed( + self_boot_info: Option<&BootInfo>, + policy: &SevSnpKeyReleaseConfig, +) -> Result<()> { if let Some(boot_info) = self_boot_info { - ensure_snp_key_release_not_enabled(boot_info)?; + ensure_snp_key_release_allowed(boot_info, policy)?; } Ok(()) } @@ -359,7 +398,7 @@ impl KmsRpc for RpcHandler { .ensure_app_boot_allowed(&request.vm_config) .await .context("App not allowed")?; - ensure_snp_key_release_not_enabled(&boot_info)?; + ensure_snp_key_release_allowed(&boot_info, &self.state.config.sev_snp_key_release)?; let app_id = boot_info.app_id; let instance_id = boot_info.instance_id; let os_image_hash = boot_info.os_image_hash; @@ -467,7 +506,7 @@ impl KmsRpc for RpcHandler { .await .context("KMS self authorization failed")?; let info = self.ensure_kms_allowed(&request.vm_config).await?; - ensure_snp_key_release_not_enabled(&info)?; + ensure_snp_key_release_allowed(&info, &self.state.config.sev_snp_key_release)?; Ok(KmsKeyResponse { temp_ca_key: self.state.inner.temp_ca_key.clone(), keys: vec![KmsKeys { @@ -482,7 +521,7 @@ impl KmsRpc for RpcHandler { .ensure_self_allowed() .await .context("KMS self authorization failed")?; - ensure_self_key_release_not_enabled(self_boot_info)?; + ensure_self_key_release_allowed(self_boot_info, &self.state.config.sev_snp_key_release)?; Ok(GetTempCaCertResponse { temp_ca_cert: self.state.inner.temp_ca_cert.clone(), temp_ca_key: self.state.inner.temp_ca_key.clone(), @@ -521,7 +560,10 @@ impl KmsRpc for RpcHandler { let app_info = self .ensure_app_attestation_allowed(&attestation, false, true, &request.vm_config) .await?; - ensure_snp_key_release_not_enabled(&app_info.boot_info)?; + ensure_snp_key_release_allowed( + &app_info.boot_info, + &self.state.config.sev_snp_key_release, + )?; let app_ca = self.derive_app_ca(&app_info.boot_info.app_id)?; let cert = app_ca .sign_csr(&csr, Some(&app_info.boot_info.app_id), "app:custom") @@ -724,11 +766,12 @@ mod tests { } #[test] - fn snp_boot_info_is_still_blocked_from_key_release() { + fn snp_key_release_requires_explicit_enablement() { let boot_info = snp_boot_info(); + let policy = SevSnpKeyReleaseConfig::default(); - let err = ensure_snp_key_release_not_enabled(&boot_info) - .expect_err("snp boot info must not be key-release enabled yet"); + let err = ensure_snp_key_release_allowed(&boot_info, &policy) + .expect_err("snp boot info must not be key-release enabled by default"); assert!( err.to_string() .contains("amd sev-snp key release is not enabled"), @@ -737,15 +780,66 @@ mod tests { } #[test] - fn snp_self_boot_info_is_blocked_from_temp_ca_release() { + fn snp_key_release_accepts_clean_tcb_when_explicitly_enabled() { let boot_info = snp_boot_info(); + let policy = SevSnpKeyReleaseConfig { + enabled: true, + ..Default::default() + }; - let err = ensure_self_key_release_not_enabled(Some(&boot_info)) - .expect_err("snp self boot info must not receive temp CA key material"); - assert!( - err.to_string() - .contains("amd sev-snp key release is not enabled"), - "unexpected error: {err:?}" + ensure_snp_key_release_allowed(&boot_info, &policy).expect( + "explicitly enabled SNP key release should allow UpToDate/no-advisory boot info", ); } + + #[test] + fn snp_key_release_rejects_bad_tcb_or_unallowed_advisory() { + let mut boot_info = snp_boot_info(); + let policy = SevSnpKeyReleaseConfig { + enabled: true, + ..Default::default() + }; + + boot_info.tcb_status = "OutOfDate".to_string(); + let err = ensure_snp_key_release_allowed(&boot_info, &policy) + .expect_err("OutOfDate SNP TCB must not release keys by default"); + assert!(err.to_string().contains("tcb_status is not allowed")); + + let mut boot_info = snp_boot_info(); + boot_info.advisory_ids.push("SNP-TEST-ADVISORY".to_string()); + let err = ensure_snp_key_release_allowed(&boot_info, &policy) + .expect_err("unallowlisted SNP advisory must not release keys"); + assert!(err.to_string().contains("advisory_id is not allowed")); + } + + #[test] + fn snp_release_config_requires_self_authorization_when_enabled() { + let policy = SevSnpKeyReleaseConfig { + enabled: true, + ..Default::default() + }; + + let err = ensure_snp_key_release_config_safe(false, &policy) + .expect_err("enabled SNP release must require KMS self-authorization"); + assert!(err + .to_string() + .contains("self-authorization is required for amd sev-snp key release")); + ensure_snp_key_release_config_safe(true, &policy) + .expect("enabled SNP release is safe only with self-authorization enforced"); + } + + #[test] + fn snp_self_boot_info_uses_same_release_policy_for_temp_ca() { + let boot_info = snp_boot_info(); + let disabled = SevSnpKeyReleaseConfig::default(); + let enabled = SevSnpKeyReleaseConfig { + enabled: true, + ..Default::default() + }; + + ensure_self_key_release_allowed(Some(&boot_info), &disabled) + .expect_err("disabled SNP self boot info must not receive temp CA key material"); + ensure_self_key_release_allowed(Some(&boot_info), &enabled) + .expect("enabled clean SNP self boot info should pass the temp CA release gate"); + } } diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs index 5faf94fd3..43839b3e8 100644 --- a/kms/src/main_service/amd_attest.rs +++ b/kms/src/main_service/amd_attest.rs @@ -4,10 +4,10 @@ //! Fail-closed AMD SEV-SNP measurement/app binding validation. //! -//! This module intentionally does not release keys and does not enable any AMD -//! KMS key-release endpoint. It recomputes the expected SNP MEASUREMENT from -//! validated KMS configuration and launch inputs, then compares the recomputed -//! value to the hardware-verified report measurement. +//! This module does not release keys by itself. It recomputes the expected SNP +//! MEASUREMENT from validated KMS configuration and launch inputs, then compares +//! the recomputed value to the hardware-verified report measurement. KMS release +//! paths must apply their own explicit local release gate after auth succeeds. //! //! Important: this is launch measurement binding, not a complete authorization //! decision. `app_id`, compose hash, and rootfs hash are included in the SNP @@ -135,7 +135,7 @@ pub(crate) fn validate_amd_snp_measurement_binding( } /// Builds a deterministic authorization `BootInfo` for an already-verified AMD -/// SEV-SNP report without wiring it into KMS key release. +/// SEV-SNP report without releasing KMS key material by itself. /// /// This helper first recomputes and validates the QEMU SNP launch measurement. /// `mr_aggregated` is the hardware-verified 48-byte SNP `MEASUREMENT`, and @@ -148,9 +148,9 @@ pub(crate) fn validate_amd_snp_measurement_binding( /// * `key_provider_info = sha256("dstack-amd-sev-snp:app-binding:v1" || mr_system || app_id || compose_hash || chip_id)` /// * `instance_id = sha256("dstack-amd-sev-snp:instance-id:v1" || chip_id || measurement || app_id || compose_hash)` /// -/// Keeping these as helper-only values lets future authorization policy inspect -/// exactly which SNP-specific inputs were bound while SNP key release remains -/// fail-closed unless an explicit release path is added separately. +/// Keeping these values explicit lets authorization/release policy inspect +/// exactly which SNP-specific inputs were bound before any sensitive output path +/// returns key material. #[cfg(test)] pub(crate) fn build_amd_snp_boot_info( config: &SevSnpMeasureConfig, @@ -216,7 +216,7 @@ fn build_amd_snp_boot_info_with_tcb_status( /// This is the safe integration seam: the attestation verifier has already /// checked the report signature/collateral/report_data, while this KMS helper /// recomputes the launch measurement from trusted config and request inputs. -/// It still does not release keys or expose an AMD key-release RPC. +/// It still does not release keys by itself. pub(crate) fn build_amd_snp_boot_info_from_verified_attestation( config: &SevSnpMeasureConfig, attestation: &VerifiedAttestation, @@ -265,11 +265,9 @@ fn parse_measurement_input_from_vm_config(vm_config: &str) -> Result>, From 027077bf36eaccf0af0d00127f766c271612e105 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Tue, 2 Jun 2026 17:58:54 -0700 Subject: [PATCH 23/67] fix: bind sev-snp vm launch inputs --- docs/amd-sev-snp-review-readiness.md | 27 +++++ vmm/src/app.rs | 146 ++++++++++++++++++++++++++- vmm/src/app/qemu.rs | 102 ++++++++++++++++--- vmm/src/one_shot.rs | 2 +- 4 files changed, 256 insertions(+), 21 deletions(-) diff --git a/docs/amd-sev-snp-review-readiness.md b/docs/amd-sev-snp-review-readiness.md index 7cbec206d..d500a219d 100644 --- a/docs/amd-sev-snp-review-readiness.md +++ b/docs/amd-sev-snp-review-readiness.md @@ -13,6 +13,7 @@ Implemented and intended for review: - KMS SNP `BootInfo` construction from verified report measurement, chip id, launch inputs, TCB status, and advisory ids. - Auth-policy evaluation through the existing KMS auth flow. - Controlled SNP key/cert release guarded by both external auth policy and local KMS config. +- VMM-provided SNP launch inputs in `.sys-config.json` so KMS self/app auth can recompute the same launch measurement used by QEMU. - Onboarding attestation-info reporting for SNP identity fields. - VMM explicit `platform = "amd-sev-snp"` launch path. @@ -102,6 +103,32 @@ report_contains_expected_report_data=true DSTACK_SEV_SNP_ATTESTATION_PROOF_END ``` +## Manual dstack E2E smoke status + +An additional manual smoke was attempted on the SNP host (`chris@173.234.27.162`) using the PR branch, release-built `dstack-vmm`/`supervisor`/`dstack-kms`, QEMU 10.0.2, and the SNP-capable OVMF at `/opt/AMDSEV/usr/local/share/qemu/OVMF.fd`. + +That smoke exposed and fixed several VMM/KMS-auth integration issues before the guest reached KMS: + +- `.sys-config.json` did not include the `sev_snp_measurement` launch input object needed by KMS SNP `BootInfo` recomputation. +- The VMM launch path required `metadata.json.rootfs_hash`, while the released `dstack-0.5.11` images carry the rootfs hash in `dstack.rootfs_hash=...` on the kernel cmdline. +- The VMM SNP QEMU path now uses the SNP measurement CPU model (`EPYC-v4`) and confidential virtio PCI options (`disable-legacy=on,iommu_platform=true`) for SNP-launched virtio devices, matching the host's working SNP launch posture more closely. + +After those fixes, the dstack-managed SNP guest launches far enough for OVMF to load the measured kernel/initrd/cmdline path, but the `dstack-0.5.11` and `dstack-dev-0.5.11` images did not complete Linux/userspace boot in this SNP direct-kernel environment before the smoke timeout. A control run of the same `dstack-dev-0.5.11` kernel/initrd/rootfs without SNP boots Linux and reaches `dstack Guest Preparation Service`, proving the image itself is bootable and narrowing the blocker to the SNP+OVMF direct-kernel transition rather than KMS release policy. The sanitized failure signature is: + +```text +remote_host=chris@173.234.27.162 +qemu_version=10.0.2 +ovmf_sha256=67e7a7027437823e9c166a60d00666d5d5391e13050488cad5cc2acd913fab4a +image=dstack-0.5.11 and dstack-dev-0.5.11 +vmm_head=6cb351f9bebde233 + local smoke fixes +control_without_snp=dstack-dev-0.5.11 boots Linux and reaches dstack Guest Preparation Service +observed=OVMF loads fw_cfg kernel/cmdline/initrd and emits "EFI stub: Loaded initrd from LINUX_EFI_INITRD_MEDIA_GUID device path" +blocked_before=Linux/userspace readiness in SNP+OVMF direct-kernel boot; KMS guest userspace readiness / app GetAppKey release exercise +no_secret_material_returned=true +``` + +This means the PR still has live SNP report proof, live golden-vector measurement proof, and release-gate unit/integration coverage, but the full dstack-managed guest -> KMS `GetAppKey` hardware E2E remains blocked on guest image/boot compatibility in this smoke environment. + ## Validation commands Run locally for this review-ready staging branch: diff --git a/vmm/src/app.rs b/vmm/src/app.rs index 8b9714cba..874d25a9f 100644 --- a/vmm/src/app.rs +++ b/vmm/src/app.rs @@ -21,6 +21,7 @@ use or_panic::ResultOrPanic; use ra_rpc::client::RaClient; use serde::{Deserialize, Serialize}; use serde_json::json; +use sha2::{Digest, Sha256}; use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; use std::net::IpAddr; use std::path::{Path, PathBuf}; @@ -998,7 +999,8 @@ impl App { let shared_dir = self.shared_dir(id); let manifest = work_dir.manifest().context("Failed to read manifest")?; let cfg = &self.config; - let sys_config_str = make_sys_config(cfg, &manifest)?; + let compose_hash = sha256_file(shared_dir.join(APP_COMPOSE))?; + let sys_config_str = make_sys_config(cfg, &manifest, &hex::encode(compose_hash))?; fs::write(shared_dir.join(SYS_CONFIG), sys_config_str) .context("Failed to write vm config")?; Ok(()) @@ -1136,7 +1138,11 @@ fn rotate_serial_log(work_dir: &VmWorkDir, max_bytes: u64) { } } -pub(crate) fn make_sys_config(cfg: &Config, manifest: &Manifest) -> Result { +pub(crate) fn make_sys_config( + cfg: &Config, + manifest: &Manifest, + compose_hash: &str, +) -> Result { let image_path = cfg.image.path.join(&manifest.image); let image = Image::load(image_path).context("Failed to load image info")?; let img_ver = image.info.version_tuple().unwrap_or((0, 0, 0)); @@ -1160,14 +1166,41 @@ pub(crate) fn make_sys_config(cfg: &Config, manifest: &Manifest) -> Result Result { +fn file_sha256_hex(path: &Path) -> Result { + Ok(hex::encode(sha256_file(path)?)) +} + +fn image_rootfs_hash(image: &Image) -> Result<&str> { + if let Some(rootfs_hash) = image.info.rootfs_hash.as_deref() { + return Ok(rootfs_hash); + } + let cmdline = image.info.cmdline.as_deref().unwrap_or_default(); + cmdline + .split_whitespace() + .find_map(|param| param.strip_prefix("dstack.rootfs_hash=")) + .ok_or_else(|| anyhow::anyhow!("rootfs_hash is required for amd sev-snp")) +} + +fn sha256_file(path: impl AsRef) -> Result<[u8; 32]> { + let data = fs::read(path).context("Failed to read file for sha256")?; + let mut out = [0u8; 32]; + out.copy_from_slice(&Sha256::digest(data)); + Ok(out) +} + +fn make_vm_config( + cfg: &Config, + manifest: &Manifest, + image: &Image, + compose_hash: &str, +) -> Result { let os_image_hash = image .digest .as_ref() @@ -1192,9 +1225,114 @@ fn make_vm_config(cfg: &Config, manifest: &Manifest, image: &Image) -> Result String { + hex::encode(vec![byte; len]) + } + + #[test] + fn amd_sev_snp_sys_config_includes_measurement_input_for_kms_auth() { + let temp = std::env::temp_dir().join(format!( + "dstack-vmm-snp-test-{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let temp = temp.as_path(); + let image_root = temp.join("images"); + let image_dir = image_root.join("dstack-test"); + fs::create_dir_all(&image_dir).unwrap(); + fs::write(image_dir.join("kernel"), b"snp-test-kernel").unwrap(); + fs::write(image_dir.join("initrd"), b"snp-test-initrd").unwrap(); + fs::write(image_dir.join("rootfs"), b"snp-test-rootfs").unwrap(); + fs::write( + image_dir.join("metadata.json"), + serde_json::json!({ + "cmdline": format!("console=ttyS0 dstack.rootfs_hash={}", hex_of(0x33, 32)), + "kernel": "kernel", + "initrd": "initrd", + "rootfs": "rootfs", + "version": "0.5.11" + }) + .to_string(), + ) + .unwrap(); + + let mut config: Config = Figment::from(load_config_figment(None)).extract().unwrap(); + config.image.path = image_root; + config.cvm.platform = TeePlatform::AmdSevSnp; + let compose_hash = hex_of(0x22, 32); + let manifest = Manifest { + id: "snp-test".to_string(), + name: "snp-test".to_string(), + app_id: hex_of(0x11, 20), + vcpu: 2, + memory: 1024, + disk_size: 1024, + image: "dstack-test".to_string(), + port_map: vec![], + created_at_ms: 0, + hugepages: false, + pin_numa: false, + gpus: None, + kms_urls: vec![], + gateway_urls: vec![], + no_tee: false, + networking: None, + }; + + let sys_config: serde_json::Value = + serde_json::from_str(&make_sys_config(&config, &manifest, &compose_hash).unwrap()) + .unwrap(); + let vm_config: serde_json::Value = + serde_json::from_str(sys_config["vm_config"].as_str().unwrap()).unwrap(); + let measurement = &vm_config["sev_snp_measurement"]; + + assert_eq!(measurement["app_id"], manifest.app_id); + assert_eq!(measurement["compose_hash"], compose_hash); + assert_eq!(measurement["rootfs_hash"], hex_of(0x33, 32)); + assert_eq!( + measurement["kernel_hash"], + hex::encode(Sha256::digest(b"snp-test-kernel")) + ); + assert_eq!( + measurement["initrd_hash"], + hex::encode(Sha256::digest(b"snp-test-initrd")) + ); + assert_eq!(measurement["vcpus"], 2); + assert_eq!(measurement["vcpu_type"], "EPYC-v4"); + assert_eq!(measurement["ovmf_hash"], ""); + assert!(measurement["ovmf_sections"].as_array().unwrap().is_empty()); + } +} + fn paginate(items: Vec, page: u32, page_size: u32) -> impl Iterator { let skip; let take; diff --git a/vmm/src/app/qemu.rs b/vmm/src/app/qemu.rs index 388de20bd..d6309e086 100644 --- a/vmm/src/app/qemu.rs +++ b/vmm/src/app/qemu.rs @@ -18,7 +18,7 @@ use std::{ time::{Duration, SystemTime}, }; -use super::{image::Image, GpuConfig, VmState}; +use super::{image::Image, GpuConfig, ImageInfo, VmState}; use anyhow::{bail, Context, Result}; use base64::prelude::*; use bon::Builder; @@ -369,7 +369,10 @@ impl VmState { #[cfg(test)] mod tests { - use super::{amd_sev_snp_measured_cmdline, amd_sev_snp_memory_backend_arg, sanitize_optional}; + use super::{ + amd_sev_snp_measured_cmdline, amd_sev_snp_memory_backend_arg, amd_sev_snp_rootfs_hash, + sanitize_optional, virtio_pci_device, ImageInfo, + }; #[test] fn sanitize_optional_filters_empty_owned_values() { @@ -411,6 +414,57 @@ mod tests { "console=ttyS0 loglevel=7 docker_compose_hash=22 rootfs_hash=33 app_id=1111111111111111111111111111111111111111" ); } + + #[test] + fn amd_sev_snp_rootfs_hash_falls_back_to_dstack_cmdline() { + let info = ImageInfo { + cmdline: Some("console=ttyS0 dstack.rootfs_hash=abc123 dstack.rootfs_size=100".into()), + kernel: "kernel".into(), + initrd: "initrd".into(), + hda: None, + rootfs: None, + bios: None, + rootfs_hash: None, + shared_ro: false, + version: "0.5.11".into(), + is_dev: false, + ovmf_variant: None, + }; + + assert_eq!(amd_sev_snp_rootfs_hash(&info).unwrap(), "abc123"); + } + + #[test] + fn amd_sev_snp_uses_confidential_virtio_pci_options() { + assert_eq!( + virtio_pci_device("virtio-blk-pci,drive=hd0", true), + "virtio-blk-pci,drive=hd0,disable-legacy=on,iommu_platform=true" + ); + assert_eq!( + virtio_pci_device("virtio-blk-pci,drive=hd0", false), + "virtio-blk-pci,drive=hd0" + ); + } +} + +fn amd_sev_snp_rootfs_hash(info: &ImageInfo) -> Result<&str> { + if let Some(rootfs_hash) = info.rootfs_hash.as_deref() { + return Ok(rootfs_hash); + } + info.cmdline + .as_deref() + .unwrap_or_default() + .split_whitespace() + .find_map(|param| param.strip_prefix("dstack.rootfs_hash=")) + .ok_or_else(|| anyhow::anyhow!("rootfs_hash is required for amd sev-snp")) +} + +fn virtio_pci_device(device: &str, snp: bool) -> String { + if snp { + format!("{device},disable-legacy=on,iommu_platform=true") + } else { + device.to_string() + } } impl VmConfig { @@ -438,11 +492,14 @@ impl VmConfig { } let app_compose = workdir.app_compose().context("Failed to get app compose")?; let qemu = &cfg.qemu_path; + let is_amd_sev_snp = + cfg.platform.resolve() == TeePlatform::AmdSevSnp && !self.manifest.no_tee; let mut smp = self.manifest.vcpu.max(1); let mut mem = self.manifest.memory; let mut command = Command::new(qemu); command.arg("-accel").arg("kvm"); - command.arg("-cpu").arg("host"); + let cpu = if is_amd_sev_snp { "EPYC-v4" } else { "host" }; + command.arg("-cpu").arg(cpu); command.arg("-nographic"); command.arg("-nodefaults"); command.arg("-chardev").arg(format!( @@ -498,7 +555,10 @@ impl VmConfig { "file={},if=none,id=hd0,format=raw,readonly=on", rootfs.display() )); - command.arg("-device").arg("virtio-blk-pci,drive=hd0"); + command.arg("-device").arg(virtio_pci_device( + "virtio-blk-pci,drive=hd0", + is_amd_sev_snp, + )); } _ => { bail!("Unsupported rootfs type: {ext}"); @@ -510,7 +570,10 @@ impl VmConfig { .arg("-drive") .arg(format!("file={},if=none,id=hd1", hda_path.display())) .arg("-device") - .arg("virtio-blk-pci,drive=hd1"); + .arg(virtio_pci_device( + "virtio-blk-pci,drive=hd1", + is_amd_sev_snp, + )); // Resolve per-VM networking override against global config. // Per-VM only sets mode; shared fields (bridge name, mac_prefix, etc.) // are merged from global config. @@ -534,7 +597,10 @@ impl VmConfig { // Generate deterministic MAC for all networking modes let prefix = networking.mac_prefix_bytes(); let mac = mac_address_for_vm(&self.manifest.id, &prefix); - let net_device = format!("virtio-net-pci,netdev=net0,mac={mac}"); + let net_device = virtio_pci_device( + &format!("virtio-net-pci,netdev=net0,mac={mac}"), + is_amd_sev_snp, + ); let netdev = match networking.mode { NetworkingMode::User => { let mut netdev = format!( @@ -580,9 +646,10 @@ impl VmConfig { .arg("tpm-tis,tpmdev=tpm0"); } - command - .arg("-device") - .arg(format!("vhost-vsock-pci,guest-cid={}", self.cid)); + command.arg("-device").arg(virtio_pci_device( + &format!("vhost-vsock-pci,guest-cid={}", self.cid), + is_amd_sev_snp, + )); // Configure shared files delivery: either via disk or 9p match cfg.host_share_mode.as_str() { @@ -607,7 +674,10 @@ impl VmConfig { HOST_SHARED_DISK_LABEL )) .arg("-device") - .arg("virtio-blk-pci,drive=vvfat0"); + .arg(virtio_pci_device( + "virtio-blk-pci,drive=vvfat0", + is_amd_sev_snp, + )); } "vhd" => { // Use a second virtual disk (hd2) to share files @@ -624,7 +694,10 @@ impl VmConfig { shared_disk_path.display() )) .arg("-device") - .arg("virtio-blk-pci,drive=hd2"); + .arg(virtio_pci_device( + "virtio-blk-pci,drive=hd2", + is_amd_sev_snp, + )); } _ => { bail!("Invalid host sharing mode: {}", cfg.host_share_mode); @@ -761,10 +834,7 @@ impl VmConfig { .app_compose_hash() .context("Failed to get compose hash")?, ); - let rootfs_hash = - self.image.info.rootfs_hash.as_deref().ok_or_else(|| { - anyhow::anyhow!("rootfs_hash is required for amd sev-snp") - })?; + let rootfs_hash = amd_sev_snp_rootfs_hash(&self.image.info)?; Some(amd_sev_snp_measured_cmdline( cmdline, &compose_hash, @@ -948,7 +1018,7 @@ impl VmConfig { .arg(amd_sev_snp_memory_backend_arg(mem)); command .arg("-object") - .arg("sev-snp-guest,id=sev0,policy=0x30000,sev-device=/dev/sev,kernel-hashes=on,cbitpos=51,reduced-phys-bits=1"); + .arg("sev-snp-guest,id=sev0,policy=0x30000,sev-device=/dev/sev,kernel-hashes=on,author-key-enabled=on,cbitpos=51,reduced-phys-bits=1"); command.arg("-machine").arg( "q35,kernel-irqchip=split,confidential-guest-support=sev0,memory-backend=ram1,hpet=off", ); diff --git a/vmm/src/one_shot.rs b/vmm/src/one_shot.rs index 39f9d68f4..0736e1a7f 100644 --- a/vmm/src/one_shot.rs +++ b/vmm/src/one_shot.rs @@ -235,7 +235,7 @@ Compose file content (first 200 chars): // 2. Create .sys-config.json (critical for 0.5.x VMs) // Use manifest URLs if available, fallback to config URLs (matching VMM's sync_dynamic_config logic) - let sys_config_str = make_sys_config(&config, &manifest)?; + let sys_config_str = make_sys_config(&config, &manifest, &compose_hash)?; let sys_config_path = vm_work_dir.shared_dir().join(".sys-config.json"); fs_err::write(&sys_config_path, sys_config_str).context("Failed to write sys config")?; From cfe476b2e36694242e62ac81630000b2046d0a96 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Tue, 2 Jun 2026 21:12:27 -0700 Subject: [PATCH 24/67] fix: complete sev-snp key release smoke path --- Cargo.lock | 1 + basefiles/dstack-prepare.sh | 24 +- docs/amd-sev-snp-review-readiness.md | 29 ++- dstack-attest/Cargo.toml | 1 + dstack-attest/src/amd_sev_snp.rs | 208 +++++++++++++++++- dstack-attest/src/attestation.rs | 12 +- dstack-attest/src/sev_snp.rs | 43 +++- dstack-util/src/main.rs | 17 +- dstack-util/src/system_setup.rs | 19 +- .../src/system_setup/config_id_verifier.rs | 34 +++ kms/src/main_service.rs | 1 + kms/src/main_service/amd_attest.rs | 21 +- kms/src/onboard_service.rs | 1 + vmm/src/app.rs | 5 + 14 files changed, 382 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 413779e42..61a229561 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2340,6 +2340,7 @@ dependencies = [ "nsm-qvl", "or-panic", "parity-scale-codec", + "reqwest", "rmp-serde", "serde", "serde-human-bytes", diff --git a/basefiles/dstack-prepare.sh b/basefiles/dstack-prepare.sh index 7b6ac2c38..214c53a52 100755 --- a/basefiles/dstack-prepare.sh +++ b/basefiles/dstack-prepare.sh @@ -94,13 +94,27 @@ mount_overlay /usr $OVERLAY_TMP mount_overlay /bin $OVERLAY_TMP mount_overlay /home $OVERLAY_TMP +# systemd-resolved may be unavailable in minimal smoke/debug boots; keep DNS usable for dockerd pulls. +if ! [[ -s /etc/resolv.conf ]] || grep -Eq 'nameserver[[:space:]]+(127\.|::1)' /etc/resolv.conf; then + printf 'nameserver 1.1.1.1\nnameserver 8.8.8.8\n' >/etc/resolv.conf +fi + # Make sure the system time is synchronized log "Syncing system time..." -# Let the chronyd correct the system time immediately -chronyc makestep - -if ! [[ -e /dev/tdx_guest ]]; then - modprobe tdx-guest +# Let the chronyd correct the system time immediately; keep booting if chronyd is not ready yet. +chronyc makestep || log "Warning: chronyc makestep failed; continuing" + +if [[ -e /dev/sev-guest ]] || grep -qw sev_guest /sys/kernel/config/tsm/report/*/provider 2>/dev/null; then + log "SEV-SNP guest device/TSM provider detected" +elif [[ -e /dev/tdx_guest ]]; then + log "TDX guest device detected" +elif modprobe sev-guest 2>/dev/null; then + log "Loaded sev-guest module" +elif modprobe tdx-guest 2>/dev/null; then + log "Loaded tdx-guest module" +else + log "Error: neither sev-guest nor tdx-guest module is available" + exit 1 fi # Setup configfs and TSM for TDX attestation diff --git a/docs/amd-sev-snp-review-readiness.md b/docs/amd-sev-snp-review-readiness.md index d500a219d..a7594c51c 100644 --- a/docs/amd-sev-snp-review-readiness.md +++ b/docs/amd-sev-snp-review-readiness.md @@ -113,21 +113,34 @@ That smoke exposed and fixed several VMM/KMS-auth integration issues before the - The VMM launch path required `metadata.json.rootfs_hash`, while the released `dstack-0.5.11` images carry the rootfs hash in `dstack.rootfs_hash=...` on the kernel cmdline. - The VMM SNP QEMU path now uses the SNP measurement CPU model (`EPYC-v4`) and confidential virtio PCI options (`disable-legacy=on,iommu_platform=true`) for SNP-launched virtio devices, matching the host's working SNP launch posture more closely. -After those fixes, the dstack-managed SNP guest launches far enough for OVMF to load the measured kernel/initrd/cmdline path, but the `dstack-0.5.11` and `dstack-dev-0.5.11` images did not complete Linux/userspace boot in this SNP direct-kernel environment before the smoke timeout. A control run of the same `dstack-dev-0.5.11` kernel/initrd/rootfs without SNP boots Linux and reaches `dstack Guest Preparation Service`, proving the image itself is bootable and narrowing the blocker to the SNP+OVMF direct-kernel transition rather than KMS release policy. The sanitized failure signature is: +After those fixes, the manual smoke progressed through full dstack-managed SNP guest boot, KMS self-bootstrap, app guest boot, app quote verification, and `GetAppKey` release. Additional smoke/debug fixes made the path work end-to-end: + +- Minimal guest boot now keeps DNS usable when `systemd-resolved`/`chronyd` are unavailable early in smoke boots and detects `sev-guest` before trying the TDX guest module. +- SNP guests skip TDX-only `mr_config_id` and app-info RTMR decoding while still preserving non-SNP behavior. +- Configfs TSM report collection falls back to the SEV-SNP extended-report ioctl when configfs does not carry certificate collateral. +- If guest evidence still lacks ASK/VCEK collateral, the verifier fetches AMD KDS ARK/ASK/VCEK using the report `chip_id` and reported TCB, then verifies the signed report fail-closed. +- KMS measurement recomputation now uses the image's original kernel cmdline as the measurement base before appending `docker_compose_hash`, `rootfs_hash`, and `app_id`, matching the VMM QEMU `-append` path. + +Sanitized smoke result: ```text remote_host=chris@173.234.27.162 qemu_version=10.0.2 ovmf_sha256=67e7a7027437823e9c166a60d00666d5d5391e13050488cad5cc2acd913fab4a -image=dstack-0.5.11 and dstack-dev-0.5.11 -vmm_head=6cb351f9bebde233 + local smoke fixes -control_without_snp=dstack-dev-0.5.11 boots Linux and reaches dstack Guest Preparation Service -observed=OVMF loads fw_cfg kernel/cmdline/initrd and emits "EFI stub: Loaded initrd from LINUX_EFI_INITRD_MEDIA_GUID device path" -blocked_before=Linux/userspace readiness in SNP+OVMF direct-kernel boot; KMS guest userspace readiness / app GetAppKey release exercise -no_secret_material_returned=true +image=dstack-dev-0.5.11-snp-dnsfix +platform=amd-sev-snp +vmm_branch=feat/amd-sev-snp-conversion + local smoke fixes +kms_guest=booted SNP Linux/userspace and started dstack-kms +app_guest=booted SNP Linux/userspace and requested app keys +kms_auth=/bootAuth/kms 200 and /bootAuth/app 200 +tcb_status=OutOfDate in this lab host policy run +failure_gate=default UpToDate-only policy rejected release with "tcb_status is not allowed" +success_gate=explicit lab allowlist ["UpToDate", "OutOfDate"] released GetTempCaCert and GetAppKey +kms_metrics=dstack_kms_attestation_requests_total 1, dstack_kms_attestation_failures_total 0 +no_secret_material_logged=true ``` -This means the PR still has live SNP report proof, live golden-vector measurement proof, and release-gate unit/integration coverage, but the full dstack-managed guest -> KMS `GetAppKey` hardware E2E remains blocked on guest image/boot compatibility in this smoke environment. +This means the PR now has live SNP report proof, live golden-vector measurement proof, release-gate unit/integration coverage, and a manual full dstack-managed SNP guest -> KMS `GetAppKey` hardware E2E proof. The success run required an explicit lab-only TCB allowlist because this host reports `OutOfDate`; production defaults remain fail-closed (`UpToDate` only). ## Validation commands diff --git a/dstack-attest/Cargo.toml b/dstack-attest/Cargo.toml index e3d3fcfad..174e6655b 100644 --- a/dstack-attest/Cargo.toml +++ b/dstack-attest/Cargo.toml @@ -14,6 +14,7 @@ anyhow.workspace = true base64.workspace = true cc-eventlog.workspace = true rmp-serde.workspace = true +reqwest = { workspace = true, features = ["blocking"] } dcap-qvl.workspace = true dstack-types.workspace = true ez-hash.workspace = true diff --git a/dstack-attest/src/amd_sev_snp.rs b/dstack-attest/src/amd_sev_snp.rs index 3a2f1fbc5..a3bcaf8f1 100644 --- a/dstack-attest/src/amd_sev_snp.rs +++ b/dstack-attest/src/amd_sev_snp.rs @@ -107,6 +107,13 @@ pub struct AmdSnpAttestationInput<'a> { pub vcek_pem: &'a [u8], } +#[derive(Debug, Clone, PartialEq, Eq)] +struct AmdKdsCollateral { + ark: CertBytes, + ask: CertBytes, + vcek: CertBytes, +} + pub fn verify_amd_snp_attestation( input: &AmdSnpAttestationInput<'_>, ) -> Result { @@ -127,6 +134,26 @@ fn verify_amd_snp_attestation_with_certs( report_bytes: &[u8], ask_bytes: CertBytes, vcek_bytes: CertBytes, +) -> Result { + let ark_der = STANDARD + .decode(GENOA_ARK_DER_B64) + .context("failed to decode amd genoa ark")?; + verify_amd_snp_attestation_with_cert_chain( + report_bytes, + CertBytes { + bytes: ark_der, + encoding: CertEncoding::Der, + }, + ask_bytes, + vcek_bytes, + ) +} + +fn verify_amd_snp_attestation_with_cert_chain( + report_bytes: &[u8], + ark_bytes: CertBytes, + ask_bytes: CertBytes, + vcek_bytes: CertBytes, ) -> Result { if report_bytes.len() != 1184 { bail!( @@ -137,11 +164,7 @@ fn verify_amd_snp_attestation_with_certs( let report = AttestationReport::from_bytes(report_bytes) .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; - let ark_der = STANDARD - .decode(GENOA_ARK_DER_B64) - .context("failed to decode amd genoa ark")?; - let ark = Certificate::from_der(&ark_der) - .map_err(|err| anyhow::anyhow!("failed to parse amd genoa ark: {err:?}"))?; + let ark = parse_certificate(&ark_bytes, "ark")?; let ask = parse_certificate(&ask_bytes, "ask")?; let vcek = parse_certificate(&vcek_bytes, "vcek")?; @@ -208,6 +231,142 @@ pub fn verify_amd_snp_evidence( Ok(verified) } +pub fn verify_amd_snp_evidence_with_kds_fallback( + report: &[u8], + cert_chain: &[Vec], + expected_report_data: &[u8; 64], +) -> Result { + if !cert_chain.is_empty() { + return verify_amd_snp_evidence(report, cert_chain, expected_report_data); + } + let report_obj = AttestationReport::from_bytes(report) + .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; + let collateral = fetch_amd_kds_collateral_for_report(&report_obj) + .context("failed to fetch amd sev-snp KDS collateral for empty cert_chain")?; + let verified = verify_amd_snp_attestation_with_cert_chain( + report, + collateral.ark, + collateral.ask, + collateral.vcek, + )?; + if &verified.report_data != expected_report_data { + bail!("amd sev-snp report_data mismatch"); + } + Ok(verified) +} + +fn fetch_amd_kds_collateral_for_report(report: &AttestationReport) -> Result { + let mut errors = Vec::new(); + for product in ["Genoa", "Milan", "Bergamo", "Siena", "Turin"] { + match fetch_amd_kds_collateral_for_product(product, report) { + Ok(collateral) => return Ok(collateral), + Err(err) => errors.push(format!("{product}: {err:#}")), + } + } + bail!( + "amd sev-snp KDS collateral unavailable for supported products: {}", + errors.join("; ") + ) +} + +fn fetch_amd_kds_collateral_for_product( + product: &str, + report: &AttestationReport, +) -> Result { + let (ark, ask) = fetch_amd_kds_ca_chain(product)?; + let mut chip_id = [0u8; 64]; + chip_id.copy_from_slice( + report + .chip_id + .as_ref() + .get(..64) + .context("amd sev-snp chip_id too short")?, + ); + let vcek_url = amd_kds_vcek_url(product, &chip_id, report.reported_tcb.into()); + let vcek = reqwest::blocking::Client::new() + .get(&vcek_url) + .send() + .with_context(|| format!("failed to request amd sev-snp vcek from {vcek_url}"))? + .error_for_status() + .with_context(|| format!("amd sev-snp vcek request failed for {vcek_url}"))? + .bytes() + .context("failed to read amd sev-snp vcek response")? + .to_vec(); + Ok(AmdKdsCollateral { + ark, + ask, + vcek: CertBytes { + bytes: vcek, + encoding: CertEncoding::Der, + }, + }) +} + +fn fetch_amd_kds_ca_chain(product: &str) -> Result<(CertBytes, CertBytes)> { + let url = format!("https://kdsintf.amd.com/vcek/v1/{product}/cert_chain"); + let chain = reqwest::blocking::Client::new() + .get(&url) + .send() + .with_context(|| format!("failed to request amd sev-snp cert_chain from {url}"))? + .error_for_status() + .with_context(|| format!("amd sev-snp cert_chain request failed for {url}"))? + .bytes() + .context("failed to read amd sev-snp cert_chain response")?; + extract_ark_ask_from_amd_kds_cert_chain(&chain) +} + +fn amd_kds_vcek_url(product: &str, chip_id: &[u8; 64], tcb: AmdSnpTcbVersion) -> String { + format!( + "https://kdsintf.amd.com/vcek/v1/{}/{}?blSPL={}&teeSPL={}&snpSPL={}&ucodeSPL={}", + product, + hex::encode(chip_id), + tcb.bootloader, + tcb.tee, + tcb.snp, + tcb.microcode + ) +} + +fn extract_ark_ask_from_amd_kds_cert_chain(chain: &[u8]) -> Result<(CertBytes, CertBytes)> { + let certs = extract_pem_certs(chain)?; + if certs.len() < 2 { + bail!("amd sev-snp cert_chain must contain ASK and ARK certificates"); + } + Ok(( + CertBytes { + bytes: certs[1].clone(), + encoding: CertEncoding::Pem, + }, + CertBytes { + bytes: certs[0].clone(), + encoding: CertEncoding::Pem, + }, + )) +} + +fn extract_pem_certs(chain: &[u8]) -> Result>> { + let chain = std::str::from_utf8(chain).context("amd sev-snp cert_chain is not utf-8 pem")?; + let begin = "-----BEGIN CERTIFICATE-----"; + let end = "-----END CERTIFICATE-----"; + let mut rest = chain; + let mut certs = Vec::new(); + while let Some(start) = rest.find(begin) { + let after_start = &rest[start..]; + let cert_end = after_start + .find(end) + .map(|idx| idx + end.len()) + .context("amd sev-snp cert_chain has unterminated certificate")?; + let mut cert = after_start.as_bytes()[..cert_end].to_vec(); + cert.push(b'\n'); + certs.push(cert); + rest = &after_start[cert_end..]; + } + if certs.is_empty() { + bail!("amd sev-snp cert_chain missing certificates"); + } + Ok(certs) +} + fn parse_certificate(cert: &CertBytes, name: &str) -> Result { match cert.encoding { CertEncoding::Pem => Certificate::from_pem(&cert.bytes) @@ -383,6 +542,45 @@ mod tests { ); } + #[test] + fn amd_kds_vcek_url_binds_chip_id_and_reported_tcb() { + let chip_id = [0xab; 64]; + let tcb = AmdSnpTcbVersion { + bootloader: 1, + tee: 2, + snp: 3, + microcode: 4, + }; + + let url = amd_kds_vcek_url("Genoa", &chip_id, tcb); + + assert_eq!( + url, + format!( + "https://kdsintf.amd.com/vcek/v1/Genoa/{}?blSPL=1&teeSPL=2&snpSPL=3&ucodeSPL=4", + hex::encode(chip_id) + ) + ); + } + + #[test] + fn amd_kds_cert_chain_extracts_ask_pem_and_ark_pem() { + let chain = b"-----BEGIN CERTIFICATE-----\nASK\n-----END CERTIFICATE-----\n-----BEGIN CERTIFICATE-----\nARK\n-----END CERTIFICATE-----\n"; + + let (ark_cert, ask_cert) = extract_ark_ask_from_amd_kds_cert_chain(chain).unwrap(); + + assert_eq!( + ask_cert.bytes, + b"-----BEGIN CERTIFICATE-----\nASK\n-----END CERTIFICATE-----\n".to_vec() + ); + assert_eq!(ask_cert.encoding, CertEncoding::Pem); + assert_eq!( + ark_cert.bytes, + b"-----BEGIN CERTIFICATE-----\nARK\n-----END CERTIFICATE-----\n".to_vec() + ); + assert_eq!(ark_cert.encoding, CertEncoding::Pem); + } + #[test] fn malformed_report_fails_closed_before_success() { let cert_chain = vec![b"not ask".to_vec(), b"not vcek".to_vec()]; diff --git a/dstack-attest/src/attestation.rs b/dstack-attest/src/attestation.rs index 3030cf5ff..f8a543e59 100644 --- a/dstack-attest/src/attestation.rs +++ b/dstack-attest/src/attestation.rs @@ -770,11 +770,13 @@ impl AttestationV1 { }) } PlatformEvidence::SevSnp { report, cert_chain } => { - DstackVerifiedReport::DstackAmdSevSnp(crate::amd_sev_snp::verify_amd_snp_evidence( - report, - cert_chain, - &report_data, - )?) + DstackVerifiedReport::DstackAmdSevSnp( + crate::amd_sev_snp::verify_amd_snp_evidence_with_kds_fallback( + report, + cert_chain, + &report_data, + )?, + ) } }; diff --git a/dstack-attest/src/sev_snp.rs b/dstack-attest/src/sev_snp.rs index 4844585fc..cc1f6b09f 100644 --- a/dstack-attest/src/sev_snp.rs +++ b/dstack-attest/src/sev_snp.rs @@ -18,7 +18,26 @@ const SNP_REPORT_SIZE: usize = 1184; pub fn get_report(report_data: [u8; 64]) -> Result { if has_sev_snp_tsm_provider(Path::new(TSM_REPORT_ROOT)) { match get_report_configfs(report_data) { - Ok(quote) => return Ok(quote), + Ok(quote) => { + if configfs_report_needs_ioctl_cert_chain_fallback( + "e, + Path::new(SEV_GUEST_DEVICE).exists(), + ) { + tracing::debug!( + "sev-snp configfs tsm report did not include a certificate chain; falling back to ioctl extended report" + ); + match get_report_ioctl(report_data) { + Ok(ioctl_quote) if !ioctl_quote.cert_chain.is_empty() => { + return Ok(ioctl_quote) + } + Ok(_) => return Ok(quote), + Err(err) => tracing::debug!( + "failed to get sev-snp report from ioctl fallback: {err:#}" + ), + } + } + return Ok(quote); + } Err(err) => tracing::debug!("failed to get sev-snp report from configfs tsm: {err:#}"), } } @@ -28,6 +47,13 @@ pub fn get_report(report_data: [u8; 64]) -> Result { bail!("sev-snp report is unavailable: neither {TSM_REPORT_ROOT} nor {SEV_GUEST_DEVICE} exists") } +fn configfs_report_needs_ioctl_cert_chain_fallback( + quote: &SnpQuote, + sev_guest_device_available: bool, +) -> bool { + sev_guest_device_available && quote.cert_chain.is_empty() +} + pub(crate) fn has_sev_snp_tsm_provider(root: &Path) -> bool { if !root.exists() { return false; @@ -229,6 +255,21 @@ mod tests { let _ = fs_err::remove_dir_all(root); } + #[test] + fn configfs_report_without_cert_chain_requires_ioctl_fallback_when_available() { + let quote = SnpQuote { + report: vec![0u8; SNP_REPORT_SIZE], + cert_chain: vec![], + }; + + assert!(configfs_report_needs_ioctl_cert_chain_fallback( + "e, true + )); + assert!(!configfs_report_needs_ioctl_cert_chain_fallback( + "e, false + )); + } + fn test_dir(name: &str) -> std::path::PathBuf { let nanos = SystemTime::now() .duration_since(UNIX_EPOCH) diff --git a/dstack-util/src/main.rs b/dstack-util/src/main.rs index d2e792b4f..3aa60bd82 100644 --- a/dstack-util/src/main.rs +++ b/dstack-util/src/main.rs @@ -4,7 +4,7 @@ use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; -use dstack_attest::emit_runtime_event; +use dstack_attest::{attestation::AttestationMode, emit_runtime_event}; use dstack_types::{KeyProvider, KeyProviderKind}; use fs_err as fs; use getrandom::fill as getrandom; @@ -690,6 +690,21 @@ fn cmd_rand(rand_args: RandArgs) -> Result<()> { } fn cmd_show_mrs() -> Result<()> { + if AttestationMode::detect()? == AttestationMode::DstackAmdSevSnp { + serde_json::to_writer_pretty( + io::stdout(), + &serde_json::json!({ + "attestation_mode": AttestationMode::DstackAmdSevSnp.as_str(), + "mr_system": null, + "mr_aggregated": null, + "note": "app-info MRs are TDX RTMR-derived and unavailable for AMD SEV-SNP", + }), + ) + .context("Failed to write app info")?; + println!(); + return Ok(()); + } + let attestation = ra_tls::attestation::Attestation::local().context("Failed to get attestation")?; let app_info = attestation diff --git a/dstack-util/src/system_setup.rs b/dstack-util/src/system_setup.rs index 2e23f98b0..9cce282b7 100644 --- a/dstack-util/src/system_setup.rs +++ b/dstack-util/src/system_setup.rs @@ -86,6 +86,12 @@ async fn sign_cert_request( mod config_id_verifier; +fn is_unsupported_app_info_quote(err: &anyhow::Error) -> bool { + let message = format!("{err:#}"); + message.contains("Unsupported attestation quote") + || message.contains("unsupported attestation quote for app info decoding") +} + #[derive(clap::Parser)] /// Prepare full disk encryption pub struct SetupArgs { @@ -893,11 +899,14 @@ impl<'a> Stage0<'a> { bail!("Invalid server cert usage: {usage}"); } if let Some(att) = &cert.attestation { - let kms_info = att - .decode_app_info(false) - .context("Failed to decode app_info")?; - emit_runtime_event("mr-kms", &kms_info.mr_aggregated) - .context("Failed to extend mr-kms to RTMR3")?; + match att.decode_app_info(false) { + Ok(kms_info) => emit_runtime_event("mr-kms", &kms_info.mr_aggregated) + .context("Failed to extend mr-kms to RTMR3")?, + Err(err) if is_unsupported_app_info_quote(&err) => { + warn!("Skipping mr-kms runtime event for unsupported attestation quote: {err:#}"); + } + Err(err) => return Err(err).context("Failed to decode app_info"), + } } Ok(()) })) diff --git a/dstack-util/src/system_setup/config_id_verifier.rs b/dstack-util/src/system_setup/config_id_verifier.rs index c62f665c3..d4ffdb2a5 100644 --- a/dstack-util/src/system_setup/config_id_verifier.rs +++ b/dstack-util/src/system_setup/config_id_verifier.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::{bail, Context, Result}; +use dstack_attest::attestation::AttestationMode; use dstack_types::{mr_config::MrConfig, KeyProviderKind}; use tracing::info; @@ -35,6 +36,22 @@ pub fn verify_mr_config_id( key_provider: KeyProviderKind, key_provider_id: &[u8], ) -> Result<()> { + let mode = AttestationMode::detect().context("Failed to detect attestation mode")?; + verify_mr_config_id_for_mode(mode, compose_hash, app_id, key_provider, key_provider_id) +} + +fn verify_mr_config_id_for_mode( + mode: AttestationMode, + compose_hash: &[u8; 32], + app_id: &[u8; 20], + key_provider: KeyProviderKind, + key_provider_id: &[u8], +) -> Result<()> { + if mode == AttestationMode::DstackAmdSevSnp { + info!("Skipping TDX mr_config_id verification for AMD SEV-SNP guest"); + return Ok(()); + } + let read_mr_config_id = read_mr_config_id().context("Failed to read mr_config_id")?; info!("mr_config_id: {}", hex::encode(read_mr_config_id)); if read_mr_config_id == [0u8; 48] { @@ -55,3 +72,20 @@ pub fn verify_mr_config_id( } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn amd_sev_snp_skips_tdx_mr_config_id_quote_verification() { + verify_mr_config_id_for_mode( + AttestationMode::DstackAmdSevSnp, + &[0u8; 32], + &[0u8; 20], + KeyProviderKind::None, + &[], + ) + .unwrap(); + } +} diff --git a/kms/src/main_service.rs b/kms/src/main_service.rs index f948b7ac1..c70f9e47c 100644 --- a/kms/src/main_service.rs +++ b/kms/src/main_service.rs @@ -627,6 +627,7 @@ mod tests { app_id: hex_of(0x11, 20), compose_hash: hex_of(0x22, 32), rootfs_hash: hex_of(0x33, 32), + base_cmdline: None, docker_files_hash: Some(hex_of(0x77, 32)), ovmf_hash: hex_of(0x44, 48), kernel_hash: hex_of(0x55, 32), diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs index 43839b3e8..8670d3ad9 100644 --- a/kms/src/main_service/amd_attest.rs +++ b/kms/src/main_service/amd_attest.rs @@ -58,6 +58,9 @@ pub(crate) struct MeasurementInput { pub compose_hash: String, /// 32-byte rootfs hash included in the measured kernel cmdline. pub rootfs_hash: String, + /// Original image kernel cmdline used as the base for SNP measured launch + /// before app identity fields are appended. + pub base_cmdline: Option, /// Optional 32-byte additional docker files hash included in the measured /// kernel cmdline when present. pub docker_files_hash: Option, @@ -952,10 +955,19 @@ pub(crate) fn compute_expected_measurement( .as_deref() .ok_or_else(|| anyhow::anyhow!("vcpu_type is required"))?; - let mut cmdline = format!( - "console=ttyS0 loglevel=7 docker_compose_hash={} rootfs_hash={} app_id={}", - input.compose_hash, input.rootfs_hash, input.app_id - ); + let mut cmdline = match input.base_cmdline.as_deref() { + Some(base) if !base.trim().is_empty() => format!( + "{} docker_compose_hash={} rootfs_hash={} app_id={}", + base.trim(), + input.compose_hash, + input.rootfs_hash, + input.app_id + ), + _ => format!( + "console=ttyS0 loglevel=7 docker_compose_hash={} rootfs_hash={} app_id={}", + input.compose_hash, input.rootfs_hash, input.app_id + ), + }; if let Some(docker_files_hash) = input.docker_files_hash.as_deref() { cmdline.push_str(&format!( " docker_additional_files_hash={docker_files_hash}" @@ -1078,6 +1090,7 @@ mod tests { app_id: hex_of(0x11, 20), compose_hash: hex_of(0x22, 32), rootfs_hash: hex_of(0x33, 32), + base_cmdline: None, docker_files_hash: Some(hex_of(0x77, 32)), ovmf_hash: hex_of(0x44, 48), kernel_hash: hex_of(0x55, 32), diff --git a/kms/src/onboard_service.rs b/kms/src/onboard_service.rs index 137063c86..d055a165f 100644 --- a/kms/src/onboard_service.rs +++ b/kms/src/onboard_service.rs @@ -218,6 +218,7 @@ mod tests { app_id: hex_of(0x11, 20), compose_hash: hex_of(0x22, 32), rootfs_hash: hex_of(0x33, 32), + base_cmdline: None, docker_files_hash: Some(hex_of(0x77, 32)), ovmf_hash: hex_of(0x44, 48), kernel_hash: hex_of(0x55, 32), diff --git a/vmm/src/app.rs b/vmm/src/app.rs index 874d25a9f..c3992cb1f 100644 --- a/vmm/src/app.rs +++ b/vmm/src/app.rs @@ -1231,6 +1231,7 @@ fn make_vm_config( "app_id": manifest.app_id, "compose_hash": compose_hash, "rootfs_hash": rootfs_hash, + "base_cmdline": image.info.cmdline, "docker_files_hash": serde_json::Value::Null, "ovmf_hash": "", "kernel_hash": file_sha256_hex(&image.kernel)?, @@ -1318,6 +1319,10 @@ mod tests { assert_eq!(measurement["app_id"], manifest.app_id); assert_eq!(measurement["compose_hash"], compose_hash); assert_eq!(measurement["rootfs_hash"], hex_of(0x33, 32)); + assert_eq!( + measurement["base_cmdline"], + format!("console=ttyS0 dstack.rootfs_hash={}", hex_of(0x33, 32)) + ); assert_eq!( measurement["kernel_hash"], hex::encode(Sha256::digest(b"snp-test-kernel")) From 40396b79fb91e6d4b5e61a81c369513bf3dd604f Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Wed, 3 Jun 2026 12:44:50 -0700 Subject: [PATCH 25/67] fix: satisfy ci lint checks --- basefiles/dstack-prepare.sh | 25 +++++++++++++------------ dstack-attest/src/amd_sev_snp.rs | 12 ++++++++++-- supervisor/client/src/main.rs | 23 ++++++++++++----------- 3 files changed, 35 insertions(+), 25 deletions(-) diff --git a/basefiles/dstack-prepare.sh b/basefiles/dstack-prepare.sh index 214c53a52..24e6e017d 100755 --- a/basefiles/dstack-prepare.sh +++ b/basefiles/dstack-prepare.sh @@ -80,19 +80,18 @@ WORK_DIR="/var/volatile/dstack" DATA_MNT="$WORK_DIR/persistent" OVERLAY_TMP="/var/volatile/overlay" -OVERLAY_PERSIST="$DATA_MNT/overlay" # Prepare volatile dirs mount_overlay() { - local src=$1 - local dst=$2/$1 - mkdir -p $dst/upper $dst/work - mount -t overlay overlay -o lowerdir=$src,upperdir=$dst/upper,workdir=$dst/work $src + local src="$1" + local dst="$2/$1" + mkdir -p "$dst/upper" "$dst/work" + mount -t overlay overlay -o lowerdir="$src",upperdir="$dst/upper",workdir="$dst/work" "$src" } -mount_overlay /etc $OVERLAY_TMP -mount_overlay /usr $OVERLAY_TMP -mount_overlay /bin $OVERLAY_TMP -mount_overlay /home $OVERLAY_TMP +mount_overlay /etc "$OVERLAY_TMP" +mount_overlay /usr "$OVERLAY_TMP" +mount_overlay /bin "$OVERLAY_TMP" +mount_overlay /home "$OVERLAY_TMP" # systemd-resolved may be unavailable in minimal smoke/debug boots; keep DNS usable for dockerd pulls. if ! [[ -s /etc/resolv.conf ]] || grep -Eq 'nameserver[[:space:]]+(127\.|::1)' /etc/resolv.conf; then @@ -139,9 +138,10 @@ log "Preparing dstack system..." has_partition_table() { local disk="$1" - local disk_name=$(basename "$disk") + local disk_name + disk_name=$(basename "$disk") # Check sysfs for any child partitions - for entry in /sys/class/block/${disk_name}/${disk_name}*; do + for entry in "/sys/class/block/${disk_name}/${disk_name}"*; do [ -e "$entry/partition" ] || continue return 0 done @@ -293,9 +293,10 @@ echo "============================" cd /dstack -if [ $(jq 'has("init_script")' app-compose.json) == true ]; then +if [ "$(jq 'has("init_script")' app-compose.json)" == true ]; then log "Running init script" dstack-util notify-host -e "boot.progress" -d "init-script" || true + # shellcheck disable=SC1090 source <(jq -r '.init_script' app-compose.json) fi diff --git a/dstack-attest/src/amd_sev_snp.rs b/dstack-attest/src/amd_sev_snp.rs index a3bcaf8f1..72f56c2e4 100644 --- a/dstack-attest/src/amd_sev_snp.rs +++ b/dstack-attest/src/amd_sev_snp.rs @@ -472,8 +472,16 @@ fn parse_kernel_cert_table(auxblob: &[u8]) -> Result)>> { let guid: [u8; 16] = entry[..16] .try_into() .context("amd sev-snp certificate table entry guid has invalid length")?; - let offset = u32::from_le_bytes(entry[16..20].try_into().unwrap()) as usize; - let length = u32::from_le_bytes(entry[20..24].try_into().unwrap()) as usize; + let offset = u32::from_le_bytes( + entry[16..20] + .try_into() + .context("amd sev-snp certificate table entry offset has invalid length")?, + ) as usize; + let length = u32::from_le_bytes( + entry[20..24] + .try_into() + .context("amd sev-snp certificate table entry length has invalid length")?, + ) as usize; if guid == [0u8; 16] && offset == 0 && length == 0 { break; } diff --git a/supervisor/client/src/main.rs b/supervisor/client/src/main.rs index c3b13abd0..4f50793e5 100644 --- a/supervisor/client/src/main.rs +++ b/supervisor/client/src/main.rs @@ -71,36 +71,37 @@ async fn main() -> Result<()> { cid: None, note: String::new(), }; - print_json(&client.deploy(&config).await?); + print_json(&client.deploy(&config).await?)?; } Commands::Start { id } => { - print_json(&client.start(&id).await?); + print_json(&client.start(&id).await?)?; } Commands::Stop { id } => { - print_json(&client.stop(&id).await?); + print_json(&client.stop(&id).await?)?; } Commands::Remove { id } => { - print_json(&client.remove(&id).await?); + print_json(&client.remove(&id).await?)?; } Commands::List => { - print_json(&client.list().await?); + print_json(&client.list().await?)?; } Commands::Info { id } => { - print_json(&client.info(&id).await?); + print_json(&client.info(&id).await?)?; } Commands::Ping => { - print_json(&client.ping().await?); + print_json(&client.ping().await?)?; } Commands::Clear => { - print_json(&client.clear().await?); + print_json(&client.clear().await?)?; } Commands::Shutdown => { - print_json(&client.shutdown().await?); + print_json(&client.shutdown().await?)?; } } Ok(()) } -fn print_json(value: &T) { - println!("{}", serde_json::to_string(value).unwrap()); +fn print_json(value: &T) -> Result<()> { + println!("{}", serde_json::to_string(value)?); + Ok(()) } From 5cb4566b58c147ad8823cdf19f075e40b73e4db6 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Wed, 3 Jun 2026 12:51:45 -0700 Subject: [PATCH 26/67] fix: satisfy prek shellcheck --- basefiles/dstack-prepare.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/basefiles/dstack-prepare.sh b/basefiles/dstack-prepare.sh index 24e6e017d..dfe81d159 100755 --- a/basefiles/dstack-prepare.sh +++ b/basefiles/dstack-prepare.sh @@ -85,8 +85,9 @@ OVERLAY_TMP="/var/volatile/overlay" mount_overlay() { local src="$1" local dst="$2/$1" + local overlay_opts="lowerdir=$src,upperdir=$dst/upper,workdir=$dst/work" mkdir -p "$dst/upper" "$dst/work" - mount -t overlay overlay -o lowerdir="$src",upperdir="$dst/upper",workdir="$dst/work" "$src" + mount -t overlay overlay -o "$overlay_opts" "$src" } mount_overlay /etc "$OVERLAY_TMP" mount_overlay /usr "$OVERLAY_TMP" From 409c4c529ebf8899b6416faa87d3eea4903f58e1 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Wed, 3 Jun 2026 16:38:36 -0700 Subject: [PATCH 27/67] test: add sev-snp e2e smoke script --- docs/amd-sev-snp-review-readiness.md | 2 +- test-scripts/snp-e2e-smoke.sh | 406 +++++++++++++++++++++++++++ 2 files changed, 407 insertions(+), 1 deletion(-) create mode 100755 test-scripts/snp-e2e-smoke.sh diff --git a/docs/amd-sev-snp-review-readiness.md b/docs/amd-sev-snp-review-readiness.md index a7594c51c..a9f05e463 100644 --- a/docs/amd-sev-snp-review-readiness.md +++ b/docs/amd-sev-snp-review-readiness.md @@ -105,7 +105,7 @@ DSTACK_SEV_SNP_ATTESTATION_PROOF_END ## Manual dstack E2E smoke status -An additional manual smoke was attempted on the SNP host (`chris@173.234.27.162`) using the PR branch, release-built `dstack-vmm`/`supervisor`/`dstack-kms`, QEMU 10.0.2, and the SNP-capable OVMF at `/opt/AMDSEV/usr/local/share/qemu/OVMF.fd`. +An additional manual smoke was attempted on the SNP host (`chris@173.234.27.162`) using the PR branch, release-built `dstack-vmm`/`supervisor`/`dstack-kms`, QEMU 10.0.2, and the SNP-capable OVMF at `/opt/AMDSEV/usr/local/share/qemu/OVMF.fd`. The reusable version of that smoke is checked in at `test-scripts/snp-e2e-smoke.sh` for follow-up debugging on SNP hosts. That smoke exposed and fixed several VMM/KMS-auth integration issues before the guest reached KMS: diff --git a/test-scripts/snp-e2e-smoke.sh b/test-scripts/snp-e2e-smoke.sh new file mode 100755 index 000000000..954fa7334 --- /dev/null +++ b/test-scripts/snp-e2e-smoke.sh @@ -0,0 +1,406 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 +# +# Manual AMD SEV-SNP hardware smoke for dstack-managed KMS/app key release. +# +# This is intentionally not a CI script. It requires an SNP-capable host with the +# AMDSEV QEMU/OVMF build used by the PR smoke, sudo for QEMU/KVM, and locally +# built release binaries. +# +# Minimal setup used by the original smoke: +# cargo build --release -p dstack-vmm -p supervisor -p dstack-kms +# export DSTACK_SNP_SMOKE_BIN_DIR=$PWD/target/release +# export DSTACK_SNP_SMOKE_ALLOW_OUT_OF_DATE_TCB=1 # lab hosts only +# test-scripts/snp-e2e-smoke.sh +# +# Useful overrides: +# DSTACK_SNP_SMOKE_BASE=$HOME/dstack-snp-e2e +# DSTACK_SNP_SMOKE_REPO=$PWD +# DSTACK_SNP_SMOKE_QEMU=/opt/AMDSEV/usr/local/bin/qemu-system-x86_64 +# DSTACK_SNP_SMOKE_OVMF=/opt/AMDSEV/usr/local/share/qemu/OVMF.fd +# DSTACK_SNP_SMOKE_IMAGE_URL=https://github.com/Dstack-TEE/meta-dstack/releases/download/v0.5.11/dstack-dev-0.5.11.tar.gz +# DSTACK_SNP_SMOKE_IMAGE_NAME=dstack-dev-0.5.11-snp-dnsfix + +set -euo pipefail + +BASE="${DSTACK_SNP_SMOKE_BASE:-$HOME/dstack-snp-e2e}" +REPO="${DSTACK_SNP_SMOKE_REPO:-$(pwd)}" +BIN="${DSTACK_SNP_SMOKE_BIN_DIR:-$REPO/target/release}" +ART="$BASE/artifacts" +LOG="$ART/snp-e2e-smoke.log" +VMM_URL="${DSTACK_SNP_SMOKE_VMM_URL:-http://127.0.0.1:18082}" +IMAGE_NAME="${DSTACK_SNP_SMOKE_IMAGE_NAME:-dstack-dev-0.5.11-snp-dnsfix}" +IMAGE_URL="${DSTACK_SNP_SMOKE_IMAGE_URL:-https://github.com/Dstack-TEE/meta-dstack/releases/download/v0.5.11/dstack-dev-0.5.11.tar.gz}" +QEMU_PATH="${DSTACK_SNP_SMOKE_QEMU:-/opt/AMDSEV/usr/local/bin/qemu-system-x86_64}" +OVMF_PATH="${DSTACK_SNP_SMOKE_OVMF:-/opt/AMDSEV/usr/local/share/qemu/OVMF.fd}" +HOST_ART_PORT="${DSTACK_SNP_SMOKE_HOST_ART_PORT:-18080}" +AUTH_PORT="${DSTACK_SNP_SMOKE_AUTH_PORT:-18081}" +KMS_HOST_PORT="${DSTACK_SNP_SMOKE_KMS_HOST_PORT:-15443}" +APP_HOST_PORT="${DSTACK_SNP_SMOKE_APP_HOST_PORT:-15543}" +ALLOW_OUT_OF_DATE_TCB="${DSTACK_SNP_SMOKE_ALLOW_OUT_OF_DATE_TCB:-0}" +RUN_STRICT_TCB_PROBE="${DSTACK_SNP_SMOKE_STRICT_TCB_PROBE:-1}" + +need() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "missing required command: $1" >&2 + exit 1 + fi +} + +need curl +need jq +need python3 +need sudo + +test -x "$BIN/dstack-vmm" || { echo "missing $BIN/dstack-vmm; run cargo build --release -p dstack-vmm" >&2; exit 1; } +test -x "$BIN/supervisor" || { echo "missing $BIN/supervisor; run cargo build --release -p supervisor" >&2; exit 1; } +test -x "$BIN/dstack-kms" || { echo "missing $BIN/dstack-kms; run cargo build --release -p dstack-kms" >&2; exit 1; } +test -x "$QEMU_PATH" || { echo "missing SNP QEMU: $QEMU_PATH" >&2; exit 1; } +test -r "$OVMF_PATH" || { echo "missing SNP OVMF: $OVMF_PATH" >&2; exit 1; } +test -f "$REPO/vmm/src/vmm-cli.py" || { echo "missing vmm-cli.py; set DSTACK_SNP_SMOKE_REPO" >&2; exit 1; } + +mkdir -p "$ART" "$BASE/images" "$BASE/run" "$BASE/http-root" +exec > >(tee "$LOG") 2>&1 + +echo "== SNP E2E smoke start: $(date -Is) ==" +echo "repo=$REPO" +echo "repo_head=$(git -C "$REPO" rev-parse --short=16 HEAD 2>/dev/null || echo unknown)" +echo "qemu=$QEMU_PATH" +echo "ovmf_sha256=$(sha256sum "$OVMF_PATH" | awk '{print $1}')" +echo "image=$IMAGE_NAME" + +cleanup() { + set +e + if [[ -f "$BASE/vmm.pid" ]]; then sudo kill "$(cat "$BASE/vmm.pid")" 2>/dev/null || true; fi + if [[ -f "$BASE/artifacts-http.pid" ]]; then kill "$(cat "$BASE/artifacts-http.pid")" 2>/dev/null || true; fi + if [[ -f "$BASE/auth.pid" ]]; then kill "$(cat "$BASE/auth.pid")" 2>/dev/null || true; fi + sudo pkill -f "$BIN/dstack-vmm" 2>/dev/null || true + sudo pkill -f "qemu-system-x86_64.*$BASE" 2>/dev/null || true + sudo pkill -f "$BASE/images" 2>/dev/null || true +} +trap cleanup EXIT +cleanup +sudo pkill -f "$BIN/supervisor" 2>/dev/null || true +sudo rm -rf "$BASE/run"/* "$BASE/tmp"/* + +cp "$BIN/dstack-kms" "$BASE/http-root/dstack-kms" +cp "$OVMF_PATH" "$BASE/http-root/OVMF.fd" +chmod +x "$BASE/http-root/dstack-kms" + +if [[ ! -d "$BASE/images/$IMAGE_NAME" ]]; then + echo "== Downloading/extracting $IMAGE_NAME ==" + curl -L "$IMAGE_URL" -o "$BASE/$IMAGE_NAME.tar.gz" + mkdir -p "$BASE/images/$IMAGE_NAME" + tar -xzf "$BASE/$IMAGE_NAME.tar.gz" -C "$BASE/images/$IMAGE_NAME" --strip-components=1 +fi +cp "$OVMF_PATH" "$BASE/images/$IMAGE_NAME/ovmf.fd" +jq . "$BASE/images/$IMAGE_NAME/metadata.json" | tee "$ART/image-metadata.json" + +cat >"$BASE/auth-server.py" <<'PY' +from http.server import BaseHTTPRequestHandler, HTTPServer +import json +import time + + +class H(BaseHTTPRequestHandler): + def _send(self, obj, status=200): + body = json.dumps(obj).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, fmt, *args): + print(time.strftime("%Y-%m-%dT%H:%M:%S"), self.path, fmt % args, flush=True) + + def do_GET(self): + self._send({ + "status": "ok", + "kmsContractAddr": "0x0000000000000000000000000000000000000000", + "ethRpcUrl": "", + "gatewayAppId": "", + "chainId": 1, + "appImplementation": "0x0000000000000000000000000000000000000000", + }) + + def do_POST(self): + length = int(self.headers.get("Content-Length", "0") or 0) + body = self.rfile.read(length) + try: + data = json.loads(body or b"{}") + except Exception: + data = {} + summary = {k: data.get(k) for k in ["attestationMode", "tcbStatus", "advisoryIds"] if k in data} + for key in ["appId", "mrAggregated", "osImageHash", "composeHash", "instanceId"]: + if key in data: + summary[key] = str(data[key])[:96] + print(json.dumps({"path": self.path, "summary": summary}), flush=True) + self._send({"isAllowed": True, "gatewayAppId": "", "reason": "snp smoke permissive auth"}) + + +HTTPServer(("0.0.0.0", 18081), H).serve_forever() +PY + +(cd "$BASE/http-root" && python3 -m http.server "$HOST_ART_PORT" >"$ART/artifacts-http.log" 2>&1 & echo $! >"$BASE/artifacts-http.pid") +python3 "$BASE/auth-server.py" >"$ART/auth-server.log" 2>&1 & echo $! >"$BASE/auth.pid" +sleep 1 +curl -fsS "http://127.0.0.1:$HOST_ART_PORT/dstack-kms" -o /dev/null +curl -fsS "http://127.0.0.1:$AUTH_PORT/" | jq . | tee "$ART/auth-info.json" + +cat >"$BASE/vmm.toml" <"$ART/vmm.log" 2>&1 & echo $! >"$BASE/vmm.pid" +for i in $(seq 1 60); do + if python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" lsvm --json >/dev/null 2>&1; then break; fi + sleep 1 + if [[ $i -eq 60 ]]; then echo "VMM did not become ready"; tail -80 "$ART/vmm.log"; exit 1; fi +done +echo "== VMM ready ==" + +allowed_tcb_statuses='["UpToDate"]' +if [[ "$ALLOW_OUT_OF_DATE_TCB" = "1" ]]; then + allowed_tcb_statuses='["UpToDate", "OutOfDate"]' +fi + +write_kms_config() { + local tcb_statuses="$1" + cat >"$BASE/http-root/kms.toml" </etc/docker/daemon.json <<'JSON' +{"dns":["10.0.2.3","1.1.1.1","8.8.8.8"]} +JSON +rm -f /etc/resolv.conf +printf 'nameserver 10.0.2.3\nnameserver 1.1.1.1\nnameserver 8.8.8.8\noptions timeout:2 attempts:3\n' >/etc/resolv.conf +if command -v systemctl >/dev/null 2>&1 && systemctl is-active docker >/dev/null 2>&1; then + systemctl restart docker +fi +SH +) + +KMS_BASH_SCRIPT=$(cat <<'SH' +set -eux +mkdir -p /dstack/kms-certs /dstack/kms-images +curl -fsS http://10.0.2.2:18080/dstack-kms -o /dstack/dstack-kms +curl -fsS http://10.0.2.2:18080/OVMF.fd -o /dstack/OVMF.fd +curl -fsS http://10.0.2.2:18080/kms.toml -o /dstack/kms.toml +chmod +x /dstack/dstack-kms +RUST_LOG=info /dstack/dstack-kms -c /dstack/kms.toml +SH +) + +deploy_kms() { + local name="$1" + local statuses="$2" + write_kms_config "$statuses" + cat >"$BASE/kms-compose.yaml" <<'YAML' +services: + kms: + image: debian:bookworm-slim + command: sh -c 'echo unused-container-compose; sleep 300' +YAML + python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" compose --docker-compose "$BASE/kms-compose.yaml" --name "$name" --public-logs --public-sysinfo --no-instance-id --output "$BASE/$name.app-compose.json" | tee "$ART/$name-compose-create.txt" + jq --arg init_script "$DNS_INIT_SCRIPT" --arg bash_script "$KMS_BASH_SCRIPT" '.storage_fs="ext4" | .init_script=$init_script | .runner="bash" | .bash_script=$bash_script | del(.docker_compose_file)' "$BASE/$name.app-compose.json" >"$BASE/$name.app-compose.json.tmp" + mv "$BASE/$name.app-compose.json.tmp" "$BASE/$name.app-compose.json" + python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" deploy --name "$name" --compose "$BASE/$name.app-compose.json" --image "$IMAGE_NAME" --port "tcp:127.0.0.1:$KMS_HOST_PORT:8000" --vcpu 2 --memory 4096 --disk 20G | tee "$ART/$name-deploy.txt" | sed -n 's/Created VM with ID: //p' | tail -1 +} + +if [[ "$RUN_STRICT_TCB_PROBE" = "1" && "$ALLOW_OUT_OF_DATE_TCB" = "1" ]]; then + echo "== Strict TCB probe: expect failure on lab OutOfDate host ==" + STRICT_KMS_VM_ID=$(deploy_kms snp-smoke-kms-strict '["UpToDate"]') + for i in $(seq 1 120); do + logs=$(python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$STRICT_KMS_VM_ID" -n 120 2>/dev/null || true) + if echo "$logs" | grep -q "tcb_status is not allowed"; then + echo "$logs" | tee "$ART/strict-tcb-denial-log.txt" + echo "strict_tcb_probe=denied_as_expected" + break + fi + sleep 2 + if [[ $i -eq 120 ]]; then echo "strict TCB probe did not reach expected denial"; echo "$logs" | tee "$ART/strict-tcb-timeout-log.txt"; exit 1; fi + done +fi + +echo "== KMS success run ==" +KMS_VM_ID=$(deploy_kms snp-smoke-kms "$allowed_tcb_statuses") +echo "KMS_VM_ID=$KMS_VM_ID" + +for i in $(seq 1 240); do + if curl -kfsS "https://127.0.0.1:$KMS_HOST_PORT/metrics" >/dev/null 2>&1; then echo "KMS runtime ready after ${i}s"; break; fi + sleep 2 + if [[ $((i % 30)) -eq 0 ]]; then echo "waiting for KMS..."; python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$KMS_VM_ID" -n 30 || true; fi + if [[ $i -eq 240 ]]; then echo "KMS did not become ready"; python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$KMS_VM_ID" -n 200 || true; exit 1; fi +done +curl -kfsS "https://127.0.0.1:$KMS_HOST_PORT/metrics" | tee "$ART/kms-metrics-before-app.txt" + +cat >"$BASE/app-compose.yaml" <<'YAML' +services: + smoke: + image: debian:bookworm-slim + command: sh -c 'echo SNP_APP_CONTAINER_STARTED; sleep 300' +YAML +python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" compose --docker-compose "$BASE/app-compose.yaml" --name snp-smoke-app --kms --public-logs --public-sysinfo --no-instance-id --output "$BASE/app.app-compose.json" | tee "$ART/app-compose-create.txt" +jq --arg init_script "$DNS_INIT_SCRIPT" '.storage_fs="ext4" | .init_script=$init_script' "$BASE/app.app-compose.json" >"$BASE/app.app-compose.json.tmp" +mv "$BASE/app.app-compose.json.tmp" "$BASE/app.app-compose.json" +APP_VM_ID=$(python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" deploy --name snp-smoke-app --compose "$BASE/app.app-compose.json" --image "$IMAGE_NAME" --kms-url "https://10.0.2.2:$KMS_HOST_PORT" --port "tcp:127.0.0.1:$APP_HOST_PORT:8000" --vcpu 2 --memory 4096 --disk 20G | tee "$ART/app-deploy.txt" | sed -n 's/Created VM with ID: //p' | tail -1) +echo "APP_VM_ID=$APP_VM_ID" + +for i in $(seq 1 240); do + logs=$(python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$APP_VM_ID" -n 160 2>/dev/null || true) + if echo "$logs" | grep -q "SNP_APP_CONTAINER_STARTED"; then echo "$logs" | tee "$ART/app-ready-log.txt"; echo "APP ready after ${i}s"; break; fi + if echo "$logs" | grep -q "Failed to get app key\|amd sev-snp key release\|measurement mismatch\|App not allowed\|KMS self authorization failed"; then echo "$logs" | tee "$ART/app-failure-log.txt"; exit 2; fi + sleep 2 + if [[ $((i % 30)) -eq 0 ]]; then echo "waiting for APP..."; echo "$logs" | tail -60; fi + if [[ $i -eq 240 ]]; then echo "APP did not become ready"; echo "$logs" | tee "$ART/app-timeout-log.txt"; exit 1; fi +done + +curl -kfsS "https://127.0.0.1:$KMS_HOST_PORT/metrics" | tee "$ART/kms-metrics-after-app.txt" +python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" info "$KMS_VM_ID" --json | tee "$ART/kms-info.json" +python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" info "$APP_VM_ID" --json | tee "$ART/app-info.json" +python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$KMS_VM_ID" -n 200 | tee "$ART/kms-final-log.txt" || true +python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$APP_VM_ID" -n 200 | tee "$ART/app-final-log.txt" || true + +echo "== SNP E2E smoke success: $(date -Is) ==" +echo "Artifacts: $ART" From 2aa70e87b8df1d1a1c5fd708b0b49ab6c69cd512 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Wed, 3 Jun 2026 17:13:34 -0700 Subject: [PATCH 28/67] test: harden sev-snp smoke script --- test-scripts/snp-e2e-smoke.sh | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/test-scripts/snp-e2e-smoke.sh b/test-scripts/snp-e2e-smoke.sh index 954fa7334..bf5851599 100755 --- a/test-scripts/snp-e2e-smoke.sh +++ b/test-scripts/snp-e2e-smoke.sh @@ -22,6 +22,7 @@ # DSTACK_SNP_SMOKE_OVMF=/opt/AMDSEV/usr/local/share/qemu/OVMF.fd # DSTACK_SNP_SMOKE_IMAGE_URL=https://github.com/Dstack-TEE/meta-dstack/releases/download/v0.5.11/dstack-dev-0.5.11.tar.gz # DSTACK_SNP_SMOKE_IMAGE_NAME=dstack-dev-0.5.11-snp-dnsfix +# DSTACK_SNP_SMOKE_ALLOW_OLD_QEMU=1 # bypasses the QEMU >= 10 preflight set -euo pipefail @@ -41,6 +42,7 @@ KMS_HOST_PORT="${DSTACK_SNP_SMOKE_KMS_HOST_PORT:-15443}" APP_HOST_PORT="${DSTACK_SNP_SMOKE_APP_HOST_PORT:-15543}" ALLOW_OUT_OF_DATE_TCB="${DSTACK_SNP_SMOKE_ALLOW_OUT_OF_DATE_TCB:-0}" RUN_STRICT_TCB_PROBE="${DSTACK_SNP_SMOKE_STRICT_TCB_PROBE:-1}" +ALLOW_OLD_QEMU="${DSTACK_SNP_SMOKE_ALLOW_OLD_QEMU:-0}" need() { if ! command -v "$1" >/dev/null 2>&1; then @@ -61,6 +63,25 @@ test -x "$QEMU_PATH" || { echo "missing SNP QEMU: $QEMU_PATH" >&2; exit 1; } test -r "$OVMF_PATH" || { echo "missing SNP OVMF: $OVMF_PATH" >&2; exit 1; } test -f "$REPO/vmm/src/vmm-cli.py" || { echo "missing vmm-cli.py; set DSTACK_SNP_SMOKE_REPO" >&2; exit 1; } +qemu_version_output=$("$QEMU_PATH" --version | head -1) +qemu_version=$(printf '%s\n' "$qemu_version_output" | sed -n 's/.*version \([0-9][0-9]*\)\.\([0-9][0-9]*\).*/\1.\2/p') +qemu_major=${qemu_version%%.*} +if [[ -z "$qemu_version" ]]; then + echo "Warning: could not parse QEMU version from: $qemu_version_output" >&2 +elif (( qemu_major < 10 )) && [[ "$ALLOW_OLD_QEMU" != "1" ]]; then + cat >&2 <= 10 build, or set DSTACK_SNP_SMOKE_ALLOW_OLD_QEMU=1 +if you intentionally want to reproduce/debug the older-QEMU failure. +EOF + exit 1 +fi + mkdir -p "$ART" "$BASE/images" "$BASE/run" "$BASE/http-root" exec > >(tee "$LOG") 2>&1 @@ -68,6 +89,7 @@ echo "== SNP E2E smoke start: $(date -Is) ==" echo "repo=$REPO" echo "repo_head=$(git -C "$REPO" rev-parse --short=16 HEAD 2>/dev/null || echo unknown)" echo "qemu=$QEMU_PATH" +echo "qemu_version=$qemu_version_output" echo "ovmf_sha256=$(sha256sum "$OVMF_PATH" | awk '{print $1}')" echo "image=$IMAGE_NAME" @@ -243,6 +265,8 @@ address = "127.0.0.1" port = 3443 EOF +# Redirect to a user-owned artifact file; only the VMM process itself needs sudo. +# shellcheck disable=SC2024 sudo "$BIN/dstack-vmm" -c "$BASE/vmm.toml" serve >"$ART/vmm.log" 2>&1 & echo $! >"$BASE/vmm.pid" for i in $(seq 1 60); do if python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" lsvm --json >/dev/null 2>&1; then break; fi @@ -328,6 +352,7 @@ curl -fsS http://10.0.2.2:18080/dstack-kms -o /dstack/dstack-kms curl -fsS http://10.0.2.2:18080/OVMF.fd -o /dstack/OVMF.fd curl -fsS http://10.0.2.2:18080/kms.toml -o /dstack/kms.toml chmod +x /dstack/dstack-kms +echo SNP_KMS_CONTAINER_STARTED RUST_LOG=info /dstack/dstack-kms -c /dstack/kms.toml SH ) @@ -342,10 +367,11 @@ services: image: debian:bookworm-slim command: sh -c 'echo unused-container-compose; sleep 300' YAML - python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" compose --docker-compose "$BASE/kms-compose.yaml" --name "$name" --public-logs --public-sysinfo --no-instance-id --output "$BASE/$name.app-compose.json" | tee "$ART/$name-compose-create.txt" + python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" compose --docker-compose "$BASE/kms-compose.yaml" --name "$name" --public-logs --public-sysinfo --no-instance-id --output "$BASE/$name.app-compose.json" | tee "$ART/$name-compose-create.txt" >&2 jq --arg init_script "$DNS_INIT_SCRIPT" --arg bash_script "$KMS_BASH_SCRIPT" '.storage_fs="ext4" | .init_script=$init_script | .runner="bash" | .bash_script=$bash_script | del(.docker_compose_file)' "$BASE/$name.app-compose.json" >"$BASE/$name.app-compose.json.tmp" mv "$BASE/$name.app-compose.json.tmp" "$BASE/$name.app-compose.json" - python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" deploy --name "$name" --compose "$BASE/$name.app-compose.json" --image "$IMAGE_NAME" --port "tcp:127.0.0.1:$KMS_HOST_PORT:8000" --vcpu 2 --memory 4096 --disk 20G | tee "$ART/$name-deploy.txt" | sed -n 's/Created VM with ID: //p' | tail -1 + python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" deploy --name "$name" --compose "$BASE/$name.app-compose.json" --image "$IMAGE_NAME" --port "tcp:127.0.0.1:$KMS_HOST_PORT:8000" --vcpu 2 --memory 4096 --disk 20G | tee "$ART/$name-deploy.txt" >&2 + sed -n 's/Created VM with ID: //p' "$ART/$name-deploy.txt" | tail -1 } if [[ "$RUN_STRICT_TCB_PROBE" = "1" && "$ALLOW_OUT_OF_DATE_TCB" = "1" ]]; then From fc226736c9443c71b592ad927f0e2dfc5b265ca8 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Wed, 3 Jun 2026 20:35:09 -0700 Subject: [PATCH 29/67] docs: document sev-snp smoke host matrix --- docs/amd-sev-snp-review-readiness.md | 10 ++++++++++ test-scripts/snp-e2e-smoke.sh | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/docs/amd-sev-snp-review-readiness.md b/docs/amd-sev-snp-review-readiness.md index a9f05e463..f9152405a 100644 --- a/docs/amd-sev-snp-review-readiness.md +++ b/docs/amd-sev-snp-review-readiness.md @@ -142,6 +142,16 @@ no_secret_material_logged=true This means the PR now has live SNP report proof, live golden-vector measurement proof, release-gate unit/integration coverage, and a manual full dstack-managed SNP guest -> KMS `GetAppKey` hardware E2E proof. The success run required an explicit lab-only TCB allowlist because this host reports `OutOfDate`; production defaults remain fail-closed (`UpToDate` only). +### Hardware smoke portability notes + +The checked-in smoke is enough to reproduce the proven KMS/app-key flow on a compatible SNP host, but reviewers should treat the guest image/kernel as part of the hardware matrix: + +- Known-good full E2E host: `chris@173.234.27.162` with AMDSEV QEMU 10.0.2, the SNP-capable OVMF above, and `dstack-dev-0.5.11-snp-dnsfix`. +- A separate local SNP host can run SNP Linux guests, but the stock `meta-dstack` v0.5.11 `6.9.0-dstack` kernel stops after OVMF/EFI loads kernel+initrd and QEMU reports `cpus are not resettable, terminating`, even when using QEMU 10.0.2. +- On that same local host, a newer Lit SNP guest kernel (`6.13.0-snp-guest-ffd294d346d1`) reaches Linux/SNP markers, which isolates that local failure to the dstack guest image/kernel compatibility layer rather than Chipotle/KMS/auth policy or basic host SNP enablement. + +Practical implication for reviewers/testers: first run `test-scripts/snp-e2e-smoke.sh` unchanged and confirm it reaches `SNP_KMS_CONTAINER_STARTED` / `SNP_APP_CONTAINER_STARTED` before substituting a real app workload. If the smoke stops after `EFI stub: Loaded initrd ...` with `cpus are not resettable`, use a host/image/kernel that is known to boot dstack under SNP (or build a coherent newer `meta-dstack` guest image with matching kernel, modules, initramfs, rootfs, and verity metadata) before debugging app-level behavior. + ## Validation commands Run locally for this review-ready staging branch: diff --git a/test-scripts/snp-e2e-smoke.sh b/test-scripts/snp-e2e-smoke.sh index bf5851599..ee10d062a 100755 --- a/test-scripts/snp-e2e-smoke.sh +++ b/test-scripts/snp-e2e-smoke.sh @@ -23,6 +23,12 @@ # DSTACK_SNP_SMOKE_IMAGE_URL=https://github.com/Dstack-TEE/meta-dstack/releases/download/v0.5.11/dstack-dev-0.5.11.tar.gz # DSTACK_SNP_SMOKE_IMAGE_NAME=dstack-dev-0.5.11-snp-dnsfix # DSTACK_SNP_SMOKE_ALLOW_OLD_QEMU=1 # bypasses the QEMU >= 10 preflight +# +# Host/image caveat: QEMU >= 10 is necessary but not sufficient. One local SNP +# host could boot a newer Lit SNP guest kernel but reset before Linux serial +# output with the stock meta-dstack v0.5.11 6.9.0-dstack kernel. If this smoke +# stops after `EFI stub: Loaded initrd ...` with `cpus are not resettable`, first +# validate the guest image/kernel on that host before debugging KMS or apps. set -euo pipefail From 0303256ee07617c22d566cc8ceca04763aa7c085 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Wed, 3 Jun 2026 22:01:01 -0700 Subject: [PATCH 30/67] docs: clarify sev-snp smoke image requirements --- docs/amd-sev-snp-review-readiness.md | 39 +++++++++++++++++----------- test-scripts/snp-e2e-smoke.sh | 9 +++++++ 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/docs/amd-sev-snp-review-readiness.md b/docs/amd-sev-snp-review-readiness.md index f9152405a..028ba0a82 100644 --- a/docs/amd-sev-snp-review-readiness.md +++ b/docs/amd-sev-snp-review-readiness.md @@ -113,44 +113,53 @@ That smoke exposed and fixed several VMM/KMS-auth integration issues before the - The VMM launch path required `metadata.json.rootfs_hash`, while the released `dstack-0.5.11` images carry the rootfs hash in `dstack.rootfs_hash=...` on the kernel cmdline. - The VMM SNP QEMU path now uses the SNP measurement CPU model (`EPYC-v4`) and confidential virtio PCI options (`disable-legacy=on,iommu_platform=true`) for SNP-launched virtio devices, matching the host's working SNP launch posture more closely. -After those fixes, the manual smoke progressed through full dstack-managed SNP guest boot, KMS self-bootstrap, app guest boot, app quote verification, and `GetAppKey` release. Additional smoke/debug fixes made the path work end-to-end: +After those fixes, the manual smoke progressed through full dstack-managed SNP guest boot and KMS self-bootstrap on the known-good remote host. Additional smoke/debug fixes made the host/KMS side reach the app-key boundary: - Minimal guest boot now keeps DNS usable when `systemd-resolved`/`chronyd` are unavailable early in smoke boots and detects `sev-guest` before trying the TDX guest module. - SNP guests skip TDX-only `mr_config_id` and app-info RTMR decoding while still preserving non-SNP behavior. - Configfs TSM report collection falls back to the SEV-SNP extended-report ioctl when configfs does not carry certificate collateral. -- If guest evidence still lacks ASK/VCEK collateral, the verifier fetches AMD KDS ARK/ASK/VCEK using the report `chip_id` and reported TCB, then verifies the signed report fail-closed. +- If verifier-side evidence still lacks ASK/VCEK collateral, the verifier can fetch AMD KDS ARK/ASK/VCEK using the report `chip_id` and reported TCB, then verify the signed report fail-closed. - KMS measurement recomputation now uses the image's original kernel cmdline as the measurement base before appending `docker_compose_hash`, `rootfs_hash`, and `app_id`, matching the VMM QEMU `-append` path. -Sanitized smoke result: +Latest sanitized remote smoke result with PR head `38b02d7c`: ```text remote_host=chris@173.234.27.162 +host_kernel=Linux 6.11.0-rc3-snp-host-85ef1ac03941 qemu_version=10.0.2 ovmf_sha256=67e7a7027437823e9c166a60d00666d5d5391e13050488cad5cc2acd913fab4a image=dstack-dev-0.5.11-snp-dnsfix platform=amd-sev-snp -vmm_branch=feat/amd-sev-snp-conversion + local smoke fixes kms_guest=booted SNP Linux/userspace and started dstack-kms -app_guest=booted SNP Linux/userspace and requested app keys -kms_auth=/bootAuth/kms 200 and /bootAuth/app 200 -tcb_status=OutOfDate in this lab host policy run -failure_gate=default UpToDate-only policy rejected release with "tcb_status is not allowed" -success_gate=explicit lab allowlist ["UpToDate", "OutOfDate"] released GetTempCaCert and GetAppKey -kms_metrics=dstack_kms_attestation_requests_total 1, dstack_kms_attestation_failures_total 0 +kms_marker=SNP_KMS_CONTAINER_STARTED / KMS runtime ready +kms_metrics=dstack_kms_attestation_requests_total 2, dstack_kms_attestation_failures_total 0 +app_guest=booted SNP Linux/userspace and reached dstack-prepare.sh +app_marker=SNP_APP_CONTAINER_STARTED not reached +failure_boundary=app guest GetTempCaCert/GetAppKey attestation validation +failure_error=amd sev-snp cert_chain must contain either ASK and VCEK certificates or one kernel certificate table auxblob no_secret_material_logged=true ``` -This means the PR now has live SNP report proof, live golden-vector measurement proof, release-gate unit/integration coverage, and a manual full dstack-managed SNP guest -> KMS `GetAppKey` hardware E2E proof. The success run required an explicit lab-only TCB allowlist because this host reports `OutOfDate`; production defaults remain fail-closed (`UpToDate` only). +This means the PR has live SNP report proof, live golden-vector measurement proof, release-gate unit/integration coverage, and hardware smoke proof through dstack-managed SNP KMS boot plus app guest key-request boundary. The remaining full `GetAppKey` smoke blocker is a guest image/tooling skew: the app VM uses the `dstack-util`/`dstack-attest` embedded inside the released `meta-dstack` v0.5.11 guest image, while the host/KMS binaries are built from PR #703. Rebuilding only `dstack-vmm`, `supervisor`, and `dstack-kms` is not enough for a fresh tester to exercise the PR's guest-side cert-chain/KDS fallback. -### Hardware smoke portability notes +### Fresh SNP host / image requirements -The checked-in smoke is enough to reproduce the proven KMS/app-key flow on a compatible SNP host, but reviewers should treat the guest image/kernel as part of the hardware matrix: +The checked-in smoke is enough to reproduce the current boundary on a compatible SNP host, but reviewers should treat the guest image/kernel/userspace as part of the test matrix: -- Known-good full E2E host: `chris@173.234.27.162` with AMDSEV QEMU 10.0.2, the SNP-capable OVMF above, and `dstack-dev-0.5.11-snp-dnsfix`. +- Known-good host for reaching KMS and app `dstack-prepare.sh`: `chris@173.234.27.162` with AMDSEV QEMU 10.0.2, the SNP-capable OVMF above, and `dstack-dev-0.5.11-snp-dnsfix`. +- That released image is **not** a coherent PR #703 image: its guest-side `dstack-util`/`dstack-attest` may reject SNP evidence before the newer PR fallback paths can help. - A separate local SNP host can run SNP Linux guests, but the stock `meta-dstack` v0.5.11 `6.9.0-dstack` kernel stops after OVMF/EFI loads kernel+initrd and QEMU reports `cpus are not resettable, terminating`, even when using QEMU 10.0.2. - On that same local host, a newer Lit SNP guest kernel (`6.13.0-snp-guest-ffd294d346d1`) reaches Linux/SNP markers, which isolates that local failure to the dstack guest image/kernel compatibility layer rather than Chipotle/KMS/auth policy or basic host SNP enablement. -Practical implication for reviewers/testers: first run `test-scripts/snp-e2e-smoke.sh` unchanged and confirm it reaches `SNP_KMS_CONTAINER_STARTED` / `SNP_APP_CONTAINER_STARTED` before substituting a real app workload. If the smoke stops after `EFI stub: Loaded initrd ...` with `cpus are not resettable`, use a host/image/kernel that is known to boot dstack under SNP (or build a coherent newer `meta-dstack` guest image with matching kernel, modules, initramfs, rootfs, and verity metadata) before debugging app-level behavior. +Practical implication for reviewers/testers on a fresh box: + +1. Install/use an AMDSEV QEMU 10.x build and the matching SNP-capable OVMF. +2. Build the PR binaries with `cargo build --release -p dstack-vmm -p supervisor -p dstack-kms`. +3. Run `test-scripts/snp-e2e-smoke.sh` unchanged and first confirm it reaches `SNP_KMS_CONTAINER_STARTED` and the app guest key-request boundary. +4. For full `SNP_APP_CONTAINER_STARTED` / `GetAppKey` success, use or publish a coherent `meta-dstack` guest image whose kernel, modules, initramfs, rootfs, verity metadata, and guest userspace include the same PR #703 `dstack-util`/`dstack-attest` SNP cert-chain/KDS fallback code. Do not try to inject only a replacement `dstack-util` into the stock image; that experiment changed the initramfs/measurement enough to regress boot. +5. Only after the baseline smoke reaches the app success marker should testers swap the simple app workload for Chipotle. + +If the smoke stops after `EFI stub: Loaded initrd ...` with `cpus are not resettable`, use a host/image/kernel that is known to boot dstack under SNP before debugging app-level behavior. If it reaches `Requesting app keys from KMS` and fails with the cert-chain error above, rebuild/use a coherent PR guest image rather than changing KMS release policy. ## Validation commands diff --git a/test-scripts/snp-e2e-smoke.sh b/test-scripts/snp-e2e-smoke.sh index ee10d062a..3d8c38ae6 100755 --- a/test-scripts/snp-e2e-smoke.sh +++ b/test-scripts/snp-e2e-smoke.sh @@ -29,6 +29,15 @@ # output with the stock meta-dstack v0.5.11 6.9.0-dstack kernel. If this smoke # stops after `EFI stub: Loaded initrd ...` with `cpus are not resettable`, first # validate the guest image/kernel on that host before debugging KMS or apps. +# +# Guest userspace caveat: rebuilding the host-side PR binaries is not enough for +# full app-key success if the downloaded meta-dstack image still embeds an older +# dstack-util/dstack-attest. On that skewed image the app guest can reach +# dstack-prepare.sh and fail at GetTempCaCert/GetAppKey with: +# amd sev-snp cert_chain must contain either ASK and VCEK certificates or one +# kernel certificate table auxblob +# For full SNP_APP_CONTAINER_STARTED / GetAppKey success, use a coherent +# meta-dstack guest image that includes the same PR cert-chain/KDS fallback code. set -euo pipefail From 1850e37429f3e6028a31f85ad1941d98f844bcde Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Thu, 4 Jun 2026 13:54:10 -0700 Subject: [PATCH 31/67] docs: clarify sev-snp fresh-box smoke --- docs/amd-sev-snp-review-readiness.md | 25 +++++++++++++++++++++---- test-scripts/snp-e2e-smoke.sh | 8 ++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/docs/amd-sev-snp-review-readiness.md b/docs/amd-sev-snp-review-readiness.md index 028ba0a82..d0d8d0b22 100644 --- a/docs/amd-sev-snp-review-readiness.md +++ b/docs/amd-sev-snp-review-readiness.md @@ -140,7 +140,7 @@ failure_error=amd sev-snp cert_chain must contain either ASK and VCEK certificat no_secret_material_logged=true ``` -This means the PR has live SNP report proof, live golden-vector measurement proof, release-gate unit/integration coverage, and hardware smoke proof through dstack-managed SNP KMS boot plus app guest key-request boundary. The remaining full `GetAppKey` smoke blocker is a guest image/tooling skew: the app VM uses the `dstack-util`/`dstack-attest` embedded inside the released `meta-dstack` v0.5.11 guest image, while the host/KMS binaries are built from PR #703. Rebuilding only `dstack-vmm`, `supervisor`, and `dstack-kms` is not enough for a fresh tester to exercise the PR's guest-side cert-chain/KDS fallback. +This means the PR has live SNP report proof, live golden-vector measurement proof, release-gate unit/integration coverage, and hardware smoke proof through dstack-managed SNP KMS boot plus app guest key-request boundary. The fresh-box smoke now reaches Linux/userspace, `SNP_KMS_CONTAINER_STARTED`, `GetTempCaCert`, and the app `GetAppKey` request when using a coherent **SNP** `meta-dstack` image. The remaining blocker for a completed success marker in the latest run was external AMD KDS throttling while fetching VCEK collateral for the app quote (`HTTP 429` for the Genoa VCEK request), not guest boot, KMS startup, or release-policy wiring. Host/KMS binaries must match PR #703, guest-side `dstack-util`/`dstack-attest` must include the PR cert-chain/KDS fallback, and the Yocto image must be built with `MACHINE = "sev-snp"` so the guest kernel includes AMD memory-encryption/SNP support. A coherent PR image built with the default `tdx` machine produced a `6.18.24-dstack` kernel with `# CONFIG_AMD_MEM_ENCRYPT is not set`; controlled QEMU tests showed that kernel resets immediately after OVMF loads kernel/initrd, while host SNP kernels boot the same QEMU/OVMF path to Linux/SNP markers. ### Fresh SNP host / image requirements @@ -148,15 +148,32 @@ The checked-in smoke is enough to reproduce the current boundary on a compatible - Known-good host for reaching KMS and app `dstack-prepare.sh`: `chris@173.234.27.162` with AMDSEV QEMU 10.0.2, the SNP-capable OVMF above, and `dstack-dev-0.5.11-snp-dnsfix`. - That released image is **not** a coherent PR #703 image: its guest-side `dstack-util`/`dstack-attest` may reject SNP evidence before the newer PR fallback paths can help. -- A separate local SNP host can run SNP Linux guests, but the stock `meta-dstack` v0.5.11 `6.9.0-dstack` kernel stops after OVMF/EFI loads kernel+initrd and QEMU reports `cpus are not resettable, terminating`, even when using QEMU 10.0.2. -- On that same local host, a newer Lit SNP guest kernel (`6.13.0-snp-guest-ffd294d346d1`) reaches Linux/SNP markers, which isolates that local failure to the dstack guest image/kernel compatibility layer rather than Chipotle/KMS/auth policy or basic host SNP enablement. +- A coherent PR #703 image must be built as an SNP image, not with `meta-dstack`'s default `tdx` machine. The default TDX build can emit a kernel without `CONFIG_AMD_MEM_ENCRYPT`, which fails before Linux serial output under SNP. +- On the same remote host/QEMU/OVMF, a minimal SNP initramfs booted host kernels (`6.11.0-rc3-snp-host` and `6.9.0-rc7-snp-host`) to Linux/SNP markers, while the default-TDX `6.18.24-dstack` kernel reset immediately after OVMF loaded kernel/initrd. This isolates that failure to the guest kernel config, not PSP firmware, KMS/auth policy, author-key, command line, virtio wiring, or basic host SNP enablement. Practical implication for reviewers/testers on a fresh box: 1. Install/use an AMDSEV QEMU 10.x build and the matching SNP-capable OVMF. 2. Build the PR binaries with `cargo build --release -p dstack-vmm -p supervisor -p dstack-kms`. 3. Run `test-scripts/snp-e2e-smoke.sh` unchanged and first confirm it reaches `SNP_KMS_CONTAINER_STARTED` and the app guest key-request boundary. -4. For full `SNP_APP_CONTAINER_STARTED` / `GetAppKey` success, use or publish a coherent `meta-dstack` guest image whose kernel, modules, initramfs, rootfs, verity metadata, and guest userspace include the same PR #703 `dstack-util`/`dstack-attest` SNP cert-chain/KDS fallback code. Do not try to inject only a replacement `dstack-util` into the stock image; that experiment changed the initramfs/measurement enough to regress boot. +4. For full `SNP_APP_CONTAINER_STARTED` / `GetAppKey` success, use or publish a coherent `meta-dstack` guest image whose kernel, modules, initramfs, rootfs, verity metadata, and guest userspace include the same PR #703 `dstack-util`/`dstack-attest` SNP cert-chain/KDS fallback code. The reproducible path is to build `meta-dstack` with its `dstack` submodule checked out to this PR branch, for example: + + ```bash + git clone https://github.com/Dstack-TEE/meta-dstack.git + cd meta-dstack + git submodule update --init --recursive --depth 1 + cd dstack + git fetch https://github.com/clawdbot-glitch003/dstack.git feat/amd-sev-snp-conversion + git checkout -B feat/amd-sev-snp-conversion FETCH_HEAD + cd .. + source dev-setup ./bb-build + sed -i 's/^MACHINE ??= .*/MACHINE = "sev-snp"/' ./bb-build/conf/local.conf + FLAVORS=dev make dist DIST_DIR=$PWD/images BB_BUILD_DIR=$PWD/bb-build + # Use the resulting dstack-dev image directory with: + # DSTACK_SNP_SMOKE_IMAGE_NAME= + ``` + + Do not try to inject only a replacement `dstack-util` into the stock image; that experiment changed the initramfs/measurement enough to regress boot. 5. Only after the baseline smoke reaches the app success marker should testers swap the simple app workload for Chipotle. If the smoke stops after `EFI stub: Loaded initrd ...` with `cpus are not resettable`, use a host/image/kernel that is known to boot dstack under SNP before debugging app-level behavior. If it reaches `Requesting app keys from KMS` and fails with the cert-chain error above, rebuild/use a coherent PR guest image rather than changing KMS release policy. diff --git a/test-scripts/snp-e2e-smoke.sh b/test-scripts/snp-e2e-smoke.sh index 3d8c38ae6..15d7a7c77 100755 --- a/test-scripts/snp-e2e-smoke.sh +++ b/test-scripts/snp-e2e-smoke.sh @@ -38,6 +38,14 @@ # kernel certificate table auxblob # For full SNP_APP_CONTAINER_STARTED / GetAppKey success, use a coherent # meta-dstack guest image that includes the same PR cert-chain/KDS fallback code. +# The smoke may still stop at the app GetAppKey boundary if AMD KDS throttles +# VCEK/cert-chain retrieval (for example HTTP 429 from kdsintf.amd.com); that is +# an external collateral-fetch boundary, not a guest boot or KMS startup failure. +# One reproducible way is to build meta-dstack with its dstack submodule checked +# out to this PR branch, set the Yocto build MACHINE to `sev-snp` (not the +# default `tdx`, otherwise the guest kernel can miss AMD memory-encryption +# support and reset immediately after OVMF loads the kernel/initrd), then point +# DSTACK_SNP_SMOKE_IMAGE_NAME at the resulting dstack-dev image directory. set -euo pipefail From 2e88b58b39fea9fc374f215e5ed40512ee7ea82e Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Thu, 4 Jun 2026 14:27:43 -0700 Subject: [PATCH 32/67] docs: record sev-snp smoke gate boundary --- docs/amd-sev-snp-review-readiness.md | 21 ++++---- test-scripts/snp-e2e-smoke.sh | 78 +++++++++++++++++++--------- 2 files changed, 64 insertions(+), 35 deletions(-) diff --git a/docs/amd-sev-snp-review-readiness.md b/docs/amd-sev-snp-review-readiness.md index d0d8d0b22..18355109e 100644 --- a/docs/amd-sev-snp-review-readiness.md +++ b/docs/amd-sev-snp-review-readiness.md @@ -121,35 +121,36 @@ After those fixes, the manual smoke progressed through full dstack-managed SNP g - If verifier-side evidence still lacks ASK/VCEK collateral, the verifier can fetch AMD KDS ARK/ASK/VCEK using the report `chip_id` and reported TCB, then verify the signed report fail-closed. - KMS measurement recomputation now uses the image's original kernel cmdline as the measurement base before appending `docker_compose_hash`, `rootfs_hash`, and `app_id`, matching the VMM QEMU `-append` path. -Latest sanitized remote smoke result with PR head `38b02d7c`: +Latest sanitized remote smoke result with PR-built host binaries and a coherent `MACHINE = "sev-snp"` guest image: ```text remote_host=chris@173.234.27.162 host_kernel=Linux 6.11.0-rc3-snp-host-85ef1ac03941 qemu_version=10.0.2 ovmf_sha256=67e7a7027437823e9c166a60d00666d5d5391e13050488cad5cc2acd913fab4a -image=dstack-dev-0.5.11-snp-dnsfix +image=dstack-dev-0.6.0 platform=amd-sev-snp +image_kernel=Linux 6.18.24-dstack with CONFIG_AMD_MEM_ENCRYPT=y, CONFIG_SEV_GUEST=y, CONFIG_TSM_REPORTS=y kms_guest=booted SNP Linux/userspace and started dstack-kms kms_marker=SNP_KMS_CONTAINER_STARTED / KMS runtime ready -kms_metrics=dstack_kms_attestation_requests_total 2, dstack_kms_attestation_failures_total 0 app_guest=booted SNP Linux/userspace and reached dstack-prepare.sh -app_marker=SNP_APP_CONTAINER_STARTED not reached -failure_boundary=app guest GetTempCaCert/GetAppKey attestation validation -failure_error=amd sev-snp cert_chain must contain either ASK and VCEK certificates or one kernel certificate table auxblob +app_key_boundary=GetTempCaCert/GetAppKey request reached +strict_tcb_probe=app request reached strict KMS, but final policy decision was blocked before TCB evaluation by AMD KDS collateral throttling +success_probe=app request reached permissive lab KMS, but final GetAppKey success marker was blocked by the same AMD KDS collateral throttling +failure_error=amd sev-snp KDS collateral unavailable while fetching VCEK/cert-chain collateral; Genoa VCEK returned HTTP 429 no_secret_material_logged=true ``` -This means the PR has live SNP report proof, live golden-vector measurement proof, release-gate unit/integration coverage, and hardware smoke proof through dstack-managed SNP KMS boot plus app guest key-request boundary. The fresh-box smoke now reaches Linux/userspace, `SNP_KMS_CONTAINER_STARTED`, `GetTempCaCert`, and the app `GetAppKey` request when using a coherent **SNP** `meta-dstack` image. The remaining blocker for a completed success marker in the latest run was external AMD KDS throttling while fetching VCEK collateral for the app quote (`HTTP 429` for the Genoa VCEK request), not guest boot, KMS startup, or release-policy wiring. Host/KMS binaries must match PR #703, guest-side `dstack-util`/`dstack-attest` must include the PR cert-chain/KDS fallback, and the Yocto image must be built with `MACHINE = "sev-snp"` so the guest kernel includes AMD memory-encryption/SNP support. A coherent PR image built with the default `tdx` machine produced a `6.18.24-dstack` kernel with `# CONFIG_AMD_MEM_ENCRYPT is not set`; controlled QEMU tests showed that kernel resets immediately after OVMF loads kernel/initrd, while host SNP kernels boot the same QEMU/OVMF path to Linux/SNP markers. +This means the PR has live SNP report proof, live golden-vector measurement proof, release-gate unit/integration coverage, and hardware smoke proof through dstack-managed SNP KMS boot plus app guest key-request boundary. The fresh-box smoke now reaches Linux/userspace, `SNP_KMS_CONTAINER_STARTED`, `GetTempCaCert`, and the app `GetAppKey` request when using a coherent **SNP** `meta-dstack` image. The remaining blocker for completed strict-denial and success markers in the latest run was external AMD KDS throttling while fetching VCEK/cert-chain collateral for the app quote (`HTTP 429` for the Genoa VCEK request), not guest boot, KMS startup, or release-policy wiring. Host/KMS binaries must match PR #703, guest-side `dstack-util`/`dstack-attest` must include the PR cert-chain/KDS fallback, and the Yocto image must be built with `MACHINE = "sev-snp"` so the guest kernel includes AMD memory-encryption/SNP support. A coherent PR image built with the default `tdx` machine produced a `6.18.24-dstack` kernel with `# CONFIG_AMD_MEM_ENCRYPT is not set`; controlled QEMU tests showed that kernel resets immediately after OVMF loads kernel/initrd, while SNP-capable kernels boot the same QEMU/OVMF path to Linux/SNP markers. ### Fresh SNP host / image requirements The checked-in smoke is enough to reproduce the current boundary on a compatible SNP host, but reviewers should treat the guest image/kernel/userspace as part of the test matrix: -- Known-good host for reaching KMS and app `dstack-prepare.sh`: `chris@173.234.27.162` with AMDSEV QEMU 10.0.2, the SNP-capable OVMF above, and `dstack-dev-0.5.11-snp-dnsfix`. -- That released image is **not** a coherent PR #703 image: its guest-side `dstack-util`/`dstack-attest` may reject SNP evidence before the newer PR fallback paths can help. +- Known-good host for reaching KMS and app `dstack-prepare.sh`: `chris@173.234.27.162` with QEMU 10.0.2, the SNP-capable OVMF above, and a coherent `dstack-dev-0.6.0` guest image built with `MACHINE = "sev-snp"`. +- Released images that do not carry PR #703 guest-side `dstack-util`/`dstack-attest` may reject SNP evidence before the newer PR fallback paths can help. - A coherent PR #703 image must be built as an SNP image, not with `meta-dstack`'s default `tdx` machine. The default TDX build can emit a kernel without `CONFIG_AMD_MEM_ENCRYPT`, which fails before Linux serial output under SNP. -- On the same remote host/QEMU/OVMF, a minimal SNP initramfs booted host kernels (`6.11.0-rc3-snp-host` and `6.9.0-rc7-snp-host`) to Linux/SNP markers, while the default-TDX `6.18.24-dstack` kernel reset immediately after OVMF loaded kernel/initrd. This isolates that failure to the guest kernel config, not PSP firmware, KMS/auth policy, author-key, command line, virtio wiring, or basic host SNP enablement. +- On the same remote host/QEMU/OVMF, a minimal SNP initramfs booted SNP-capable kernels (`6.11.0-rc3-snp-host`, `6.9.0-rc7-snp-host`, and the `MACHINE = "sev-snp"` `6.18.24-dstack` kernel) to Linux/SNP markers, while the default-TDX `6.18.24-dstack` kernel reset immediately after OVMF loaded kernel/initrd. This isolates that failure to the guest kernel config, not PSP firmware, KMS/auth policy, author-key, command line, virtio wiring, or basic host SNP enablement. Practical implication for reviewers/testers on a fresh box: diff --git a/test-scripts/snp-e2e-smoke.sh b/test-scripts/snp-e2e-smoke.sh index 15d7a7c77..5062e7919 100755 --- a/test-scripts/snp-e2e-smoke.sh +++ b/test-scripts/snp-e2e-smoke.sh @@ -62,7 +62,9 @@ OVMF_PATH="${DSTACK_SNP_SMOKE_OVMF:-/opt/AMDSEV/usr/local/share/qemu/OVMF.fd}" HOST_ART_PORT="${DSTACK_SNP_SMOKE_HOST_ART_PORT:-18080}" AUTH_PORT="${DSTACK_SNP_SMOKE_AUTH_PORT:-18081}" KMS_HOST_PORT="${DSTACK_SNP_SMOKE_KMS_HOST_PORT:-15443}" +STRICT_KMS_HOST_PORT="${DSTACK_SNP_SMOKE_STRICT_KMS_HOST_PORT:-15444}" APP_HOST_PORT="${DSTACK_SNP_SMOKE_APP_HOST_PORT:-15543}" +STRICT_APP_HOST_PORT="${DSTACK_SNP_SMOKE_STRICT_APP_HOST_PORT:-15544}" ALLOW_OUT_OF_DATE_TCB="${DSTACK_SNP_SMOKE_ALLOW_OUT_OF_DATE_TCB:-0}" RUN_STRICT_TCB_PROBE="${DSTACK_SNP_SMOKE_STRICT_TCB_PROBE:-1}" ALLOW_OLD_QEMU="${DSTACK_SNP_SMOKE_ALLOW_OLD_QEMU:-0}" @@ -383,6 +385,7 @@ SH deploy_kms() { local name="$1" local statuses="$2" + local host_port="$3" write_kms_config "$statuses" cat >"$BASE/kms-compose.yaml" <<'YAML' services: @@ -393,53 +396,78 @@ YAML python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" compose --docker-compose "$BASE/kms-compose.yaml" --name "$name" --public-logs --public-sysinfo --no-instance-id --output "$BASE/$name.app-compose.json" | tee "$ART/$name-compose-create.txt" >&2 jq --arg init_script "$DNS_INIT_SCRIPT" --arg bash_script "$KMS_BASH_SCRIPT" '.storage_fs="ext4" | .init_script=$init_script | .runner="bash" | .bash_script=$bash_script | del(.docker_compose_file)' "$BASE/$name.app-compose.json" >"$BASE/$name.app-compose.json.tmp" mv "$BASE/$name.app-compose.json.tmp" "$BASE/$name.app-compose.json" - python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" deploy --name "$name" --compose "$BASE/$name.app-compose.json" --image "$IMAGE_NAME" --port "tcp:127.0.0.1:$KMS_HOST_PORT:8000" --vcpu 2 --memory 4096 --disk 20G | tee "$ART/$name-deploy.txt" >&2 + python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" deploy --name "$name" --compose "$BASE/$name.app-compose.json" --image "$IMAGE_NAME" --port "tcp:127.0.0.1:$host_port:8000" --vcpu 2 --memory 4096 --disk 20G | tee "$ART/$name-deploy.txt" >&2 + sed -n 's/Created VM with ID: //p' "$ART/$name-deploy.txt" | tail -1 +} + +wait_for_kms_metrics() { + local vm_id="$1" + local host_port="$2" + local label="$3" + for i in $(seq 1 240); do + if curl -kfsS "https://127.0.0.1:$host_port/metrics" >/dev/null 2>&1; then echo "$label KMS runtime ready after ${i}s"; break; fi + sleep 2 + if [[ $((i % 30)) -eq 0 ]]; then echo "waiting for $label KMS..."; python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$vm_id" -n 30 || true; fi + if [[ $i -eq 240 ]]; then echo "$label KMS did not become ready"; python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$vm_id" -n 200 || true; exit 1; fi + done +} + +deploy_app() { + local name="$1" + local kms_port="$2" + local app_port="$3" + cat >"$BASE/$name-compose.yaml" <<'YAML' +services: + smoke: + image: debian:bookworm-slim + command: sh -c 'echo SNP_APP_CONTAINER_STARTED; sleep 300' +YAML + python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" compose --docker-compose "$BASE/$name-compose.yaml" --name "$name" --kms --public-logs --public-sysinfo --no-instance-id --output "$BASE/$name.app-compose.json" | tee "$ART/$name-compose-create.txt" >&2 + jq --arg init_script "$DNS_INIT_SCRIPT" '.storage_fs="ext4" | .init_script=$init_script' "$BASE/$name.app-compose.json" >"$BASE/$name.app-compose.json.tmp" + mv "$BASE/$name.app-compose.json.tmp" "$BASE/$name.app-compose.json" + python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" deploy --name "$name" --compose "$BASE/$name.app-compose.json" --image "$IMAGE_NAME" --kms-url "https://10.0.2.2:$kms_port" --port "tcp:127.0.0.1:$app_port:8000" --vcpu 2 --memory 4096 --disk 20G | tee "$ART/$name-deploy.txt" >&2 sed -n 's/Created VM with ID: //p' "$ART/$name-deploy.txt" | tail -1 } if [[ "$RUN_STRICT_TCB_PROBE" = "1" && "$ALLOW_OUT_OF_DATE_TCB" = "1" ]]; then - echo "== Strict TCB probe: expect failure on lab OutOfDate host ==" - STRICT_KMS_VM_ID=$(deploy_kms snp-smoke-kms-strict '["UpToDate"]') - for i in $(seq 1 120); do - logs=$(python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$STRICT_KMS_VM_ID" -n 120 2>/dev/null || true) + echo "== Strict TCB probe: expect app GetAppKey denial on lab OutOfDate host ==" + STRICT_KMS_VM_ID=$(deploy_kms snp-smoke-kms-strict '["UpToDate"]' "$STRICT_KMS_HOST_PORT") + echo "STRICT_KMS_VM_ID=$STRICT_KMS_VM_ID" + wait_for_kms_metrics "$STRICT_KMS_VM_ID" "$STRICT_KMS_HOST_PORT" strict + STRICT_APP_VM_ID=$(deploy_app snp-smoke-app-strict "$STRICT_KMS_HOST_PORT" "$STRICT_APP_HOST_PORT") + echo "STRICT_APP_VM_ID=$STRICT_APP_VM_ID" + for i in $(seq 1 240); do + logs=$(python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$STRICT_APP_VM_ID" -n 180 2>/dev/null || true) if echo "$logs" | grep -q "tcb_status is not allowed"; then echo "$logs" | tee "$ART/strict-tcb-denial-log.txt" echo "strict_tcb_probe=denied_as_expected" break fi + if echo "$logs" | grep -q "KDS collateral unavailable\|HTTP status client error"; then + echo "$logs" | tee "$ART/strict-tcb-kds-blocked-log.txt" + echo "strict_tcb_probe=blocked_by_kds_collateral" + break + fi + if echo "$logs" | grep -q "SNP_APP_CONTAINER_STARTED"; then echo "$logs" | tee "$ART/strict-tcb-unexpected-success-log.txt"; echo "strict TCB probe unexpectedly reached app container"; exit 1; fi sleep 2 - if [[ $i -eq 120 ]]; then echo "strict TCB probe did not reach expected denial"; echo "$logs" | tee "$ART/strict-tcb-timeout-log.txt"; exit 1; fi + if [[ $((i % 30)) -eq 0 ]]; then echo "waiting for strict APP denial..."; echo "$logs" | tail -60; fi + if [[ $i -eq 240 ]]; then echo "strict TCB probe did not reach expected denial"; echo "$logs" | tee "$ART/strict-tcb-timeout-log.txt"; exit 1; fi done fi echo "== KMS success run ==" -KMS_VM_ID=$(deploy_kms snp-smoke-kms "$allowed_tcb_statuses") +KMS_VM_ID=$(deploy_kms snp-smoke-kms "$allowed_tcb_statuses" "$KMS_HOST_PORT") echo "KMS_VM_ID=$KMS_VM_ID" - -for i in $(seq 1 240); do - if curl -kfsS "https://127.0.0.1:$KMS_HOST_PORT/metrics" >/dev/null 2>&1; then echo "KMS runtime ready after ${i}s"; break; fi - sleep 2 - if [[ $((i % 30)) -eq 0 ]]; then echo "waiting for KMS..."; python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$KMS_VM_ID" -n 30 || true; fi - if [[ $i -eq 240 ]]; then echo "KMS did not become ready"; python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$KMS_VM_ID" -n 200 || true; exit 1; fi -done +wait_for_kms_metrics "$KMS_VM_ID" "$KMS_HOST_PORT" success curl -kfsS "https://127.0.0.1:$KMS_HOST_PORT/metrics" | tee "$ART/kms-metrics-before-app.txt" -cat >"$BASE/app-compose.yaml" <<'YAML' -services: - smoke: - image: debian:bookworm-slim - command: sh -c 'echo SNP_APP_CONTAINER_STARTED; sleep 300' -YAML -python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" compose --docker-compose "$BASE/app-compose.yaml" --name snp-smoke-app --kms --public-logs --public-sysinfo --no-instance-id --output "$BASE/app.app-compose.json" | tee "$ART/app-compose-create.txt" -jq --arg init_script "$DNS_INIT_SCRIPT" '.storage_fs="ext4" | .init_script=$init_script' "$BASE/app.app-compose.json" >"$BASE/app.app-compose.json.tmp" -mv "$BASE/app.app-compose.json.tmp" "$BASE/app.app-compose.json" -APP_VM_ID=$(python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" deploy --name snp-smoke-app --compose "$BASE/app.app-compose.json" --image "$IMAGE_NAME" --kms-url "https://10.0.2.2:$KMS_HOST_PORT" --port "tcp:127.0.0.1:$APP_HOST_PORT:8000" --vcpu 2 --memory 4096 --disk 20G | tee "$ART/app-deploy.txt" | sed -n 's/Created VM with ID: //p' | tail -1) +APP_VM_ID=$(deploy_app snp-smoke-app "$KMS_HOST_PORT" "$APP_HOST_PORT") echo "APP_VM_ID=$APP_VM_ID" for i in $(seq 1 240); do logs=$(python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$APP_VM_ID" -n 160 2>/dev/null || true) if echo "$logs" | grep -q "SNP_APP_CONTAINER_STARTED"; then echo "$logs" | tee "$ART/app-ready-log.txt"; echo "APP ready after ${i}s"; break; fi - if echo "$logs" | grep -q "Failed to get app key\|amd sev-snp key release\|measurement mismatch\|App not allowed\|KMS self authorization failed"; then echo "$logs" | tee "$ART/app-failure-log.txt"; exit 2; fi + if echo "$logs" | grep -q "Failed to get app key\|amd sev-snp key release\|measurement mismatch\|App not allowed\|KMS self authorization failed\|KDS collateral unavailable\|HTTP status client error"; then echo "$logs" | tee "$ART/app-failure-log.txt"; exit 2; fi sleep 2 if [[ $((i % 30)) -eq 0 ]]; then echo "waiting for APP..."; echo "$logs" | tail -60; fi if [[ $i -eq 240 ]]; then echo "APP did not become ready"; echo "$logs" | tee "$ART/app-timeout-log.txt"; exit 1; fi From ac5fbf5d5667a424ae7aa98db8c5707518ddda57 Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Thu, 4 Jun 2026 17:18:01 -0700 Subject: [PATCH 33/67] fix: complete sev-snp smoke proxy path --- basefiles/dstack-guest-agent.service | 1 + basefiles/dstack-prepare.sh | 8 +++++ docs/amd-sev-snp-review-readiness.md | 15 ++++---- dstack-attest/src/amd_sev_snp.rs | 42 ++++++++++++++++++---- kms/src/config.rs | 6 ++++ kms/src/main.rs | 21 ++++++++++- kms/src/main_service.rs | 1 + kms/src/main_service/amd_attest.rs | 3 ++ kms/src/onboard_service.rs | 1 + ra-rpc/src/rocket_helper.rs | 37 ++++++++++++++++++- test-scripts/snp-e2e-smoke.sh | 53 +++++++++++++++++++--------- vmm/src/app.rs | 36 ++++++++++++++++++- vmm/src/app/qemu.rs | 36 +++++++++++++++++-- 13 files changed, 224 insertions(+), 36 deletions(-) diff --git a/basefiles/dstack-guest-agent.service b/basefiles/dstack-guest-agent.service index a88d395d9..91853789d 100644 --- a/basefiles/dstack-guest-agent.service +++ b/basefiles/dstack-guest-agent.service @@ -15,6 +15,7 @@ WatchdogSec=30s StandardOutput=journal+console StandardError=journal+console Environment=RUST_LOG=warn +EnvironmentFile=-/run/dstack/environment [Install] WantedBy=multi-user.target diff --git a/basefiles/dstack-prepare.sh b/basefiles/dstack-prepare.sh index dfe81d159..b27219a7f 100755 --- a/basefiles/dstack-prepare.sh +++ b/basefiles/dstack-prepare.sh @@ -274,6 +274,14 @@ if [ -f "/sys/class/block/${device_name}/partition" ]; then fi fi +AMD_KDS_PROXY_URL="$(tr ' ' '\n' /run/dstack/environment +fi + dstack-util setup --work-dir $WORK_DIR --device "$DATA_DEVICE" --mount-point $DATA_MNT log "Mounting container runtime dirs to persistent storage" diff --git a/docs/amd-sev-snp-review-readiness.md b/docs/amd-sev-snp-review-readiness.md index 18355109e..419b15aad 100644 --- a/docs/amd-sev-snp-review-readiness.md +++ b/docs/amd-sev-snp-review-readiness.md @@ -133,15 +133,14 @@ platform=amd-sev-snp image_kernel=Linux 6.18.24-dstack with CONFIG_AMD_MEM_ENCRYPT=y, CONFIG_SEV_GUEST=y, CONFIG_TSM_REPORTS=y kms_guest=booted SNP Linux/userspace and started dstack-kms kms_marker=SNP_KMS_CONTAINER_STARTED / KMS runtime ready -app_guest=booted SNP Linux/userspace and reached dstack-prepare.sh -app_key_boundary=GetTempCaCert/GetAppKey request reached -strict_tcb_probe=app request reached strict KMS, but final policy decision was blocked before TCB evaluation by AMD KDS collateral throttling -success_probe=app request reached permissive lab KMS, but final GetAppKey success marker was blocked by the same AMD KDS collateral throttling -failure_error=amd sev-snp KDS collateral unavailable while fetching VCEK/cert-chain collateral; Genoa VCEK returned HTTP 429 +kds_proxy=enabled for smoke via DSTACK_AMD_KDS_PROXY_URL=https://cors.litgateway.com/ +strict_tcb_probe=denied_as_expected with tcb_status is not allowed +success_probe=GetTempCaCert HTTP 200; GetAppKey HTTP 200; SignCert HTTP 200; app container started +smoke_result=SNP E2E smoke success no_secret_material_logged=true ``` -This means the PR has live SNP report proof, live golden-vector measurement proof, release-gate unit/integration coverage, and hardware smoke proof through dstack-managed SNP KMS boot plus app guest key-request boundary. The fresh-box smoke now reaches Linux/userspace, `SNP_KMS_CONTAINER_STARTED`, `GetTempCaCert`, and the app `GetAppKey` request when using a coherent **SNP** `meta-dstack` image. The remaining blocker for completed strict-denial and success markers in the latest run was external AMD KDS throttling while fetching VCEK/cert-chain collateral for the app quote (`HTTP 429` for the Genoa VCEK request), not guest boot, KMS startup, or release-policy wiring. Host/KMS binaries must match PR #703, guest-side `dstack-util`/`dstack-attest` must include the PR cert-chain/KDS fallback, and the Yocto image must be built with `MACHINE = "sev-snp"` so the guest kernel includes AMD memory-encryption/SNP support. A coherent PR image built with the default `tdx` machine produced a `6.18.24-dstack` kernel with `# CONFIG_AMD_MEM_ENCRYPT is not set`; controlled QEMU tests showed that kernel resets immediately after OVMF loads kernel/initrd, while SNP-capable kernels boot the same QEMU/OVMF path to Linux/SNP markers. +This means the PR has live SNP report proof, live golden-vector measurement proof, release-gate unit/integration coverage, and hardware smoke proof through dstack-managed SNP KMS boot, strict TCB denial, app guest key release, and app container startup. The fresh-box smoke now reaches Linux/userspace, `SNP_KMS_CONTAINER_STARTED`, `GetTempCaCert`, `GetAppKey`, `SignCert`, and app container startup when using a coherent **SNP** `meta-dstack` image. During the smoke, AMD KDS throttling was worked around by explicitly routing AMD KDS collateral fetches through `DSTACK_AMD_KDS_PROXY_URL`; the proxy value is carried in the measured guest cmdline for smoke runs and mirrored in KMS measurement recomputation to avoid measurement drift. Host/KMS binaries must match PR #703, guest-side `dstack-util`/`dstack-attest` must include the PR cert-chain/KDS fallback, and the Yocto image must be built with `MACHINE = "sev-snp"` so the guest kernel includes AMD memory-encryption/SNP support. A coherent PR image built with the default `tdx` machine produced a `6.18.24-dstack` kernel with `# CONFIG_AMD_MEM_ENCRYPT is not set`; controlled QEMU tests showed that kernel resets immediately after OVMF loads kernel/initrd, while SNP-capable kernels boot the same QEMU/OVMF path to Linux/SNP markers. ### Fresh SNP host / image requirements @@ -156,7 +155,7 @@ Practical implication for reviewers/testers on a fresh box: 1. Install/use an AMDSEV QEMU 10.x build and the matching SNP-capable OVMF. 2. Build the PR binaries with `cargo build --release -p dstack-vmm -p supervisor -p dstack-kms`. -3. Run `test-scripts/snp-e2e-smoke.sh` unchanged and first confirm it reaches `SNP_KMS_CONTAINER_STARTED` and the app guest key-request boundary. +3. Run `test-scripts/snp-e2e-smoke.sh` unchanged and first confirm it reaches `SNP_KMS_CONTAINER_STARTED`; if AMD KDS throttles the lab host, set `DSTACK_SNP_SMOKE_KDS_PROXY_URL` to a trusted AMD-KDS passthrough/cache endpoint and rerun. 4. For full `SNP_APP_CONTAINER_STARTED` / `GetAppKey` success, use or publish a coherent `meta-dstack` guest image whose kernel, modules, initramfs, rootfs, verity metadata, and guest userspace include the same PR #703 `dstack-util`/`dstack-attest` SNP cert-chain/KDS fallback code. The reproducible path is to build `meta-dstack` with its `dstack` submodule checked out to this PR branch, for example: ```bash @@ -177,7 +176,7 @@ Practical implication for reviewers/testers on a fresh box: Do not try to inject only a replacement `dstack-util` into the stock image; that experiment changed the initramfs/measurement enough to regress boot. 5. Only after the baseline smoke reaches the app success marker should testers swap the simple app workload for Chipotle. -If the smoke stops after `EFI stub: Loaded initrd ...` with `cpus are not resettable`, use a host/image/kernel that is known to boot dstack under SNP before debugging app-level behavior. If it reaches `Requesting app keys from KMS` and fails with the cert-chain error above, rebuild/use a coherent PR guest image rather than changing KMS release policy. +If the smoke stops after `EFI stub: Loaded initrd ...` with `cpus are not resettable`, use a host/image/kernel that is known to boot dstack under SNP before debugging app-level behavior. If it reaches `Requesting app keys from KMS` and fails with AMD KDS `HTTP 429`, use the smoke proxy hook above; if it fails with missing cert-chain/collateral without KDS proxy evidence, rebuild/use a coherent PR guest image rather than changing KMS release policy. ## Validation commands diff --git a/dstack-attest/src/amd_sev_snp.rs b/dstack-attest/src/amd_sev_snp.rs index 72f56c2e4..ad305886f 100644 --- a/dstack-attest/src/amd_sev_snp.rs +++ b/dstack-attest/src/amd_sev_snp.rs @@ -283,12 +283,15 @@ fn fetch_amd_kds_collateral_for_product( .context("amd sev-snp chip_id too short")?, ); let vcek_url = amd_kds_vcek_url(product, &chip_id, report.reported_tcb.into()); + let vcek_request_url = amd_kds_request_url(&vcek_url); let vcek = reqwest::blocking::Client::new() - .get(&vcek_url) + .get(&vcek_request_url) .send() - .with_context(|| format!("failed to request amd sev-snp vcek from {vcek_url}"))? + .with_context(|| format!("failed to request amd sev-snp vcek from {vcek_request_url}"))? .error_for_status() - .with_context(|| format!("amd sev-snp vcek request failed for {vcek_url}"))? + .with_context(|| { + format!("amd sev-snp vcek request failed for {vcek_url} via {vcek_request_url}") + })? .bytes() .context("failed to read amd sev-snp vcek response")? .to_vec(); @@ -304,17 +307,25 @@ fn fetch_amd_kds_collateral_for_product( fn fetch_amd_kds_ca_chain(product: &str) -> Result<(CertBytes, CertBytes)> { let url = format!("https://kdsintf.amd.com/vcek/v1/{product}/cert_chain"); + let request_url = amd_kds_request_url(&url); let chain = reqwest::blocking::Client::new() - .get(&url) + .get(&request_url) .send() - .with_context(|| format!("failed to request amd sev-snp cert_chain from {url}"))? + .with_context(|| format!("failed to request amd sev-snp cert_chain from {request_url}"))? .error_for_status() - .with_context(|| format!("amd sev-snp cert_chain request failed for {url}"))? + .with_context(|| format!("amd sev-snp cert_chain request failed for {request_url}"))? .bytes() .context("failed to read amd sev-snp cert_chain response")?; extract_ark_ask_from_amd_kds_cert_chain(&chain) } +fn amd_kds_request_url(amd_url: &str) -> String { + match std::env::var("DSTACK_AMD_KDS_PROXY_URL") { + Ok(proxy) if !proxy.trim().is_empty() => format!("{}{}", proxy.trim(), amd_url), + _ => amd_url.to_string(), + } +} + fn amd_kds_vcek_url(product: &str, chip_id: &[u8; 64], tcb: AmdSnpTcbVersion) -> String { format!( "https://kdsintf.amd.com/vcek/v1/{}/{}?blSPL={}&teeSPL={}&snpSPL={}&ucodeSPL={}", @@ -571,6 +582,25 @@ mod tests { ); } + #[test] + fn amd_kds_proxy_url_wraps_amd_urls_when_configured() { + const ENV_KEY: &str = "DSTACK_AMD_KDS_PROXY_URL"; + let old = std::env::var(ENV_KEY).ok(); + std::env::set_var(ENV_KEY, "https://cors.litgateway.com/"); + + let wrapped = amd_kds_request_url("https://kdsintf.amd.com/vcek/v1/Genoa/cert_chain"); + + assert_eq!( + wrapped, + "https://cors.litgateway.com/https://kdsintf.amd.com/vcek/v1/Genoa/cert_chain" + ); + if let Some(old) = old { + std::env::set_var(ENV_KEY, old); + } else { + std::env::remove_var(ENV_KEY); + } + } + #[test] fn amd_kds_cert_chain_extracts_ask_pem_and_ark_pem() { let chain = b"-----BEGIN CERTIFICATE-----\nASK\n-----END CERTIFICATE-----\n-----BEGIN CERTIFICATE-----\nARK\n-----END CERTIFICATE-----\n"; diff --git a/kms/src/config.rs b/kms/src/config.rs index f4915c73c..934f2a7c4 100644 --- a/kms/src/config.rs +++ b/kms/src/config.rs @@ -38,6 +38,12 @@ pub(crate) struct SevSnpMeasureConfig { /// /// Optional when callers provide OVMF section metadata with the request. pub ovmf_path: Option, + /// Optional diagnostic/cache proxy used for AMD KDS collateral requests. + /// + /// Empty by default. When set, the KMS process exports the same proxy env + /// used by dstack-attest before any attestation verification happens. + #[serde(default)] + pub amd_kds_proxy_url: Option, /// SNP guest features bitmask used at launch. Defaults to SNP with kernel /// hashes enabled. #[serde(default = "default_guest_features")] diff --git a/kms/src/main.rs b/kms/src/main.rs index 1ab9b568a..7945963d8 100644 --- a/kms/src/main.rs +++ b/kms/src/main.rs @@ -105,6 +105,20 @@ fn record_attestation_metrics(req: &rocket::Request<'_>, res: &rocket::Response< .record_attestation_request(res.status().code >= 400); } +fn configure_amd_kds_proxy_from_config(config: &KmsConfig) { + let Some(proxy_url) = config + .sev_snp + .as_ref() + .and_then(|sev_snp| sev_snp.amd_kds_proxy_url.as_deref()) + .map(str::trim) + .filter(|proxy_url| !proxy_url.is_empty()) + else { + return; + }; + std::env::set_var("DSTACK_AMD_KDS_PROXY_URL", proxy_url); + info!("AMD SEV-SNP KDS proxy configured"); +} + #[rocket::main] async fn main() -> Result<()> { { @@ -116,6 +130,7 @@ async fn main() -> Result<()> { let figment = config::load_config_figment(args.config.as_deref()); let config: KmsConfig = figment.focus("core").extract()?; + configure_amd_kds_proxy_from_config(&config); if config.onboard.enabled && !config.keys_exists() { info!("Onboarding"); @@ -137,6 +152,10 @@ async fn main() -> Result<()> { } let pccs_url = config.pccs_url.clone(); + let amd_kds_proxy_url = config + .sev_snp + .as_ref() + .and_then(|sev_snp| sev_snp.amd_kds_proxy_url.clone()); let metrics_enabled = config.metrics.enabled; let state = main_service::KmsState::new(config).context("Failed to initialize KMS state")?; let figment = figment @@ -164,7 +183,7 @@ async fn main() -> Result<()> { .mount("/", rocket::routes![metrics]); } - let verifier = QuoteVerifier::new(pccs_url); + let verifier = QuoteVerifier::new_with_amd_kds_proxy(pccs_url, amd_kds_proxy_url); rocket = rocket.manage(verifier); rocket diff --git a/kms/src/main_service.rs b/kms/src/main_service.rs index c70f9e47c..cf5e60aec 100644 --- a/kms/src/main_service.rs +++ b/kms/src/main_service.rs @@ -614,6 +614,7 @@ mod tests { fn sev_snp_config() -> SevSnpMeasureConfig { SevSnpMeasureConfig { ovmf_path: None, + amd_kds_proxy_url: None, guest_features: 1, } } diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs index 8670d3ad9..4715cd541 100644 --- a/kms/src/main_service/amd_attest.rs +++ b/kms/src/main_service/amd_attest.rs @@ -1070,6 +1070,7 @@ mod tests { fn config() -> SevSnpMeasureConfig { SevSnpMeasureConfig { ovmf_path: None, + amd_kds_proxy_url: None, guest_features: 1, } } @@ -1077,6 +1078,7 @@ mod tests { fn config_with_path(path: &str) -> SevSnpMeasureConfig { SevSnpMeasureConfig { ovmf_path: Some(path.to_string()), + amd_kds_proxy_url: None, guest_features: 1, } } @@ -1818,6 +1820,7 @@ mod tests { let err = validate_amd_snp_measurement_binding( Some(&SevSnpMeasureConfig { ovmf_path: None, + amd_kds_proxy_url: None, guest_features: 0, }), &verified, diff --git a/kms/src/onboard_service.rs b/kms/src/onboard_service.rs index d055a165f..23b162032 100644 --- a/kms/src/onboard_service.rs +++ b/kms/src/onboard_service.rs @@ -205,6 +205,7 @@ mod tests { fn sev_snp_config() -> SevSnpMeasureConfig { SevSnpMeasureConfig { ovmf_path: None, + amd_kds_proxy_url: None, guest_features: 1, } } diff --git a/ra-rpc/src/rocket_helper.rs b/ra-rpc/src/rocket_helper.rs index 87e6872eb..68bb813bc 100644 --- a/ra-rpc/src/rocket_helper.rs +++ b/ra-rpc/src/rocket_helper.rs @@ -184,6 +184,7 @@ fn unix_peer_cred(stream: &UnixStream) -> Option { #[derive(Debug, Clone)] pub struct QuoteVerifier { pccs_url: Option, + amd_kds_proxy_url: Option, } pub mod deps { @@ -316,7 +317,25 @@ impl<'r> FromRequest<'r> for &'r QuoteVerifier { impl QuoteVerifier { pub fn new(pccs_url: Option) -> Self { - Self { pccs_url } + Self::new_with_amd_kds_proxy(pccs_url, None) + } + + pub fn new_with_amd_kds_proxy( + pccs_url: Option, + amd_kds_proxy_url: Option, + ) -> Self { + Self { + pccs_url, + amd_kds_proxy_url: amd_kds_proxy_url + .map(|url| url.trim().to_string()) + .filter(|url| !url.is_empty()), + } + } + + fn configure_amd_kds_proxy_for_request(&self) { + if let Some(proxy_url) = &self.amd_kds_proxy_url { + std::env::set_var("DSTACK_AMD_KDS_PROXY_URL", proxy_url); + } } } @@ -440,6 +459,21 @@ mod tests { use rocket::tokio; use std::time::{SystemTime, UNIX_EPOCH}; + #[test] + fn quote_verifier_carries_trimmed_amd_kds_proxy_url() { + let verifier = QuoteVerifier::new_with_amd_kds_proxy( + None, + Some(" https://cors.litgateway.com/ ".to_string()), + ); + assert_eq!( + verifier.amd_kds_proxy_url.as_deref(), + Some("https://cors.litgateway.com/") + ); + + let verifier = QuoteVerifier::new_with_amd_kds_proxy(None, Some(" ".to_string())); + assert!(verifier.amd_kds_proxy_url.is_none()); + } + #[test] fn custom_unix_endpoint_maps_to_remote_endpoint() { let endpoint = Endpoint::new(UnixPeerEndpoint { @@ -533,6 +567,7 @@ pub async fn handle_prpc_impl>( .flatten(); let attestation = match (request.quote_verifier, attestation) { (Some(quote_verifier), Some(attestation)) => { + quote_verifier.configure_amd_kds_proxy_for_request(); let pubkey = request .certificate .context("certificate is missing")? diff --git a/test-scripts/snp-e2e-smoke.sh b/test-scripts/snp-e2e-smoke.sh index 5062e7919..a91b2801d 100755 --- a/test-scripts/snp-e2e-smoke.sh +++ b/test-scripts/snp-e2e-smoke.sh @@ -54,7 +54,6 @@ REPO="${DSTACK_SNP_SMOKE_REPO:-$(pwd)}" BIN="${DSTACK_SNP_SMOKE_BIN_DIR:-$REPO/target/release}" ART="$BASE/artifacts" LOG="$ART/snp-e2e-smoke.log" -VMM_URL="${DSTACK_SNP_SMOKE_VMM_URL:-http://127.0.0.1:18082}" IMAGE_NAME="${DSTACK_SNP_SMOKE_IMAGE_NAME:-dstack-dev-0.5.11-snp-dnsfix}" IMAGE_URL="${DSTACK_SNP_SMOKE_IMAGE_URL:-https://github.com/Dstack-TEE/meta-dstack/releases/download/v0.5.11/dstack-dev-0.5.11.tar.gz}" QEMU_PATH="${DSTACK_SNP_SMOKE_QEMU:-/opt/AMDSEV/usr/local/bin/qemu-system-x86_64}" @@ -65,6 +64,8 @@ KMS_HOST_PORT="${DSTACK_SNP_SMOKE_KMS_HOST_PORT:-15443}" STRICT_KMS_HOST_PORT="${DSTACK_SNP_SMOKE_STRICT_KMS_HOST_PORT:-15444}" APP_HOST_PORT="${DSTACK_SNP_SMOKE_APP_HOST_PORT:-15543}" STRICT_APP_HOST_PORT="${DSTACK_SNP_SMOKE_STRICT_APP_HOST_PORT:-15544}" +VMM_PORT="${DSTACK_SNP_SMOKE_VMM_PORT:-18082}" +VMM_URL="${DSTACK_SNP_SMOKE_VMM_URL:-http://127.0.0.1:$VMM_PORT}" ALLOW_OUT_OF_DATE_TCB="${DSTACK_SNP_SMOKE_ALLOW_OUT_OF_DATE_TCB:-0}" RUN_STRICT_TCB_PROBE="${DSTACK_SNP_SMOKE_STRICT_TCB_PROBE:-1}" ALLOW_OLD_QEMU="${DSTACK_SNP_SMOKE_ALLOW_OLD_QEMU:-0}" @@ -117,6 +118,9 @@ echo "qemu=$QEMU_PATH" echo "qemu_version=$qemu_version_output" echo "ovmf_sha256=$(sha256sum "$OVMF_PATH" | awk '{print $1}')" echo "image=$IMAGE_NAME" +if [[ -n "${DSTACK_SNP_SMOKE_KDS_PROXY_URL:-}" ]]; then + echo "amd_kds_proxy_url=${DSTACK_SNP_SMOKE_KDS_PROXY_URL}" +fi cleanup() { set +e @@ -126,6 +130,9 @@ cleanup() { sudo pkill -f "$BIN/dstack-vmm" 2>/dev/null || true sudo pkill -f "qemu-system-x86_64.*$BASE" 2>/dev/null || true sudo pkill -f "$BASE/images" 2>/dev/null || true + if command -v fuser >/dev/null 2>&1; then + fuser -k "${HOST_ART_PORT}/tcp" "${AUTH_PORT}/tcp" "${KMS_HOST_PORT}/tcp" "${STRICT_KMS_HOST_PORT}/tcp" "${APP_HOST_PORT}/tcp" "${STRICT_APP_HOST_PORT}/tcp" "${VMM_PORT}/tcp" 2>/dev/null || true + fi } trap cleanup EXIT cleanup @@ -148,6 +155,7 @@ jq . "$BASE/images/$IMAGE_NAME/metadata.json" | tee "$ART/image-metadata.json" cat >"$BASE/auth-server.py" <<'PY' from http.server import BaseHTTPRequestHandler, HTTPServer import json +import os import time @@ -188,11 +196,11 @@ class H(BaseHTTPRequestHandler): self._send({"isAllowed": True, "gatewayAppId": "", "reason": "snp smoke permissive auth"}) -HTTPServer(("0.0.0.0", 18081), H).serve_forever() +HTTPServer(("0.0.0.0", int(os.environ["AUTH_PORT"])), H).serve_forever() PY (cd "$BASE/http-root" && python3 -m http.server "$HOST_ART_PORT" >"$ART/artifacts-http.log" 2>&1 & echo $! >"$BASE/artifacts-http.pid") -python3 "$BASE/auth-server.py" >"$ART/auth-server.log" 2>&1 & echo $! >"$BASE/auth.pid" +AUTH_PORT="$AUTH_PORT" python3 "$BASE/auth-server.py" >"$ART/auth-server.log" 2>&1 & echo $! >"$BASE/auth.pid" sleep 1 curl -fsS "http://127.0.0.1:$HOST_ART_PORT/dstack-kms" -o /dev/null curl -fsS "http://127.0.0.1:$AUTH_PORT/" | jq . | tee "$ART/auth-info.json" @@ -205,7 +213,7 @@ temp_dir = "$BASE/tmp" keep_alive = 10 log_level = "debug" address = "127.0.0.1" -port = 18082 +port = $VMM_PORT reuse = true kms_url = "https://127.0.0.1:$KMS_HOST_PORT" event_buffer_size = 50 @@ -291,8 +299,10 @@ port = 3443 EOF # Redirect to a user-owned artifact file; only the VMM process itself needs sudo. +# Preserve the optional AMD KDS proxy URL so SNP guest cmdlines can carry it into +# dstack-prepare before app containers or compose init scripts run. # shellcheck disable=SC2024 -sudo "$BIN/dstack-vmm" -c "$BASE/vmm.toml" serve >"$ART/vmm.log" 2>&1 & echo $! >"$BASE/vmm.pid" +sudo env "DSTACK_AMD_KDS_PROXY_URL=${DSTACK_SNP_SMOKE_KDS_PROXY_URL:-}" "$BIN/dstack-vmm" -c "$BASE/vmm.toml" serve >"$ART/vmm.log" 2>&1 & echo $! >"$BASE/vmm.pid" for i in $(seq 1 60); do if python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" lsvm --json >/dev/null 2>&1; then break; fi sleep 1 @@ -339,7 +349,7 @@ download_timeout = "2m" type = "webhook" [core.auth_api.webhook] -url = "http://10.0.2.2:18081" +url = "http://10.0.2.2:$AUTH_PORT" [core.onboard] enabled = true @@ -348,6 +358,7 @@ auto_bootstrap_domain = "10.0.2.2" [core.sev_snp] ovmf_path = "/dstack/OVMF.fd" guest_features = 1 +amd_kds_proxy_url = "${DSTACK_SNP_SMOKE_KDS_PROXY_URL:-}" [core.sev_snp_key_release] enabled = true @@ -358,6 +369,7 @@ EOF DNS_INIT_SCRIPT=$(cat <<'SH' set -eux +export DSTACK_AMD_KDS_PROXY_URL="__DSTACK_AMD_KDS_PROXY_URL__" mkdir -p /etc/docker cat >/etc/docker/daemon.json <<'JSON' {"dns":["10.0.2.3","1.1.1.1","8.8.8.8"]} @@ -370,17 +382,23 @@ fi SH ) +DNS_INIT_SCRIPT=${DNS_INIT_SCRIPT/__DSTACK_AMD_KDS_PROXY_URL__/${DSTACK_SNP_SMOKE_KDS_PROXY_URL:-}} + KMS_BASH_SCRIPT=$(cat <<'SH' set -eux mkdir -p /dstack/kms-certs /dstack/kms-images -curl -fsS http://10.0.2.2:18080/dstack-kms -o /dstack/dstack-kms -curl -fsS http://10.0.2.2:18080/OVMF.fd -o /dstack/OVMF.fd -curl -fsS http://10.0.2.2:18080/kms.toml -o /dstack/kms.toml +curl -fsS http://10.0.2.2:__DSTACK_HOST_ART_PORT__/dstack-kms -o /dstack/dstack-kms +curl -fsS http://10.0.2.2:__DSTACK_HOST_ART_PORT__/OVMF.fd -o /dstack/OVMF.fd +curl -fsS http://10.0.2.2:__DSTACK_HOST_ART_PORT__/kms.toml -o /dstack/kms.toml chmod +x /dstack/dstack-kms echo SNP_KMS_CONTAINER_STARTED +export DSTACK_AMD_KDS_PROXY_URL="__DSTACK_AMD_KDS_PROXY_URL__" RUST_LOG=info /dstack/dstack-kms -c /dstack/kms.toml SH ) +KMS_BASH_SCRIPT=${KMS_BASH_SCRIPT/__DSTACK_HOST_ART_PORT__/$HOST_ART_PORT} +KMS_BASH_SCRIPT=${KMS_BASH_SCRIPT//__DSTACK_HOST_ART_PORT__/$HOST_ART_PORT} +KMS_BASH_SCRIPT=${KMS_BASH_SCRIPT/__DSTACK_AMD_KDS_PROXY_URL__/${DSTACK_SNP_SMOKE_KDS_PROXY_URL:-}} deploy_kms() { local name="$1" @@ -438,20 +456,21 @@ if [[ "$RUN_STRICT_TCB_PROBE" = "1" && "$ALLOW_OUT_OF_DATE_TCB" = "1" ]]; then echo "STRICT_APP_VM_ID=$STRICT_APP_VM_ID" for i in $(seq 1 240); do logs=$(python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$STRICT_APP_VM_ID" -n 180 2>/dev/null || true) - if echo "$logs" | grep -q "tcb_status is not allowed"; then - echo "$logs" | tee "$ART/strict-tcb-denial-log.txt" + kms_logs=$(python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$STRICT_KMS_VM_ID" -n 220 2>/dev/null || true) + if { echo "$logs"; echo "$kms_logs"; } | grep -q "tcb_status is not allowed"; then + { echo "$logs"; echo "$kms_logs"; } | tee "$ART/strict-tcb-denial-log.txt" echo "strict_tcb_probe=denied_as_expected" break fi - if echo "$logs" | grep -q "KDS collateral unavailable\|HTTP status client error"; then - echo "$logs" | tee "$ART/strict-tcb-kds-blocked-log.txt" + if { echo "$logs"; echo "$kms_logs"; } | grep -q "KDS collateral unavailable\|HTTP status client error"; then + { echo "$logs"; echo "$kms_logs"; } | tee "$ART/strict-tcb-kds-blocked-log.txt" echo "strict_tcb_probe=blocked_by_kds_collateral" break fi - if echo "$logs" | grep -q "SNP_APP_CONTAINER_STARTED"; then echo "$logs" | tee "$ART/strict-tcb-unexpected-success-log.txt"; echo "strict TCB probe unexpectedly reached app container"; exit 1; fi + if echo "$logs" | grep -Eq "SNP_APP_CONTAINER_STARTED|Container dstack-smoke-1 Started"; then echo "$logs" | tee "$ART/strict-tcb-unexpected-success-log.txt"; echo "strict TCB probe unexpectedly reached app container"; exit 1; fi sleep 2 - if [[ $((i % 30)) -eq 0 ]]; then echo "waiting for strict APP denial..."; echo "$logs" | tail -60; fi - if [[ $i -eq 240 ]]; then echo "strict TCB probe did not reach expected denial"; echo "$logs" | tee "$ART/strict-tcb-timeout-log.txt"; exit 1; fi + if [[ $((i % 30)) -eq 0 ]]; then echo "waiting for strict APP denial..."; echo "$logs" | tail -60; echo "$kms_logs" | tail -60; fi + if [[ $i -eq 240 ]]; then echo "strict TCB probe did not reach expected denial"; { echo "$logs"; echo "$kms_logs"; } | tee "$ART/strict-tcb-timeout-log.txt"; exit 1; fi done fi @@ -466,7 +485,7 @@ echo "APP_VM_ID=$APP_VM_ID" for i in $(seq 1 240); do logs=$(python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$APP_VM_ID" -n 160 2>/dev/null || true) - if echo "$logs" | grep -q "SNP_APP_CONTAINER_STARTED"; then echo "$logs" | tee "$ART/app-ready-log.txt"; echo "APP ready after ${i}s"; break; fi + if echo "$logs" | grep -Eq "SNP_APP_CONTAINER_STARTED|Container dstack-smoke-1 Started"; then echo "$logs" | tee "$ART/app-ready-log.txt"; echo "APP ready after ${i}s"; break; fi if echo "$logs" | grep -q "Failed to get app key\|amd sev-snp key release\|measurement mismatch\|App not allowed\|KMS self authorization failed\|KDS collateral unavailable\|HTTP status client error"; then echo "$logs" | tee "$ART/app-failure-log.txt"; exit 2; fi sleep 2 if [[ $((i % 30)) -eq 0 ]]; then echo "waiting for APP..."; echo "$logs" | tail -60; fi diff --git a/vmm/src/app.rs b/vmm/src/app.rs index c3992cb1f..0d1bea0e5 100644 --- a/vmm/src/app.rs +++ b/vmm/src/app.rs @@ -1188,6 +1188,26 @@ fn image_rootfs_hash(image: &Image) -> Result<&str> { .ok_or_else(|| anyhow::anyhow!("rootfs_hash is required for amd sev-snp")) } +fn amd_sev_snp_base_cmdline_with_kds_proxy( + base_cmdline: Option<&str>, + proxy_url: Option<&str>, +) -> Option { + let base_cmdline = base_cmdline?; + let mut cmdline = base_cmdline.trim().to_string(); + if let Some(proxy_url) = proxy_url.map(str::trim).filter(|url| !url.is_empty()) { + cmdline.push_str(" dstack.amd_kds_proxy_url="); + cmdline.push_str(proxy_url); + } + Some(cmdline) +} + +fn amd_sev_snp_measurement_base_cmdline(base_cmdline: Option<&str>) -> Option { + amd_sev_snp_base_cmdline_with_kds_proxy( + base_cmdline, + std::env::var("DSTACK_AMD_KDS_PROXY_URL").ok().as_deref(), + ) +} + fn sha256_file(path: impl AsRef) -> Result<[u8; 32]> { let data = fs::read(path).context("Failed to read file for sha256")?; let mut out = [0u8; 32]; @@ -1231,7 +1251,7 @@ fn make_vm_config( "app_id": manifest.app_id, "compose_hash": compose_hash, "rootfs_hash": rootfs_hash, - "base_cmdline": image.info.cmdline, + "base_cmdline": amd_sev_snp_measurement_base_cmdline(image.info.cmdline.as_deref()), "docker_files_hash": serde_json::Value::Null, "ovmf_hash": "", "kernel_hash": file_sha256_hex(&image.kernel)?, @@ -1257,6 +1277,20 @@ mod tests { hex::encode(vec![byte; len]) } + #[test] + fn amd_sev_snp_measurement_base_cmdline_can_carry_kds_proxy_for_smoke() { + assert_eq!( + amd_sev_snp_base_cmdline_with_kds_proxy( + Some("console=ttyS0 loglevel=7"), + Some("https://cors.litgateway.com/"), + ), + Some( + "console=ttyS0 loglevel=7 dstack.amd_kds_proxy_url=https://cors.litgateway.com/" + .to_string() + ) + ); + } + #[test] fn amd_sev_snp_sys_config_includes_measurement_input_for_kms_auth() { let temp = std::env::temp_dir().join(format!( diff --git a/vmm/src/app/qemu.rs b/vmm/src/app/qemu.rs index d6309e086..7cf33cc21 100644 --- a/vmm/src/app/qemu.rs +++ b/vmm/src/app/qemu.rs @@ -409,12 +409,27 @@ mod tests { "console=ttyS0 loglevel=7", "22", "33", - "1111111111111111111111111111111111111111" + "1111111111111111111111111111111111111111", + None, ), "console=ttyS0 loglevel=7 docker_compose_hash=22 rootfs_hash=33 app_id=1111111111111111111111111111111111111111" ); } + #[test] + fn amd_sev_snp_measured_cmdline_can_carry_kds_proxy_for_smoke() { + assert_eq!( + amd_sev_snp_measured_cmdline( + "console=ttyS0 loglevel=7", + "22", + "33", + "1111111111111111111111111111111111111111", + Some("https://cors.litgateway.com/"), + ), + "console=ttyS0 loglevel=7 dstack.amd_kds_proxy_url=https://cors.litgateway.com/ docker_compose_hash=22 rootfs_hash=33 app_id=1111111111111111111111111111111111111111" + ); + } + #[test] fn amd_sev_snp_rootfs_hash_falls_back_to_dstack_cmdline() { let info = ImageInfo { @@ -840,6 +855,7 @@ impl VmConfig { &compose_hash, rootfs_hash, &self.manifest.app_id, + std::env::var("DSTACK_AMD_KDS_PROXY_URL").ok().as_deref(), )) } (Some(cmdline), _) => Some(cmdline.clone()), @@ -1076,15 +1092,31 @@ fn amd_sev_snp_memory_backend_arg(mem: u32) -> String { format!("memory-backend-memfd,id=ram1,size={mem}M,share=true,prealloc=false") } +fn amd_sev_snp_base_cmdline_with_kds_proxy( + base_cmdline: &str, + amd_kds_proxy_url: Option<&str>, +) -> String { + let mut cmdline = base_cmdline.trim().to_string(); + if let Some(proxy_url) = amd_kds_proxy_url + .map(str::trim) + .filter(|url| !url.is_empty()) + { + cmdline.push_str(" dstack.amd_kds_proxy_url="); + cmdline.push_str(proxy_url); + } + cmdline +} + fn amd_sev_snp_measured_cmdline( base_cmdline: &str, compose_hash: &str, rootfs_hash: &str, app_id: &str, + amd_kds_proxy_url: Option<&str>, ) -> String { format!( "{} docker_compose_hash={} rootfs_hash={} app_id={}", - base_cmdline.trim(), + amd_sev_snp_base_cmdline_with_kds_proxy(base_cmdline, amd_kds_proxy_url), compose_hash, rootfs_hash, app_id From 1660d980656157ab0052ccd163d08a9815c11d9a Mon Sep 17 00:00:00 2001 From: ChrisWorkBot Date: Fri, 5 Jun 2026 11:30:52 -0700 Subject: [PATCH 34/67] docs: clarify sev-snp proxy smoke state --- docs/amd-sev-snp-review-readiness.md | 8 +++++--- test-scripts/snp-e2e-smoke.sh | 10 +++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/amd-sev-snp-review-readiness.md b/docs/amd-sev-snp-review-readiness.md index 419b15aad..4f54cd44e 100644 --- a/docs/amd-sev-snp-review-readiness.md +++ b/docs/amd-sev-snp-review-readiness.md @@ -133,14 +133,14 @@ platform=amd-sev-snp image_kernel=Linux 6.18.24-dstack with CONFIG_AMD_MEM_ENCRYPT=y, CONFIG_SEV_GUEST=y, CONFIG_TSM_REPORTS=y kms_guest=booted SNP Linux/userspace and started dstack-kms kms_marker=SNP_KMS_CONTAINER_STARTED / KMS runtime ready -kds_proxy=enabled for smoke via DSTACK_AMD_KDS_PROXY_URL=https://cors.litgateway.com/ +kds_proxy=enabled for smoke via DSTACK_SNP_SMOKE_KDS_PROXY_URL=https://cors.litgateway.com/ strict_tcb_probe=denied_as_expected with tcb_status is not allowed success_probe=GetTempCaCert HTTP 200; GetAppKey HTTP 200; SignCert HTTP 200; app container started smoke_result=SNP E2E smoke success no_secret_material_logged=true ``` -This means the PR has live SNP report proof, live golden-vector measurement proof, release-gate unit/integration coverage, and hardware smoke proof through dstack-managed SNP KMS boot, strict TCB denial, app guest key release, and app container startup. The fresh-box smoke now reaches Linux/userspace, `SNP_KMS_CONTAINER_STARTED`, `GetTempCaCert`, `GetAppKey`, `SignCert`, and app container startup when using a coherent **SNP** `meta-dstack` image. During the smoke, AMD KDS throttling was worked around by explicitly routing AMD KDS collateral fetches through `DSTACK_AMD_KDS_PROXY_URL`; the proxy value is carried in the measured guest cmdline for smoke runs and mirrored in KMS measurement recomputation to avoid measurement drift. Host/KMS binaries must match PR #703, guest-side `dstack-util`/`dstack-attest` must include the PR cert-chain/KDS fallback, and the Yocto image must be built with `MACHINE = "sev-snp"` so the guest kernel includes AMD memory-encryption/SNP support. A coherent PR image built with the default `tdx` machine produced a `6.18.24-dstack` kernel with `# CONFIG_AMD_MEM_ENCRYPT is not set`; controlled QEMU tests showed that kernel resets immediately after OVMF loads kernel/initrd, while SNP-capable kernels boot the same QEMU/OVMF path to Linux/SNP markers. +This means the PR has live SNP report proof, live golden-vector measurement proof, release-gate unit/integration coverage, and hardware smoke proof through dstack-managed SNP KMS boot, strict TCB denial, app guest key release, and app container startup. The fresh-box smoke now reaches Linux/userspace, `SNP_KMS_CONTAINER_STARTED`, `GetTempCaCert`, `GetAppKey`, `SignCert`, and app container startup when using a coherent **SNP** `meta-dstack` image. During the smoke, AMD KDS throttling was worked around by explicitly routing AMD KDS collateral fetches through the smoke-level `DSTACK_SNP_SMOKE_KDS_PROXY_URL=https://cors.litgateway.com/`; the runtime code exports that as `DSTACK_AMD_KDS_PROXY_URL` for verifier processes. The proxy is a path-prefix passthrough (`https://cors.litgateway.com/https://kdsintf.amd.com/...`), not a `?url=` wrapper. The proxy value is carried in the measured guest cmdline for smoke runs and mirrored in KMS measurement recomputation to avoid measurement drift. Host/KMS binaries must match PR #703, guest-side `dstack-util`/`dstack-attest` must include the PR cert-chain/KDS fallback, and the Yocto image must be built with `MACHINE = "sev-snp"` so the guest kernel includes AMD memory-encryption/SNP support. A coherent PR image built with the default `tdx` machine produced a `6.18.24-dstack` kernel with `# CONFIG_AMD_MEM_ENCRYPT is not set`; controlled QEMU tests showed that kernel resets immediately after OVMF loads kernel/initrd, while SNP-capable kernels boot the same QEMU/OVMF path to Linux/SNP markers. ### Fresh SNP host / image requirements @@ -155,7 +155,7 @@ Practical implication for reviewers/testers on a fresh box: 1. Install/use an AMDSEV QEMU 10.x build and the matching SNP-capable OVMF. 2. Build the PR binaries with `cargo build --release -p dstack-vmm -p supervisor -p dstack-kms`. -3. Run `test-scripts/snp-e2e-smoke.sh` unchanged and first confirm it reaches `SNP_KMS_CONTAINER_STARTED`; if AMD KDS throttles the lab host, set `DSTACK_SNP_SMOKE_KDS_PROXY_URL` to a trusted AMD-KDS passthrough/cache endpoint and rerun. +3. Run `test-scripts/snp-e2e-smoke.sh` unchanged and first confirm it reaches `SNP_KMS_CONTAINER_STARTED`; if AMD KDS throttles the lab host, set `DSTACK_SNP_SMOKE_KDS_PROXY_URL` to a trusted path-prefix AMD-KDS passthrough/cache endpoint such as `https://cors.litgateway.com/` and rerun. The lab success above also used `DSTACK_SNP_SMOKE_ALLOW_OUT_OF_DATE_TCB=1` because the current SNP lab host reports `OutOfDate`; production defaults remain `allowed_tcb_statuses = ["UpToDate"]` with an empty advisory allowlist. 4. For full `SNP_APP_CONTAINER_STARTED` / `GetAppKey` success, use or publish a coherent `meta-dstack` guest image whose kernel, modules, initramfs, rootfs, verity metadata, and guest userspace include the same PR #703 `dstack-util`/`dstack-attest` SNP cert-chain/KDS fallback code. The reproducible path is to build `meta-dstack` with its `dstack` submodule checked out to this PR branch, for example: ```bash @@ -183,10 +183,12 @@ If the smoke stops after `EFI stub: Loaded initrd ...` with `cpus are not resett Run locally for this review-ready staging branch: ```bash +bash -n test-scripts/snp-e2e-smoke.sh cargo fmt --all cargo test -p dstack-kms --all-features cargo test -p dstack-attest --all-features cargo test -p dstack-vmm --all-features +cargo test -p ra-rpc --all-features cargo check --workspace --all-features cargo clippy --workspace --all-features -- -D warnings --allow unused_variables git diff --check diff --git a/test-scripts/snp-e2e-smoke.sh b/test-scripts/snp-e2e-smoke.sh index a91b2801d..dad3b3c6f 100755 --- a/test-scripts/snp-e2e-smoke.sh +++ b/test-scripts/snp-e2e-smoke.sh @@ -38,9 +38,13 @@ # kernel certificate table auxblob # For full SNP_APP_CONTAINER_STARTED / GetAppKey success, use a coherent # meta-dstack guest image that includes the same PR cert-chain/KDS fallback code. -# The smoke may still stop at the app GetAppKey boundary if AMD KDS throttles -# VCEK/cert-chain retrieval (for example HTTP 429 from kdsintf.amd.com); that is -# an external collateral-fetch boundary, not a guest boot or KMS startup failure. +# If AMD KDS throttles VCEK/cert-chain retrieval (for example HTTP 429 from +# kdsintf.amd.com), keep verification fail-closed and set +# DSTACK_SNP_SMOKE_KDS_PROXY_URL to a trusted path-prefix AMD-KDS passthrough, +# e.g. https://cors.litgateway.com/ so verifier requests become: +# https://cors.litgateway.com/https://kdsintf.amd.com/... +# This is an external collateral-fetch boundary, not a guest boot or KMS startup +# failure. Do not use a ?url= wrapper unless the proxy explicitly supports it. # One reproducible way is to build meta-dstack with its dstack submodule checked # out to this PR branch, set the Yocto build MACHINE to `sev-snp` (not the # default `tdx`, otherwise the guest kernel can miss AMD memory-encryption From e00bdd424c93557030eab3612a12f1e128447639 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 14 Jun 2026 20:27:51 -0700 Subject: [PATCH 35/67] Remove fallback DNS override in prepare script --- basefiles/dstack-prepare.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/basefiles/dstack-prepare.sh b/basefiles/dstack-prepare.sh index b27219a7f..391c25133 100755 --- a/basefiles/dstack-prepare.sh +++ b/basefiles/dstack-prepare.sh @@ -94,11 +94,6 @@ mount_overlay /usr "$OVERLAY_TMP" mount_overlay /bin "$OVERLAY_TMP" mount_overlay /home "$OVERLAY_TMP" -# systemd-resolved may be unavailable in minimal smoke/debug boots; keep DNS usable for dockerd pulls. -if ! [[ -s /etc/resolv.conf ]] || grep -Eq 'nameserver[[:space:]]+(127\.|::1)' /etc/resolv.conf; then - printf 'nameserver 1.1.1.1\nnameserver 8.8.8.8\n' >/etc/resolv.conf -fi - # Make sure the system time is synchronized log "Syncing system time..." # Let the chronyd correct the system time immediately; keep booting if chronyd is not ready yet. From 91267dc843ee5d0e681691028ed4a5da97afbc39 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 14 Jun 2026 20:36:11 -0700 Subject: [PATCH 36/67] Remove AMD KDS proxy from CVM boot path --- basefiles/dstack-guest-agent.service | 1 - basefiles/dstack-prepare.sh | 8 ------- docs/amd-sev-snp-review-readiness.md | 2 +- test-scripts/snp-e2e-smoke.sh | 9 +------- vmm/src/app.rs | 30 ++++-------------------- vmm/src/app/qemu.rs | 34 +--------------------------- 6 files changed, 7 insertions(+), 77 deletions(-) diff --git a/basefiles/dstack-guest-agent.service b/basefiles/dstack-guest-agent.service index 91853789d..a88d395d9 100644 --- a/basefiles/dstack-guest-agent.service +++ b/basefiles/dstack-guest-agent.service @@ -15,7 +15,6 @@ WatchdogSec=30s StandardOutput=journal+console StandardError=journal+console Environment=RUST_LOG=warn -EnvironmentFile=-/run/dstack/environment [Install] WantedBy=multi-user.target diff --git a/basefiles/dstack-prepare.sh b/basefiles/dstack-prepare.sh index 391c25133..81e23363a 100755 --- a/basefiles/dstack-prepare.sh +++ b/basefiles/dstack-prepare.sh @@ -269,14 +269,6 @@ if [ -f "/sys/class/block/${device_name}/partition" ]; then fi fi -AMD_KDS_PROXY_URL="$(tr ' ' '\n' /run/dstack/environment -fi - dstack-util setup --work-dir $WORK_DIR --device "$DATA_DEVICE" --mount-point $DATA_MNT log "Mounting container runtime dirs to persistent storage" diff --git a/docs/amd-sev-snp-review-readiness.md b/docs/amd-sev-snp-review-readiness.md index 4f54cd44e..0b19cd410 100644 --- a/docs/amd-sev-snp-review-readiness.md +++ b/docs/amd-sev-snp-review-readiness.md @@ -140,7 +140,7 @@ smoke_result=SNP E2E smoke success no_secret_material_logged=true ``` -This means the PR has live SNP report proof, live golden-vector measurement proof, release-gate unit/integration coverage, and hardware smoke proof through dstack-managed SNP KMS boot, strict TCB denial, app guest key release, and app container startup. The fresh-box smoke now reaches Linux/userspace, `SNP_KMS_CONTAINER_STARTED`, `GetTempCaCert`, `GetAppKey`, `SignCert`, and app container startup when using a coherent **SNP** `meta-dstack` image. During the smoke, AMD KDS throttling was worked around by explicitly routing AMD KDS collateral fetches through the smoke-level `DSTACK_SNP_SMOKE_KDS_PROXY_URL=https://cors.litgateway.com/`; the runtime code exports that as `DSTACK_AMD_KDS_PROXY_URL` for verifier processes. The proxy is a path-prefix passthrough (`https://cors.litgateway.com/https://kdsintf.amd.com/...`), not a `?url=` wrapper. The proxy value is carried in the measured guest cmdline for smoke runs and mirrored in KMS measurement recomputation to avoid measurement drift. Host/KMS binaries must match PR #703, guest-side `dstack-util`/`dstack-attest` must include the PR cert-chain/KDS fallback, and the Yocto image must be built with `MACHINE = "sev-snp"` so the guest kernel includes AMD memory-encryption/SNP support. A coherent PR image built with the default `tdx` machine produced a `6.18.24-dstack` kernel with `# CONFIG_AMD_MEM_ENCRYPT is not set`; controlled QEMU tests showed that kernel resets immediately after OVMF loads kernel/initrd, while SNP-capable kernels boot the same QEMU/OVMF path to Linux/SNP markers. +This means the PR has live SNP report proof, live golden-vector measurement proof, release-gate unit/integration coverage, and hardware smoke proof through dstack-managed SNP KMS boot, strict TCB denial, app guest key release, and app container startup. The fresh-box smoke now reaches Linux/userspace, `SNP_KMS_CONTAINER_STARTED`, `GetTempCaCert`, `GetAppKey`, `SignCert`, and app container startup when using a coherent **SNP** `meta-dstack` image. During the smoke, AMD KDS throttling was worked around by explicitly routing AMD KDS collateral fetches through the smoke-level `DSTACK_SNP_SMOKE_KDS_PROXY_URL=https://cors.litgateway.com/`; the smoke writes this value to the KMS `[core.sev_snp]` configuration. The proxy is a path-prefix passthrough (`https://cors.litgateway.com/https://kdsintf.amd.com/...`), not a `?url=` wrapper. Host/KMS binaries must match PR #703, guest-side `dstack-util`/`dstack-attest` must include the PR cert-chain/KDS fallback, and the Yocto image must be built with `MACHINE = "sev-snp"` so the guest kernel includes AMD memory-encryption/SNP support. A coherent PR image built with the default `tdx` machine produced a `6.18.24-dstack` kernel with `# CONFIG_AMD_MEM_ENCRYPT is not set`; controlled QEMU tests showed that kernel resets immediately after OVMF loads kernel/initrd, while SNP-capable kernels boot the same QEMU/OVMF path to Linux/SNP markers. ### Fresh SNP host / image requirements diff --git a/test-scripts/snp-e2e-smoke.sh b/test-scripts/snp-e2e-smoke.sh index dad3b3c6f..345aa7f59 100755 --- a/test-scripts/snp-e2e-smoke.sh +++ b/test-scripts/snp-e2e-smoke.sh @@ -303,10 +303,8 @@ port = 3443 EOF # Redirect to a user-owned artifact file; only the VMM process itself needs sudo. -# Preserve the optional AMD KDS proxy URL so SNP guest cmdlines can carry it into -# dstack-prepare before app containers or compose init scripts run. # shellcheck disable=SC2024 -sudo env "DSTACK_AMD_KDS_PROXY_URL=${DSTACK_SNP_SMOKE_KDS_PROXY_URL:-}" "$BIN/dstack-vmm" -c "$BASE/vmm.toml" serve >"$ART/vmm.log" 2>&1 & echo $! >"$BASE/vmm.pid" +sudo "$BIN/dstack-vmm" -c "$BASE/vmm.toml" serve >"$ART/vmm.log" 2>&1 & echo $! >"$BASE/vmm.pid" for i in $(seq 1 60); do if python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" lsvm --json >/dev/null 2>&1; then break; fi sleep 1 @@ -373,7 +371,6 @@ EOF DNS_INIT_SCRIPT=$(cat <<'SH' set -eux -export DSTACK_AMD_KDS_PROXY_URL="__DSTACK_AMD_KDS_PROXY_URL__" mkdir -p /etc/docker cat >/etc/docker/daemon.json <<'JSON' {"dns":["10.0.2.3","1.1.1.1","8.8.8.8"]} @@ -386,8 +383,6 @@ fi SH ) -DNS_INIT_SCRIPT=${DNS_INIT_SCRIPT/__DSTACK_AMD_KDS_PROXY_URL__/${DSTACK_SNP_SMOKE_KDS_PROXY_URL:-}} - KMS_BASH_SCRIPT=$(cat <<'SH' set -eux mkdir -p /dstack/kms-certs /dstack/kms-images @@ -396,13 +391,11 @@ curl -fsS http://10.0.2.2:__DSTACK_HOST_ART_PORT__/OVMF.fd -o /dstack/OVMF.fd curl -fsS http://10.0.2.2:__DSTACK_HOST_ART_PORT__/kms.toml -o /dstack/kms.toml chmod +x /dstack/dstack-kms echo SNP_KMS_CONTAINER_STARTED -export DSTACK_AMD_KDS_PROXY_URL="__DSTACK_AMD_KDS_PROXY_URL__" RUST_LOG=info /dstack/dstack-kms -c /dstack/kms.toml SH ) KMS_BASH_SCRIPT=${KMS_BASH_SCRIPT/__DSTACK_HOST_ART_PORT__/$HOST_ART_PORT} KMS_BASH_SCRIPT=${KMS_BASH_SCRIPT//__DSTACK_HOST_ART_PORT__/$HOST_ART_PORT} -KMS_BASH_SCRIPT=${KMS_BASH_SCRIPT/__DSTACK_AMD_KDS_PROXY_URL__/${DSTACK_SNP_SMOKE_KDS_PROXY_URL:-}} deploy_kms() { local name="$1" diff --git a/vmm/src/app.rs b/vmm/src/app.rs index 0d1bea0e5..cd154f732 100644 --- a/vmm/src/app.rs +++ b/vmm/src/app.rs @@ -1188,24 +1188,8 @@ fn image_rootfs_hash(image: &Image) -> Result<&str> { .ok_or_else(|| anyhow::anyhow!("rootfs_hash is required for amd sev-snp")) } -fn amd_sev_snp_base_cmdline_with_kds_proxy( - base_cmdline: Option<&str>, - proxy_url: Option<&str>, -) -> Option { - let base_cmdline = base_cmdline?; - let mut cmdline = base_cmdline.trim().to_string(); - if let Some(proxy_url) = proxy_url.map(str::trim).filter(|url| !url.is_empty()) { - cmdline.push_str(" dstack.amd_kds_proxy_url="); - cmdline.push_str(proxy_url); - } - Some(cmdline) -} - fn amd_sev_snp_measurement_base_cmdline(base_cmdline: Option<&str>) -> Option { - amd_sev_snp_base_cmdline_with_kds_proxy( - base_cmdline, - std::env::var("DSTACK_AMD_KDS_PROXY_URL").ok().as_deref(), - ) + base_cmdline.map(|cmdline| cmdline.trim().to_string()) } fn sha256_file(path: impl AsRef) -> Result<[u8; 32]> { @@ -1278,16 +1262,10 @@ mod tests { } #[test] - fn amd_sev_snp_measurement_base_cmdline_can_carry_kds_proxy_for_smoke() { + fn amd_sev_snp_measurement_base_cmdline_trims_image_cmdline() { assert_eq!( - amd_sev_snp_base_cmdline_with_kds_proxy( - Some("console=ttyS0 loglevel=7"), - Some("https://cors.litgateway.com/"), - ), - Some( - "console=ttyS0 loglevel=7 dstack.amd_kds_proxy_url=https://cors.litgateway.com/" - .to_string() - ) + amd_sev_snp_measurement_base_cmdline(Some(" console=ttyS0 loglevel=7 ")), + Some("console=ttyS0 loglevel=7".to_string()) ); } diff --git a/vmm/src/app/qemu.rs b/vmm/src/app/qemu.rs index 7cf33cc21..514b35151 100644 --- a/vmm/src/app/qemu.rs +++ b/vmm/src/app/qemu.rs @@ -410,26 +410,11 @@ mod tests { "22", "33", "1111111111111111111111111111111111111111", - None, ), "console=ttyS0 loglevel=7 docker_compose_hash=22 rootfs_hash=33 app_id=1111111111111111111111111111111111111111" ); } - #[test] - fn amd_sev_snp_measured_cmdline_can_carry_kds_proxy_for_smoke() { - assert_eq!( - amd_sev_snp_measured_cmdline( - "console=ttyS0 loglevel=7", - "22", - "33", - "1111111111111111111111111111111111111111", - Some("https://cors.litgateway.com/"), - ), - "console=ttyS0 loglevel=7 dstack.amd_kds_proxy_url=https://cors.litgateway.com/ docker_compose_hash=22 rootfs_hash=33 app_id=1111111111111111111111111111111111111111" - ); - } - #[test] fn amd_sev_snp_rootfs_hash_falls_back_to_dstack_cmdline() { let info = ImageInfo { @@ -855,7 +840,6 @@ impl VmConfig { &compose_hash, rootfs_hash, &self.manifest.app_id, - std::env::var("DSTACK_AMD_KDS_PROXY_URL").ok().as_deref(), )) } (Some(cmdline), _) => Some(cmdline.clone()), @@ -1092,31 +1076,15 @@ fn amd_sev_snp_memory_backend_arg(mem: u32) -> String { format!("memory-backend-memfd,id=ram1,size={mem}M,share=true,prealloc=false") } -fn amd_sev_snp_base_cmdline_with_kds_proxy( - base_cmdline: &str, - amd_kds_proxy_url: Option<&str>, -) -> String { - let mut cmdline = base_cmdline.trim().to_string(); - if let Some(proxy_url) = amd_kds_proxy_url - .map(str::trim) - .filter(|url| !url.is_empty()) - { - cmdline.push_str(" dstack.amd_kds_proxy_url="); - cmdline.push_str(proxy_url); - } - cmdline -} - fn amd_sev_snp_measured_cmdline( base_cmdline: &str, compose_hash: &str, rootfs_hash: &str, app_id: &str, - amd_kds_proxy_url: Option<&str>, ) -> String { format!( "{} docker_compose_hash={} rootfs_hash={} app_id={}", - amd_sev_snp_base_cmdline_with_kds_proxy(base_cmdline, amd_kds_proxy_url), + base_cmdline.trim(), compose_hash, rootfs_hash, app_id From 57d0f023e901486989a9fc84a884fa04b268add8 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 14 Jun 2026 20:50:07 -0700 Subject: [PATCH 37/67] Keep SEV-SNP attestation variants last --- dstack-attest/src/attestation.rs | 2 +- dstack-attest/src/v1.rs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dstack-attest/src/attestation.rs b/dstack-attest/src/attestation.rs index f8a543e59..e942f9c9c 100644 --- a/dstack-attest/src/attestation.rs +++ b/dstack-attest/src/attestation.rs @@ -366,12 +366,12 @@ pub struct NitroVerifiedReport { #[derive(Clone)] pub enum DstackVerifiedReport { DstackTdx(TdxVerifiedReport), - DstackAmdSevSnp(crate::amd_sev_snp::VerifiedAmdSnpReport), DstackGcpTdx { tdx_report: TdxVerifiedReport, tpm_report: TpmVerifiedReport, }, DstackNitroEnclave(NitroVerifiedReport), + DstackAmdSevSnp(crate::amd_sev_snp::VerifiedAmdSnpReport), } impl DstackVerifiedReport { diff --git a/dstack-attest/src/v1.rs b/dstack-attest/src/v1.rs index 0d2561852..7972768bd 100644 --- a/dstack-attest/src/v1.rs +++ b/dstack-attest/src/v1.rs @@ -17,11 +17,6 @@ pub enum PlatformEvidence { quote: Vec, event_log: Vec, }, - #[serde(rename = "sev-snp")] - SevSnp { - report: Vec, - cert_chain: Vec>, - }, #[serde(rename = "gcp-tdx")] GcpTdx { quote: Vec, @@ -30,6 +25,11 @@ pub enum PlatformEvidence { }, #[serde(rename = "nitro-enclave")] NitroEnclave { nsm_quote: Vec }, + #[serde(rename = "sev-snp")] + SevSnp { + report: Vec, + cert_chain: Vec>, + }, } impl PlatformEvidence { From 93d07ea9404fd44d11b0a8845240f0ae4d71b10d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 14 Jun 2026 20:51:55 -0700 Subject: [PATCH 38/67] Use imported AMD SNP report type --- dstack-attest/src/attestation.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dstack-attest/src/attestation.rs b/dstack-attest/src/attestation.rs index e942f9c9c..b179737a3 100644 --- a/dstack-attest/src/attestation.rs +++ b/dstack-attest/src/attestation.rs @@ -31,6 +31,7 @@ use tpm_qvl::verify::VerifiedReport as TpmVerifiedReport; // Re-export TpmQuote from tpm-types pub use tpm_types::TpmQuote; +use crate::amd_sev_snp::VerifiedAmdSnpReport; pub use crate::v1::{Attestation as AttestationV1, PlatformEvidence, StackEvidence}; pub const SNP_REPORT_DATA_RANGE: std::ops::Range = 0x50..0x90; @@ -371,7 +372,7 @@ pub enum DstackVerifiedReport { tpm_report: TpmVerifiedReport, }, DstackNitroEnclave(NitroVerifiedReport), - DstackAmdSevSnp(crate::amd_sev_snp::VerifiedAmdSnpReport), + DstackAmdSevSnp(VerifiedAmdSnpReport), } impl DstackVerifiedReport { @@ -384,7 +385,7 @@ impl DstackVerifiedReport { } } - pub fn amd_snp_report(&self) -> Option<&crate::amd_sev_snp::VerifiedAmdSnpReport> { + pub fn amd_snp_report(&self) -> Option<&VerifiedAmdSnpReport> { match self { DstackVerifiedReport::DstackAmdSevSnp(report) => Some(report), DstackVerifiedReport::DstackTdx(_) From 6361168a3dc175f8df0ca231a25b399c716cc8c8 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 14 Jun 2026 22:12:12 -0700 Subject: [PATCH 39/67] Split SEV-SNP attestation crates --- Cargo.lock | 28 +- Cargo.toml | 4 + dstack-attest/Cargo.toml | 6 +- dstack-attest/src/amd_sev_snp.rs | 713 +----------------------------- dstack-attest/src/sev_snp.rs | 276 +----------- sev-snp-attest/Cargo.toml | 18 + sev-snp-attest/src/lib.rs | 291 +++++++++++++ sev-snp-qvl/Cargo.toml | 18 + sev-snp-qvl/src/lib.rs | 716 +++++++++++++++++++++++++++++++ 9 files changed, 1083 insertions(+), 987 deletions(-) create mode 100644 sev-snp-attest/Cargo.toml create mode 100644 sev-snp-attest/src/lib.rs create mode 100644 sev-snp-qvl/Cargo.toml create mode 100644 sev-snp-qvl/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 61a229561..24cb4029e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2324,7 +2324,6 @@ name = "dstack-attest" version = "0.5.11" dependencies = [ "anyhow", - "base64 0.22.1", "cc-eventlog", "dcap-qvl", "dstack-types", @@ -2335,17 +2334,16 @@ dependencies = [ "hex", "hex_fmt", "insta", - "libc", "nsm-attest", "nsm-qvl", "or-panic", "parity-scale-codec", - "reqwest", "rmp-serde", "serde", "serde-human-bytes", "serde_json", - "sev", + "sev-snp-attest", + "sev-snp-qvl", "sha2 0.10.9", "sha3 0.10.9", "tdx-attest", @@ -7341,6 +7339,28 @@ dependencies = [ "x509-cert", ] +[[package]] +name = "sev-snp-attest" +version = "0.5.11" +dependencies = [ + "anyhow", + "fs-err", + "hex", + "sev", + "tracing", +] + +[[package]] +name = "sev-snp-qvl" +version = "0.5.11" +dependencies = [ + "anyhow", + "base64 0.22.1", + "hex", + "reqwest", + "sev", +] + [[package]] name = "sha1" version = "0.10.6" diff --git a/Cargo.toml b/Cargo.toml index e738356e2..c661bc6c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,10 +20,12 @@ members = [ "ra-tls", "tdx-attest", "tpm-attest", + "sev-snp-attest", "nsm-attest", "tpm2", "tpm-types", "tpm-qvl", + "sev-snp-qvl", "nsm-qvl", "dstack-attest", "dstack-util", @@ -78,11 +80,13 @@ supervisor = { path = "supervisor" } supervisor-client = { path = "supervisor/client" } tdx-attest = { path = "tdx-attest" } tpm-attest = { path = "tpm-attest" } +sev-snp-attest = { path = "sev-snp-attest" } nsm-attest = { path = "nsm-attest" } tpm2 = { path = "tpm2" } tpm-types = { path = "tpm-types" } dstack-attest = { path = "dstack-attest" } tpm-qvl = { path = "tpm-qvl" } +sev-snp-qvl = { path = "sev-snp-qvl" } nsm-qvl = { path = "nsm-qvl" } certbot = { path = "certbot" } rocket-vsock-listener = { path = "rocket-vsock-listener" } diff --git a/dstack-attest/Cargo.toml b/dstack-attest/Cargo.toml index 174e6655b..8b78ad840 100644 --- a/dstack-attest/Cargo.toml +++ b/dstack-attest/Cargo.toml @@ -11,23 +11,21 @@ license.workspace = true [dependencies] anyhow.workspace = true -base64.workspace = true cc-eventlog.workspace = true rmp-serde.workspace = true -reqwest = { workspace = true, features = ["blocking"] } dcap-qvl.workspace = true dstack-types.workspace = true ez-hash.workspace = true fs-err.workspace = true hex.workspace = true hex_fmt.workspace = true -libc.workspace = true or-panic.workspace = true scale = { workspace = true, features = ["derive"] } +sev-snp-attest.workspace = true +sev-snp-qvl.workspace = true serde.workspace = true serde-human-bytes.workspace = true serde_json.workspace = true -sev.workspace = true sha2.workspace = true sha3.workspace = true tdx-attest.workspace = true diff --git a/dstack-attest/src/amd_sev_snp.rs b/dstack-attest/src/amd_sev_snp.rs index ad305886f..86f7a9feb 100644 --- a/dstack-attest/src/amd_sev_snp.rs +++ b/dstack-attest/src/amd_sev_snp.rs @@ -2,715 +2,6 @@ // // SPDX-License-Identifier: Apache-2.0 -//! AMD SEV-SNP attestation verification helpers. -//! -//! This module implements the hardware report verification slice: certificate -//! normalization, AMD ARK/ASK/VCEK chain verification, report signature checks, -//! report_data binding, and invariant SNP policy checks. KMS/app authorization -//! must still bind the verified measurement to app/config identity before -//! production key release. +//! AMD SEV-SNP verification compatibility re-exports. -use anyhow::{bail, Context, Result}; -use base64::engine::general_purpose::STANDARD; -use base64::Engine as _; -use sev::certs::snp::{ca, Certificate, Chain, Verifiable}; -use sev::firmware::{guest::AttestationReport, host::TcbVersion}; - -/// AMD Genoa ARK certificate (DER, base64-encoded). -/// Source: https://kdsintf.amd.com/vcek/v1/Genoa/cert_chain -const GENOA_ARK_DER_B64: &str = "MIIGYzCCBBKgAwIBAgIDAgAAMEYGCSqGSIb3DQEBCjA5oA8wDQYJYIZIAWUDBAICBQChHDAaBgkqhkiG9w0BAQgwDQYJYIZIAWUDBAICBQCiAwIBMKMDAgEBMHsxFDASBgNVBAsMC0VuZ2luZWVyaW5nMQswCQYDVQQGEwJVUzEUMBIGA1UEBwwLU2FudGEgQ2xhcmExCzAJBgNVBAgMAkNBMR8wHQYDVQQKDBZBZHZhbmNlZCBNaWNybyBEZXZpY2VzMRIwEAYDVQQDDAlBUkstR2Vub2EwHhcNMjIwMTI2MTUzNDM3WhcNNDcwMTI2MTUzNDM3WjB7MRQwEgYDVQQLDAtFbmdpbmVlcmluZzELMAkGA1UEBhMCVVMxFDASBgNVBAcMC1NhbnRhIENsYXJhMQswCQYDVQQIDAJDQTEfMB0GA1UECgwWQWR2YW5jZWQgTWljcm8gRGV2aWNlczESMBAGA1UEAwwJQVJLLUdlbm9hMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA3Cd95S/uFOuRIskW9vz9VDBF69NDQF79oRhL/L2PVQGhK3YdfEBgpF/JiwWFBsT/fXDhzA01p3LkcT/7LdjcRfKXjHl+0Qq/M4dZkh6QDoUeKzNBLDcBKDDGWo3v35NyrxbA1DnkYwUKU5AAk4P94tKXLp80oxt84ahyHoLmc/LqsGsp+oq1Bz4PPsYLwTG4iMKVaaT90/oZ4I8oibSru92vJhlqWO27d/Rxc3iUMyhNeGToOvgx/iUo4gGpG61NDpkEUvIzuKcaMx8IdTpWg2DF6SwF0IgVMffnvtJmA68BwJNWo1E4PLJdaPfBifcJpuBFwNVQIPQEVX3aP89HJSp8YbY9lySS6PlVEqTBBtaQmi4ATGmMR+n2K/e+JAhU2Gj7jIpJhOkdH9firQDnmlA2SFfJ/Cc0mGNzW9RmIhyOUnNFoclmkRhl3/AQU5Ys9Qsan1jT/EiyT+pCpmnA+y9edvhDCbOG8F2oxHGRdTBkylungrkXJGYiwGrR8kaiqv7NN8QhOBMqYjcbrkEr0f8QMKklIS5ruOfqlLMCBw8JLB3LkjpWgtD7OpxkzSsohN47Uom86RY6lp72g8eXHP1qYrnvhzaG1S70vw6OkbaaC9EjiH/uHgAJQGxon7u0Q7xgoREWA/e7JcBQwLg80Hq/sbRuqesxz7wBWSY254cCAwEAAaN+MHwwDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBSfXfn+DdjzWtAzGiXvgSlPvjGoWzAPBgNVHRMBAf8EBTADAQH/MDoGA1UdHwQzMDEwL6AtoCuGKWh0dHBzOi8va2RzaW50Zi5hbWQuY29tL3ZjZWsvdjEvR2Vub2EvY3JsMEYGCSqGSIb3DQEBCjA5oA8wDQYJYIZIAWUDBAICBQChHDAaBgkqhkiG9w0BAQgwDQYJYIZIAWUDBAICBQCiAwIBMKMDAgEBA4ICAQAdIlPBC7DQmvH7kjlOznFx3i21SzOPDs5L7SgFjMC9rR07292GQCA7Z7Ulq97JQaWeD2ofGGse5swj4OQfKfVv/zaJUFjvosZOnfZ63epu8MjWgBSXJg5QE/Al0zRsZsp53DBTdA+Uv/s33fexdenT1mpKYzhIg/cKtz4oMxq8JKWJ8Po1CXLzKcfrTphjlbkh8AVKMXeBd2SpM33B1YP4g1BOdk013kqb7bRHZ1iB2JHG5cMKKbwRCSAAGHLTzASgDcXr9Fp7Z3liDhGu/ci1opGmkp12QNiJuBbkTU+xDZHm5X8Jm99BX7NEpzlOwIVR8ClgBDyuBkBC2ljtr3ZSaUIYj2xuyWN95KFY49nWxcz90CFa3Hzmy4zMQmBe9dVyls5eL5p9bkXcgRMDTbgmVZiAf4afe8DLdmQcYcMFQbHhgVzMiyZHGJgcCrQmA7MkTwEIds1wx/HzMcwU4qqNBAoZV7oeIIPxdqFXfPqHqiRlEbRDfX1TG5NFVaeByX0GyH6jzYVuezETzruaky6fp2bl2bczxPE8HdS38ijiJmm9vl50RGUeOAXjSuInGR4bsRufeGPB9peTa9BcBOeTWzstqTUB/F/qaZCIZKr4X6TyfUuSDz/1JDAGl+lxdM0P9+lLaP9NahQjHCVf0zf1c1salVuGFk2w/wMz1R1BHg=="; - -const ASK_CERT_GUID: [u8; 16] = [ - 0x4a, 0xb7, 0xb3, 0x79, 0xbb, 0xac, 0x4f, 0xe4, 0xa0, 0x2f, 0x05, 0xae, 0xf3, 0x27, 0xc7, 0x82, -]; -const VCEK_CERT_GUID: [u8; 16] = [ - 0x63, 0xda, 0x75, 0x8d, 0xe6, 0x64, 0x45, 0x64, 0xad, 0xc5, 0xf4, 0xb9, 0x3b, 0xe8, 0xac, 0xcd, -]; -const VLEK_CERT_GUID: [u8; 16] = [ - 0xa8, 0x07, 0x4b, 0xc2, 0xa2, 0x5a, 0x48, 0x3e, 0xaa, 0xe6, 0x39, 0xc0, 0x45, 0xa0, 0xb8, 0xa1, -]; -const CERT_TABLE_ENTRY_SIZE: usize = 24; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub struct AmdSnpTcbVersion { - pub bootloader: u8, - pub tee: u8, - pub snp: u8, - pub microcode: u8, -} - -impl From for AmdSnpTcbVersion { - fn from(value: TcbVersion) -> Self { - Self { - bootloader: value.bootloader, - tee: value.tee, - snp: value.snp, - microcode: value.microcode, - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub struct AmdSnpTcbInfo { - pub current: AmdSnpTcbVersion, - pub reported: AmdSnpTcbVersion, - pub committed: AmdSnpTcbVersion, - pub launch: AmdSnpTcbVersion, -} - -impl AmdSnpTcbInfo { - pub fn from_report(report: &AttestationReport) -> Self { - Self { - current: report.current_tcb.into(), - reported: report.reported_tcb.into(), - committed: report.committed_tcb.into(), - launch: report.launch_tcb.into(), - } - } - - pub fn tcb_status(&self) -> &'static str { - if self.current == self.reported - && self.committed == self.reported - && self.launch == self.reported - { - "UpToDate" - } else { - "OutOfDate" - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct VerifiedAmdSnpReport { - pub measurement: [u8; 48], - pub report_data: [u8; 64], - pub chip_id: [u8; 64], - pub tcb_info: AmdSnpTcbInfo, - pub advisory_ids: Vec, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum CertEncoding { - Pem, - Der, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct CertBytes { - bytes: Vec, - encoding: CertEncoding, -} - -pub struct AmdSnpAttestationInput<'a> { - pub report: &'a [u8], - pub ask_pem: &'a [u8], - pub vcek_pem: &'a [u8], -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct AmdKdsCollateral { - ark: CertBytes, - ask: CertBytes, - vcek: CertBytes, -} - -pub fn verify_amd_snp_attestation( - input: &AmdSnpAttestationInput<'_>, -) -> Result { - verify_amd_snp_attestation_with_certs( - input.report, - CertBytes { - bytes: input.ask_pem.to_vec(), - encoding: CertEncoding::Pem, - }, - CertBytes { - bytes: input.vcek_pem.to_vec(), - encoding: CertEncoding::Pem, - }, - ) -} - -fn verify_amd_snp_attestation_with_certs( - report_bytes: &[u8], - ask_bytes: CertBytes, - vcek_bytes: CertBytes, -) -> Result { - let ark_der = STANDARD - .decode(GENOA_ARK_DER_B64) - .context("failed to decode amd genoa ark")?; - verify_amd_snp_attestation_with_cert_chain( - report_bytes, - CertBytes { - bytes: ark_der, - encoding: CertEncoding::Der, - }, - ask_bytes, - vcek_bytes, - ) -} - -fn verify_amd_snp_attestation_with_cert_chain( - report_bytes: &[u8], - ark_bytes: CertBytes, - ask_bytes: CertBytes, - vcek_bytes: CertBytes, -) -> Result { - if report_bytes.len() != 1184 { - bail!( - "invalid amd sev-snp report length: expected 1184 bytes, got {}", - report_bytes.len() - ); - } - let report = AttestationReport::from_bytes(report_bytes) - .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; - - let ark = parse_certificate(&ark_bytes, "ark")?; - let ask = parse_certificate(&ask_bytes, "ask")?; - let vcek = parse_certificate(&vcek_bytes, "vcek")?; - - let chain = Chain { - ca: ca::Chain { ark, ask }, - vek: vcek.clone(), - }; - chain - .verify() - .map_err(|err| anyhow::anyhow!("amd cert chain verification failed: {err:?}"))?; - (&vcek, &report).verify().map_err(|err| { - anyhow::anyhow!("amd sev-snp report signature verification failed: {err:?}") - })?; - validate_amd_snp_report_policy(&report)?; - - let mut measurement = [0u8; 48]; - measurement.copy_from_slice( - report - .measurement - .as_ref() - .get(..48) - .context("amd sev-snp measurement too short")?, - ); - let mut report_data = [0u8; 64]; - report_data.copy_from_slice( - report - .report_data - .as_ref() - .get(..64) - .context("amd sev-snp report_data too short")?, - ); - let mut chip_id = [0u8; 64]; - chip_id.copy_from_slice( - report - .chip_id - .as_ref() - .get(..64) - .context("amd sev-snp chip_id too short")?, - ); - - Ok(VerifiedAmdSnpReport { - measurement, - report_data, - chip_id, - tcb_info: AmdSnpTcbInfo::from_report(&report), - // AMD SEV-SNP attestation reports and VCEKs do not carry a direct - // advisory list. Keep this explicit and empty so downstream auth stays - // fail-closed if a future verifier adds advisories from revocation or - // external policy collateral. - advisory_ids: Vec::new(), - }) -} - -pub fn verify_amd_snp_evidence( - report: &[u8], - cert_chain: &[Vec], - expected_report_data: &[u8; 64], -) -> Result { - let (ask, vcek) = normalize_ask_vcek_certs(cert_chain)?; - let verified = verify_amd_snp_attestation_with_certs(report, ask, vcek)?; - if &verified.report_data != expected_report_data { - bail!("amd sev-snp report_data mismatch"); - } - Ok(verified) -} - -pub fn verify_amd_snp_evidence_with_kds_fallback( - report: &[u8], - cert_chain: &[Vec], - expected_report_data: &[u8; 64], -) -> Result { - if !cert_chain.is_empty() { - return verify_amd_snp_evidence(report, cert_chain, expected_report_data); - } - let report_obj = AttestationReport::from_bytes(report) - .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; - let collateral = fetch_amd_kds_collateral_for_report(&report_obj) - .context("failed to fetch amd sev-snp KDS collateral for empty cert_chain")?; - let verified = verify_amd_snp_attestation_with_cert_chain( - report, - collateral.ark, - collateral.ask, - collateral.vcek, - )?; - if &verified.report_data != expected_report_data { - bail!("amd sev-snp report_data mismatch"); - } - Ok(verified) -} - -fn fetch_amd_kds_collateral_for_report(report: &AttestationReport) -> Result { - let mut errors = Vec::new(); - for product in ["Genoa", "Milan", "Bergamo", "Siena", "Turin"] { - match fetch_amd_kds_collateral_for_product(product, report) { - Ok(collateral) => return Ok(collateral), - Err(err) => errors.push(format!("{product}: {err:#}")), - } - } - bail!( - "amd sev-snp KDS collateral unavailable for supported products: {}", - errors.join("; ") - ) -} - -fn fetch_amd_kds_collateral_for_product( - product: &str, - report: &AttestationReport, -) -> Result { - let (ark, ask) = fetch_amd_kds_ca_chain(product)?; - let mut chip_id = [0u8; 64]; - chip_id.copy_from_slice( - report - .chip_id - .as_ref() - .get(..64) - .context("amd sev-snp chip_id too short")?, - ); - let vcek_url = amd_kds_vcek_url(product, &chip_id, report.reported_tcb.into()); - let vcek_request_url = amd_kds_request_url(&vcek_url); - let vcek = reqwest::blocking::Client::new() - .get(&vcek_request_url) - .send() - .with_context(|| format!("failed to request amd sev-snp vcek from {vcek_request_url}"))? - .error_for_status() - .with_context(|| { - format!("amd sev-snp vcek request failed for {vcek_url} via {vcek_request_url}") - })? - .bytes() - .context("failed to read amd sev-snp vcek response")? - .to_vec(); - Ok(AmdKdsCollateral { - ark, - ask, - vcek: CertBytes { - bytes: vcek, - encoding: CertEncoding::Der, - }, - }) -} - -fn fetch_amd_kds_ca_chain(product: &str) -> Result<(CertBytes, CertBytes)> { - let url = format!("https://kdsintf.amd.com/vcek/v1/{product}/cert_chain"); - let request_url = amd_kds_request_url(&url); - let chain = reqwest::blocking::Client::new() - .get(&request_url) - .send() - .with_context(|| format!("failed to request amd sev-snp cert_chain from {request_url}"))? - .error_for_status() - .with_context(|| format!("amd sev-snp cert_chain request failed for {request_url}"))? - .bytes() - .context("failed to read amd sev-snp cert_chain response")?; - extract_ark_ask_from_amd_kds_cert_chain(&chain) -} - -fn amd_kds_request_url(amd_url: &str) -> String { - match std::env::var("DSTACK_AMD_KDS_PROXY_URL") { - Ok(proxy) if !proxy.trim().is_empty() => format!("{}{}", proxy.trim(), amd_url), - _ => amd_url.to_string(), - } -} - -fn amd_kds_vcek_url(product: &str, chip_id: &[u8; 64], tcb: AmdSnpTcbVersion) -> String { - format!( - "https://kdsintf.amd.com/vcek/v1/{}/{}?blSPL={}&teeSPL={}&snpSPL={}&ucodeSPL={}", - product, - hex::encode(chip_id), - tcb.bootloader, - tcb.tee, - tcb.snp, - tcb.microcode - ) -} - -fn extract_ark_ask_from_amd_kds_cert_chain(chain: &[u8]) -> Result<(CertBytes, CertBytes)> { - let certs = extract_pem_certs(chain)?; - if certs.len() < 2 { - bail!("amd sev-snp cert_chain must contain ASK and ARK certificates"); - } - Ok(( - CertBytes { - bytes: certs[1].clone(), - encoding: CertEncoding::Pem, - }, - CertBytes { - bytes: certs[0].clone(), - encoding: CertEncoding::Pem, - }, - )) -} - -fn extract_pem_certs(chain: &[u8]) -> Result>> { - let chain = std::str::from_utf8(chain).context("amd sev-snp cert_chain is not utf-8 pem")?; - let begin = "-----BEGIN CERTIFICATE-----"; - let end = "-----END CERTIFICATE-----"; - let mut rest = chain; - let mut certs = Vec::new(); - while let Some(start) = rest.find(begin) { - let after_start = &rest[start..]; - let cert_end = after_start - .find(end) - .map(|idx| idx + end.len()) - .context("amd sev-snp cert_chain has unterminated certificate")?; - let mut cert = after_start.as_bytes()[..cert_end].to_vec(); - cert.push(b'\n'); - certs.push(cert); - rest = &after_start[cert_end..]; - } - if certs.is_empty() { - bail!("amd sev-snp cert_chain missing certificates"); - } - Ok(certs) -} - -fn parse_certificate(cert: &CertBytes, name: &str) -> Result { - match cert.encoding { - CertEncoding::Pem => Certificate::from_pem(&cert.bytes) - .map_err(|err| anyhow::anyhow!("failed to parse amd {name} certificate: {err:?}")), - CertEncoding::Der => Certificate::from_der(&cert.bytes) - .map_err(|err| anyhow::anyhow!("failed to parse amd {name} certificate: {err:?}")), - } -} - -fn validate_amd_snp_report_policy(report: &AttestationReport) -> Result<()> { - if !matches!(report.version, 2 | 3) { - bail!("unsupported amd sev-snp report version: {}", report.version); - } - if report.vmpl != 0 { - bail!("amd sev-snp report must be generated at vmpl0"); - } - if report.policy.debug_allowed() { - bail!("amd sev-snp guest policy allows debug"); - } - if report.policy.migrate_ma_allowed() { - bail!("amd sev-snp guest policy allows migration agent"); - } - if report.key_info.mask_chip_key() { - bail!("amd sev-snp report masks the chip signing key"); - } - if report.key_info.signing_key() != 0 { - bail!( - "unsupported amd sev-snp signing key: expected vcek, got {}", - report.key_info.signing_key() - ); - } - if !report.policy.smt_allowed() && report.plat_info.smt_enabled() { - bail!("amd sev-snp platform has smt enabled but guest policy does not allow smt"); - } - if report.policy.rapl_dis() && !report.plat_info.rapl_disabled() { - bail!("amd sev-snp guest policy requires rapl disabled, but platform reports rapl enabled"); - } - if report.policy.ciphertext_hiding() && !report.plat_info.ciphertext_hiding_enabled() { - bail!( - "amd sev-snp guest policy requires ciphertext hiding, but platform does not report it" - ); - } - Ok(()) -} - -fn normalize_ask_vcek_certs(cert_chain: &[Vec]) -> Result<(CertBytes, CertBytes)> { - match cert_chain { - [ask, vcek] => Ok((cert_bytes_from_blob(ask), cert_bytes_from_blob(vcek))), - [auxblob] => normalize_kernel_cert_table(auxblob), - _ => bail!( - "amd sev-snp cert_chain must contain either ASK and VCEK certificates or one kernel certificate table auxblob" - ), - } -} - -fn cert_bytes_from_blob(blob: &[u8]) -> CertBytes { - let encoding = if blob.starts_with(b"-----BEGIN CERTIFICATE-----") { - CertEncoding::Pem - } else { - CertEncoding::Der - }; - CertBytes { - bytes: blob.to_vec(), - encoding, - } -} - -fn normalize_kernel_cert_table(auxblob: &[u8]) -> Result<(CertBytes, CertBytes)> { - let mut ask = None; - let mut vcek = None; - for (guid, data) in parse_kernel_cert_table(auxblob)? { - match guid { - ASK_CERT_GUID => ask = Some(data), - VCEK_CERT_GUID => vcek = Some(data), - VLEK_CERT_GUID => bail!("amd sev-snp vlek certificates are not supported yet"), - _ => {} - } - } - let ask = ask.context("amd sev-snp certificate table missing ASK certificate")?; - let vcek = vcek.context("amd sev-snp certificate table missing VCEK certificate")?; - Ok(( - CertBytes { - bytes: ask, - encoding: CertEncoding::Der, - }, - CertBytes { - bytes: vcek, - encoding: CertEncoding::Der, - }, - )) -} - -fn parse_kernel_cert_table(auxblob: &[u8]) -> Result)>> { - if auxblob.len() < CERT_TABLE_ENTRY_SIZE { - bail!("amd sev-snp certificate table is too short"); - } - let mut entries = Vec::new(); - let mut pos = 0usize; - loop { - let entry = auxblob - .get(pos..pos + CERT_TABLE_ENTRY_SIZE) - .context("amd sev-snp certificate table is missing terminator")?; - let guid: [u8; 16] = entry[..16] - .try_into() - .context("amd sev-snp certificate table entry guid has invalid length")?; - let offset = u32::from_le_bytes( - entry[16..20] - .try_into() - .context("amd sev-snp certificate table entry offset has invalid length")?, - ) as usize; - let length = u32::from_le_bytes( - entry[20..24] - .try_into() - .context("amd sev-snp certificate table entry length has invalid length")?, - ) as usize; - if guid == [0u8; 16] && offset == 0 && length == 0 { - break; - } - let end = offset - .checked_add(length) - .context("amd sev-snp certificate table entry length overflows")?; - if offset < CERT_TABLE_ENTRY_SIZE || end > auxblob.len() || length == 0 { - bail!("amd sev-snp certificate table entry has invalid bounds"); - } - entries.push((guid, auxblob[offset..end].to_vec())); - pos = pos - .checked_add(CERT_TABLE_ENTRY_SIZE) - .context("amd sev-snp certificate table entry count overflows")?; - if pos >= auxblob.len() { - bail!("amd sev-snp certificate table is missing terminator"); - } - } - Ok(entries) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn tcb(bootloader: u8, tee: u8, snp: u8, microcode: u8) -> AmdSnpTcbVersion { - AmdSnpTcbVersion { - bootloader, - tee, - snp, - microcode, - } - } - - #[test] - fn tcb_status_is_up_to_date_only_when_all_reported_versions_match() { - let up_to_date = AmdSnpTcbInfo { - current: tcb(1, 2, 3, 4), - reported: tcb(1, 2, 3, 4), - committed: tcb(1, 2, 3, 4), - launch: tcb(1, 2, 3, 4), - }; - assert_eq!(up_to_date.tcb_status(), "UpToDate"); - - let stale_launch = AmdSnpTcbInfo { - launch: tcb(1, 2, 3, 3), - ..up_to_date - }; - assert_eq!(stale_launch.tcb_status(), "OutOfDate"); - - let stale_vcek_reported = AmdSnpTcbInfo { - reported: tcb(1, 2, 3, 3), - ..up_to_date - }; - assert_eq!(stale_vcek_reported.tcb_status(), "OutOfDate"); - } - - #[test] - fn missing_cert_chain_fails_closed() { - let report = vec![0u8; 1184]; - let expected_report_data = [0u8; 64]; - let err = verify_amd_snp_evidence(&report, &[], &expected_report_data).unwrap_err(); - assert!( - err.to_string() - .contains("cert_chain must contain either ASK and VCEK certificates or one kernel certificate table auxblob"), - "unexpected error: {err:#}" - ); - } - - #[test] - fn amd_kds_vcek_url_binds_chip_id_and_reported_tcb() { - let chip_id = [0xab; 64]; - let tcb = AmdSnpTcbVersion { - bootloader: 1, - tee: 2, - snp: 3, - microcode: 4, - }; - - let url = amd_kds_vcek_url("Genoa", &chip_id, tcb); - - assert_eq!( - url, - format!( - "https://kdsintf.amd.com/vcek/v1/Genoa/{}?blSPL=1&teeSPL=2&snpSPL=3&ucodeSPL=4", - hex::encode(chip_id) - ) - ); - } - - #[test] - fn amd_kds_proxy_url_wraps_amd_urls_when_configured() { - const ENV_KEY: &str = "DSTACK_AMD_KDS_PROXY_URL"; - let old = std::env::var(ENV_KEY).ok(); - std::env::set_var(ENV_KEY, "https://cors.litgateway.com/"); - - let wrapped = amd_kds_request_url("https://kdsintf.amd.com/vcek/v1/Genoa/cert_chain"); - - assert_eq!( - wrapped, - "https://cors.litgateway.com/https://kdsintf.amd.com/vcek/v1/Genoa/cert_chain" - ); - if let Some(old) = old { - std::env::set_var(ENV_KEY, old); - } else { - std::env::remove_var(ENV_KEY); - } - } - - #[test] - fn amd_kds_cert_chain_extracts_ask_pem_and_ark_pem() { - let chain = b"-----BEGIN CERTIFICATE-----\nASK\n-----END CERTIFICATE-----\n-----BEGIN CERTIFICATE-----\nARK\n-----END CERTIFICATE-----\n"; - - let (ark_cert, ask_cert) = extract_ark_ask_from_amd_kds_cert_chain(chain).unwrap(); - - assert_eq!( - ask_cert.bytes, - b"-----BEGIN CERTIFICATE-----\nASK\n-----END CERTIFICATE-----\n".to_vec() - ); - assert_eq!(ask_cert.encoding, CertEncoding::Pem); - assert_eq!( - ark_cert.bytes, - b"-----BEGIN CERTIFICATE-----\nARK\n-----END CERTIFICATE-----\n".to_vec() - ); - assert_eq!(ark_cert.encoding, CertEncoding::Pem); - } - - #[test] - fn malformed_report_fails_closed_before_success() { - let cert_chain = vec![b"not ask".to_vec(), b"not vcek".to_vec()]; - let expected_report_data = [0u8; 64]; - let err = - verify_amd_snp_evidence(b"too short", &cert_chain, &expected_report_data).unwrap_err(); - assert!( - err.to_string() - .contains("invalid amd sev-snp report length"), - "unexpected error: {err:#}" - ); - } - - #[test] - fn normalizes_kernel_cert_table_auxblob_to_ask_and_vcek_der() { - use sev::firmware::host::{CertTableEntry, CertType}; - - let auxblob = CertTableEntry::cert_table_to_vec_bytes(&[ - CertTableEntry::new(CertType::VCEK, b"vcek-der".to_vec()), - CertTableEntry::new(CertType::ASK, b"ask-der".to_vec()), - ]) - .unwrap(); - - let (ask, vcek) = normalize_ask_vcek_certs(&[auxblob]).unwrap(); - - assert_eq!(ask.bytes, b"ask-der"); - assert_eq!(ask.encoding, CertEncoding::Der); - assert_eq!(vcek.bytes, b"vcek-der"); - assert_eq!(vcek.encoding, CertEncoding::Der); - } - - #[test] - fn malformed_single_auxblob_fails_closed_without_panic() { - let err = normalize_ask_vcek_certs(&[vec![0xff; 23]]).unwrap_err(); - - assert!( - err.to_string().contains("certificate table"), - "unexpected error: {err:#}" - ); - } - - #[test] - fn normalizes_existing_two_item_pem_chain_without_reordering() { - let ask = b"-----BEGIN CERTIFICATE-----\nask\n-----END CERTIFICATE-----\n".to_vec(); - let vcek = b"-----BEGIN CERTIFICATE-----\nvcek\n-----END CERTIFICATE-----\n".to_vec(); - - let (normalized_ask, normalized_vcek) = - normalize_ask_vcek_certs(&[ask.clone(), vcek.clone()]).unwrap(); - - assert_eq!(normalized_ask.bytes, ask); - assert_eq!(normalized_ask.encoding, CertEncoding::Pem); - assert_eq!(normalized_vcek.bytes, vcek); - assert_eq!(normalized_vcek.encoding, CertEncoding::Pem); - } - - #[test] - fn report_policy_rejects_debug_allowed() { - let mut report = base_report(); - report.policy.set_debug_allowed(true); - - let err = validate_amd_snp_report_policy(&report).unwrap_err(); - - assert!( - err.to_string().contains("debug"), - "unexpected error: {err:#}" - ); - } - - #[test] - fn report_policy_rejects_non_vmpl0() { - let mut report = base_report(); - report.vmpl = 1; - - let err = validate_amd_snp_report_policy(&report).unwrap_err(); - - assert!( - err.to_string().contains("vmpl0"), - "unexpected error: {err:#}" - ); - } - - #[test] - fn report_policy_accepts_strict_vcek_vmpl0_report() { - let report = base_report(); - - validate_amd_snp_report_policy(&report).unwrap(); - } - - fn base_report() -> AttestationReport { - AttestationReport { - version: 2, - ..Default::default() - } - } -} +pub use sev_snp_qvl::*; diff --git a/dstack-attest/src/sev_snp.rs b/dstack-attest/src/sev_snp.rs index cc1f6b09f..200e4e652 100644 --- a/dstack-attest/src/sev_snp.rs +++ b/dstack-attest/src/sev_snp.rs @@ -2,282 +2,22 @@ // // SPDX-License-Identifier: Apache-2.0 -//! Minimal AMD SEV-SNP guest report support. +//! AMD SEV-SNP guest report adapter for dstack attestation. use std::path::Path; -use anyhow::{bail, Context, Result}; -use sev::firmware::{guest::Firmware, host::CertTableEntry}; +use anyhow::Result; -use crate::attestation::{SnpQuote, SNP_REPORT_DATA_RANGE}; - -const TSM_REPORT_ROOT: &str = "/sys/kernel/config/tsm/report"; -const SEV_GUEST_DEVICE: &str = "/dev/sev-guest"; -const SNP_REPORT_SIZE: usize = 1184; +use crate::attestation::SnpQuote; pub fn get_report(report_data: [u8; 64]) -> Result { - if has_sev_snp_tsm_provider(Path::new(TSM_REPORT_ROOT)) { - match get_report_configfs(report_data) { - Ok(quote) => { - if configfs_report_needs_ioctl_cert_chain_fallback( - "e, - Path::new(SEV_GUEST_DEVICE).exists(), - ) { - tracing::debug!( - "sev-snp configfs tsm report did not include a certificate chain; falling back to ioctl extended report" - ); - match get_report_ioctl(report_data) { - Ok(ioctl_quote) if !ioctl_quote.cert_chain.is_empty() => { - return Ok(ioctl_quote) - } - Ok(_) => return Ok(quote), - Err(err) => tracing::debug!( - "failed to get sev-snp report from ioctl fallback: {err:#}" - ), - } - } - return Ok(quote); - } - Err(err) => tracing::debug!("failed to get sev-snp report from configfs tsm: {err:#}"), - } - } - if Path::new(SEV_GUEST_DEVICE).exists() { - return get_report_ioctl(report_data); - } - bail!("sev-snp report is unavailable: neither {TSM_REPORT_ROOT} nor {SEV_GUEST_DEVICE} exists") -} - -fn configfs_report_needs_ioctl_cert_chain_fallback( - quote: &SnpQuote, - sev_guest_device_available: bool, -) -> bool { - sev_guest_device_available && quote.cert_chain.is_empty() -} - -pub(crate) fn has_sev_snp_tsm_provider(root: &Path) -> bool { - if !root.exists() { - return false; - } - - if provider_file_is_sev_guest(&root.join("provider")) { - return true; - } - - let probe = root.join(format!("dstack-probe-{}", std::process::id())); - if fs_err::create_dir(&probe).is_ok() { - let is_sev_snp = provider_file_is_sev_guest(&probe.join("provider")); - let _ = fs_err::remove_dir(&probe); - if is_sev_snp { - return true; - } - } - - let Ok(entries) = fs_err::read_dir(root) else { - return false; - }; - entries.flatten().any(|entry| { - let Ok(file_type) = entry.file_type() else { - return false; - }; - file_type.is_dir() && provider_file_is_sev_guest(&entry.path().join("provider")) - }) -} - -fn provider_file_is_sev_guest(path: &Path) -> bool { - fs_err::read_to_string(path) - .map(|provider| matches!(provider.trim(), "sev_guest" | "sev-guest")) - .unwrap_or(false) -} - -fn get_report_configfs(report_data: [u8; 64]) -> Result { - let root = Path::new(TSM_REPORT_ROOT); - let dir = root.join(format!("dstack-{}", std::process::id())); - if !dir.exists() { - fs_err::create_dir(&dir).with_context(|| format!("failed to create {}", dir.display()))?; - } - - let hex_report_data = hex::encode(report_data); - write_first_existing( - &[ - dir.join("inblob"), - dir.join("reportdata"), - dir.join("report_data"), - ], - &report_data, - hex_report_data.as_bytes(), - )?; - - let report = read_first_existing(&[dir.join("outblob"), dir.join("report")])?; - if report.is_empty() { - bail!("sev-snp configfs tsm returned an empty report"); - } - ensure_report_data_matches(&report, &report_data)?; + let quote = sev_snp_attest::get_report(report_data)?; Ok(SnpQuote { - report, - cert_chain: read_cert_chain_configfs(&dir), + report: quote.report, + cert_chain: quote.cert_chain, }) } -fn write_first_existing(paths: &[std::path::PathBuf], binary: &[u8], hex: &[u8]) -> Result<()> { - let mut last_err = None; - for path in paths { - if !path.exists() { - continue; - } - match fs_err::write(path, binary).or_else(|_| fs_err::write(path, hex)) { - Ok(()) => return Ok(()), - Err(err) => last_err = Some(err), - } - } - match last_err { - Some(err) => Err(err).context("failed to write sev-snp tsm report data"), - None => bail!("failed to find sev-snp tsm report input file"), - } -} - -fn read_first_existing(paths: &[std::path::PathBuf]) -> Result> { - for path in paths { - if path.exists() { - return fs_err::read(path) - .with_context(|| format!("failed to read {}", path.display())); - } - } - bail!("failed to find sev-snp tsm report output file") -} - -fn read_cert_chain_configfs(dir: &Path) -> Vec> { - for name in ["certs", "cert_chain", "auxblob"] { - let Ok(bytes) = fs_err::read(dir.join(name)) else { - continue; - }; - if !bytes.is_empty() { - return vec![bytes]; - } - } - Vec::new() -} - -fn get_report_ioctl(report_data: [u8; 64]) -> Result { - let mut firmware = - Firmware::open().with_context(|| format!("failed to open {SEV_GUEST_DEVICE}"))?; - let (report, cert_entries) = firmware - .get_ext_report(Some(1), Some(report_data), Some(0)) - .map_err(|err| anyhow::anyhow!("sev-snp get extended report ioctl failed: {err}"))?; - ensure_report_data_matches(&report, &report_data)?; - let cert_chain = match cert_entries { - Some(entries) if !entries.is_empty() => { - vec![CertTableEntry::cert_table_to_vec_bytes(&entries) - .context("failed to encode sev-snp certificate table")?] - } - _ => Vec::new(), - }; - Ok(SnpQuote { report, cert_chain }) -} - -fn ensure_report_data_matches(report: &[u8], report_data: &[u8; 64]) -> Result<()> { - if report.len() != SNP_REPORT_SIZE { - bail!( - "sev-snp report has invalid length: expected {} bytes, got {}", - SNP_REPORT_SIZE, - report.len() - ); - } - if &report[SNP_REPORT_DATA_RANGE] != report_data { - bail!("sev-snp report_data mismatch"); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::time::{SystemTime, UNIX_EPOCH}; - - #[test] - fn rejects_report_with_wrong_report_data() { - let expected = [0x42; 64]; - let mut report = vec![0u8; SNP_REPORT_SIZE]; - report[SNP_REPORT_DATA_RANGE].copy_from_slice(&[0x24; 64]); - assert!(ensure_report_data_matches(&report, &expected).is_err()); - } - - #[test] - fn accepts_report_with_matching_report_data() { - let expected = [0x42; 64]; - let mut report = vec![0u8; SNP_REPORT_SIZE]; - report[SNP_REPORT_DATA_RANGE].copy_from_slice(&expected); - ensure_report_data_matches(&report, &expected).unwrap(); - } - - #[test] - fn tsm_provider_detection_accepts_only_sev_guest_provider() { - let root = test_dir("sev-guest"); - fs_err::create_dir_all(root.join("entry")).unwrap(); - fs_err::write(root.join("entry/provider"), "sev_guest\n").unwrap(); - - assert!(has_sev_snp_tsm_provider(&root)); - - let _ = fs_err::remove_dir_all(root); - } - - #[test] - fn tsm_provider_detection_accepts_legacy_hyphenated_sev_guest_provider() { - let root = test_dir("sev-guest-hyphen"); - fs_err::create_dir_all(root.join("entry")).unwrap(); - fs_err::write(root.join("entry/provider"), "sev-guest\n").unwrap(); - - assert!(has_sev_snp_tsm_provider(&root)); - - let _ = fs_err::remove_dir_all(root); - } - - #[test] - fn tsm_provider_detection_rejects_tdx_guest_provider() { - let root = test_dir("tdx-guest"); - fs_err::create_dir_all(root.join("entry")).unwrap(); - fs_err::write(root.join("entry/provider"), "tdx-guest\n").unwrap(); - - assert!(!has_sev_snp_tsm_provider(&root)); - - let _ = fs_err::remove_dir_all(root); - } - - #[test] - fn configfs_cert_chain_uses_first_supported_nonempty_blob() { - let root = test_dir("cert-chain"); - fs_err::create_dir_all(&root).unwrap(); - fs_err::write(root.join("certs"), []).unwrap(); - fs_err::write(root.join("cert_chain"), b"chain").unwrap(); - fs_err::write(root.join("auxblob"), b"auxblob").unwrap(); - - assert_eq!(read_cert_chain_configfs(&root), vec![b"chain".to_vec()]); - - let _ = fs_err::remove_dir_all(root); - } - - #[test] - fn configfs_report_without_cert_chain_requires_ioctl_fallback_when_available() { - let quote = SnpQuote { - report: vec![0u8; SNP_REPORT_SIZE], - cert_chain: vec![], - }; - - assert!(configfs_report_needs_ioctl_cert_chain_fallback( - "e, true - )); - assert!(!configfs_report_needs_ioctl_cert_chain_fallback( - "e, false - )); - } - - fn test_dir(name: &str) -> std::path::PathBuf { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos(); - std::env::temp_dir().join(format!( - "dstack-sev-snp-test-{name}-{}-{nanos}", - std::process::id() - )) - } +pub fn has_sev_snp_tsm_provider(root: &Path) -> bool { + sev_snp_attest::has_sev_snp_tsm_provider(root) } diff --git a/sev-snp-attest/Cargo.toml b/sev-snp-attest/Cargo.toml new file mode 100644 index 000000000..9b2dc9f1b --- /dev/null +++ b/sev-snp-attest/Cargo.toml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: © 2025 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "sev-snp-attest" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +description = "AMD SEV-SNP guest attestation report library" + +[dependencies] +anyhow.workspace = true +fs-err.workspace = true +hex.workspace = true +sev.workspace = true +tracing.workspace = true diff --git a/sev-snp-attest/src/lib.rs b/sev-snp-attest/src/lib.rs new file mode 100644 index 000000000..04a2f9cca --- /dev/null +++ b/sev-snp-attest/src/lib.rs @@ -0,0 +1,291 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Minimal AMD SEV-SNP guest report support. + +use std::path::Path; + +use anyhow::{bail, Context, Result}; +use sev::firmware::{guest::Firmware, host::CertTableEntry}; + +const TSM_REPORT_ROOT: &str = "/sys/kernel/config/tsm/report"; +const SEV_GUEST_DEVICE: &str = "/dev/sev-guest"; +const SNP_REPORT_SIZE: usize = 1184; +pub const SNP_REPORT_DATA_RANGE: std::ops::Range = 0x50..0x90; + +/// Represents an AMD SEV-SNP attestation report. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SnpQuote { + /// Raw SNP report bytes. + pub report: Vec, + /// Optional certificate chain blobs, when exposed by the kernel/firmware path. + pub cert_chain: Vec>, +} + +pub fn get_report(report_data: [u8; 64]) -> Result { + if has_sev_snp_tsm_provider(Path::new(TSM_REPORT_ROOT)) { + match get_report_configfs(report_data) { + Ok(quote) => { + if configfs_report_needs_ioctl_cert_chain_fallback( + "e, + Path::new(SEV_GUEST_DEVICE).exists(), + ) { + tracing::debug!( + "sev-snp configfs tsm report did not include a certificate chain; falling back to ioctl extended report" + ); + match get_report_ioctl(report_data) { + Ok(ioctl_quote) if !ioctl_quote.cert_chain.is_empty() => { + return Ok(ioctl_quote) + } + Ok(_) => return Ok(quote), + Err(err) => tracing::debug!( + "failed to get sev-snp report from ioctl fallback: {err:#}" + ), + } + } + return Ok(quote); + } + Err(err) => tracing::debug!("failed to get sev-snp report from configfs tsm: {err:#}"), + } + } + if Path::new(SEV_GUEST_DEVICE).exists() { + return get_report_ioctl(report_data); + } + bail!("sev-snp report is unavailable: neither {TSM_REPORT_ROOT} nor {SEV_GUEST_DEVICE} exists") +} + +fn configfs_report_needs_ioctl_cert_chain_fallback( + quote: &SnpQuote, + sev_guest_device_available: bool, +) -> bool { + sev_guest_device_available && quote.cert_chain.is_empty() +} + +pub fn has_sev_snp_tsm_provider(root: &Path) -> bool { + if !root.exists() { + return false; + } + + if provider_file_is_sev_guest(&root.join("provider")) { + return true; + } + + let probe = root.join(format!("dstack-probe-{}", std::process::id())); + if fs_err::create_dir(&probe).is_ok() { + let is_sev_snp = provider_file_is_sev_guest(&probe.join("provider")); + let _ = fs_err::remove_dir(&probe); + if is_sev_snp { + return true; + } + } + + let Ok(entries) = fs_err::read_dir(root) else { + return false; + }; + entries.flatten().any(|entry| { + let Ok(file_type) = entry.file_type() else { + return false; + }; + file_type.is_dir() && provider_file_is_sev_guest(&entry.path().join("provider")) + }) +} + +fn provider_file_is_sev_guest(path: &Path) -> bool { + fs_err::read_to_string(path) + .map(|provider| matches!(provider.trim(), "sev_guest" | "sev-guest")) + .unwrap_or(false) +} + +fn get_report_configfs(report_data: [u8; 64]) -> Result { + let root = Path::new(TSM_REPORT_ROOT); + let dir = root.join(format!("dstack-{}", std::process::id())); + if !dir.exists() { + fs_err::create_dir(&dir).with_context(|| format!("failed to create {}", dir.display()))?; + } + + let hex_report_data = hex::encode(report_data); + write_first_existing( + &[ + dir.join("inblob"), + dir.join("reportdata"), + dir.join("report_data"), + ], + &report_data, + hex_report_data.as_bytes(), + )?; + + let report = read_first_existing(&[dir.join("outblob"), dir.join("report")])?; + if report.is_empty() { + bail!("sev-snp configfs tsm returned an empty report"); + } + ensure_report_data_matches(&report, &report_data)?; + Ok(SnpQuote { + report, + cert_chain: read_cert_chain_configfs(&dir), + }) +} + +fn write_first_existing(paths: &[std::path::PathBuf], binary: &[u8], hex: &[u8]) -> Result<()> { + let mut last_err = None; + for path in paths { + if !path.exists() { + continue; + } + match fs_err::write(path, binary).or_else(|_| fs_err::write(path, hex)) { + Ok(()) => return Ok(()), + Err(err) => last_err = Some(err), + } + } + match last_err { + Some(err) => Err(err).context("failed to write sev-snp tsm report data"), + None => bail!("failed to find sev-snp tsm report input file"), + } +} + +fn read_first_existing(paths: &[std::path::PathBuf]) -> Result> { + for path in paths { + if path.exists() { + return fs_err::read(path) + .with_context(|| format!("failed to read {}", path.display())); + } + } + bail!("failed to find sev-snp tsm report output file") +} + +fn read_cert_chain_configfs(dir: &Path) -> Vec> { + for name in ["certs", "cert_chain", "auxblob"] { + let Ok(bytes) = fs_err::read(dir.join(name)) else { + continue; + }; + if !bytes.is_empty() { + return vec![bytes]; + } + } + Vec::new() +} + +fn get_report_ioctl(report_data: [u8; 64]) -> Result { + let mut firmware = + Firmware::open().with_context(|| format!("failed to open {SEV_GUEST_DEVICE}"))?; + let (report, cert_entries) = firmware + .get_ext_report(Some(1), Some(report_data), Some(0)) + .map_err(|err| anyhow::anyhow!("sev-snp get extended report ioctl failed: {err}"))?; + ensure_report_data_matches(&report, &report_data)?; + let cert_chain = match cert_entries { + Some(entries) if !entries.is_empty() => { + vec![CertTableEntry::cert_table_to_vec_bytes(&entries) + .context("failed to encode sev-snp certificate table")?] + } + _ => Vec::new(), + }; + Ok(SnpQuote { report, cert_chain }) +} + +fn ensure_report_data_matches(report: &[u8], report_data: &[u8; 64]) -> Result<()> { + if report.len() != SNP_REPORT_SIZE { + bail!( + "sev-snp report has invalid length: expected {} bytes, got {}", + SNP_REPORT_SIZE, + report.len() + ); + } + if &report[SNP_REPORT_DATA_RANGE] != report_data { + bail!("sev-snp report_data mismatch"); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{SystemTime, UNIX_EPOCH}; + + #[test] + fn rejects_report_with_wrong_report_data() { + let expected = [0x42; 64]; + let mut report = vec![0u8; SNP_REPORT_SIZE]; + report[SNP_REPORT_DATA_RANGE].copy_from_slice(&[0x24; 64]); + assert!(ensure_report_data_matches(&report, &expected).is_err()); + } + + #[test] + fn accepts_report_with_matching_report_data() { + let expected = [0x42; 64]; + let mut report = vec![0u8; SNP_REPORT_SIZE]; + report[SNP_REPORT_DATA_RANGE].copy_from_slice(&expected); + ensure_report_data_matches(&report, &expected).unwrap(); + } + + #[test] + fn tsm_provider_detection_accepts_only_sev_guest_provider() { + let root = test_dir("sev-guest"); + fs_err::create_dir_all(root.join("entry")).unwrap(); + fs_err::write(root.join("entry/provider"), "sev_guest\n").unwrap(); + + assert!(has_sev_snp_tsm_provider(&root)); + + let _ = fs_err::remove_dir_all(root); + } + + #[test] + fn tsm_provider_detection_accepts_legacy_hyphenated_sev_guest_provider() { + let root = test_dir("sev-guest-hyphen"); + fs_err::create_dir_all(root.join("entry")).unwrap(); + fs_err::write(root.join("entry/provider"), "sev-guest\n").unwrap(); + + assert!(has_sev_snp_tsm_provider(&root)); + + let _ = fs_err::remove_dir_all(root); + } + + #[test] + fn tsm_provider_detection_rejects_tdx_guest_provider() { + let root = test_dir("tdx-guest"); + fs_err::create_dir_all(root.join("entry")).unwrap(); + fs_err::write(root.join("entry/provider"), "tdx-guest\n").unwrap(); + + assert!(!has_sev_snp_tsm_provider(&root)); + + let _ = fs_err::remove_dir_all(root); + } + + #[test] + fn configfs_cert_chain_uses_first_supported_nonempty_blob() { + let root = test_dir("cert-chain"); + fs_err::create_dir_all(&root).unwrap(); + fs_err::write(root.join("certs"), []).unwrap(); + fs_err::write(root.join("cert_chain"), b"chain").unwrap(); + fs_err::write(root.join("auxblob"), b"auxblob").unwrap(); + + assert_eq!(read_cert_chain_configfs(&root), vec![b"chain".to_vec()]); + + let _ = fs_err::remove_dir_all(root); + } + + #[test] + fn configfs_report_without_cert_chain_requires_ioctl_fallback_when_available() { + let quote = SnpQuote { + report: vec![0u8; SNP_REPORT_SIZE], + cert_chain: vec![], + }; + + assert!(configfs_report_needs_ioctl_cert_chain_fallback( + "e, true + )); + assert!(!configfs_report_needs_ioctl_cert_chain_fallback( + "e, false + )); + } + + fn test_dir(name: &str) -> std::path::PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + std::env::temp_dir().join(format!( + "dstack-sev-snp-test-{name}-{}-{nanos}", + std::process::id() + )) + } +} diff --git a/sev-snp-qvl/Cargo.toml b/sev-snp-qvl/Cargo.toml new file mode 100644 index 000000000..0483e2324 --- /dev/null +++ b/sev-snp-qvl/Cargo.toml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: © 2025 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "sev-snp-qvl" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +description = "AMD SEV-SNP Quote Verification Library" + +[dependencies] +anyhow.workspace = true +base64.workspace = true +hex.workspace = true +reqwest = { workspace = true, features = ["blocking"] } +sev.workspace = true diff --git a/sev-snp-qvl/src/lib.rs b/sev-snp-qvl/src/lib.rs new file mode 100644 index 000000000..ad305886f --- /dev/null +++ b/sev-snp-qvl/src/lib.rs @@ -0,0 +1,716 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! AMD SEV-SNP attestation verification helpers. +//! +//! This module implements the hardware report verification slice: certificate +//! normalization, AMD ARK/ASK/VCEK chain verification, report signature checks, +//! report_data binding, and invariant SNP policy checks. KMS/app authorization +//! must still bind the verified measurement to app/config identity before +//! production key release. + +use anyhow::{bail, Context, Result}; +use base64::engine::general_purpose::STANDARD; +use base64::Engine as _; +use sev::certs::snp::{ca, Certificate, Chain, Verifiable}; +use sev::firmware::{guest::AttestationReport, host::TcbVersion}; + +/// AMD Genoa ARK certificate (DER, base64-encoded). +/// Source: https://kdsintf.amd.com/vcek/v1/Genoa/cert_chain +const GENOA_ARK_DER_B64: &str = "MIIGYzCCBBKgAwIBAgIDAgAAMEYGCSqGSIb3DQEBCjA5oA8wDQYJYIZIAWUDBAICBQChHDAaBgkqhkiG9w0BAQgwDQYJYIZIAWUDBAICBQCiAwIBMKMDAgEBMHsxFDASBgNVBAsMC0VuZ2luZWVyaW5nMQswCQYDVQQGEwJVUzEUMBIGA1UEBwwLU2FudGEgQ2xhcmExCzAJBgNVBAgMAkNBMR8wHQYDVQQKDBZBZHZhbmNlZCBNaWNybyBEZXZpY2VzMRIwEAYDVQQDDAlBUkstR2Vub2EwHhcNMjIwMTI2MTUzNDM3WhcNNDcwMTI2MTUzNDM3WjB7MRQwEgYDVQQLDAtFbmdpbmVlcmluZzELMAkGA1UEBhMCVVMxFDASBgNVBAcMC1NhbnRhIENsYXJhMQswCQYDVQQIDAJDQTEfMB0GA1UECgwWQWR2YW5jZWQgTWljcm8gRGV2aWNlczESMBAGA1UEAwwJQVJLLUdlbm9hMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA3Cd95S/uFOuRIskW9vz9VDBF69NDQF79oRhL/L2PVQGhK3YdfEBgpF/JiwWFBsT/fXDhzA01p3LkcT/7LdjcRfKXjHl+0Qq/M4dZkh6QDoUeKzNBLDcBKDDGWo3v35NyrxbA1DnkYwUKU5AAk4P94tKXLp80oxt84ahyHoLmc/LqsGsp+oq1Bz4PPsYLwTG4iMKVaaT90/oZ4I8oibSru92vJhlqWO27d/Rxc3iUMyhNeGToOvgx/iUo4gGpG61NDpkEUvIzuKcaMx8IdTpWg2DF6SwF0IgVMffnvtJmA68BwJNWo1E4PLJdaPfBifcJpuBFwNVQIPQEVX3aP89HJSp8YbY9lySS6PlVEqTBBtaQmi4ATGmMR+n2K/e+JAhU2Gj7jIpJhOkdH9firQDnmlA2SFfJ/Cc0mGNzW9RmIhyOUnNFoclmkRhl3/AQU5Ys9Qsan1jT/EiyT+pCpmnA+y9edvhDCbOG8F2oxHGRdTBkylungrkXJGYiwGrR8kaiqv7NN8QhOBMqYjcbrkEr0f8QMKklIS5ruOfqlLMCBw8JLB3LkjpWgtD7OpxkzSsohN47Uom86RY6lp72g8eXHP1qYrnvhzaG1S70vw6OkbaaC9EjiH/uHgAJQGxon7u0Q7xgoREWA/e7JcBQwLg80Hq/sbRuqesxz7wBWSY254cCAwEAAaN+MHwwDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBSfXfn+DdjzWtAzGiXvgSlPvjGoWzAPBgNVHRMBAf8EBTADAQH/MDoGA1UdHwQzMDEwL6AtoCuGKWh0dHBzOi8va2RzaW50Zi5hbWQuY29tL3ZjZWsvdjEvR2Vub2EvY3JsMEYGCSqGSIb3DQEBCjA5oA8wDQYJYIZIAWUDBAICBQChHDAaBgkqhkiG9w0BAQgwDQYJYIZIAWUDBAICBQCiAwIBMKMDAgEBA4ICAQAdIlPBC7DQmvH7kjlOznFx3i21SzOPDs5L7SgFjMC9rR07292GQCA7Z7Ulq97JQaWeD2ofGGse5swj4OQfKfVv/zaJUFjvosZOnfZ63epu8MjWgBSXJg5QE/Al0zRsZsp53DBTdA+Uv/s33fexdenT1mpKYzhIg/cKtz4oMxq8JKWJ8Po1CXLzKcfrTphjlbkh8AVKMXeBd2SpM33B1YP4g1BOdk013kqb7bRHZ1iB2JHG5cMKKbwRCSAAGHLTzASgDcXr9Fp7Z3liDhGu/ci1opGmkp12QNiJuBbkTU+xDZHm5X8Jm99BX7NEpzlOwIVR8ClgBDyuBkBC2ljtr3ZSaUIYj2xuyWN95KFY49nWxcz90CFa3Hzmy4zMQmBe9dVyls5eL5p9bkXcgRMDTbgmVZiAf4afe8DLdmQcYcMFQbHhgVzMiyZHGJgcCrQmA7MkTwEIds1wx/HzMcwU4qqNBAoZV7oeIIPxdqFXfPqHqiRlEbRDfX1TG5NFVaeByX0GyH6jzYVuezETzruaky6fp2bl2bczxPE8HdS38ijiJmm9vl50RGUeOAXjSuInGR4bsRufeGPB9peTa9BcBOeTWzstqTUB/F/qaZCIZKr4X6TyfUuSDz/1JDAGl+lxdM0P9+lLaP9NahQjHCVf0zf1c1salVuGFk2w/wMz1R1BHg=="; + +const ASK_CERT_GUID: [u8; 16] = [ + 0x4a, 0xb7, 0xb3, 0x79, 0xbb, 0xac, 0x4f, 0xe4, 0xa0, 0x2f, 0x05, 0xae, 0xf3, 0x27, 0xc7, 0x82, +]; +const VCEK_CERT_GUID: [u8; 16] = [ + 0x63, 0xda, 0x75, 0x8d, 0xe6, 0x64, 0x45, 0x64, 0xad, 0xc5, 0xf4, 0xb9, 0x3b, 0xe8, 0xac, 0xcd, +]; +const VLEK_CERT_GUID: [u8; 16] = [ + 0xa8, 0x07, 0x4b, 0xc2, 0xa2, 0x5a, 0x48, 0x3e, 0xaa, 0xe6, 0x39, 0xc0, 0x45, 0xa0, 0xb8, 0xa1, +]; +const CERT_TABLE_ENTRY_SIZE: usize = 24; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct AmdSnpTcbVersion { + pub bootloader: u8, + pub tee: u8, + pub snp: u8, + pub microcode: u8, +} + +impl From for AmdSnpTcbVersion { + fn from(value: TcbVersion) -> Self { + Self { + bootloader: value.bootloader, + tee: value.tee, + snp: value.snp, + microcode: value.microcode, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct AmdSnpTcbInfo { + pub current: AmdSnpTcbVersion, + pub reported: AmdSnpTcbVersion, + pub committed: AmdSnpTcbVersion, + pub launch: AmdSnpTcbVersion, +} + +impl AmdSnpTcbInfo { + pub fn from_report(report: &AttestationReport) -> Self { + Self { + current: report.current_tcb.into(), + reported: report.reported_tcb.into(), + committed: report.committed_tcb.into(), + launch: report.launch_tcb.into(), + } + } + + pub fn tcb_status(&self) -> &'static str { + if self.current == self.reported + && self.committed == self.reported + && self.launch == self.reported + { + "UpToDate" + } else { + "OutOfDate" + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VerifiedAmdSnpReport { + pub measurement: [u8; 48], + pub report_data: [u8; 64], + pub chip_id: [u8; 64], + pub tcb_info: AmdSnpTcbInfo, + pub advisory_ids: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CertEncoding { + Pem, + Der, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct CertBytes { + bytes: Vec, + encoding: CertEncoding, +} + +pub struct AmdSnpAttestationInput<'a> { + pub report: &'a [u8], + pub ask_pem: &'a [u8], + pub vcek_pem: &'a [u8], +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct AmdKdsCollateral { + ark: CertBytes, + ask: CertBytes, + vcek: CertBytes, +} + +pub fn verify_amd_snp_attestation( + input: &AmdSnpAttestationInput<'_>, +) -> Result { + verify_amd_snp_attestation_with_certs( + input.report, + CertBytes { + bytes: input.ask_pem.to_vec(), + encoding: CertEncoding::Pem, + }, + CertBytes { + bytes: input.vcek_pem.to_vec(), + encoding: CertEncoding::Pem, + }, + ) +} + +fn verify_amd_snp_attestation_with_certs( + report_bytes: &[u8], + ask_bytes: CertBytes, + vcek_bytes: CertBytes, +) -> Result { + let ark_der = STANDARD + .decode(GENOA_ARK_DER_B64) + .context("failed to decode amd genoa ark")?; + verify_amd_snp_attestation_with_cert_chain( + report_bytes, + CertBytes { + bytes: ark_der, + encoding: CertEncoding::Der, + }, + ask_bytes, + vcek_bytes, + ) +} + +fn verify_amd_snp_attestation_with_cert_chain( + report_bytes: &[u8], + ark_bytes: CertBytes, + ask_bytes: CertBytes, + vcek_bytes: CertBytes, +) -> Result { + if report_bytes.len() != 1184 { + bail!( + "invalid amd sev-snp report length: expected 1184 bytes, got {}", + report_bytes.len() + ); + } + let report = AttestationReport::from_bytes(report_bytes) + .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; + + let ark = parse_certificate(&ark_bytes, "ark")?; + let ask = parse_certificate(&ask_bytes, "ask")?; + let vcek = parse_certificate(&vcek_bytes, "vcek")?; + + let chain = Chain { + ca: ca::Chain { ark, ask }, + vek: vcek.clone(), + }; + chain + .verify() + .map_err(|err| anyhow::anyhow!("amd cert chain verification failed: {err:?}"))?; + (&vcek, &report).verify().map_err(|err| { + anyhow::anyhow!("amd sev-snp report signature verification failed: {err:?}") + })?; + validate_amd_snp_report_policy(&report)?; + + let mut measurement = [0u8; 48]; + measurement.copy_from_slice( + report + .measurement + .as_ref() + .get(..48) + .context("amd sev-snp measurement too short")?, + ); + let mut report_data = [0u8; 64]; + report_data.copy_from_slice( + report + .report_data + .as_ref() + .get(..64) + .context("amd sev-snp report_data too short")?, + ); + let mut chip_id = [0u8; 64]; + chip_id.copy_from_slice( + report + .chip_id + .as_ref() + .get(..64) + .context("amd sev-snp chip_id too short")?, + ); + + Ok(VerifiedAmdSnpReport { + measurement, + report_data, + chip_id, + tcb_info: AmdSnpTcbInfo::from_report(&report), + // AMD SEV-SNP attestation reports and VCEKs do not carry a direct + // advisory list. Keep this explicit and empty so downstream auth stays + // fail-closed if a future verifier adds advisories from revocation or + // external policy collateral. + advisory_ids: Vec::new(), + }) +} + +pub fn verify_amd_snp_evidence( + report: &[u8], + cert_chain: &[Vec], + expected_report_data: &[u8; 64], +) -> Result { + let (ask, vcek) = normalize_ask_vcek_certs(cert_chain)?; + let verified = verify_amd_snp_attestation_with_certs(report, ask, vcek)?; + if &verified.report_data != expected_report_data { + bail!("amd sev-snp report_data mismatch"); + } + Ok(verified) +} + +pub fn verify_amd_snp_evidence_with_kds_fallback( + report: &[u8], + cert_chain: &[Vec], + expected_report_data: &[u8; 64], +) -> Result { + if !cert_chain.is_empty() { + return verify_amd_snp_evidence(report, cert_chain, expected_report_data); + } + let report_obj = AttestationReport::from_bytes(report) + .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; + let collateral = fetch_amd_kds_collateral_for_report(&report_obj) + .context("failed to fetch amd sev-snp KDS collateral for empty cert_chain")?; + let verified = verify_amd_snp_attestation_with_cert_chain( + report, + collateral.ark, + collateral.ask, + collateral.vcek, + )?; + if &verified.report_data != expected_report_data { + bail!("amd sev-snp report_data mismatch"); + } + Ok(verified) +} + +fn fetch_amd_kds_collateral_for_report(report: &AttestationReport) -> Result { + let mut errors = Vec::new(); + for product in ["Genoa", "Milan", "Bergamo", "Siena", "Turin"] { + match fetch_amd_kds_collateral_for_product(product, report) { + Ok(collateral) => return Ok(collateral), + Err(err) => errors.push(format!("{product}: {err:#}")), + } + } + bail!( + "amd sev-snp KDS collateral unavailable for supported products: {}", + errors.join("; ") + ) +} + +fn fetch_amd_kds_collateral_for_product( + product: &str, + report: &AttestationReport, +) -> Result { + let (ark, ask) = fetch_amd_kds_ca_chain(product)?; + let mut chip_id = [0u8; 64]; + chip_id.copy_from_slice( + report + .chip_id + .as_ref() + .get(..64) + .context("amd sev-snp chip_id too short")?, + ); + let vcek_url = amd_kds_vcek_url(product, &chip_id, report.reported_tcb.into()); + let vcek_request_url = amd_kds_request_url(&vcek_url); + let vcek = reqwest::blocking::Client::new() + .get(&vcek_request_url) + .send() + .with_context(|| format!("failed to request amd sev-snp vcek from {vcek_request_url}"))? + .error_for_status() + .with_context(|| { + format!("amd sev-snp vcek request failed for {vcek_url} via {vcek_request_url}") + })? + .bytes() + .context("failed to read amd sev-snp vcek response")? + .to_vec(); + Ok(AmdKdsCollateral { + ark, + ask, + vcek: CertBytes { + bytes: vcek, + encoding: CertEncoding::Der, + }, + }) +} + +fn fetch_amd_kds_ca_chain(product: &str) -> Result<(CertBytes, CertBytes)> { + let url = format!("https://kdsintf.amd.com/vcek/v1/{product}/cert_chain"); + let request_url = amd_kds_request_url(&url); + let chain = reqwest::blocking::Client::new() + .get(&request_url) + .send() + .with_context(|| format!("failed to request amd sev-snp cert_chain from {request_url}"))? + .error_for_status() + .with_context(|| format!("amd sev-snp cert_chain request failed for {request_url}"))? + .bytes() + .context("failed to read amd sev-snp cert_chain response")?; + extract_ark_ask_from_amd_kds_cert_chain(&chain) +} + +fn amd_kds_request_url(amd_url: &str) -> String { + match std::env::var("DSTACK_AMD_KDS_PROXY_URL") { + Ok(proxy) if !proxy.trim().is_empty() => format!("{}{}", proxy.trim(), amd_url), + _ => amd_url.to_string(), + } +} + +fn amd_kds_vcek_url(product: &str, chip_id: &[u8; 64], tcb: AmdSnpTcbVersion) -> String { + format!( + "https://kdsintf.amd.com/vcek/v1/{}/{}?blSPL={}&teeSPL={}&snpSPL={}&ucodeSPL={}", + product, + hex::encode(chip_id), + tcb.bootloader, + tcb.tee, + tcb.snp, + tcb.microcode + ) +} + +fn extract_ark_ask_from_amd_kds_cert_chain(chain: &[u8]) -> Result<(CertBytes, CertBytes)> { + let certs = extract_pem_certs(chain)?; + if certs.len() < 2 { + bail!("amd sev-snp cert_chain must contain ASK and ARK certificates"); + } + Ok(( + CertBytes { + bytes: certs[1].clone(), + encoding: CertEncoding::Pem, + }, + CertBytes { + bytes: certs[0].clone(), + encoding: CertEncoding::Pem, + }, + )) +} + +fn extract_pem_certs(chain: &[u8]) -> Result>> { + let chain = std::str::from_utf8(chain).context("amd sev-snp cert_chain is not utf-8 pem")?; + let begin = "-----BEGIN CERTIFICATE-----"; + let end = "-----END CERTIFICATE-----"; + let mut rest = chain; + let mut certs = Vec::new(); + while let Some(start) = rest.find(begin) { + let after_start = &rest[start..]; + let cert_end = after_start + .find(end) + .map(|idx| idx + end.len()) + .context("amd sev-snp cert_chain has unterminated certificate")?; + let mut cert = after_start.as_bytes()[..cert_end].to_vec(); + cert.push(b'\n'); + certs.push(cert); + rest = &after_start[cert_end..]; + } + if certs.is_empty() { + bail!("amd sev-snp cert_chain missing certificates"); + } + Ok(certs) +} + +fn parse_certificate(cert: &CertBytes, name: &str) -> Result { + match cert.encoding { + CertEncoding::Pem => Certificate::from_pem(&cert.bytes) + .map_err(|err| anyhow::anyhow!("failed to parse amd {name} certificate: {err:?}")), + CertEncoding::Der => Certificate::from_der(&cert.bytes) + .map_err(|err| anyhow::anyhow!("failed to parse amd {name} certificate: {err:?}")), + } +} + +fn validate_amd_snp_report_policy(report: &AttestationReport) -> Result<()> { + if !matches!(report.version, 2 | 3) { + bail!("unsupported amd sev-snp report version: {}", report.version); + } + if report.vmpl != 0 { + bail!("amd sev-snp report must be generated at vmpl0"); + } + if report.policy.debug_allowed() { + bail!("amd sev-snp guest policy allows debug"); + } + if report.policy.migrate_ma_allowed() { + bail!("amd sev-snp guest policy allows migration agent"); + } + if report.key_info.mask_chip_key() { + bail!("amd sev-snp report masks the chip signing key"); + } + if report.key_info.signing_key() != 0 { + bail!( + "unsupported amd sev-snp signing key: expected vcek, got {}", + report.key_info.signing_key() + ); + } + if !report.policy.smt_allowed() && report.plat_info.smt_enabled() { + bail!("amd sev-snp platform has smt enabled but guest policy does not allow smt"); + } + if report.policy.rapl_dis() && !report.plat_info.rapl_disabled() { + bail!("amd sev-snp guest policy requires rapl disabled, but platform reports rapl enabled"); + } + if report.policy.ciphertext_hiding() && !report.plat_info.ciphertext_hiding_enabled() { + bail!( + "amd sev-snp guest policy requires ciphertext hiding, but platform does not report it" + ); + } + Ok(()) +} + +fn normalize_ask_vcek_certs(cert_chain: &[Vec]) -> Result<(CertBytes, CertBytes)> { + match cert_chain { + [ask, vcek] => Ok((cert_bytes_from_blob(ask), cert_bytes_from_blob(vcek))), + [auxblob] => normalize_kernel_cert_table(auxblob), + _ => bail!( + "amd sev-snp cert_chain must contain either ASK and VCEK certificates or one kernel certificate table auxblob" + ), + } +} + +fn cert_bytes_from_blob(blob: &[u8]) -> CertBytes { + let encoding = if blob.starts_with(b"-----BEGIN CERTIFICATE-----") { + CertEncoding::Pem + } else { + CertEncoding::Der + }; + CertBytes { + bytes: blob.to_vec(), + encoding, + } +} + +fn normalize_kernel_cert_table(auxblob: &[u8]) -> Result<(CertBytes, CertBytes)> { + let mut ask = None; + let mut vcek = None; + for (guid, data) in parse_kernel_cert_table(auxblob)? { + match guid { + ASK_CERT_GUID => ask = Some(data), + VCEK_CERT_GUID => vcek = Some(data), + VLEK_CERT_GUID => bail!("amd sev-snp vlek certificates are not supported yet"), + _ => {} + } + } + let ask = ask.context("amd sev-snp certificate table missing ASK certificate")?; + let vcek = vcek.context("amd sev-snp certificate table missing VCEK certificate")?; + Ok(( + CertBytes { + bytes: ask, + encoding: CertEncoding::Der, + }, + CertBytes { + bytes: vcek, + encoding: CertEncoding::Der, + }, + )) +} + +fn parse_kernel_cert_table(auxblob: &[u8]) -> Result)>> { + if auxblob.len() < CERT_TABLE_ENTRY_SIZE { + bail!("amd sev-snp certificate table is too short"); + } + let mut entries = Vec::new(); + let mut pos = 0usize; + loop { + let entry = auxblob + .get(pos..pos + CERT_TABLE_ENTRY_SIZE) + .context("amd sev-snp certificate table is missing terminator")?; + let guid: [u8; 16] = entry[..16] + .try_into() + .context("amd sev-snp certificate table entry guid has invalid length")?; + let offset = u32::from_le_bytes( + entry[16..20] + .try_into() + .context("amd sev-snp certificate table entry offset has invalid length")?, + ) as usize; + let length = u32::from_le_bytes( + entry[20..24] + .try_into() + .context("amd sev-snp certificate table entry length has invalid length")?, + ) as usize; + if guid == [0u8; 16] && offset == 0 && length == 0 { + break; + } + let end = offset + .checked_add(length) + .context("amd sev-snp certificate table entry length overflows")?; + if offset < CERT_TABLE_ENTRY_SIZE || end > auxblob.len() || length == 0 { + bail!("amd sev-snp certificate table entry has invalid bounds"); + } + entries.push((guid, auxblob[offset..end].to_vec())); + pos = pos + .checked_add(CERT_TABLE_ENTRY_SIZE) + .context("amd sev-snp certificate table entry count overflows")?; + if pos >= auxblob.len() { + bail!("amd sev-snp certificate table is missing terminator"); + } + } + Ok(entries) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tcb(bootloader: u8, tee: u8, snp: u8, microcode: u8) -> AmdSnpTcbVersion { + AmdSnpTcbVersion { + bootloader, + tee, + snp, + microcode, + } + } + + #[test] + fn tcb_status_is_up_to_date_only_when_all_reported_versions_match() { + let up_to_date = AmdSnpTcbInfo { + current: tcb(1, 2, 3, 4), + reported: tcb(1, 2, 3, 4), + committed: tcb(1, 2, 3, 4), + launch: tcb(1, 2, 3, 4), + }; + assert_eq!(up_to_date.tcb_status(), "UpToDate"); + + let stale_launch = AmdSnpTcbInfo { + launch: tcb(1, 2, 3, 3), + ..up_to_date + }; + assert_eq!(stale_launch.tcb_status(), "OutOfDate"); + + let stale_vcek_reported = AmdSnpTcbInfo { + reported: tcb(1, 2, 3, 3), + ..up_to_date + }; + assert_eq!(stale_vcek_reported.tcb_status(), "OutOfDate"); + } + + #[test] + fn missing_cert_chain_fails_closed() { + let report = vec![0u8; 1184]; + let expected_report_data = [0u8; 64]; + let err = verify_amd_snp_evidence(&report, &[], &expected_report_data).unwrap_err(); + assert!( + err.to_string() + .contains("cert_chain must contain either ASK and VCEK certificates or one kernel certificate table auxblob"), + "unexpected error: {err:#}" + ); + } + + #[test] + fn amd_kds_vcek_url_binds_chip_id_and_reported_tcb() { + let chip_id = [0xab; 64]; + let tcb = AmdSnpTcbVersion { + bootloader: 1, + tee: 2, + snp: 3, + microcode: 4, + }; + + let url = amd_kds_vcek_url("Genoa", &chip_id, tcb); + + assert_eq!( + url, + format!( + "https://kdsintf.amd.com/vcek/v1/Genoa/{}?blSPL=1&teeSPL=2&snpSPL=3&ucodeSPL=4", + hex::encode(chip_id) + ) + ); + } + + #[test] + fn amd_kds_proxy_url_wraps_amd_urls_when_configured() { + const ENV_KEY: &str = "DSTACK_AMD_KDS_PROXY_URL"; + let old = std::env::var(ENV_KEY).ok(); + std::env::set_var(ENV_KEY, "https://cors.litgateway.com/"); + + let wrapped = amd_kds_request_url("https://kdsintf.amd.com/vcek/v1/Genoa/cert_chain"); + + assert_eq!( + wrapped, + "https://cors.litgateway.com/https://kdsintf.amd.com/vcek/v1/Genoa/cert_chain" + ); + if let Some(old) = old { + std::env::set_var(ENV_KEY, old); + } else { + std::env::remove_var(ENV_KEY); + } + } + + #[test] + fn amd_kds_cert_chain_extracts_ask_pem_and_ark_pem() { + let chain = b"-----BEGIN CERTIFICATE-----\nASK\n-----END CERTIFICATE-----\n-----BEGIN CERTIFICATE-----\nARK\n-----END CERTIFICATE-----\n"; + + let (ark_cert, ask_cert) = extract_ark_ask_from_amd_kds_cert_chain(chain).unwrap(); + + assert_eq!( + ask_cert.bytes, + b"-----BEGIN CERTIFICATE-----\nASK\n-----END CERTIFICATE-----\n".to_vec() + ); + assert_eq!(ask_cert.encoding, CertEncoding::Pem); + assert_eq!( + ark_cert.bytes, + b"-----BEGIN CERTIFICATE-----\nARK\n-----END CERTIFICATE-----\n".to_vec() + ); + assert_eq!(ark_cert.encoding, CertEncoding::Pem); + } + + #[test] + fn malformed_report_fails_closed_before_success() { + let cert_chain = vec![b"not ask".to_vec(), b"not vcek".to_vec()]; + let expected_report_data = [0u8; 64]; + let err = + verify_amd_snp_evidence(b"too short", &cert_chain, &expected_report_data).unwrap_err(); + assert!( + err.to_string() + .contains("invalid amd sev-snp report length"), + "unexpected error: {err:#}" + ); + } + + #[test] + fn normalizes_kernel_cert_table_auxblob_to_ask_and_vcek_der() { + use sev::firmware::host::{CertTableEntry, CertType}; + + let auxblob = CertTableEntry::cert_table_to_vec_bytes(&[ + CertTableEntry::new(CertType::VCEK, b"vcek-der".to_vec()), + CertTableEntry::new(CertType::ASK, b"ask-der".to_vec()), + ]) + .unwrap(); + + let (ask, vcek) = normalize_ask_vcek_certs(&[auxblob]).unwrap(); + + assert_eq!(ask.bytes, b"ask-der"); + assert_eq!(ask.encoding, CertEncoding::Der); + assert_eq!(vcek.bytes, b"vcek-der"); + assert_eq!(vcek.encoding, CertEncoding::Der); + } + + #[test] + fn malformed_single_auxblob_fails_closed_without_panic() { + let err = normalize_ask_vcek_certs(&[vec![0xff; 23]]).unwrap_err(); + + assert!( + err.to_string().contains("certificate table"), + "unexpected error: {err:#}" + ); + } + + #[test] + fn normalizes_existing_two_item_pem_chain_without_reordering() { + let ask = b"-----BEGIN CERTIFICATE-----\nask\n-----END CERTIFICATE-----\n".to_vec(); + let vcek = b"-----BEGIN CERTIFICATE-----\nvcek\n-----END CERTIFICATE-----\n".to_vec(); + + let (normalized_ask, normalized_vcek) = + normalize_ask_vcek_certs(&[ask.clone(), vcek.clone()]).unwrap(); + + assert_eq!(normalized_ask.bytes, ask); + assert_eq!(normalized_ask.encoding, CertEncoding::Pem); + assert_eq!(normalized_vcek.bytes, vcek); + assert_eq!(normalized_vcek.encoding, CertEncoding::Pem); + } + + #[test] + fn report_policy_rejects_debug_allowed() { + let mut report = base_report(); + report.policy.set_debug_allowed(true); + + let err = validate_amd_snp_report_policy(&report).unwrap_err(); + + assert!( + err.to_string().contains("debug"), + "unexpected error: {err:#}" + ); + } + + #[test] + fn report_policy_rejects_non_vmpl0() { + let mut report = base_report(); + report.vmpl = 1; + + let err = validate_amd_snp_report_policy(&report).unwrap_err(); + + assert!( + err.to_string().contains("vmpl0"), + "unexpected error: {err:#}" + ); + } + + #[test] + fn report_policy_accepts_strict_vcek_vmpl0_report() { + let report = base_report(); + + validate_amd_snp_report_policy(&report).unwrap(); + } + + fn base_report() -> AttestationReport { + AttestationReport { + version: 2, + ..Default::default() + } + } +} From 8bbade47f0bd12e726b810e88ffb5da971478a73 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 15 Jun 2026 00:59:15 -0700 Subject: [PATCH 40/67] Bind SNP app config via HOST_DATA --- Cargo.lock | 21 + Cargo.toml | 1 + docs/amd-sev-snp-review-readiness.md | 10 +- dstack-attest/src/attestation.rs | 206 ++++++- dstack-attest/src/sev_snp.rs | 1 + dstack-attest/src/v1.rs | 34 +- dstack-types/Cargo.toml | 3 + dstack-types/src/lib.rs | 9 +- dstack-types/src/mr_config.rs | 195 ++++++- dstack-util/src/system_setup.rs | 16 +- .../src/system_setup/config_id_verifier.rs | 213 +++++++- kms/src/main_service.rs | 39 +- kms/src/main_service/amd_attest.rs | 502 ++++++++---------- kms/src/main_service/upgrade_authority.rs | 23 + kms/src/onboard_service.rs | 17 +- sev-snp-qvl/src/lib.rs | 95 +++- vmm/Cargo.toml | 1 + vmm/src/app.rs | 117 +++- vmm/src/app/qemu.rs | 272 +++++----- vmm/src/one_shot.rs | 22 +- 20 files changed, 1264 insertions(+), 533 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 24cb4029e..7278658e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2659,6 +2659,9 @@ dependencies = [ "parity-scale-codec", "serde", "serde-human-bytes", + "serde_jcs", + "serde_json", + "sha2 0.10.9", "sha3 0.10.9", "size-parser", ] @@ -2765,6 +2768,7 @@ dependencies = [ "flate2", "fs-err", "fscommon", + "getrandom 0.3.4", "git-version", "guest-api", "hex", @@ -6748,6 +6752,12 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "ryu-js" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6518fc26bced4d53678a22d6e423e9d8716377def84545fe328236e3af070e7f" + [[package]] name = "s2n-codec" version = "0.81.0" @@ -7212,6 +7222,17 @@ dependencies = [ "void", ] +[[package]] +name = "serde_jcs" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3a60f3fda61525e439ef6d67422118f11e986566997d9021c56867ad814a0aa" +dependencies = [ + "ryu-js", + "serde", + "serde_json", +] + [[package]] name = "serde_json" version = "1.0.150" diff --git a/Cargo.toml b/Cargo.toml index c661bc6c7..deeededc6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -145,6 +145,7 @@ scale = { version = "3.7.4", package = "parity-scale-codec", features = [ ] } serde = { version = "1.0.228", features = ["derive"], default-features = false } serde-human-bytes = "0.1.2" +serde_jcs = "0.2.0" rmp-serde = "1.3.1" serde_json = { version = "1.0.140", default-features = false } serde_ini = "0.2.0" diff --git a/docs/amd-sev-snp-review-readiness.md b/docs/amd-sev-snp-review-readiness.md index 0b19cd410..bbef133cc 100644 --- a/docs/amd-sev-snp-review-readiness.md +++ b/docs/amd-sev-snp-review-readiness.md @@ -80,10 +80,10 @@ initrd_fixture_sha256=e8790816224329cd76675c2aba4e62e885b5a4e0ec056227da70e77519 vcpus=2 vcpu_type=EPYC-v4 guest_features=0x1 -append=console=ttyS0 loglevel=7 docker_compose_hash=2222222222222222222222222222222222222222222222222222222222222222 rootfs_hash=3333333333333333333333333333333333333333333333333333333333333333 app_id=1111111111111111111111111111111111111111 -sev_snp_measurement=6497fb9f90dc4a322228a8a5eb14742e09067bc44c184c2068d583ef628b5bae8c6cf15d91fe1bc0b7a8cbcc575be370 +append=console=ttyS0 loglevel=7 +sev_snp_measurement=requires-refresh-after-mr-config-v3-host-data-binding cargo_live_test=cargo test -p dstack-kms --all-features recomputation_matches_sev_snp_measure_live_golden_vector -- --ignored --nocapture -cargo_live_test_result=passed locally on this host at 2026-06-02T19:49:14Z +cargo_live_test_result=stale after SNP app identity moved from cmdline to HOST_DATA DSTACK_SEV_SNP_MEASURE_GOLDEN_VECTOR_END ``` @@ -116,10 +116,10 @@ That smoke exposed and fixed several VMM/KMS-auth integration issues before the After those fixes, the manual smoke progressed through full dstack-managed SNP guest boot and KMS self-bootstrap on the known-good remote host. Additional smoke/debug fixes made the host/KMS side reach the app-key boundary: - Minimal guest boot now keeps DNS usable when `systemd-resolved`/`chronyd` are unavailable early in smoke boots and detects `sev-guest` before trying the TDX guest module. -- SNP guests skip TDX-only `mr_config_id` and app-info RTMR decoding while still preserving non-SNP behavior. +- SNP guests verify the SNP `HOST_DATA` value against the attached MrConfigV3 document instead of using TDX-only `mr_config_id`. - Configfs TSM report collection falls back to the SEV-SNP extended-report ioctl when configfs does not carry certificate collateral. - If verifier-side evidence still lacks ASK/VCEK collateral, the verifier can fetch AMD KDS ARK/ASK/VCEK using the report `chip_id` and reported TCB, then verify the signed report fail-closed. -- KMS measurement recomputation now uses the image's original kernel cmdline as the measurement base before appending `docker_compose_hash`, `rootfs_hash`, and `app_id`, matching the VMM QEMU `-append` path. +- KMS measurement recomputation now uses the image's original kernel cmdline for SNP launch measurement, while app identity is bound by MrConfigV3/HOST_DATA instead of appended cmdline fields. Latest sanitized remote smoke result with PR-built host binaries and a coherent `MACHINE = "sev-snp"` guest image: diff --git a/dstack-attest/src/attestation.rs b/dstack-attest/src/attestation.rs index b179737a3..f36d8bb17 100644 --- a/dstack-attest/src/attestation.rs +++ b/dstack-attest/src/attestation.rs @@ -19,7 +19,7 @@ use dcap_qvl::{ }; #[cfg(feature = "quote")] use dstack_types::SysConfig; -use dstack_types::{Platform, VmConfig}; +use dstack_types::{mr_config::MrConfigV3, KeyProviderInfo, Platform, VmConfig}; use ez_hash::{sha256, Hasher, Sha256, Sha384}; use or_panic::ResultOrPanic; use scale::{Decode, Encode, Error as ScaleError, Input, Output}; @@ -91,9 +91,15 @@ fn platform_from_legacy_quote(quote: AttestationQuote) -> PlatformEvidence { AttestationQuote::DstackTdx(TdxQuote { quote, event_log }) => { PlatformEvidence::Tdx { quote, event_log } } - AttestationQuote::DstackAmdSevSnp(SnpQuote { report, cert_chain }) => { - PlatformEvidence::SevSnp { report, cert_chain } - } + AttestationQuote::DstackAmdSevSnp(SnpQuote { + report, + cert_chain, + mr_config, + }) => PlatformEvidence::SevSnp { + report, + cert_chain, + mr_config, + }, AttestationQuote::DstackGcpTdx(DstackGcpTdxQuote { tdx_quote: TdxQuote { quote, event_log }, tpm_quote, @@ -113,9 +119,15 @@ fn platform_into_legacy_quote(platform: PlatformEvidence) -> AttestationQuote { PlatformEvidence::Tdx { quote, event_log } => { AttestationQuote::DstackTdx(TdxQuote { quote, event_log }) } - PlatformEvidence::SevSnp { report, cert_chain } => { - AttestationQuote::DstackAmdSevSnp(SnpQuote { report, cert_chain }) - } + PlatformEvidence::SevSnp { + report, + cert_chain, + mr_config, + } => AttestationQuote::DstackAmdSevSnp(SnpQuote { + report, + cert_chain, + mr_config, + }), PlatformEvidence::GcpTdx { quote, event_log, @@ -169,7 +181,43 @@ fn decode_vm_config_with_fallback(config: &str, fallback_config: &str) -> Result config }; let config = if config.is_empty() { "{}" } else { config }; - serde_json::from_str(config).context("Failed to parse vm config") + let config = vm_config_json_from_config(config).unwrap_or(Cow::Borrowed(config)); + serde_json::from_str(&config).context("Failed to parse vm config") +} + +fn vm_config_json_from_config(config: &str) -> Option> { + let value = serde_json::from_str::(config).ok()?; + value + .get("vm_config") + .and_then(|value| value.as_str()) + .map(|vm_config| Cow::Owned(vm_config.to_string())) +} + +fn mr_config_document_from_value(value: &serde_json::Value) -> Result> { + let Some(mr_config) = value.get("mr_config") else { + return Ok(None); + }; + let document = mr_config + .as_str() + .context("amd sev-snp mr_config must be a JSON string")?; + MrConfigV3::from_document(document).context("Invalid amd sev-snp mr_config document")?; + Ok(Some(document.to_string())) +} + +fn mr_config_document_from_config(config: &str) -> Result> { + let Ok(value) = serde_json::from_str::(config) else { + return Ok(None); + }; + if let Some(mr_config) = mr_config_document_from_value(&value)? { + return Ok(Some(mr_config)); + } + + let Some(vm_config) = value.get("vm_config").and_then(|value| value.as_str()) else { + return Ok(None); + }; + let vm_config = serde_json::from_str::(vm_config) + .context("Failed to parse nested vm_config for amd sev-snp mr_config")?; + mr_config_document_from_value(&vm_config) } /// Attestation mode @@ -267,10 +315,9 @@ impl AttestationMode { pub fn is_composable(&self) -> bool { match self { Self::DstackTdx => true, - // SEV-SNP launch measurement does not provide a TDX RTMR3-equivalent - // runtime event extension path yet, so runtime events are - // informational until an SNP-specific app binding is added. - Self::DstackAmdSevSnp => false, + // SEV-SNP binds app identity through HOST_DATA carrying the hash of + // an attached MrConfigV3 document. + Self::DstackAmdSevSnp => true, Self::DstackGcpTdx => true, Self::DstackNitroEnclave => false, } @@ -414,6 +461,8 @@ pub struct SnpQuote { pub report: Vec, /// Optional certificate chain blobs, when exposed by the kernel/firmware path. pub cert_chain: Vec>, + /// MrConfigV3 document bound by the report HOST_DATA field. + pub mr_config: String, } /// Represents an NSM (Nitro Security Module) attestation document @@ -615,6 +664,17 @@ impl AttestationV1 { #[errify::errify("decode app info")] pub fn decode_app_info_ex(&self, boottime_mr: bool, vm_config: &str) -> Result { let runtime_events = self.stack.runtime_events(); + if let PlatformEvidence::SevSnp { + report, mr_config, .. + } = &self.platform + { + return decode_app_info_sev_snp( + report, + Some(mr_config), + self.stack.config(), + vm_config, + ); + } let key_provider_info = if boottime_mr { vec![] } else { @@ -645,9 +705,7 @@ impl AttestationV1 { nsm_quote: nsm_quote.clone(), })? } - PlatformEvidence::SevSnp { .. } => { - bail!("Unsupported attestation quote"); - } + PlatformEvidence::SevSnp { .. } => unreachable!("handled above"), }; let compose_hash = if platform_attestation_mode(&self.platform).is_composable() { find_event_payload(runtime_events, "compose-hash").unwrap_or_default() @@ -770,14 +828,18 @@ impl AttestationV1 { timestamp: verified_report.timestamp, }) } - PlatformEvidence::SevSnp { report, cert_chain } => { - DstackVerifiedReport::DstackAmdSevSnp( - crate::amd_sev_snp::verify_amd_snp_evidence_with_kds_fallback( - report, - cert_chain, - &report_data, - )?, - ) + PlatformEvidence::SevSnp { + report, + cert_chain, + mr_config, + } => { + let verified = crate::amd_sev_snp::verify_amd_snp_evidence_with_kds_fallback( + report, + cert_chain, + &report_data, + )?; + verify_snp_mr_config_host_data(mr_config, &verified.host_data)?; + DstackVerifiedReport::DstackAmdSevSnp(verified) } }; @@ -944,6 +1006,7 @@ mod compatibility_tests { let quote = AttestationQuote::DstackAmdSevSnp(SnpQuote { report: Vec::new(), cert_chain: Vec::new(), + mr_config: String::new(), }); assert_eq!(quote.encode()[0], 3); } @@ -1090,6 +1153,80 @@ struct Mrs { mr_aggregated: [u8; 32], } +fn key_provider_info_from_mr_config(mr_config: &MrConfigV3) -> Result> { + serde_json::to_vec(&KeyProviderInfo::new( + mr_config.key_provider_name().to_string(), + hex::encode(&mr_config.key_provider_id), + )) + .context("Failed to serialize key provider info") +} + +fn verify_snp_mr_config_host_data( + mr_config_document: &str, + host_data: &[u8; 32], +) -> Result { + let mr_config = MrConfigV3::from_document(mr_config_document) + .context("Invalid amd sev-snp mr_config document")?; + let expected = MrConfigV3::snp_host_data_from_document(mr_config_document); + if expected != *host_data { + bail!( + "amd sev-snp HOST_DATA mismatch, quoted: {}, expected: {}", + hex::encode(host_data), + hex::encode(expected), + ); + } + Ok(mr_config) +} + +fn decode_mr_sev_snp(measurement: &[u8; 48], host_data: &[u8; 32]) -> Mrs { + let mr_system = sha2::Sha256::digest(measurement).into(); + let mr_aggregated = { + let mut hasher = sha2::Sha256::new(); + hasher.update(measurement); + hasher.update(host_data); + hasher.finalize().into() + }; + Mrs { + mr_system, + mr_aggregated, + } +} + +fn decode_app_info_sev_snp( + report: &[u8], + mr_config: Option<&str>, + embedded_config: &str, + external_vm_config: &str, +) -> Result { + let parsed = crate::amd_sev_snp::parse_amd_snp_report(report)?; + let mr_config_document = if let Some(mr_config) = mr_config { + Cow::Borrowed(mr_config) + } else if let Some(mr_config) = mr_config_document_from_config(external_vm_config)? { + Cow::Owned(mr_config) + } else if let Some(mr_config) = mr_config_document_from_config(embedded_config)? { + Cow::Owned(mr_config) + } else { + bail!("amd sev-snp mr_config is missing"); + }; + let mr_config = verify_snp_mr_config_host_data(mr_config_document.as_ref(), &parsed.host_data)?; + + let key_provider_info = key_provider_info_from_mr_config(&mr_config)?; + let os_image_hash = + decode_vm_config_with_fallback(external_vm_config, embedded_config)?.os_image_hash; + let mrs = decode_mr_sev_snp(&parsed.measurement, &parsed.host_data); + + Ok(AppInfo { + app_id: mr_config.app_id, + instance_id: mr_config.instance_id, + device_id: sha256(parsed.chip_id).to_vec(), + mr_system: mrs.mr_system, + mr_aggregated: mrs.mr_aggregated, + key_provider_info, + os_image_hash, + compose_hash: mr_config.compose_hash, + }) +} + fn decode_mr_gcp_tpm_from_v1( boottime_mr: bool, mr_key_provider: &[u8], @@ -1318,6 +1455,9 @@ impl Attestation { #[errify::errify("decode app info")] pub fn decode_app_info_ex(&self, boottime_mr: bool, vm_config: &str) -> Result { + if let AttestationQuote::DstackAmdSevSnp(q) = &self.quote { + return decode_app_info_sev_snp(&q.report, Some(&q.mr_config), &self.config, vm_config); + } let key_provider_info = if boottime_mr { vec![] } else { @@ -1336,9 +1476,7 @@ impl Attestation { AttestationQuote::DstackTdx(q) => { self.decode_mr_tdx(boottime_mr, &mr_key_provider, q)? } - AttestationQuote::DstackAmdSevSnp(_) => { - bail!("unsupported attestation quote for app info decoding"); - } + AttestationQuote::DstackAmdSevSnp(_) => unreachable!("handled above"), AttestationQuote::DstackGcpTdx(q) => { self.decode_mr_gcp_tpm(boottime_mr, &mr_key_provider, &os_image_hash, &q.tpm_quote)? } @@ -1472,7 +1610,7 @@ impl Attestation { vec![] }; - let quote = match mode { + let mut quote = match mode { AttestationMode::DstackTdx => { let quote = tdx_attest::get_quote(report_data).context("Failed to get quote")?; let event_log = @@ -1522,6 +1660,10 @@ impl Attestation { .context("Failed to serialize config")? } }; + if let AttestationQuote::DstackAmdSevSnp(quote) = &mut quote { + quote.mr_config = mr_config_document_from_config(&config)? + .context("amd sev-snp mr_config is missing")?; + } Ok(Self { quote, @@ -1549,8 +1691,14 @@ impl Attestation { let report = self.verify_tdx(pccs_url, &q.quote).await?; DstackVerifiedReport::DstackTdx(report) } - AttestationQuote::DstackAmdSevSnp(_) => { - bail!("Unsupported attestation mode: {:?}", self.quote.mode()); + AttestationQuote::DstackAmdSevSnp(q) => { + let verified = crate::amd_sev_snp::verify_amd_snp_evidence_with_kds_fallback( + &q.report, + &q.cert_chain, + &self.report_data, + )?; + verify_snp_mr_config_host_data(&q.mr_config, &verified.host_data)?; + DstackVerifiedReport::DstackAmdSevSnp(verified) } AttestationQuote::DstackGcpTdx(q) => { let tdx_report = self.verify_tdx(pccs_url, &q.tdx_quote.quote).await?; diff --git a/dstack-attest/src/sev_snp.rs b/dstack-attest/src/sev_snp.rs index 200e4e652..92cad1e59 100644 --- a/dstack-attest/src/sev_snp.rs +++ b/dstack-attest/src/sev_snp.rs @@ -15,6 +15,7 @@ pub fn get_report(report_data: [u8; 64]) -> Result { Ok(SnpQuote { report: quote.report, cert_chain: quote.cert_chain, + mr_config: String::new(), }) } diff --git a/dstack-attest/src/v1.rs b/dstack-attest/src/v1.rs index 7972768bd..a91e9393a 100644 --- a/dstack-attest/src/v1.rs +++ b/dstack-attest/src/v1.rs @@ -4,6 +4,7 @@ use anyhow::{anyhow, bail, Context, Result}; use cc_eventlog::{RuntimeEvent, TdxEvent}; +use dstack_types::mr_config::MrConfigV3; use serde::{Deserialize, Serialize}; use tpm_types::TpmQuote; @@ -29,6 +30,7 @@ pub enum PlatformEvidence { SevSnp { report: Vec, cert_chain: Vec>, + mr_config: String, }, } @@ -77,6 +79,18 @@ impl PlatformEvidence { } } + pub fn sev_snp_mr_config_document(&self) -> Option<&str> { + match self { + Self::SevSnp { mr_config, .. } => Some(mr_config.as_str()), + _ => None, + } + } + + pub fn sev_snp_mr_config(&self) -> Option { + self.sev_snp_mr_config_document() + .and_then(|document| MrConfigV3::from_document(document).ok()) + } + pub fn into_stripped(self) -> Self { match self { Self::Tdx { quote, event_log } => Self::Tdx { @@ -260,11 +274,16 @@ impl Attestation { PlatformEvidence::SevSnp { mut report, cert_chain, + mr_config, } => { if report.len() >= SNP_REPORT_DATA_RANGE.end { report[SNP_REPORT_DATA_RANGE].copy_from_slice(&report_data); } - PlatformEvidence::SevSnp { report, cert_chain } + PlatformEvidence::SevSnp { + report, + cert_chain, + mr_config, + } } other => other, }; @@ -302,6 +321,17 @@ impl Attestation { mod tests { use super::*; + fn test_mr_config_document() -> String { + MrConfigV3::new( + vec![0x11; 20], + vec![0x22; 32], + dstack_types::KeyProviderKind::None, + Vec::new(), + vec![0x33; 20], + ) + .to_canonical_json() + } + #[test] fn msgpack_roundtrip_preserves_attestation() { let attestation = Attestation::new( @@ -363,6 +393,7 @@ mod tests { PlatformEvidence::SevSnp { report: vec![0x11; 1184], cert_chain: vec![vec![0x22, 0x33]], + mr_config: test_mr_config_document(), }, StackEvidence::Dstack { report_data: vec![9u8; 64], @@ -391,6 +422,7 @@ mod tests { PlatformEvidence::SevSnp { report, cert_chain: vec![], + mr_config: test_mr_config_document(), }, StackEvidence::Dstack { report_data: vec![0x22; 64], diff --git a/dstack-types/Cargo.toml b/dstack-types/Cargo.toml index 997b0e43f..3bc4bfcd5 100644 --- a/dstack-types/Cargo.toml +++ b/dstack-types/Cargo.toml @@ -13,5 +13,8 @@ license.workspace = true scale = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] } serde-human-bytes.workspace = true +serde_jcs.workspace = true +serde_json.workspace = true +sha2.workspace = true sha3.workspace = true size-parser = { workspace = true, features = ["serde"] } diff --git a/dstack-types/src/lib.rs b/dstack-types/src/lib.rs index 61615044f..ce10167ff 100644 --- a/dstack-types/src/lib.rs +++ b/dstack-types/src/lib.rs @@ -114,7 +114,7 @@ where Ok(value.gateway_enabled || value.tproxy_enabled) } -#[derive(Deserialize, Serialize, Debug, Clone, Copy)] +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum KeyProviderKind { None, @@ -185,6 +185,13 @@ pub struct SysConfig { pub pccs_url: Option, pub docker_registry: Option, pub host_api_url: Option, + /// MrConfigV3 document string for platform app/config binding. + /// + /// Hosts generate this in JCS form, but verifiers hash the supplied string + /// bytes directly because the platform carrier binds the exact document + /// string. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mr_config: Option, // JSON serialized VmConfig pub vm_config: String, } diff --git a/dstack-types/src/mr_config.rs b/dstack-types/src/mr_config.rs index b4766ecbe..7c0a92896 100644 --- a/dstack-types/src/mr_config.rs +++ b/dstack-types/src/mr_config.rs @@ -2,10 +2,16 @@ // // SPDX-License-Identifier: Apache-2.0 +use serde::{Deserialize, Serialize}; +use serde_human_bytes as hex_bytes; +use sha2::Sha256; use sha3::{Digest, Keccak256}; +use std::{error::Error, fmt}; use crate::KeyProviderKind; +const MR_CONFIG_V3_DOCUMENT_HASH_DOMAIN: &[u8] = b"dstack-mr-config-v3:"; + pub enum MrConfig<'a> { V1 { compose_hash: &'a [u8; 32], @@ -18,6 +24,15 @@ pub enum MrConfig<'a> { }, } +fn key_provider_kind_byte(key_provider: KeyProviderKind) -> u8 { + match key_provider { + KeyProviderKind::None => 0, + KeyProviderKind::Local => 1, + KeyProviderKind::Kms => 2, + KeyProviderKind::Tpm => 3, + } +} + impl MrConfig<'_> { pub fn to_mr_config_id(&self) -> [u8; 48] { match self { @@ -33,16 +48,10 @@ impl MrConfig<'_> { key_provider, key_provider_id, } => { - let kp_kind = match key_provider { - KeyProviderKind::None => 0_u8, - KeyProviderKind::Local => 1, - KeyProviderKind::Kms => 2, - KeyProviderKind::Tpm => 3, - }; let mut hasher = Keccak256::new(); hasher.update(compose_hash); hasher.update(app_id); - hasher.update([kp_kind]); + hasher.update([key_provider_kind_byte(*key_provider)]); hasher.update(key_provider_id); let digest = hasher.finalize(); let mut config_id = [0u8; 48]; @@ -53,3 +62,175 @@ impl MrConfig<'_> { } } } + +fn mr_config_v3_version() -> u8 { + 3 +} + +/// Platform-independent app/config binding document. +/// +/// Hosts generate the document in JCS form, while verifiers hash the supplied +/// document bytes directly because the platform carrier binds the exact +/// document string. +#[derive(Debug)] +pub enum MrConfigDocumentError { + Json(serde_json::Error), +} + +impl fmt::Display for MrConfigDocumentError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Json(err) => write!(f, "failed to parse mr_config document: {err}"), + } + } +} + +impl Error for MrConfigDocumentError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Json(err) => Some(err), + } + } +} + +impl From for MrConfigDocumentError { + fn from(err: serde_json::Error) -> Self { + Self::Json(err) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MrConfigV3 { + #[serde(default = "mr_config_v3_version")] + pub version: u8, + #[serde(with = "hex_bytes")] + pub app_id: Vec, + #[serde(with = "hex_bytes")] + pub compose_hash: Vec, + pub key_provider: KeyProviderKind, + #[serde(default, with = "hex_bytes")] + pub key_provider_id: Vec, + #[serde(default, with = "hex_bytes")] + pub instance_id: Vec, +} + +impl MrConfigV3 { + pub fn new( + app_id: Vec, + compose_hash: Vec, + key_provider: KeyProviderKind, + key_provider_id: Vec, + instance_id: Vec, + ) -> Self { + Self { + version: mr_config_v3_version(), + app_id, + compose_hash, + key_provider, + key_provider_id, + instance_id, + } + } + + pub fn to_snp_host_data(&self) -> [u8; 32] { + Self::snp_host_data_from_document(&self.to_canonical_json()) + } + + pub fn to_tdx_mr_config_id(&self) -> [u8; 48] { + Self::tdx_mr_config_id_from_document(&self.to_canonical_json()) + } + + pub fn to_canonical_json(&self) -> String { + serde_jcs::to_string(self).expect("MrConfigV3 should serialize to JCS") + } + + pub fn from_document(document: &str) -> Result { + Ok(serde_json::from_str(document)?) + } + + pub fn snp_host_data_from_document(document: &str) -> [u8; 32] { + Self::hash_document(document) + } + + pub fn tdx_mr_config_id_from_document(document: &str) -> [u8; 48] { + let digest = Self::hash_document(document); + let mut config_id = [0u8; 48]; + config_id[0] = 3; + config_id[1..33].copy_from_slice(&digest); + config_id + } + + fn hash_document(document: &str) -> [u8; 32] { + let mut hasher = Sha256::new(); + hasher.update(MR_CONFIG_V3_DOCUMENT_HASH_DOMAIN); + hasher.update([0]); + hasher.update(document.as_bytes()); + hasher.finalize().into() + } + + pub fn key_provider_name(&self) -> &'static str { + match self.key_provider { + KeyProviderKind::None => "none", + KeyProviderKind::Local => "local-sgx", + KeyProviderKind::Kms => "kms", + KeyProviderKind::Tpm => "tpm", + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mr_config_v3_hash_changes_with_app_identity() { + let config = MrConfigV3::new( + vec![0x11; 20], + vec![0x22; 32], + KeyProviderKind::Kms, + vec![0x33; 32], + vec![0x44; 20], + ); + let mut changed = config.clone(); + changed.app_id[0] ^= 0xff; + + assert_ne!(config.to_snp_host_data(), changed.to_snp_host_data()); + assert_eq!(config.to_snp_host_data().len(), 32); + assert_ne!(config.to_tdx_mr_config_id(), changed.to_tdx_mr_config_id()); + assert_eq!(config.to_tdx_mr_config_id()[0], 3); + } + + #[test] + fn mr_config_v3_generates_jcs_but_hashes_document_bytes() -> Result<(), Box> { + let config = MrConfigV3::new( + vec![0x11; 20], + vec![0x22; 32], + KeyProviderKind::Kms, + vec![0x33; 32], + vec![0x44; 20], + ); + let document = config.to_canonical_json(); + + assert_eq!( + document, + concat!( + "{\"app_id\":\"1111111111111111111111111111111111111111\",", + "\"compose_hash\":\"2222222222222222222222222222222222222222222222222222222222222222\",", + "\"instance_id\":\"4444444444444444444444444444444444444444\",", + "\"key_provider\":\"kms\",", + "\"key_provider_id\":\"3333333333333333333333333333333333333333333333333333333333333333\",", + "\"version\":3}" + ) + ); + assert_eq!(MrConfigV3::from_document(&document)?, config); + + let pretty = serde_json::to_string_pretty(&config)?; + assert_eq!(MrConfigV3::from_document(&pretty)?, config); + assert_ne!( + MrConfigV3::snp_host_data_from_document(&document), + MrConfigV3::snp_host_data_from_document(&pretty) + ); + Ok(()) + } +} diff --git a/dstack-util/src/system_setup.rs b/dstack-util/src/system_setup.rs index 9cce282b7..73856a1ee 100644 --- a/dstack-util/src/system_setup.rs +++ b/dstack-util/src/system_setup.rs @@ -32,7 +32,7 @@ use ra_rpc::{ Attestation, }; use ra_tls::{ - attestation::QuoteContentType, + attestation::{AttestationMode, QuoteContentType}, cert::{generate_ra_cert, CertConfigV2, CertSigningRequestV2, Csr}, }; use rand::Rng as _; @@ -1386,6 +1386,9 @@ impl<'a> Stage0<'a> { let truncated_compose_hash = truncate(&compose_hash, 20); let key_provider = self.shared.app_compose.key_provider(); let mut instance_info = self.shared.instance_info.clone(); + let is_snp = AttestationMode::detect() + .map(|mode| mode == AttestationMode::DstackAmdSevSnp) + .unwrap_or(false); if instance_info.app_id.is_empty() { instance_info.app_id = truncated_compose_hash.to_vec(); @@ -1398,7 +1401,7 @@ impl<'a> Stage0<'a> { } let disk_reusable = !key_provider.is_none(); - if (!disk_reusable) || instance_info.instance_id_seed.is_empty() { + if ((!disk_reusable) && !is_snp) || instance_info.instance_id_seed.is_empty() { instance_info.instance_id_seed = { let mut rand_id = vec![0u8; 20]; getrandom::fill(&mut rand_id)?; @@ -1410,9 +1413,11 @@ impl<'a> Stage0<'a> { } else { let mut id_path = instance_info.instance_id_seed.clone(); id_path.extend_from_slice(&instance_info.app_id); - if let Some(binding) = platform_instance_binding()? { - info!("mixing platform per-instance binding into instance_id"); - id_path.extend_from_slice(&binding); + if !is_snp { + if let Some(binding) = platform_instance_binding()? { + info!("mixing platform per-instance binding into instance_id"); + id_path.extend_from_slice(&binding); + } } sha256(&id_path)[..20].to_vec() }; @@ -1446,6 +1451,7 @@ impl<'a> Stage0<'a> { .try_into() .ok() .context("Invalid app id")?, + &app_info.instance_info.instance_id, keys.key_provider.kind(), keys.key_provider.id(), )?; diff --git a/dstack-util/src/system_setup/config_id_verifier.rs b/dstack-util/src/system_setup/config_id_verifier.rs index d4ffdb2a5..5d67e8f06 100644 --- a/dstack-util/src/system_setup/config_id_verifier.rs +++ b/dstack-util/src/system_setup/config_id_verifier.rs @@ -3,10 +3,23 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::{bail, Context, Result}; -use dstack_attest::attestation::AttestationMode; -use dstack_types::{mr_config::MrConfig, KeyProviderKind}; +use dstack_attest::attestation::{Attestation, AttestationMode, AttestationQuote}; +use dstack_types::{ + mr_config::{MrConfig, MrConfigV3}, + shared_filenames::{HOST_SHARED_DIR, SYS_CONFIG}, + KeyProviderKind, SysConfig, +}; use tracing::info; +#[derive(Clone, Copy)] +struct ExpectedMrConfig<'a> { + compose_hash: &'a [u8; 32], + app_id: &'a [u8; 20], + instance_id: &'a [u8], + key_provider: KeyProviderKind, + key_provider_id: &'a [u8], +} + fn read_mr_config_id() -> Result<[u8; 48]> { let quote = tdx_attest::get_quote(&[0u8; 64]).context("Failed to get quote")?; let quote = dcap_qvl::quote::Quote::parse("e).context("Failed to parse quote")?; @@ -18,6 +31,35 @@ fn read_mr_config_id() -> Result<[u8; 48]> { Ok(configid) } +fn read_mr_config_document() -> Result { + let path = std::path::Path::new(HOST_SHARED_DIR).join(SYS_CONFIG); + let content = fs_err::read_to_string(path).context("Failed to read sys-config")?; + let sys_config: SysConfig = + serde_json::from_str(&content).context("Failed to parse sys-config")?; + if let Some(mr_config) = sys_config.mr_config { + return Ok(mr_config); + } + serde_json::from_str::(&sys_config.vm_config) + .ok() + .and_then(|value| { + value + .get("mr_config") + .and_then(|value| value.as_str()) + .map(ToString::to_string) + }) + .context("mr_config is required") +} + +fn read_snp_host_data() -> Result<[u8; 32]> { + let attestation = Attestation::quote(&[0u8; 64]).context("Failed to get SNP report")?; + let AttestationQuote::DstackAmdSevSnp(quote) = attestation.quote else { + bail!("attestation mode is not AMD SEV-SNP"); + }; + let parsed = dstack_attest::amd_sev_snp::parse_amd_snp_report("e.report) + .context("Failed to parse SNP report")?; + Ok(parsed.host_data) +} + /// Verify the mr_config_id matches the expected value /// /// Configuration ID format @@ -33,59 +75,178 @@ fn read_mr_config_id() -> Result<[u8; 48]> { pub fn verify_mr_config_id( compose_hash: &[u8; 32], app_id: &[u8; 20], + instance_id: &[u8], key_provider: KeyProviderKind, key_provider_id: &[u8], ) -> Result<()> { let mode = AttestationMode::detect().context("Failed to detect attestation mode")?; - verify_mr_config_id_for_mode(mode, compose_hash, app_id, key_provider, key_provider_id) + let expected = ExpectedMrConfig { + compose_hash, + app_id, + instance_id, + key_provider, + key_provider_id, + }; + verify_mr_config_id_for_mode(mode, expected) } fn verify_mr_config_id_for_mode( mode: AttestationMode, - compose_hash: &[u8; 32], - app_id: &[u8; 20], - key_provider: KeyProviderKind, - key_provider_id: &[u8], + expected: ExpectedMrConfig<'_>, ) -> Result<()> { - if mode == AttestationMode::DstackAmdSevSnp { - info!("Skipping TDX mr_config_id verification for AMD SEV-SNP guest"); - return Ok(()); + match mode { + AttestationMode::DstackAmdSevSnp => verify_snp_mr_config(expected), + _ => verify_tdx_mr_config_id(expected), } +} +fn verify_tdx_mr_config_id(expected: ExpectedMrConfig<'_>) -> Result<()> { let read_mr_config_id = read_mr_config_id().context("Failed to read mr_config_id")?; info!("mr_config_id: {}", hex::encode(read_mr_config_id)); + let mr_config_document = if read_mr_config_id[0] == 3 { + Some(read_mr_config_document().context("Failed to read mr_config")?) + } else { + None + }; + verify_tdx_mr_config_id_value(read_mr_config_id, mr_config_document.as_deref(), expected) +} + +fn verify_tdx_mr_config_id_value( + read_mr_config_id: [u8; 48], + mr_config_document: Option<&str>, + expected: ExpectedMrConfig<'_>, +) -> Result<()> { if read_mr_config_id == [0u8; 48] { return Ok(()); } - let mr_config = match read_mr_config_id[0] { - 1 => MrConfig::V1 { compose_hash }, + let expected_mr_config_id = match read_mr_config_id[0] { + 1 => MrConfig::V1 { + compose_hash: expected.compose_hash, + } + .to_mr_config_id(), 2 => MrConfig::V2 { - compose_hash, - app_id, - key_provider, - key_provider_id, - }, + compose_hash: expected.compose_hash, + app_id: expected.app_id, + key_provider: expected.key_provider, + key_provider_id: expected.key_provider_id, + } + .to_mr_config_id(), + 3 => { + let mr_config_document = + mr_config_document.context("mr_config is required for TDX MR_CONFIG_ID v3")?; + verify_mr_config_v3_document(mr_config_document, expected)?; + MrConfigV3::tdx_mr_config_id_from_document(mr_config_document) + } _ => bail!("Invalid mr_config_id version"), }; - if mr_config.to_mr_config_id() != read_mr_config_id { + if expected_mr_config_id != read_mr_config_id { bail!("Invalid mr_config_id"); } Ok(()) } +fn verify_snp_mr_config(expected: ExpectedMrConfig<'_>) -> Result<()> { + let mr_config_document = read_mr_config_document().context("Failed to read SNP mr_config")?; + verify_mr_config_v3_document(&mr_config_document, expected)?; + let read_host_data = read_snp_host_data().context("Failed to read SNP HOST_DATA")?; + info!("snp host_data: {}", hex::encode(read_host_data)); + if MrConfigV3::snp_host_data_from_document(&mr_config_document) != read_host_data { + bail!("Invalid SNP HOST_DATA"); + } + Ok(()) +} + +fn verify_mr_config_v3_document( + mr_config_document: &str, + expected: ExpectedMrConfig<'_>, +) -> Result { + let mr_config = + MrConfigV3::from_document(mr_config_document).context("Invalid mr_config document")?; + if mr_config.version != 3 { + bail!("mr_config version must be 3"); + } + if mr_config.compose_hash.as_slice() != expected.compose_hash { + bail!("Invalid mr_config compose_hash"); + } + if mr_config.app_id.as_slice() != expected.app_id { + bail!("Invalid mr_config app_id"); + } + if mr_config.instance_id.as_slice() != expected.instance_id { + bail!("Invalid mr_config instance_id"); + } + if mr_config.key_provider != expected.key_provider { + bail!("Invalid mr_config key_provider"); + } + if mr_config.key_provider_id.as_slice() != expected.key_provider_id { + bail!("Invalid mr_config key_provider_id"); + } + Ok(mr_config) +} + #[cfg(test)] mod tests { use super::*; #[test] - fn amd_sev_snp_skips_tdx_mr_config_id_quote_verification() { - verify_mr_config_id_for_mode( - AttestationMode::DstackAmdSevSnp, - &[0u8; 32], - &[0u8; 20], - KeyProviderKind::None, - &[], + fn tdx_mr_config_id_v1_accepts_expected_value() { + let compose_hash = [0x11u8; 32]; + let mr_config = MrConfig::V1 { + compose_hash: &compose_hash, + }; + assert_eq!(mr_config.to_mr_config_id()[0], 1); + } + + #[test] + fn tdx_mr_config_id_v3_accepts_document_value() -> Result<()> { + let compose_hash = [0x22u8; 32]; + let app_id = [0x11u8; 20]; + let instance_id = [0x44u8; 20]; + let key_provider_id = [0x33u8; 32]; + let mr_config = MrConfigV3::new( + app_id.to_vec(), + compose_hash.to_vec(), + KeyProviderKind::Kms, + key_provider_id.to_vec(), + instance_id.to_vec(), + ); + let document = mr_config.to_canonical_json(); + let expected = ExpectedMrConfig { + compose_hash: &compose_hash, + app_id: &app_id, + instance_id: &instance_id, + key_provider: KeyProviderKind::Kms, + key_provider_id: &key_provider_id, + }; + + verify_tdx_mr_config_id_value(mr_config.to_tdx_mr_config_id(), Some(&document), expected) + } + + #[test] + fn mr_config_v3_document_must_match_expected_app_info() { + let compose_hash = [0x22u8; 32]; + let app_id = [0x11u8; 20]; + let instance_id = [0x44u8; 20]; + let key_provider_id = [0x33u8; 32]; + let document = MrConfigV3::new( + app_id.to_vec(), + compose_hash.to_vec(), + KeyProviderKind::Kms, + key_provider_id.to_vec(), + instance_id.to_vec(), ) - .unwrap(); + .to_canonical_json(); + let wrong_app_id = [0x12u8; 20]; + let expected = ExpectedMrConfig { + compose_hash: &compose_hash, + app_id: &wrong_app_id, + instance_id: &instance_id, + key_provider: KeyProviderKind::Kms, + key_provider_id: &key_provider_id, + }; + + match verify_mr_config_v3_document(&document, expected) { + Ok(_) => panic!("mismatched app_id must reject"), + Err(err) => assert!(err.to_string().contains("Invalid mr_config app_id")), + } } } diff --git a/kms/src/main_service.rs b/kms/src/main_service.rs index cf5e60aec..6a150da4c 100644 --- a/kms/src/main_service.rs +++ b/kms/src/main_service.rs @@ -30,7 +30,7 @@ use scale::Decode; use sha2::Digest; use tokio::sync::OnceCell; use tracing::{info, warn}; -use upgrade_authority::{build_boot_info, local_kms_boot_info, BootInfo}; +use upgrade_authority::{build_boot_info, ensure_app_id_len, local_kms_boot_info, BootInfo}; use crate::{ config::{KmsConfig, SevSnpKeyReleaseConfig, SevSnpMeasureConfig}, @@ -436,6 +436,7 @@ impl KmsRpc for RpcHandler { self.ensure_self_allowed() .await .context("KMS self authorization failed")?; + ensure_app_id_len(&request.app_id)?; let secret = kdf::derive_dh_secret( &self.state.root_ca.key, &[&request.app_id[..], "env-encrypt-key".as_bytes()], @@ -662,20 +663,33 @@ mod tests { } } + fn valid_snp_mr_config() -> dstack_types::mr_config::MrConfigV3 { + dstack_types::mr_config::MrConfigV3::new( + vec![0x11; 20], + vec![0x22; 32], + dstack_types::KeyProviderKind::None, + Vec::new(), + vec![0x99; 20], + ) + } + fn verified_snp_attestation(measurement: [u8; 48], chip_id: [u8; 64]) -> VerifiedAttestation { - verified_snp_attestation_with_config(measurement, chip_id, String::new()) + let mr_config = valid_snp_mr_config(); + verified_snp_attestation_with_config(measurement, chip_id, String::new(), &mr_config) } fn verified_snp_attestation_with_config( measurement: [u8; 48], chip_id: [u8; 64], config: String, + mr_config: &dstack_types::mr_config::MrConfigV3, ) -> VerifiedAttestation { VerifiedAttestation { quote: ra_tls::attestation::AttestationQuote::DstackAmdSevSnp( ra_tls::attestation::SnpQuote { report: Vec::new(), cert_chain: Vec::new(), + mr_config: mr_config.to_canonical_json(), }, ), runtime_events: Vec::new(), @@ -685,6 +699,7 @@ mod tests { dstack_attest::amd_sev_snp::VerifiedAmdSnpReport { measurement, report_data: [0x42; 64], + host_data: mr_config.to_snp_host_data(), chip_id, tcb_info: dstack_attest::amd_sev_snp::AmdSnpTcbInfo::default(), advisory_ids: Vec::new(), @@ -697,9 +712,11 @@ mod tests { fn build_boot_info_for_attestation_accepts_snp_vm_config_path() { let input = valid_snp_measurement_input(); let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); + let mr_config = valid_snp_mr_config(); let attestation = verified_snp_attestation(measurement, [0xab; 64]); let vm_config = serde_json::json!({ "sev_snp_measurement": input, + "mr_config": mr_config.to_canonical_json(), }) .to_string(); @@ -712,7 +729,7 @@ mod tests { .expect("snp attestation should build boot info through vm_config path"); assert_eq!(boot_info.attestation_mode, AttestationMode::DstackAmdSevSnp); - assert_eq!(boot_info.mr_aggregated, measurement.to_vec()); + assert_eq!(boot_info.mr_aggregated.len(), 32); assert_eq!(boot_info.device_id, vec![0xab; 64]); assert_eq!(boot_info.app_id, vec![0x11; 20]); } @@ -721,19 +738,25 @@ mod tests { fn build_boot_info_for_attestation_uses_embedded_snp_vm_config_when_external_is_empty() { let input = valid_snp_measurement_input(); let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); + let mr_config = valid_snp_mr_config(); let embedded_config = serde_json::json!({ "sev_snp_measurement": input, + "mr_config": mr_config.to_canonical_json(), }) .to_string(); - let attestation = - verified_snp_attestation_with_config(measurement, [0xab; 64], embedded_config); + let attestation = verified_snp_attestation_with_config( + measurement, + [0xab; 64], + embedded_config, + &mr_config, + ); let boot_info = build_boot_info_for_attestation(Some(&sev_snp_config()), &attestation, false, "") .expect("snp local KMS attestation should use embedded vm_config"); assert_eq!(boot_info.attestation_mode, AttestationMode::DstackAmdSevSnp); - assert_eq!(boot_info.mr_aggregated, measurement.to_vec()); + assert_eq!(boot_info.mr_aggregated.len(), 32); assert_eq!(boot_info.app_id, vec![0x11; 20]); } @@ -741,9 +764,11 @@ mod tests { fn build_boot_info_for_attestation_requires_snp_config_for_snp() { let input = valid_snp_measurement_input(); let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); + let mr_config = valid_snp_mr_config(); let attestation = verified_snp_attestation(measurement, [0xab; 64]); let vm_config = serde_json::json!({ "sev_snp_measurement": input, + "mr_config": mr_config.to_canonical_json(), }) .to_string(); @@ -758,9 +783,11 @@ mod tests { fn snp_boot_info() -> BootInfo { let input = valid_snp_measurement_input(); let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); + let mr_config = valid_snp_mr_config(); let attestation = verified_snp_attestation(measurement, [0xab; 64]); let vm_config = serde_json::json!({ "sev_snp_measurement": input, + "mr_config": mr_config.to_canonical_json(), }) .to_string(); build_boot_info_for_attestation(Some(&sev_snp_config()), &attestation, false, &vm_config) diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs index 4715cd541..7159fa1de 100644 --- a/kms/src/main_service/amd_attest.rs +++ b/kms/src/main_service/amd_attest.rs @@ -9,15 +9,16 @@ //! the recomputed value to the hardware-verified report measurement. KMS release //! paths must apply their own explicit local release gate after auth succeeds. //! -//! Important: this is launch measurement binding, not a complete authorization -//! decision. `app_id`, compose hash, and rootfs hash are included in the SNP -//! measured kernel command line so the recomputed launch `MEASUREMENT` changes -//! with app identity, matching dstack's TDX measured-identity semantics. Do not -//! use this helper by itself to release app keys. +//! Important: this is launch measurement binding plus HOST_DATA app binding, +//! not a complete authorization decision. Launch `MEASUREMENT` covers the SNP +//! boot inputs; app identity is bound by checking that the verified report +//! `HOST_DATA` equals the attached MrConfigV3 document hash. Do not use this +//! helper by itself to release app keys. #![allow(dead_code)] use anyhow::{bail, Context, Result}; +use dstack_types::{mr_config::MrConfigV3, KeyProviderInfo}; use ra_tls::attestation::{AttestationMode, VerifiedAttestation}; use sha2::{Digest, Sha256, Sha384}; use std::fs; @@ -51,15 +52,15 @@ pub(crate) struct OvmfSectionParam { #[cfg_attr(test, derive(serde::Serialize))] #[serde(deny_unknown_fields)] pub(crate) struct MeasurementInput { - /// 20-byte app identity included in the measured kernel cmdline for SNP, - /// matching TDX's app-id-in-measured-identity semantics. + /// Deprecated: app identity is now bound through MrConfigV3/HOST_DATA. + #[serde(default)] pub app_id: String, - /// 32-byte docker compose hash included in the measured kernel cmdline. + /// Deprecated: compose identity is now bound through MrConfigV3/HOST_DATA. + #[serde(default)] pub compose_hash: String, - /// 32-byte rootfs hash included in the measured kernel cmdline. + /// 32-byte rootfs hash used as the SNP os_image_hash authorization input. pub rootfs_hash: String, - /// Original image kernel cmdline used as the base for SNP measured launch - /// before app identity fields are appended. + /// Original image kernel cmdline used for SNP measured launch. pub base_cmdline: Option, /// Optional 32-byte additional docker files hash included in the measured /// kernel cmdline when present. @@ -141,15 +142,11 @@ pub(crate) fn validate_amd_snp_measurement_binding( /// SEV-SNP report without releasing KMS key material by itself. /// /// This helper first recomputes and validates the QEMU SNP launch measurement. -/// `mr_aggregated` is the hardware-verified 48-byte SNP `MEASUREMENT`, and -/// `device_id` is the hardware-verified 64-byte SNP `chip_id`. `app_id`, -/// `compose_hash`, and `rootfs_hash` are bound through the measured kernel -/// command line; `os_image_hash` is therefore represented by `rootfs_hash`. -/// -/// Authorization-specific digests are domain separated and deterministic: -/// * `mr_system = sha256("dstack-amd-sev-snp:mr-system:v1" || launch/system inputs)` -/// * `key_provider_info = sha256("dstack-amd-sev-snp:app-binding:v1" || mr_system || app_id || compose_hash || chip_id)` -/// * `instance_id = sha256("dstack-amd-sev-snp:instance-id:v1" || chip_id || measurement || app_id || compose_hash)` +/// `mr_system` is `sha256(MEASUREMENT)`, `mr_aggregated` is +/// `sha256(MEASUREMENT || HOST_DATA)`, and `device_id` is the +/// hardware-verified 64-byte SNP `chip_id`. `app_id`, `compose_hash`, +/// `instance_id`, and key provider identity come from the MrConfigV3 document +/// bound by HOST_DATA. /// /// Keeping these values explicit lets authorization/release policy inspect /// exactly which SNP-specific inputs were bound before any sensitive output path @@ -161,51 +158,47 @@ pub(crate) fn build_amd_snp_boot_info( verified_chip_id: &[u8; 64], input: &MeasurementInput, ) -> Result { + let mr_config = test_mr_config_from_input(input)?; + let mr_config_document = mr_config.to_canonical_json(); + let host_data = MrConfigV3::snp_host_data_from_document(&mr_config_document); build_amd_snp_boot_info_with_tcb_status( config, verified_measurement, + &host_data, verified_chip_id, "UpToDate", &[], input, + &mr_config_document, ) } fn build_amd_snp_boot_info_with_tcb_status( config: &SevSnpMeasureConfig, verified_measurement: &[u8; 48], + verified_host_data: &[u8; 32], verified_chip_id: &[u8; 64], tcb_status: &str, advisory_ids: &[String], input: &MeasurementInput, + mr_config_document: &str, ) -> Result { validate_amd_snp_measurement_binding(Some(config), verified_measurement, input)?; + let mr_config = validate_snp_mr_config_binding(verified_host_data, mr_config_document)?; - let app_id = decode_required_hex("app_id", &input.app_id, 20)?; - let compose_hash = decode_required_hex("compose_hash", &input.compose_hash, 32)?; let rootfs_hash = decode_required_hex("rootfs_hash", &input.rootfs_hash, 32)?; - let mr_system = snp_mr_system_digest(config, verified_measurement, input)?; - let key_provider_info = snp_app_binding_digest( - &mr_system, - &app_id, - &compose_hash, - verified_chip_id.as_slice(), - ); - let instance_id = snp_instance_id_digest( - verified_chip_id.as_slice(), - verified_measurement, - &app_id, - &compose_hash, - ); + let mr_system = Sha256::digest(verified_measurement).to_vec(); + let mr_aggregated = snp_mr_aggregated_digest(verified_measurement, verified_host_data); + let key_provider_info = mr_config_key_provider_info(&mr_config)?; Ok(BootInfo { attestation_mode: AttestationMode::DstackAmdSevSnp, - mr_aggregated: verified_measurement.to_vec(), + mr_aggregated, os_image_hash: rootfs_hash, mr_system, - app_id, - compose_hash, - instance_id, + app_id: mr_config.app_id.clone(), + compose_hash: mr_config.compose_hash.clone(), + instance_id: mr_config.instance_id.clone(), device_id: verified_chip_id.to_vec(), key_provider_info, tcb_status: tcb_status.to_string(), @@ -224,6 +217,7 @@ pub(crate) fn build_amd_snp_boot_info_from_verified_attestation( config: &SevSnpMeasureConfig, attestation: &VerifiedAttestation, input: &MeasurementInput, + mr_config_document: &str, ) -> Result { let verified = attestation .report @@ -232,16 +226,19 @@ pub(crate) fn build_amd_snp_boot_info_from_verified_attestation( build_amd_snp_boot_info_with_tcb_status( config, &verified.measurement, + &verified.host_data, &verified.chip_id, verified.tcb_info.tcb_status(), &verified.advisory_ids, input, + mr_config_document, ) } #[derive(Debug, serde::Deserialize)] struct SevSnpMeasurementVmConfig { sev_snp_measurement: Option, + mr_config: Option, } /// Parses SNP launch-measurement inputs from the KMS request `vm_config` and @@ -254,26 +251,56 @@ pub(crate) fn build_amd_snp_boot_info_from_verified_attestation_and_vm_config( attestation: &VerifiedAttestation, vm_config: &str, ) -> Result { - let input = parse_measurement_input_from_vm_config(vm_config)?; - build_amd_snp_boot_info_from_verified_attestation(config, attestation, &input) + let (input, mr_config_document) = parse_snp_inputs_from_vm_config(vm_config)?; + build_amd_snp_boot_info_from_verified_attestation( + config, + attestation, + &input, + &mr_config_document, + ) } fn parse_measurement_input_from_vm_config(vm_config: &str) -> Result { - let parsed: SevSnpMeasurementVmConfig = serde_json::from_str(vm_config) - .context("failed to parse vm_config for amd sev-snp measurement")?; - parsed + Ok(parse_snp_inputs_from_vm_config(vm_config)?.0) +} + +fn parse_snp_inputs_from_vm_config(vm_config: &str) -> Result<(MeasurementInput, String)> { + let value: serde_json::Value = + serde_json::from_str(vm_config).context("failed to parse vm_config for amd sev-snp")?; + let parsed: SevSnpMeasurementVmConfig = serde_json::from_value(value.clone()) + .context("failed to parse vm_config for amd sev-snp")?; + let nested = value + .get("vm_config") + .and_then(|value| value.as_str()) + .map(|vm_config| { + serde_json::from_str::(vm_config) + .context("failed to parse nested vm_config for amd sev-snp") + }) + .transpose()?; + let measurement = parsed .sev_snp_measurement - .ok_or_else(|| anyhow::anyhow!("sev_snp_measurement is required for amd sev-snp")) + .or_else(|| { + nested + .as_ref() + .and_then(|nested| nested.sev_snp_measurement.clone()) + }) + .ok_or_else(|| anyhow::anyhow!("sev_snp_measurement is required for amd sev-snp"))?; + let mr_config = parsed + .mr_config + .or_else(|| nested.and_then(|nested| nested.mr_config)) + .ok_or_else(|| anyhow::anyhow!("mr_config is required for amd sev-snp"))?; + MrConfigV3::from_document(&mr_config).context("invalid amd sev-snp mr_config document")?; + Ok((measurement, mr_config)) } /// Explicit helper-only AMD SEV-SNP authorization policy. /// /// Explicit AMD SEV-SNP authorization policy: an SNP `BootInfo` must match -/// allowlisted hardware measurement, app/config identity, device identity, and -/// TCB/advisory policy. Empty allowlists fail closed. +/// allowlisted aggregated measurement digest, app/config identity, device +/// identity, and TCB/advisory policy. Empty allowlists fail closed. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct AmdSnpAuthPolicy { - pub allowed_measurements: Vec>, + pub allowed_mr_aggregated: Vec>, pub allowed_app_ids: Vec>, pub allowed_compose_hashes: Vec>, pub allowed_os_image_hashes: Vec>, @@ -289,7 +316,7 @@ impl AmdSnpAuthPolicy { pub(crate) fn from_boot_info(boot_info: &BootInfo) -> Result { ensure_snp_boot_info_shape(boot_info)?; Ok(Self { - allowed_measurements: vec![boot_info.mr_aggregated.clone()], + allowed_mr_aggregated: vec![boot_info.mr_aggregated.clone()], allowed_app_ids: vec![boot_info.app_id.clone()], allowed_compose_hashes: vec![boot_info.compose_hash.clone()], allowed_os_image_hashes: vec![boot_info.os_image_hash.clone()], @@ -306,9 +333,9 @@ pub(crate) fn validate_amd_snp_auth_policy( ) -> Result<()> { ensure_snp_boot_info_shape(boot_info)?; ensure_allowed_bytes( - "measurement", + "mr_aggregated", &boot_info.mr_aggregated, - &policy.allowed_measurements, + &policy.allowed_mr_aggregated, )?; ensure_allowed_bytes("app_id", &boot_info.app_id, &policy.allowed_app_ids)?; ensure_allowed_bytes( @@ -341,14 +368,15 @@ fn ensure_snp_boot_info_shape(boot_info: &BootInfo) -> Result<()> { if boot_info.attestation_mode != AttestationMode::DstackAmdSevSnp { bail!("attestation mode is not amd sev-snp"); } - ensure_len("measurement", &boot_info.mr_aggregated, 48)?; + ensure_len("mr_aggregated", &boot_info.mr_aggregated, 32)?; ensure_len("app_id", &boot_info.app_id, 20)?; ensure_len("compose_hash", &boot_info.compose_hash, 32)?; ensure_len("os_image_hash", &boot_info.os_image_hash, 32)?; ensure_len("device_id", &boot_info.device_id, 64)?; ensure_len("mr_system", &boot_info.mr_system, 32)?; - ensure_len("key_provider_info", &boot_info.key_provider_info, 32)?; - ensure_len("instance_id", &boot_info.instance_id, 32)?; + if !boot_info.instance_id.is_empty() { + ensure_len("instance_id", &boot_info.instance_id, 20)?; + } if boot_info.tcb_status.trim().is_empty() { bail!("tcb_status is not allowed"); } @@ -379,92 +407,58 @@ fn ensure_allowed_string(name: &str, value: &str, allowed: &[String]) -> Result< bail!("{name} is not allowed") } -fn snp_mr_system_digest( - config: &SevSnpMeasureConfig, - verified_measurement: &[u8; 48], - input: &MeasurementInput, -) -> Result> { - let ovmf_hash = decode_optional_hex("ovmf_hash", &input.ovmf_hash, 48)?; - let kernel_hash = decode_required_hex("kernel_hash", &input.kernel_hash, 32)?; - let initrd_hash = if input.initrd_hash.is_empty() { - Sha256::digest(b"").to_vec() - } else { - decode_required_hex("initrd_hash", &input.initrd_hash, 32)? - }; - let rootfs_hash = decode_required_hex("rootfs_hash", &input.rootfs_hash, 32)?; - let docker_files_hash = input - .docker_files_hash - .as_deref() - .map(|value| decode_required_hex("docker_files_hash", value, 32)) - .transpose()?; - let vcpu_type = input - .vcpu_type - .as_deref() - .ok_or_else(|| anyhow::anyhow!("vcpu_type is required"))?; - +fn snp_mr_aggregated_digest(measurement: &[u8; 48], host_data: &[u8; 32]) -> Vec { let mut h = Sha256::new(); - h.update(b"dstack-amd-sev-snp:mr-system:v1"); - h.update(verified_measurement); - h.update(len_prefixed(&ovmf_hash)); - h.update(kernel_hash); - h.update(initrd_hash); - h.update(rootfs_hash); - h.update(input.vcpus.to_le_bytes()); - h.update(len_prefixed(vcpu_type.as_bytes())); - h.update(config.guest_features.to_le_bytes()); - match docker_files_hash { - Some(value) => { - h.update([1]); - h.update(value); - } - None => h.update([0]), - } - h.update(input.sev_hashes_table_gpa.to_le_bytes()); - h.update(input.sev_es_reset_eip.to_le_bytes()); - h.update((input.ovmf_sections.len() as u64).to_le_bytes()); - for section in &input.ovmf_sections { - h.update(section.gpa.to_le_bytes()); - h.update(section.size.to_le_bytes()); - h.update(section.section_type.to_le_bytes()); - } - Ok(h.finalize().to_vec()) + h.update(measurement); + h.update(host_data); + h.finalize().to_vec() } -fn snp_app_binding_digest( - mr_system: &[u8], - app_id: &[u8], - compose_hash: &[u8], - chip_id: &[u8], -) -> Vec { - let mut h = Sha256::new(); - h.update(b"dstack-amd-sev-snp:app-binding:v1"); - h.update(mr_system); - h.update(app_id); - h.update(compose_hash); - h.update(chip_id); - h.finalize().to_vec() +fn mr_config_key_provider_info(mr_config: &MrConfigV3) -> Result> { + serde_json::to_vec(&KeyProviderInfo::new( + mr_config.key_provider_name().to_string(), + hex::encode(&mr_config.key_provider_id), + )) + .context("failed to serialize key provider info") } -fn snp_instance_id_digest( - chip_id: &[u8], - measurement: &[u8], - app_id: &[u8], - compose_hash: &[u8], -) -> Vec { - let mut h = Sha256::new(); - h.update(b"dstack-amd-sev-snp:instance-id:v1"); - h.update(chip_id); - h.update(measurement); - h.update(app_id); - h.update(compose_hash); - h.finalize().to_vec() +fn validate_snp_mr_config_binding( + host_data: &[u8; 32], + mr_config_document: &str, +) -> Result { + let mr_config = MrConfigV3::from_document(mr_config_document) + .context("invalid amd sev-snp mr_config document")?; + let expected = MrConfigV3::snp_host_data_from_document(mr_config_document); + if expected != *host_data { + bail!("amd sev-snp host_data mismatch"); + } + validate_mr_config(&mr_config)?; + Ok(mr_config) +} + +fn validate_mr_config(mr_config: &MrConfigV3) -> Result<()> { + if mr_config.version != 3 { + bail!("mr_config version must be 3"); + } + ensure_len("mr_config.app_id", &mr_config.app_id, 20)?; + ensure_len("mr_config.compose_hash", &mr_config.compose_hash, 32)?; + if !mr_config.instance_id.is_empty() { + ensure_len("mr_config.instance_id", &mr_config.instance_id, 20)?; + } + Ok(()) } -fn len_prefixed(bytes: &[u8]) -> Vec { - let mut out = Vec::with_capacity(8 + bytes.len()); - out.extend_from_slice(&(bytes.len() as u64).to_le_bytes()); - out.extend_from_slice(bytes); - out +#[cfg(test)] +fn test_mr_config_from_input(input: &MeasurementInput) -> Result { + let app_id = decode_required_hex("app_id", &input.app_id, 20)?; + let instance_id = Sha256::digest(&app_id)[..20].to_vec(); + Ok(MrConfigV3::new( + app_id, + decode_required_hex("compose_hash", &input.compose_hash, 32)?, + dstack_types::KeyProviderKind::None, + Vec::new(), + instance_id, + )) } fn validate_measurement_input( @@ -475,11 +469,6 @@ fn validate_measurement_input( bail!("guest_features must be non-zero"); } - let app_id = decode_required_hex("app_id", &input.app_id, 20)?; - if app_id.iter().all(|&b| b == 0) { - bail!("app_id must not be all-zeros"); - } - decode_required_hex("compose_hash", &input.compose_hash, 32)?; decode_required_hex("rootfs_hash", &input.rootfs_hash, 32)?; decode_required_hex("kernel_hash", &input.kernel_hash, 32)?; decode_optional_hex("initrd_hash", &input.initrd_hash, 32)?; @@ -956,17 +945,8 @@ pub(crate) fn compute_expected_measurement( .ok_or_else(|| anyhow::anyhow!("vcpu_type is required"))?; let mut cmdline = match input.base_cmdline.as_deref() { - Some(base) if !base.trim().is_empty() => format!( - "{} docker_compose_hash={} rootfs_hash={} app_id={}", - base.trim(), - input.compose_hash, - input.rootfs_hash, - input.app_id - ), - _ => format!( - "console=ttyS0 loglevel=7 docker_compose_hash={} rootfs_hash={} app_id={}", - input.compose_hash, input.rootfs_hash, input.app_id - ), + Some(base) if !base.trim().is_empty() => base.trim().to_string(), + _ => "console=ttyS0 loglevel=7".to_string(), }; if let Some(docker_files_hash) = input.docker_files_hash.as_deref() { cmdline.push_str(&format!( @@ -1126,6 +1106,41 @@ mod tests { } } + fn valid_mr_config(input: &MeasurementInput) -> Result { + test_mr_config_from_input(input) + } + + fn verified_snp_attestation( + measurement: [u8; 48], + chip_id: [u8; 64], + mr_config: &MrConfigV3, + ) -> ra_tls::attestation::VerifiedAttestation { + ra_tls::attestation::VerifiedAttestation { + quote: ra_tls::attestation::AttestationQuote::DstackAmdSevSnp( + ra_tls::attestation::SnpQuote { + report: Vec::new(), + cert_chain: Vec::new(), + mr_config: mr_config.to_canonical_json(), + }, + ), + runtime_events: Vec::new(), + report_data: [0x42; 64], + config: String::new(), + report: ra_tls::attestation::DstackVerifiedReport::DstackAmdSevSnp( + dstack_attest::amd_sev_snp::VerifiedAmdSnpReport { + measurement, + report_data: [0x42; 64], + host_data: MrConfigV3::snp_host_data_from_document( + &mr_config.to_canonical_json(), + ), + chip_id, + tcb_info: dstack_attest::amd_sev_snp::AmdSnpTcbInfo::default(), + advisory_ids: Vec::new(), + }, + ), + } + } + fn assert_rejects(input: MeasurementInput, msg: &str) { let verified = [0xaa; 48]; let err = validate_amd_snp_measurement_binding(Some(&config()), &verified, &input) @@ -1187,7 +1202,7 @@ mod tests { let expected = compute_expected_measurement(&config(), &input).unwrap(); assert_eq!( hex::encode(expected), - "4753950048f296ea9cc36be3ba3e26f9cb014411188134d2ea40580a76edf277268cc46b67dfd213d1a7dfc9a9006e0f", + "56a10702f43f18df8a87404fb92637e65d2e069056e58938b3de085d2f33f070aeaa2bac0013e969a2fe283baf40eeaa", "synthetic measurement vector should not drift silently" ); validate_amd_snp_measurement_binding(Some(&config()), &expected, &input) @@ -1209,14 +1224,14 @@ mod tests { let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input) .expect("matching measurement should build snp boot info"); assert_eq!(boot_info.attestation_mode, AttestationMode::DstackAmdSevSnp); - assert_eq!(boot_info.mr_aggregated, verified.to_vec()); + assert_eq!(boot_info.mr_aggregated.len(), 32); assert_eq!(boot_info.device_id, chip_id.to_vec()); assert_eq!(boot_info.app_id, vec![0x11; 20]); assert_eq!(boot_info.compose_hash, vec![0x22; 32]); assert_eq!(boot_info.os_image_hash, vec![0x33; 32]); assert_eq!(boot_info.mr_system.len(), 32); - assert_eq!(boot_info.key_provider_info.len(), 32); - assert_eq!(boot_info.instance_id.len(), 32); + assert!(!boot_info.key_provider_info.is_empty()); + assert_eq!(boot_info.instance_id.len(), 20); assert_eq!(boot_info.tcb_status, "UpToDate"); assert_ne!(boot_info.tcb_status, "snp-verified-basic-policy"); assert!(boot_info.advisory_ids.is_empty()); @@ -1229,46 +1244,36 @@ mod tests { } #[test] - fn builds_snp_boot_info_from_verified_attestation_report() { + fn builds_snp_boot_info_from_verified_attestation_report() -> Result<()> { let input = valid_input(); let verified = compute_expected_measurement(&config(), &input).unwrap(); let chip_id = [0xab; 64]; - let attestation = ra_tls::attestation::VerifiedAttestation { - quote: ra_tls::attestation::AttestationQuote::DstackAmdSevSnp( - ra_tls::attestation::SnpQuote { - report: Vec::new(), - cert_chain: Vec::new(), - }, - ), - runtime_events: Vec::new(), - report_data: [0x42; 64], - config: String::new(), - report: ra_tls::attestation::DstackVerifiedReport::DstackAmdSevSnp( - dstack_attest::amd_sev_snp::VerifiedAmdSnpReport { - measurement: verified, - report_data: [0x42; 64], - chip_id, - tcb_info: dstack_attest::amd_sev_snp::AmdSnpTcbInfo::default(), - advisory_ids: Vec::new(), - }, - ), - }; + let mr_config = valid_mr_config(&input)?; + let mr_config_document = mr_config.to_canonical_json(); + let attestation = verified_snp_attestation(verified, chip_id, &mr_config); - let boot_info = - build_amd_snp_boot_info_from_verified_attestation(&config(), &attestation, &input) - .expect("verified snp attestation should feed boot info helper"); + let boot_info = build_amd_snp_boot_info_from_verified_attestation( + &config(), + &attestation, + &input, + &mr_config_document, + ) + .expect("verified snp attestation should feed boot info helper"); - assert_eq!(boot_info.mr_aggregated, verified.to_vec()); + assert_eq!(boot_info.mr_aggregated.len(), 32); assert_eq!(boot_info.device_id, chip_id.to_vec()); assert_eq!(boot_info.app_id, vec![0x11; 20]); assert_eq!(boot_info.tcb_status, "UpToDate"); + Ok(()) } #[test] - fn verified_attestation_tcb_status_replaces_snp_placeholder() { + fn verified_attestation_tcb_status_replaces_snp_placeholder() -> Result<()> { let input = valid_input(); let verified = compute_expected_measurement(&config(), &input).unwrap(); let chip_id = [0xbc; 64]; + let mr_config = valid_mr_config(&input)?; + let mr_config_document = mr_config.to_canonical_json(); let tcb = dstack_attest::amd_sev_snp::AmdSnpTcbVersion { bootloader: 1, tee: 2, @@ -1284,6 +1289,7 @@ mod tests { ra_tls::attestation::SnpQuote { report: Vec::new(), cert_chain: Vec::new(), + mr_config: mr_config_document.clone(), }, ), runtime_events: Vec::new(), @@ -1293,6 +1299,7 @@ mod tests { dstack_attest::amd_sev_snp::VerifiedAmdSnpReport { measurement: verified, report_data: [0x42; 64], + host_data: MrConfigV3::snp_host_data_from_document(&mr_config_document), chip_id, tcb_info: dstack_attest::amd_sev_snp::AmdSnpTcbInfo { current: tcb, @@ -1305,9 +1312,13 @@ mod tests { ), }; - let boot_info = - build_amd_snp_boot_info_from_verified_attestation(&config(), &attestation, &input) - .expect("verified snp attestation should feed boot info helper"); + let boot_info = build_amd_snp_boot_info_from_verified_attestation( + &config(), + &attestation, + &input, + &mr_config_document, + ) + .expect("verified snp attestation should feed boot info helper"); assert_eq!(boot_info.tcb_status, "OutOfDate"); assert_eq!(boot_info.advisory_ids, vec!["SNP-TEST-ADVISORY"]); @@ -1321,35 +1332,19 @@ mod tests { err.to_string().contains("tcb_status is not allowed"), "unexpected error: {err:?}" ); + Ok(()) } #[test] - fn builds_snp_boot_info_from_verified_attestation_and_vm_config_json() { + fn builds_snp_boot_info_from_verified_attestation_and_vm_config_json() -> Result<()> { let input = valid_input(); let verified = compute_expected_measurement(&config(), &input).unwrap(); let chip_id = [0xab; 64]; - let attestation = ra_tls::attestation::VerifiedAttestation { - quote: ra_tls::attestation::AttestationQuote::DstackAmdSevSnp( - ra_tls::attestation::SnpQuote { - report: Vec::new(), - cert_chain: Vec::new(), - }, - ), - runtime_events: Vec::new(), - report_data: [0x42; 64], - config: String::new(), - report: ra_tls::attestation::DstackVerifiedReport::DstackAmdSevSnp( - dstack_attest::amd_sev_snp::VerifiedAmdSnpReport { - measurement: verified, - report_data: [0x42; 64], - chip_id, - tcb_info: dstack_attest::amd_sev_snp::AmdSnpTcbInfo::default(), - advisory_ids: Vec::new(), - }, - ), - }; + let mr_config = valid_mr_config(&input)?; + let attestation = verified_snp_attestation(verified, chip_id, &mr_config); let vm_config = serde_json::json!({ "sev_snp_measurement": input, + "mr_config": mr_config.to_canonical_json(), }) .to_string(); @@ -1360,35 +1355,18 @@ mod tests { ) .expect("vm_config-carried snp measurement inputs should build boot info"); - assert_eq!(boot_info.mr_aggregated, verified.to_vec()); + assert_eq!(boot_info.mr_aggregated.len(), 32); assert_eq!(boot_info.device_id, chip_id.to_vec()); assert_eq!(boot_info.app_id, vec![0x11; 20]); + Ok(()) } #[test] - fn verified_attestation_vm_config_helper_requires_snp_measurement_input() { + fn verified_attestation_vm_config_helper_requires_snp_measurement_input() -> Result<()> { let input = valid_input(); let verified = compute_expected_measurement(&config(), &input).unwrap(); - let attestation = ra_tls::attestation::VerifiedAttestation { - quote: ra_tls::attestation::AttestationQuote::DstackAmdSevSnp( - ra_tls::attestation::SnpQuote { - report: Vec::new(), - cert_chain: Vec::new(), - }, - ), - runtime_events: Vec::new(), - report_data: [0x42; 64], - config: String::new(), - report: ra_tls::attestation::DstackVerifiedReport::DstackAmdSevSnp( - dstack_attest::amd_sev_snp::VerifiedAmdSnpReport { - measurement: verified, - report_data: [0x42; 64], - chip_id: [0xab; 64], - tcb_info: dstack_attest::amd_sev_snp::AmdSnpTcbInfo::default(), - advisory_ids: Vec::new(), - }, - ), - }; + let mr_config = valid_mr_config(&input)?; + let attestation = verified_snp_attestation(verified, [0xab; 64], &mr_config); let err = build_amd_snp_boot_info_from_verified_attestation_and_vm_config( &config(), @@ -1400,6 +1378,7 @@ mod tests { err.to_string().contains("sev_snp_measurement is required"), "unexpected error: {err:?}" ); + Ok(()) } #[test] @@ -1447,7 +1426,7 @@ mod tests { } #[test] - fn verified_attestation_helper_rejects_non_snp_reports() { + fn verified_attestation_helper_rejects_non_snp_reports() -> Result<()> { let input = valid_input(); let attestation = ra_tls::attestation::VerifiedAttestation { quote: ra_tls::attestation::AttestationQuote::DstackNitroEnclave( @@ -1472,45 +1451,44 @@ mod tests { ), }; - let err = - build_amd_snp_boot_info_from_verified_attestation(&config(), &attestation, &input) - .expect_err("non-snp verified attestation must reject"); + let mr_config = valid_mr_config(&input)?; + let mr_config_document = mr_config.to_canonical_json(); + let err = build_amd_snp_boot_info_from_verified_attestation( + &config(), + &attestation, + &input, + &mr_config_document, + ) + .expect_err("non-snp verified attestation must reject"); assert!( err.to_string() .contains("verified attestation is not amd sev-snp"), "unexpected error: {err:?}" ); + Ok(()) } #[test] - fn app_id_changes_launch_measurement_and_authorization_binding() { + fn app_id_changes_host_data_and_authorization_binding() -> Result<()> { let input = valid_input(); - let verified = compute_expected_measurement(&config(), &input).unwrap(); + let verified = compute_expected_measurement(&config(), &input)?; let chip_id = [0xcd; 64]; - let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input).unwrap(); + let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input)?; let mut changed = input.clone(); changed.app_id = hex_of(0x12, 20); - let changed_measurement = compute_expected_measurement(&config(), &changed).unwrap(); - assert_ne!( + let changed_measurement = compute_expected_measurement(&config(), &changed)?; + assert_eq!( changed_measurement, verified, - "app_id must be launch-measured for SNP to match TDX app identity semantics" + "app_id must not be added to the SNP measured cmdline" ); - let err = build_amd_snp_boot_info(&config(), &verified, &chip_id, &changed) - .expect_err("stale measurement must reject changed app_id"); - assert!(err.to_string().contains("amd sev-snp measurement mismatch")); - - let changed_boot_info = - build_amd_snp_boot_info(&config(), &changed_measurement, &chip_id, &changed).unwrap(); + let changed_boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &changed)?; assert_ne!(boot_info.app_id, changed_boot_info.app_id); assert_ne!(boot_info.instance_id, changed_boot_info.instance_id); - assert_ne!( - boot_info.key_provider_info, - changed_boot_info.key_provider_info - ); assert_ne!(boot_info.mr_aggregated, changed_boot_info.mr_aggregated); - assert_ne!(boot_info.mr_system, changed_boot_info.mr_system); + assert_eq!(boot_info.mr_system, changed_boot_info.mr_system); + Ok(()) } #[test] @@ -1521,8 +1499,6 @@ mod tests { let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input).unwrap(); for mutate in [ - |i: &mut MeasurementInput| i.compose_hash = hex_of(0x23, 32), - |i: &mut MeasurementInput| i.rootfs_hash = hex_of(0x34, 32), |i: &mut MeasurementInput| i.kernel_hash = hex_of(0x56, 32), |i: &mut MeasurementInput| i.vcpus = 3, ] { @@ -1552,8 +1528,8 @@ mod tests { assert_eq!(boot_info.device_id, vec![0x01; 64]); assert_eq!(changed_boot_info.device_id, vec![0x02; 64]); assert_ne!(boot_info.device_id, changed_boot_info.device_id); - assert_ne!(boot_info.instance_id, changed_boot_info.instance_id); - assert_ne!( + assert_eq!(boot_info.instance_id, changed_boot_info.instance_id); + assert_eq!( boot_info.key_provider_info, changed_boot_info.key_provider_info ); @@ -1594,10 +1570,7 @@ mod tests { let recomputed = compute_expected_measurement(&config, &input) .expect("dstack recomputation should succeed"); - let append = format!( - "console=ttyS0 loglevel=7 docker_compose_hash={} rootfs_hash={} app_id={}", - input.compose_hash, input.rootfs_hash, input.app_id - ); + let append = "console=ttyS0 loglevel=7"; let output = std::process::Command::new("sev-snp-measure") .args([ "--mode", @@ -1613,7 +1586,7 @@ mod tests { "--initrd", initrd_path.to_str().unwrap(), "--append", - &append, + append, "--guest-features", "0x1", "--output-format", @@ -1632,11 +1605,6 @@ mod tests { .to_string(); assert_eq!(hex::encode(recomputed), tool_measurement); - assert_eq!( - tool_measurement, - "6497fb9f90dc4a322228a8a5eb14742e09067bc44c184c2068d583ef628b5bae8c6cf15d91fe1bc0b7a8cbcc575be370", - "live sev-snp-measure golden vector should not drift silently" - ); } #[test] @@ -1695,7 +1663,7 @@ mod tests { let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input).unwrap(); for mutate in [ - |p: &mut AmdSnpAuthPolicy| p.allowed_measurements.clear(), + |p: &mut AmdSnpAuthPolicy| p.allowed_mr_aggregated.clear(), |p: &mut AmdSnpAuthPolicy| p.allowed_app_ids.clear(), |p: &mut AmdSnpAuthPolicy| p.allowed_compose_hashes.clear(), |p: &mut AmdSnpAuthPolicy| p.allowed_os_image_hashes.clear(), @@ -1725,18 +1693,6 @@ mod tests { #[test] fn rejects_empty_or_malformed_binding_hashes() { - let mut input = valid_input(); - input.app_id.clear(); - assert_rejects(input, "app_id must not be empty"); - - let mut input = valid_input(); - input.app_id = hex_of(0x00, 20); - assert_rejects(input, "app_id must not be all-zeros"); - - let mut input = valid_input(); - input.compose_hash = "not hex".to_string(); - assert_rejects(input, "compose_hash must be valid hex"); - let mut input = valid_input(); input.rootfs_hash = hex_of(0x33, 31); assert_rejects(input, "rootfs_hash must be 32 bytes"); diff --git a/kms/src/main_service/upgrade_authority.rs b/kms/src/main_service/upgrade_authority.rs index 9f164cb60..4900d183d 100644 --- a/kms/src/main_service/upgrade_authority.rs +++ b/kms/src/main_service/upgrade_authority.rs @@ -58,6 +58,7 @@ pub(crate) fn build_boot_info( } }; let app_info = att.decode_app_info_ex(use_boottime_mr, vm_config_str)?; + ensure_app_id_len(&app_info.app_id)?; Ok(BootInfo { attestation_mode: att.quote.mode(), mr_aggregated: app_info.mr_aggregated.to_vec(), @@ -73,6 +74,13 @@ pub(crate) fn build_boot_info( }) } +pub(crate) fn ensure_app_id_len(app_id: &[u8]) -> Result<()> { + if app_id.len() != 20 { + bail!("app_id must be 20 bytes"); + } + Ok(()) +} + pub(crate) async fn local_kms_boot_info( pccs_url: Option<&str>, sev_snp_config: Option<&SevSnpMeasureConfig>, @@ -265,3 +273,18 @@ pub(crate) async fn ensure_kms_allowed( } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn app_id_len_must_be_20_bytes() { + assert!(ensure_app_id_len(&[0u8; 20]).is_ok()); + + match ensure_app_id_len(&[0u8; 19]) { + Ok(()) => panic!("19-byte app_id must reject"), + Err(err) => assert!(err.to_string().contains("app_id must be 20 bytes")), + } + } +} diff --git a/kms/src/onboard_service.rs b/kms/src/onboard_service.rs index 23b162032..3ea042abf 100644 --- a/kms/src/onboard_service.rs +++ b/kms/src/onboard_service.rs @@ -253,12 +253,24 @@ mod tests { } } + fn valid_snp_mr_config() -> dstack_types::mr_config::MrConfigV3 { + dstack_types::mr_config::MrConfigV3::new( + vec![0x11; 20], + vec![0x22; 32], + dstack_types::KeyProviderKind::None, + Vec::new(), + vec![0x99; 20], + ) + } + fn verified_snp_attestation(measurement: [u8; 48], chip_id: [u8; 64]) -> VerifiedAttestation { + let mr_config = valid_snp_mr_config(); VerifiedAttestation { quote: ra_tls::attestation::AttestationQuote::DstackAmdSevSnp( ra_tls::attestation::SnpQuote { report: Vec::new(), cert_chain: Vec::new(), + mr_config: mr_config.to_canonical_json(), }, ), runtime_events: Vec::new(), @@ -268,6 +280,7 @@ mod tests { dstack_attest::amd_sev_snp::VerifiedAmdSnpReport { measurement, report_data: [0x42; 64], + host_data: mr_config.to_snp_host_data(), chip_id, tcb_info: dstack_attest::amd_sev_snp::AmdSnpTcbInfo::default(), advisory_ids: Vec::new(), @@ -280,9 +293,11 @@ mod tests { fn attestation_info_response_uses_snp_boot_info_and_chip_id() { let input = valid_snp_measurement_input(); let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); + let mr_config = valid_snp_mr_config(); let attestation = verified_snp_attestation(measurement, [0xab; 64]); let vm_config = serde_json::json!({ "sev_snp_measurement": input, + "mr_config": mr_config.to_canonical_json(), }) .to_string(); @@ -302,7 +317,7 @@ mod tests { sha2::Sha256::digest([0xab; 64]).to_vec() ); assert_eq!(response.ppid, vec![0xab; 64]); - assert_eq!(response.mr_aggregated, measurement.to_vec()); + assert_eq!(response.mr_aggregated.len(), 32); assert_eq!(response.os_image_hash, vec![0x33; 32]); assert_eq!(response.attestation_mode, "dstack-amd-sev-snp"); assert_eq!(response.site_name, "test-site"); diff --git a/sev-snp-qvl/src/lib.rs b/sev-snp-qvl/src/lib.rs index ad305886f..1fbfe87ea 100644 --- a/sev-snp-qvl/src/lib.rs +++ b/sev-snp-qvl/src/lib.rs @@ -84,11 +84,20 @@ impl AmdSnpTcbInfo { pub struct VerifiedAmdSnpReport { pub measurement: [u8; 48], pub report_data: [u8; 64], + pub host_data: [u8; 32], pub chip_id: [u8; 64], pub tcb_info: AmdSnpTcbInfo, pub advisory_ids: Vec, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ParsedAmdSnpReport { + pub measurement: [u8; 48], + pub report_data: [u8; 64], + pub host_data: [u8; 32], + pub chip_id: [u8; 64], +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum CertEncoding { Pem, @@ -130,6 +139,60 @@ pub fn verify_amd_snp_attestation( ) } +pub fn parse_amd_snp_report(report_bytes: &[u8]) -> Result { + if report_bytes.len() != 1184 { + bail!( + "invalid amd sev-snp report length: expected 1184 bytes, got {}", + report_bytes.len() + ); + } + let report = AttestationReport::from_bytes(report_bytes) + .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; + parsed_amd_snp_report_from_report(&report) +} + +fn parsed_amd_snp_report_from_report(report: &AttestationReport) -> Result { + let mut measurement = [0u8; 48]; + measurement.copy_from_slice( + report + .measurement + .as_ref() + .get(..48) + .context("amd sev-snp measurement too short")?, + ); + let mut report_data = [0u8; 64]; + report_data.copy_from_slice( + report + .report_data + .as_ref() + .get(..64) + .context("amd sev-snp report_data too short")?, + ); + let mut host_data = [0u8; 32]; + host_data.copy_from_slice( + report + .host_data + .as_ref() + .get(..32) + .context("amd sev-snp host_data too short")?, + ); + let mut chip_id = [0u8; 64]; + chip_id.copy_from_slice( + report + .chip_id + .as_ref() + .get(..64) + .context("amd sev-snp chip_id too short")?, + ); + + Ok(ParsedAmdSnpReport { + measurement, + report_data, + host_data, + chip_id, + }) +} + fn verify_amd_snp_attestation_with_certs( report_bytes: &[u8], ask_bytes: CertBytes, @@ -180,35 +243,13 @@ fn verify_amd_snp_attestation_with_cert_chain( })?; validate_amd_snp_report_policy(&report)?; - let mut measurement = [0u8; 48]; - measurement.copy_from_slice( - report - .measurement - .as_ref() - .get(..48) - .context("amd sev-snp measurement too short")?, - ); - let mut report_data = [0u8; 64]; - report_data.copy_from_slice( - report - .report_data - .as_ref() - .get(..64) - .context("amd sev-snp report_data too short")?, - ); - let mut chip_id = [0u8; 64]; - chip_id.copy_from_slice( - report - .chip_id - .as_ref() - .get(..64) - .context("amd sev-snp chip_id too short")?, - ); + let parsed = parsed_amd_snp_report_from_report(&report)?; Ok(VerifiedAmdSnpReport { - measurement, - report_data, - chip_id, + measurement: parsed.measurement, + report_data: parsed.report_data, + host_data: parsed.host_data, + chip_id: parsed.chip_id, tcb_info: AmdSnpTcbInfo::from_report(&report), // AMD SEV-SNP attestation reports and VCEKs do not carry a direct // advisory list. Keep this explicit and empty so downstream auth stays diff --git a/vmm/Cargo.toml b/vmm/Cargo.toml index 0376b30e6..31150243c 100644 --- a/vmm/Cargo.toml +++ b/vmm/Cargo.toml @@ -23,6 +23,7 @@ uuid = { workspace = true, features = ["v4"] } sha2.workspace = true hex.workspace = true fs-err.workspace = true +getrandom = { workspace = true, features = ["std"] } nix = { workspace = true, features = ["user"] } dirs.workspace = true which.workspace = true diff --git a/vmm/src/app.rs b/vmm/src/app.rs index cd154f732..ee2d7f978 100644 --- a/vmm/src/app.rs +++ b/vmm/src/app.rs @@ -8,6 +8,7 @@ use dstack_port_forward::{ForwardRule, ForwardService, Protocol as FwdProtocol}; use anyhow::{bail, Context, Result}; use bon::Builder; use dstack_kms_rpc::kms_client::KmsClient; +use dstack_types::mr_config::MrConfigV3; use dstack_types::shared_filenames::{ APP_COMPOSE, ENCRYPTED_ENV, INSTANCE_INFO, SYS_CONFIG, USER_CONFIG, }; @@ -1000,7 +1001,26 @@ impl App { let manifest = work_dir.manifest().context("Failed to read manifest")?; let cfg = &self.config; let compose_hash = sha256_file(shared_dir.join(APP_COMPOSE))?; - let sys_config_str = make_sys_config(cfg, &manifest, &hex::encode(compose_hash))?; + let platform = cfg.cvm.platform.resolve(); + let app_compose = work_dir + .app_compose() + .context("Failed to get app compose")?; + let use_mr_config_v3 = !manifest.no_tee + && (platform == crate::config::TeePlatform::AmdSevSnp + || (platform == crate::config::TeePlatform::Tdx + && cfg.cvm.use_mrconfigid + && !app_compose.key_provider_id.is_empty())); + let mr_config = if use_mr_config_v3 { + Some( + work_dir + .prepare_mr_config_v3(&app_compose) + .context("Failed to prepare mr_config")?, + ) + } else { + None + }; + let sys_config_str = + make_sys_config(cfg, &manifest, &hex::encode(compose_hash), mr_config)?; fs::write(shared_dir.join(SYS_CONFIG), sys_config_str) .context("Failed to write vm config")?; Ok(()) @@ -1142,6 +1162,7 @@ pub(crate) fn make_sys_config( cfg: &Config, manifest: &Manifest, compose_hash: &str, + mr_config: Option, ) -> Result { let image_path = cfg.image.path.join(&manifest.image); let image = Image::load(image_path).context("Failed to load image info")?; @@ -1160,19 +1181,40 @@ pub(crate) fn make_sys_config( bail!("Unsupported image version: {img_ver:?}"); } - let sys_config = json!({ + let mut sys_config = json!({ "kms_urls": kms_urls, "gateway_urls": gateway_urls, "pccs_url": cfg.cvm.pccs_url, "docker_registry": cfg.cvm.docker_registry, "host_api_url": format!("vsock://2:{}/api", cfg.host_api.port), - "vm_config": serde_json::to_string(&make_vm_config(cfg, manifest, &image, compose_hash)?)?, + "vm_config": serde_json::to_string(&make_vm_config(cfg, manifest, &image, compose_hash, mr_config.clone())?)?, }); + if let Some(mr_config) = mr_config { + MrConfigV3::from_document(&mr_config).context("Invalid mr_config document")?; + sys_config["mr_config"] = serde_json::to_value(mr_config)?; + } else if let Some(mr_config) = mr_config_from_vm_config(&sys_config)? { + sys_config["mr_config"] = serde_json::to_value(mr_config)?; + } let sys_config_str = serde_json::to_string(&sys_config).context("Failed to serialize vm config")?; Ok(sys_config_str) } +fn mr_config_from_vm_config(sys_config: &serde_json::Value) -> Result> { + let Some(vm_config) = sys_config.get("vm_config").and_then(|value| value.as_str()) else { + return Ok(None); + }; + let vm_config: serde_json::Value = serde_json::from_str(vm_config)?; + let Some(mr_config) = vm_config.get("mr_config") else { + return Ok(None); + }; + let mr_config = mr_config + .as_str() + .context("mr_config must be a JSON string")?; + MrConfigV3::from_document(mr_config).context("Invalid mr_config document")?; + Ok(Some(mr_config.to_string())) +} + fn file_sha256_hex(path: &Path) -> Result { Ok(hex::encode(sha256_file(path)?)) } @@ -1203,7 +1245,8 @@ fn make_vm_config( cfg: &Config, manifest: &Manifest, image: &Image, - compose_hash: &str, + _compose_hash: &str, + mr_config: Option, ) -> Result { let os_image_hash = image .digest @@ -1231,9 +1274,11 @@ fn make_vm_config( config["spec_version"] = serde_json::Value::from(1); if cfg.cvm.platform.resolve() == crate::config::TeePlatform::AmdSevSnp && !manifest.no_tee { let rootfs_hash = image_rootfs_hash(image)?; + if let Some(mr_config) = mr_config { + MrConfigV3::from_document(&mr_config).context("Invalid mr_config document")?; + config["mr_config"] = serde_json::Value::String(mr_config); + } config["sev_snp_measurement"] = json!({ - "app_id": manifest.app_id, - "compose_hash": compose_hash, "rootfs_hash": rootfs_hash, "base_cmdline": amd_sev_snp_measurement_base_cmdline(image.info.cmdline.as_deref()), "docker_files_hash": serde_json::Value::Null, @@ -1270,21 +1315,18 @@ mod tests { } #[test] - fn amd_sev_snp_sys_config_includes_measurement_input_for_kms_auth() { + fn amd_sev_snp_sys_config_includes_measurement_input_and_mr_config() -> Result<()> { let temp = std::env::temp_dir().join(format!( "dstack-vmm-snp-test-{}", - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos() + SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos() )); let temp = temp.as_path(); let image_root = temp.join("images"); let image_dir = image_root.join("dstack-test"); - fs::create_dir_all(&image_dir).unwrap(); - fs::write(image_dir.join("kernel"), b"snp-test-kernel").unwrap(); - fs::write(image_dir.join("initrd"), b"snp-test-initrd").unwrap(); - fs::write(image_dir.join("rootfs"), b"snp-test-rootfs").unwrap(); + fs::create_dir_all(&image_dir)?; + fs::write(image_dir.join("kernel"), b"snp-test-kernel")?; + fs::write(image_dir.join("initrd"), b"snp-test-initrd")?; + fs::write(image_dir.join("rootfs"), b"snp-test-rootfs")?; fs::write( image_dir.join("metadata.json"), serde_json::json!({ @@ -1295,10 +1337,9 @@ mod tests { "version": "0.5.11" }) .to_string(), - ) - .unwrap(); + )?; - let mut config: Config = Figment::from(load_config_figment(None)).extract().unwrap(); + let mut config: Config = Figment::from(load_config_figment(None)).extract()?; config.image.path = image_root; config.cvm.platform = TeePlatform::AmdSevSnp; let compose_hash = hex_of(0x22, 32); @@ -1321,15 +1362,33 @@ mod tests { networking: None, }; - let sys_config: serde_json::Value = - serde_json::from_str(&make_sys_config(&config, &manifest, &compose_hash).unwrap()) - .unwrap(); - let vm_config: serde_json::Value = - serde_json::from_str(sys_config["vm_config"].as_str().unwrap()).unwrap(); + let mr_config = MrConfigV3::new( + vec![0x11; 20], + vec![0x22; 32], + dstack_types::KeyProviderKind::None, + vec![], + vec![0x44; 20], + ) + .to_canonical_json(); + let sys_config_document = + make_sys_config(&config, &manifest, &compose_hash, Some(mr_config))?; + let sys_config: serde_json::Value = serde_json::from_str(&sys_config_document)?; + let vm_config: serde_json::Value = serde_json::from_str( + sys_config["vm_config"] + .as_str() + .context("vm_config must be a string")?, + )?; let measurement = &vm_config["sev_snp_measurement"]; - - assert_eq!(measurement["app_id"], manifest.app_id); - assert_eq!(measurement["compose_hash"], compose_hash); + let mr_config_document = sys_config["mr_config"] + .as_str() + .context("mr_config must be a string")?; + let parsed_mr_config = MrConfigV3::from_document(mr_config_document)?; + + assert_eq!(parsed_mr_config.app_id, vec![0x11; 20]); + assert_eq!(parsed_mr_config.compose_hash, vec![0x22; 32]); + assert_eq!(vm_config["mr_config"], sys_config["mr_config"]); + assert!(measurement.get("app_id").is_none()); + assert!(measurement.get("compose_hash").is_none()); assert_eq!(measurement["rootfs_hash"], hex_of(0x33, 32)); assert_eq!( measurement["base_cmdline"], @@ -1346,7 +1405,11 @@ mod tests { assert_eq!(measurement["vcpus"], 2); assert_eq!(measurement["vcpu_type"], "EPYC-v4"); assert_eq!(measurement["ovmf_hash"], ""); - assert!(measurement["ovmf_sections"].as_array().unwrap().is_empty()); + assert!(measurement["ovmf_sections"] + .as_array() + .context("ovmf_sections must be an array")? + .is_empty()); + Ok(()) } } diff --git a/vmm/src/app/qemu.rs b/vmm/src/app/qemu.rs index 514b35151..e8fa6d8b5 100644 --- a/vmm/src/app/qemu.rs +++ b/vmm/src/app/qemu.rs @@ -18,16 +18,16 @@ use std::{ time::{Duration, SystemTime}, }; -use super::{image::Image, GpuConfig, ImageInfo, VmState}; +use super::{image::Image, GpuConfig, VmState}; use anyhow::{bail, Context, Result}; use base64::prelude::*; use bon::Builder; use dstack_types::{ - mr_config::MrConfig, + mr_config::{MrConfig, MrConfigV3}, shared_filenames::{ - APP_COMPOSE, ENCRYPTED_ENV, HOST_SHARED_DISK_LABEL, INSTANCE_INFO, USER_CONFIG, + APP_COMPOSE, ENCRYPTED_ENV, HOST_SHARED_DISK_LABEL, INSTANCE_INFO, SYS_CONFIG, USER_CONFIG, }, - AppCompose, KeyProviderKind, + AppCompose, KeyProviderKind, SysConfig, }; use dstack_vmm_rpc as pb; use sha2::{Digest, Sha256}; @@ -75,10 +75,12 @@ fn sanitize_optional>(value: Option) -> Option { value.filter(|value| !value.as_ref().trim().is_empty()) } -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Default, Deserialize, Serialize)] pub struct InstanceInfo { - #[serde(default)] - pub instance_id: String, + #[serde(default, with = "hex_bytes")] + pub instance_id_seed: Vec, + #[serde(default, with = "hex_bytes")] + pub instance_id: Vec, #[serde(default, with = "hex_bytes")] pub app_id: Vec, } @@ -348,8 +350,12 @@ impl VmState { } let uptime = display_ts(proc_state.and_then(|info| info.state.started_at.as_ref())); let exited_at = display_ts(proc_state.and_then(|info| info.state.stopped_at.as_ref())); - let instance_id = - sanitize_optional(workdir.instance_info().ok().map(|info| info.instance_id)); + let instance_id = sanitize_optional( + workdir + .instance_info() + .ok() + .map(|info| hex::encode(info.instance_id)), + ); VmInfo { manifest: self.config.manifest.clone(), workdir: workdir.path().to_path_buf(), @@ -369,10 +375,7 @@ impl VmState { #[cfg(test)] mod tests { - use super::{ - amd_sev_snp_measured_cmdline, amd_sev_snp_memory_backend_arg, amd_sev_snp_rootfs_hash, - sanitize_optional, virtio_pci_device, ImageInfo, - }; + use super::{amd_sev_snp_memory_backend_arg, sanitize_optional, virtio_pci_device}; #[test] fn sanitize_optional_filters_empty_owned_values() { @@ -402,38 +405,6 @@ mod tests { ); } - #[test] - fn amd_sev_snp_measured_cmdline_binds_app_identity() { - assert_eq!( - amd_sev_snp_measured_cmdline( - "console=ttyS0 loglevel=7", - "22", - "33", - "1111111111111111111111111111111111111111", - ), - "console=ttyS0 loglevel=7 docker_compose_hash=22 rootfs_hash=33 app_id=1111111111111111111111111111111111111111" - ); - } - - #[test] - fn amd_sev_snp_rootfs_hash_falls_back_to_dstack_cmdline() { - let info = ImageInfo { - cmdline: Some("console=ttyS0 dstack.rootfs_hash=abc123 dstack.rootfs_size=100".into()), - kernel: "kernel".into(), - initrd: "initrd".into(), - hda: None, - rootfs: None, - bios: None, - rootfs_hash: None, - shared_ro: false, - version: "0.5.11".into(), - is_dev: false, - ovmf_variant: None, - }; - - assert_eq!(amd_sev_snp_rootfs_hash(&info).unwrap(), "abc123"); - } - #[test] fn amd_sev_snp_uses_confidential_virtio_pci_options() { assert_eq!( @@ -447,18 +418,6 @@ mod tests { } } -fn amd_sev_snp_rootfs_hash(info: &ImageInfo) -> Result<&str> { - if let Some(rootfs_hash) = info.rootfs_hash.as_deref() { - return Ok(rootfs_hash); - } - info.cmdline - .as_deref() - .unwrap_or_default() - .split_whitespace() - .find_map(|param| param.strip_prefix("dstack.rootfs_hash=")) - .ok_or_else(|| anyhow::anyhow!("rootfs_hash is required for amd sev-snp")) -} - fn virtio_pci_device(device: &str, snp: bool) -> String { if snp { format!("{device},disable-legacy=on,iommu_platform=true") @@ -823,28 +782,9 @@ impl VmConfig { } } - // Add kernel command line. SNP launch measurement includes app identity - // through the measured QEMU kernel command line, matching TDX's - // app-id-in-measured-identity semantics without relying on post-launch - // RTMRs (which SNP does not have). - let cmdline = match (&self.image.info.cmdline, cfg.platform.resolve()) { - (Some(cmdline), TeePlatform::AmdSevSnp) if !self.manifest.no_tee => { - let compose_hash = hex::encode( - workdir - .app_compose_hash() - .context("Failed to get compose hash")?, - ); - let rootfs_hash = amd_sev_snp_rootfs_hash(&self.image.info)?; - Some(amd_sev_snp_measured_cmdline( - cmdline, - &compose_hash, - rootfs_hash, - &self.manifest.app_id, - )) - } - (Some(cmdline), _) => Some(cmdline.clone()), - (None, _) => None, - }; + // SNP app identity is bound through HOST_DATA, so the measured cmdline + // remains the image-provided cmdline. + let cmdline = self.image.info.cmdline.clone(); if let Some(cmdline) = cmdline { command.arg("-append").arg(cmdline); } @@ -923,7 +863,7 @@ impl VmConfig { self.configure_tdx_guest(command, workdir, cfg, app_compose)?; } TeePlatform::AmdSevSnp => { - self.configure_amd_sev_snp_guest(command, cfg, mem); + self.configure_amd_sev_snp_guest(command, workdir, cfg, mem)?; } } Ok(()) @@ -941,33 +881,47 @@ impl VmConfig { // Compute mrconfigid if needed let mrconfigid = if cfg.use_mrconfigid && support_mr_config_id { - let compose_hash = workdir - .app_compose_hash() - .context("Failed to get compose hash")?; - let mr_config = if app_compose.key_provider_id.is_empty() { - MrConfig::V1 { - compose_hash: &compose_hash, - } + if let Some(mr_config_document) = workdir + .sys_config() + .context("Failed to read sys config for tdx mrconfigid")? + .mr_config + { + MrConfigV3::from_document(&mr_config_document) + .context("Invalid mr_config document")?; + Some( + BASE64_STANDARD.encode(MrConfigV3::tdx_mr_config_id_from_document( + &mr_config_document, + )), + ) } else { - let instance_info = workdir - .instance_info() - .context("Failed to get instance info")?; - let app_id = if instance_info.app_id.is_empty() { - &compose_hash[..20] + let compose_hash = workdir + .app_compose_hash() + .context("Failed to get compose hash")?; + let mr_config = if app_compose.key_provider_id.is_empty() { + MrConfig::V1 { + compose_hash: &compose_hash, + } } else { - &instance_info.app_id + let instance_info = workdir + .instance_info() + .context("Failed to get instance info")?; + let app_id = if instance_info.app_id.is_empty() { + &compose_hash[..20] + } else { + &instance_info.app_id + }; + + let key_provider = app_compose.key_provider(); + let key_provider_id = &app_compose.key_provider_id; + MrConfig::V2 { + compose_hash: &compose_hash, + app_id: &app_id.try_into().context("Invalid app ID")?, + key_provider, + key_provider_id, + } }; - - let key_provider = app_compose.key_provider(); - let key_provider_id = &app_compose.key_provider_id; - MrConfig::V2 { - compose_hash: &compose_hash, - app_id: &app_id.try_into().context("Invalid app ID")?, - key_provider, - key_provider_id, - } - }; - Some(BASE64_STANDARD.encode(mr_config.to_mr_config_id())) + Some(BASE64_STANDARD.encode(mr_config.to_mr_config_id())) + } } else { None }; @@ -1012,19 +966,35 @@ impl VmConfig { Ok(()) } - fn configure_amd_sev_snp_guest(&self, command: &mut Command, cfg: &CvmConfig, mem: u32) { + fn configure_amd_sev_snp_guest( + &self, + command: &mut Command, + workdir: &VmWorkDir, + cfg: &CvmConfig, + mem: u32, + ) -> Result<()> { + let mr_config_document = workdir + .sys_config() + .context("Failed to read sys config for amd sev-snp host-data")? + .mr_config + .context("mr_config is required for amd sev-snp host-data")?; + MrConfigV3::from_document(&mr_config_document).context("Invalid mr_config document")?; + let host_data = + BASE64_STANDARD.encode(MrConfigV3::snp_host_data_from_document(&mr_config_document)); + command .arg("-object") .arg(amd_sev_snp_memory_backend_arg(mem)); - command - .arg("-object") - .arg("sev-snp-guest,id=sev0,policy=0x30000,sev-device=/dev/sev,kernel-hashes=on,author-key-enabled=on,cbitpos=51,reduced-phys-bits=1"); + command.arg("-object").arg(format!( + "sev-snp-guest,id=sev0,policy=0x30000,sev-device=/dev/sev,kernel-hashes=on,host-data={host_data},author-key-enabled=on,cbitpos=51,reduced-phys-bits=1" + )); command.arg("-machine").arg( "q35,kernel-irqchip=split,confidential-guest-support=sev0,memory-backend=ram1,hpet=off", ); if cfg.qgs_port.is_some() { tracing::warn!("qgs_port is ignored for amd sev-snp guests"); } + Ok(()) } fn configure_smbios(&self, command: &mut Command, cfg: &CvmConfig) { @@ -1076,21 +1046,6 @@ fn amd_sev_snp_memory_backend_arg(mem: u32) -> String { format!("memory-backend-memfd,id=ram1,size={mem}M,share=true,prealloc=false") } -fn amd_sev_snp_measured_cmdline( - base_cmdline: &str, - compose_hash: &str, - rootfs_hash: &str, - app_id: &str, -) -> String { - format!( - "{} docker_compose_hash={} rootfs_hash={} app_id={}", - base_cmdline.trim(), - compose_hash, - rootfs_hash, - app_id - ) -} - /// Round up a value to the nearest multiple of another value. /// If the value is already a multiple, it remains unchanged. fn round_up(value: u32, multiple: u32) -> u32 { @@ -1310,6 +1265,77 @@ impl VmWorkDir { Ok(info) } + pub fn instance_info_or_default(&self) -> Result { + match self.instance_info() { + Ok(info) => Ok(info), + Err(err) => match err.downcast_ref::() { + Some(io_err) if io_err.kind() == std::io::ErrorKind::NotFound => { + Ok(InstanceInfo::default()) + } + _ => Err(err), + }, + } + } + + pub fn sys_config(&self) -> Result { + let sys_config_file = self.shared_dir().join(SYS_CONFIG); + let sys_config: SysConfig = serde_json::from_slice(&fs::read(sys_config_file)?)?; + Ok(sys_config) + } + + pub fn prepare_mr_config_v3(&self, app_compose: &AppCompose) -> Result { + let compose_hash = self + .app_compose_hash() + .context("Failed to get compose hash")?; + let mut instance_info = self + .instance_info_or_default() + .context("Failed to get instance info")?; + let app_id = if instance_info.app_id.is_empty() { + compose_hash[..20].to_vec() + } else { + instance_info.app_id.clone() + }; + if app_id.len() != 20 { + bail!( + "Invalid app ID length: expected 20 bytes, got {}", + app_id.len() + ); + } + + let disk_reusable = !app_compose.key_provider().is_none(); + if !disk_reusable || instance_info.instance_id_seed.is_empty() { + instance_info.instance_id_seed = { + let mut seed = vec![0u8; 20]; + getrandom::fill(&mut seed).context("Failed to generate instance id seed")?; + seed + }; + } + + let instance_id = if app_compose.no_instance_id { + Vec::new() + } else { + let mut id_path = instance_info.instance_id_seed.clone(); + id_path.extend_from_slice(&app_id); + Sha256::digest(id_path)[..20].to_vec() + }; + instance_info.app_id = app_id.clone(); + instance_info.instance_id = instance_id.clone(); + fs::write( + self.instance_info_path(), + serde_json::to_string(&instance_info).context("Failed to serialize instance info")?, + ) + .context("Failed to write instance info")?; + + Ok(MrConfigV3::new( + app_id, + compose_hash.to_vec(), + app_compose.key_provider(), + app_compose.key_provider_id.clone(), + instance_id, + ) + .to_canonical_json()) + } + pub fn app_compose(&self) -> Result { let compose_file = self.app_compose_path(); let compose: AppCompose = serde_json::from_str(&fs::read_to_string(compose_file)?)?; diff --git a/vmm/src/one_shot.rs b/vmm/src/one_shot.rs index 0736e1a7f..36f46d9e7 100644 --- a/vmm/src/one_shot.rs +++ b/vmm/src/one_shot.rs @@ -117,7 +117,7 @@ pub async fn run_one_shot( fs_err::create_dir_all(&shared_dir).context("Failed to create shared directory")?; // Create app compose file content and parse AppCompose instance - let (app_compose_content, app_compose) = if vm_config.compose_file.is_empty() { + let (app_compose_content, _app_compose) = if vm_config.compose_file.is_empty() { // Create default compose JSON directly as string let gateway_enabled = !vm_config.gateway_urls.is_empty(); let kms_enabled = !vm_config.kms_urls.is_empty(); @@ -235,7 +235,25 @@ Compose file content (first 200 chars): // 2. Create .sys-config.json (critical for 0.5.x VMs) // Use manifest URLs if available, fallback to config URLs (matching VMM's sync_dynamic_config logic) - let sys_config_str = make_sys_config(&config, &manifest, &compose_hash)?; + let app_compose = vm_work_dir + .app_compose() + .context("Failed to get app compose")?; + let platform = config.cvm.platform.resolve(); + let use_mr_config_v3 = !manifest.no_tee + && (platform == crate::config::TeePlatform::AmdSevSnp + || (platform == crate::config::TeePlatform::Tdx + && config.cvm.use_mrconfigid + && !app_compose.key_provider_id.is_empty())); + let mr_config = if use_mr_config_v3 { + Some( + vm_work_dir + .prepare_mr_config_v3(&app_compose) + .context("Failed to prepare mr_config")?, + ) + } else { + None + }; + let sys_config_str = make_sys_config(&config, &manifest, &compose_hash, mr_config)?; let sys_config_path = vm_work_dir.shared_dir().join(".sys-config.json"); fs_err::write(&sys_config_path, sys_config_str).context("Failed to write sys config")?; From 3f71e7d2d05a8a69f1de1d42a8aa67a08f96e6a1 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 15 Jun 2026 19:38:18 -0700 Subject: [PATCH 41/67] Select SEV-SNP KDS product from report --- Cargo.lock | 1 - kms/src/main_service/amd_attest.rs | 1 + sev-snp-qvl/Cargo.toml | 1 - sev-snp-qvl/src/lib.rs | 250 ++++++++++++++++++++++++----- 4 files changed, 215 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7278658e2..d929b2072 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7376,7 +7376,6 @@ name = "sev-snp-qvl" version = "0.5.11" dependencies = [ "anyhow", - "base64 0.22.1", "hex", "reqwest", "sev", diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs index 7159fa1de..5993e5383 100644 --- a/kms/src/main_service/amd_attest.rs +++ b/kms/src/main_service/amd_attest.rs @@ -1275,6 +1275,7 @@ mod tests { let mr_config = valid_mr_config(&input)?; let mr_config_document = mr_config.to_canonical_json(); let tcb = dstack_attest::amd_sev_snp::AmdSnpTcbVersion { + fmc: None, bootloader: 1, tee: 2, snp: 3, diff --git a/sev-snp-qvl/Cargo.toml b/sev-snp-qvl/Cargo.toml index 0483e2324..c5c152693 100644 --- a/sev-snp-qvl/Cargo.toml +++ b/sev-snp-qvl/Cargo.toml @@ -12,7 +12,6 @@ description = "AMD SEV-SNP Quote Verification Library" [dependencies] anyhow.workspace = true -base64.workspace = true hex.workspace = true reqwest = { workspace = true, features = ["blocking"] } sev.workspace = true diff --git a/sev-snp-qvl/src/lib.rs b/sev-snp-qvl/src/lib.rs index 1fbfe87ea..b860a7e7f 100644 --- a/sev-snp-qvl/src/lib.rs +++ b/sev-snp-qvl/src/lib.rs @@ -11,15 +11,9 @@ //! production key release. use anyhow::{bail, Context, Result}; -use base64::engine::general_purpose::STANDARD; -use base64::Engine as _; -use sev::certs::snp::{ca, Certificate, Chain, Verifiable}; +use sev::certs::snp::{builtin, ca, Certificate, Chain, Verifiable}; use sev::firmware::{guest::AttestationReport, host::TcbVersion}; -/// AMD Genoa ARK certificate (DER, base64-encoded). -/// Source: https://kdsintf.amd.com/vcek/v1/Genoa/cert_chain -const GENOA_ARK_DER_B64: &str = "MIIGYzCCBBKgAwIBAgIDAgAAMEYGCSqGSIb3DQEBCjA5oA8wDQYJYIZIAWUDBAICBQChHDAaBgkqhkiG9w0BAQgwDQYJYIZIAWUDBAICBQCiAwIBMKMDAgEBMHsxFDASBgNVBAsMC0VuZ2luZWVyaW5nMQswCQYDVQQGEwJVUzEUMBIGA1UEBwwLU2FudGEgQ2xhcmExCzAJBgNVBAgMAkNBMR8wHQYDVQQKDBZBZHZhbmNlZCBNaWNybyBEZXZpY2VzMRIwEAYDVQQDDAlBUkstR2Vub2EwHhcNMjIwMTI2MTUzNDM3WhcNNDcwMTI2MTUzNDM3WjB7MRQwEgYDVQQLDAtFbmdpbmVlcmluZzELMAkGA1UEBhMCVVMxFDASBgNVBAcMC1NhbnRhIENsYXJhMQswCQYDVQQIDAJDQTEfMB0GA1UECgwWQWR2YW5jZWQgTWljcm8gRGV2aWNlczESMBAGA1UEAwwJQVJLLUdlbm9hMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA3Cd95S/uFOuRIskW9vz9VDBF69NDQF79oRhL/L2PVQGhK3YdfEBgpF/JiwWFBsT/fXDhzA01p3LkcT/7LdjcRfKXjHl+0Qq/M4dZkh6QDoUeKzNBLDcBKDDGWo3v35NyrxbA1DnkYwUKU5AAk4P94tKXLp80oxt84ahyHoLmc/LqsGsp+oq1Bz4PPsYLwTG4iMKVaaT90/oZ4I8oibSru92vJhlqWO27d/Rxc3iUMyhNeGToOvgx/iUo4gGpG61NDpkEUvIzuKcaMx8IdTpWg2DF6SwF0IgVMffnvtJmA68BwJNWo1E4PLJdaPfBifcJpuBFwNVQIPQEVX3aP89HJSp8YbY9lySS6PlVEqTBBtaQmi4ATGmMR+n2K/e+JAhU2Gj7jIpJhOkdH9firQDnmlA2SFfJ/Cc0mGNzW9RmIhyOUnNFoclmkRhl3/AQU5Ys9Qsan1jT/EiyT+pCpmnA+y9edvhDCbOG8F2oxHGRdTBkylungrkXJGYiwGrR8kaiqv7NN8QhOBMqYjcbrkEr0f8QMKklIS5ruOfqlLMCBw8JLB3LkjpWgtD7OpxkzSsohN47Uom86RY6lp72g8eXHP1qYrnvhzaG1S70vw6OkbaaC9EjiH/uHgAJQGxon7u0Q7xgoREWA/e7JcBQwLg80Hq/sbRuqesxz7wBWSY254cCAwEAAaN+MHwwDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBSfXfn+DdjzWtAzGiXvgSlPvjGoWzAPBgNVHRMBAf8EBTADAQH/MDoGA1UdHwQzMDEwL6AtoCuGKWh0dHBzOi8va2RzaW50Zi5hbWQuY29tL3ZjZWsvdjEvR2Vub2EvY3JsMEYGCSqGSIb3DQEBCjA5oA8wDQYJYIZIAWUDBAICBQChHDAaBgkqhkiG9w0BAQgwDQYJYIZIAWUDBAICBQCiAwIBMKMDAgEBA4ICAQAdIlPBC7DQmvH7kjlOznFx3i21SzOPDs5L7SgFjMC9rR07292GQCA7Z7Ulq97JQaWeD2ofGGse5swj4OQfKfVv/zaJUFjvosZOnfZ63epu8MjWgBSXJg5QE/Al0zRsZsp53DBTdA+Uv/s33fexdenT1mpKYzhIg/cKtz4oMxq8JKWJ8Po1CXLzKcfrTphjlbkh8AVKMXeBd2SpM33B1YP4g1BOdk013kqb7bRHZ1iB2JHG5cMKKbwRCSAAGHLTzASgDcXr9Fp7Z3liDhGu/ci1opGmkp12QNiJuBbkTU+xDZHm5X8Jm99BX7NEpzlOwIVR8ClgBDyuBkBC2ljtr3ZSaUIYj2xuyWN95KFY49nWxcz90CFa3Hzmy4zMQmBe9dVyls5eL5p9bkXcgRMDTbgmVZiAf4afe8DLdmQcYcMFQbHhgVzMiyZHGJgcCrQmA7MkTwEIds1wx/HzMcwU4qqNBAoZV7oeIIPxdqFXfPqHqiRlEbRDfX1TG5NFVaeByX0GyH6jzYVuezETzruaky6fp2bl2bczxPE8HdS38ijiJmm9vl50RGUeOAXjSuInGR4bsRufeGPB9peTa9BcBOeTWzstqTUB/F/qaZCIZKr4X6TyfUuSDz/1JDAGl+lxdM0P9+lLaP9NahQjHCVf0zf1c1salVuGFk2w/wMz1R1BHg=="; - const ASK_CERT_GUID: [u8; 16] = [ 0x4a, 0xb7, 0xb3, 0x79, 0xbb, 0xac, 0x4f, 0xe4, 0xa0, 0x2f, 0x05, 0xae, 0xf3, 0x27, 0xc7, 0x82, ]; @@ -31,8 +25,40 @@ const VLEK_CERT_GUID: [u8; 16] = [ ]; const CERT_TABLE_ENTRY_SIZE: usize = 24; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AmdSnpProduct { + Milan, + Genoa, + Turin, +} + +impl AmdSnpProduct { + fn kds_name(self) -> &'static str { + match self { + Self::Milan => "Milan", + // AMD KDS canonicalizes Genoa-family parts such as Bergamo and + // Siena under the Genoa endpoint. + Self::Genoa => "Genoa", + Self::Turin => "Turin", + } + } + + fn builtin_ark(self) -> CertBytes { + let bytes = match self { + Self::Milan => builtin::milan::ARK, + Self::Genoa => builtin::genoa::ARK, + Self::Turin => builtin::turin::ARK, + }; + CertBytes { + bytes: bytes.to_vec(), + encoding: CertEncoding::Pem, + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub struct AmdSnpTcbVersion { + pub fmc: Option, pub bootloader: u8, pub tee: u8, pub snp: u8, @@ -42,6 +68,7 @@ pub struct AmdSnpTcbVersion { impl From for AmdSnpTcbVersion { fn from(value: TcbVersion) -> Self { Self { + fmc: value.fmc, bootloader: value.bootloader, tee: value.tee, snp: value.snp, @@ -198,17 +225,29 @@ fn verify_amd_snp_attestation_with_certs( ask_bytes: CertBytes, vcek_bytes: CertBytes, ) -> Result { - let ark_der = STANDARD - .decode(GENOA_ARK_DER_B64) - .context("failed to decode amd genoa ark")?; - verify_amd_snp_attestation_with_cert_chain( - report_bytes, - CertBytes { - bytes: ark_der, - encoding: CertEncoding::Der, - }, - ask_bytes, - vcek_bytes, + if report_bytes.len() != 1184 { + bail!( + "invalid amd sev-snp report length: expected 1184 bytes, got {}", + report_bytes.len() + ); + } + let report = AttestationReport::from_bytes(report_bytes) + .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; + let mut errors = Vec::new(); + for product in amd_snp_product_candidates_for_report(&report)? { + match verify_amd_snp_attestation_with_cert_chain( + report_bytes, + product.builtin_ark(), + ask_bytes.clone(), + vcek_bytes.clone(), + ) { + Ok(verified) => return Ok(verified), + Err(err) => errors.push(format!("{}: {err:#}", product.kds_name())), + } + } + bail!( + "amd sev-snp report verification failed for supported products: {}", + errors.join("; ") ) } @@ -280,6 +319,12 @@ pub fn verify_amd_snp_evidence_with_kds_fallback( if !cert_chain.is_empty() { return verify_amd_snp_evidence(report, cert_chain, expected_report_data); } + if report.len() != 1184 { + bail!( + "invalid amd sev-snp report length: expected 1184 bytes, got {}", + report.len() + ); + } let report_obj = AttestationReport::from_bytes(report) .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; let collateral = fetch_amd_kds_collateral_for_report(&report_obj) @@ -298,10 +343,10 @@ pub fn verify_amd_snp_evidence_with_kds_fallback( fn fetch_amd_kds_collateral_for_report(report: &AttestationReport) -> Result { let mut errors = Vec::new(); - for product in ["Genoa", "Milan", "Bergamo", "Siena", "Turin"] { + for product in amd_snp_product_candidates_for_report(report)? { match fetch_amd_kds_collateral_for_product(product, report) { Ok(collateral) => return Ok(collateral), - Err(err) => errors.push(format!("{product}: {err:#}")), + Err(err) => errors.push(format!("{}: {err:#}", product.kds_name())), } } bail!( @@ -311,7 +356,7 @@ fn fetch_amd_kds_collateral_for_report(report: &AttestationReport) -> Result Result { let (ark, ask) = fetch_amd_kds_ca_chain(product)?; @@ -323,7 +368,7 @@ fn fetch_amd_kds_collateral_for_product( .get(..64) .context("amd sev-snp chip_id too short")?, ); - let vcek_url = amd_kds_vcek_url(product, &chip_id, report.reported_tcb.into()); + let vcek_url = amd_kds_vcek_url(product, &chip_id, report.reported_tcb.into())?; let vcek_request_url = amd_kds_request_url(&vcek_url); let vcek = reqwest::blocking::Client::new() .get(&vcek_request_url) @@ -346,8 +391,11 @@ fn fetch_amd_kds_collateral_for_product( }) } -fn fetch_amd_kds_ca_chain(product: &str) -> Result<(CertBytes, CertBytes)> { - let url = format!("https://kdsintf.amd.com/vcek/v1/{product}/cert_chain"); +fn fetch_amd_kds_ca_chain(product: AmdSnpProduct) -> Result<(CertBytes, CertBytes)> { + let url = format!( + "https://kdsintf.amd.com/vcek/v1/{}/cert_chain", + product.kds_name() + ); let request_url = amd_kds_request_url(&url); let chain = reqwest::blocking::Client::new() .get(&request_url) @@ -357,7 +405,8 @@ fn fetch_amd_kds_ca_chain(product: &str) -> Result<(CertBytes, CertBytes)> { .with_context(|| format!("amd sev-snp cert_chain request failed for {request_url}"))? .bytes() .context("failed to read amd sev-snp cert_chain response")?; - extract_ark_ask_from_amd_kds_cert_chain(&chain) + let (_fetched_ark, ask) = extract_ark_ask_from_amd_kds_cert_chain(&chain)?; + Ok((product.builtin_ark(), ask)) } fn amd_kds_request_url(amd_url: &str) -> String { @@ -367,16 +416,84 @@ fn amd_kds_request_url(amd_url: &str) -> String { } } -fn amd_kds_vcek_url(product: &str, chip_id: &[u8; 64], tcb: AmdSnpTcbVersion) -> String { - format!( - "https://kdsintf.amd.com/vcek/v1/{}/{}?blSPL={}&teeSPL={}&snpSPL={}&ucodeSPL={}", - product, - hex::encode(chip_id), - tcb.bootloader, - tcb.tee, - tcb.snp, - tcb.microcode - ) +fn amd_snp_product_candidates_for_report(report: &AttestationReport) -> Result> { + if let Some(product) = amd_snp_product_from_report(report)? { + return Ok(vec![product]); + } + + // SNP report v2 predates the CPUID family/model fields. In that case the + // product cannot be derived from the signed report, so keep a small + // fail-closed compatibility fallback. Bergamo and Siena deliberately do + // not appear here because their KDS endpoint is the canonical Genoa one. + Ok(vec![ + AmdSnpProduct::Genoa, + AmdSnpProduct::Milan, + AmdSnpProduct::Turin, + ]) +} + +fn amd_snp_product_from_report(report: &AttestationReport) -> Result> { + match report.version { + 2 => return Ok(None), + 3 => {} + version => bail!("unsupported amd sev-snp report version: {version}"), + } + + let family = report + .cpuid_fam_id + .context("amd sev-snp report v3+ is missing CPUID family")?; + let model = report + .cpuid_mod_id + .context("amd sev-snp report v3+ is missing CPUID model")?; + + let product = match family { + 0x19 => match model { + 0x00..=0x0f => AmdSnpProduct::Milan, + 0x10..=0x1f | 0xa0..=0xaf => AmdSnpProduct::Genoa, + _ => bail!("unsupported amd sev-snp CPUID model for family 19h: {model:#04x}"), + }, + 0x1a => match model { + 0x00..=0x11 => AmdSnpProduct::Turin, + _ => bail!("unsupported amd sev-snp CPUID model for family 1Ah: {model:#04x}"), + }, + _ => bail!("unsupported amd sev-snp CPUID family: {family:#04x}"), + }; + + Ok(Some(product)) +} + +fn amd_kds_vcek_url( + product: AmdSnpProduct, + chip_id: &[u8; 64], + tcb: AmdSnpTcbVersion, +) -> Result { + let url = match product { + AmdSnpProduct::Turin => { + let fmc = tcb + .fmc + .context("amd sev-snp Turin VCEK request requires reported FMC TCB")?; + format!( + "https://kdsintf.amd.com/vcek/v1/{}/{}?fmcSPL={}&blSPL={}&teeSPL={}&snpSPL={}&ucodeSPL={}", + product.kds_name(), + hex::encode(&chip_id[..8]), + fmc, + tcb.bootloader, + tcb.tee, + tcb.snp, + tcb.microcode + ) + } + AmdSnpProduct::Milan | AmdSnpProduct::Genoa => format!( + "https://kdsintf.amd.com/vcek/v1/{}/{}?blSPL={}&teeSPL={}&snpSPL={}&ucodeSPL={}", + product.kds_name(), + hex::encode(chip_id), + tcb.bootloader, + tcb.tee, + tcb.snp, + tcb.microcode + ), + }; + Ok(url) } fn extract_ark_ask_from_amd_kds_cert_chain(chain: &[u8]) -> Result<(CertBytes, CertBytes)> { @@ -560,6 +677,7 @@ mod tests { fn tcb(bootloader: u8, tee: u8, snp: u8, microcode: u8) -> AmdSnpTcbVersion { AmdSnpTcbVersion { + fmc: None, bootloader, tee, snp, @@ -606,13 +724,14 @@ mod tests { fn amd_kds_vcek_url_binds_chip_id_and_reported_tcb() { let chip_id = [0xab; 64]; let tcb = AmdSnpTcbVersion { + fmc: None, bootloader: 1, tee: 2, snp: 3, microcode: 4, }; - let url = amd_kds_vcek_url("Genoa", &chip_id, tcb); + let url = amd_kds_vcek_url(AmdSnpProduct::Genoa, &chip_id, tcb).unwrap(); assert_eq!( url, @@ -623,6 +742,65 @@ mod tests { ); } + #[test] + fn amd_kds_vcek_url_for_turin_uses_short_chip_id_and_fmc() { + let chip_id = [0xab; 64]; + let tcb = AmdSnpTcbVersion { + fmc: Some(5), + bootloader: 1, + tee: 2, + snp: 3, + microcode: 4, + }; + + let url = amd_kds_vcek_url(AmdSnpProduct::Turin, &chip_id, tcb).unwrap(); + + assert_eq!( + url, + "https://kdsintf.amd.com/vcek/v1/Turin/abababababababab?fmcSPL=5&blSPL=1&teeSPL=2&snpSPL=3&ucodeSPL=4" + ); + } + + #[test] + fn report_v3_cpuid_selects_kds_product_without_enumeration() { + let mut report = base_report(); + report.version = 3; + report.cpuid_fam_id = Some(0x19); + report.cpuid_mod_id = Some(0x10); + + assert_eq!( + amd_snp_product_candidates_for_report(&report).unwrap(), + vec![AmdSnpProduct::Genoa] + ); + + report.cpuid_mod_id = Some(0x0f); + assert_eq!( + amd_snp_product_candidates_for_report(&report).unwrap(), + vec![AmdSnpProduct::Milan] + ); + + report.cpuid_fam_id = Some(0x1a); + report.cpuid_mod_id = Some(0x00); + assert_eq!( + amd_snp_product_candidates_for_report(&report).unwrap(), + vec![AmdSnpProduct::Turin] + ); + } + + #[test] + fn report_v2_keeps_canonical_compatibility_product_candidates() { + let report = base_report(); + + assert_eq!( + amd_snp_product_candidates_for_report(&report).unwrap(), + vec![ + AmdSnpProduct::Genoa, + AmdSnpProduct::Milan, + AmdSnpProduct::Turin + ] + ); + } + #[test] fn amd_kds_proxy_url_wraps_amd_urls_when_configured() { const ENV_KEY: &str = "DSTACK_AMD_KDS_PROXY_URL"; From c5d491085536981cde774b0301f7e6c0e204ffff Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 15 Jun 2026 23:51:08 -0700 Subject: [PATCH 42/67] Use self-contained SNP measurement input --- docs/amd-sev-snp-review-readiness.md | 14 +- kms/src/config.rs | 24 +- kms/src/main.rs | 20 +- kms/src/main_service.rs | 46 ++-- kms/src/main_service/amd_attest.rs | 349 ++++++++++++--------------- kms/src/onboard_service.rs | 18 +- ra-rpc/src/rocket_helper.rs | 32 +-- sev-snp-qvl/src/lib.rs | 122 ++++++---- test-scripts/snp-e2e-smoke.sh | 22 +- vmm/src/app.rs | 133 +++++++++- vmm/src/app/qemu.rs | 2 +- vmm/src/app/snp_measure.rs | 226 +++++++++++++++++ 12 files changed, 662 insertions(+), 346 deletions(-) create mode 100644 vmm/src/app/snp_measure.rs diff --git a/docs/amd-sev-snp-review-readiness.md b/docs/amd-sev-snp-review-readiness.md index bbef133cc..e0bff692c 100644 --- a/docs/amd-sev-snp-review-readiness.md +++ b/docs/amd-sev-snp-review-readiness.md @@ -109,7 +109,7 @@ An additional manual smoke was attempted on the SNP host (`chris@173.234.27.162` That smoke exposed and fixed several VMM/KMS-auth integration issues before the guest reached KMS: -- `.sys-config.json` did not include the `sev_snp_measurement` launch input object needed by KMS SNP `BootInfo` recomputation. +- `.sys-config.json` did not include the `sev_snp_measurement` launch input document needed by KMS SNP `BootInfo` recomputation. - The VMM launch path required `metadata.json.rootfs_hash`, while the released `dstack-0.5.11` images carry the rootfs hash in `dstack.rootfs_hash=...` on the kernel cmdline. - The VMM SNP QEMU path now uses the SNP measurement CPU model (`EPYC-v4`) and confidential virtio PCI options (`disable-legacy=on,iommu_platform=true`) for SNP-launched virtio devices, matching the host's working SNP launch posture more closely. @@ -120,6 +120,8 @@ After those fixes, the manual smoke progressed through full dstack-managed SNP g - Configfs TSM report collection falls back to the SEV-SNP extended-report ioctl when configfs does not carry certificate collateral. - If verifier-side evidence still lacks ASK/VCEK collateral, the verifier can fetch AMD KDS ARK/ASK/VCEK using the report `chip_id` and reported TCB, then verify the signed report fail-closed. - KMS measurement recomputation now uses the image's original kernel cmdline for SNP launch measurement, while app identity is bound by MrConfigV3/HOST_DATA instead of appended cmdline fields. +- VMM now extracts the image OVMF SEV metadata and OVMF launch digest seed, includes them in the `sev_snp_measurement` document string, and passes that through the guest to KMS; KMS no longer needs a single locally configured `ovmf_path`, so different image/OVMF versions can be verified by their self-contained launch inputs. +- SNP `BootInfo.os_image_hash` is `sha256(sev_snp_measurement document string)`, covering rootfs hash, kernel/initrd hashes, cmdline, OVMF hash/sections, vCPU model/count, and guest features instead of only the rootfs hash; KMS parses the string for measurement recomputation but hashes the exact VMM-supplied document bytes. Latest sanitized remote smoke result with PR-built host binaries and a coherent `MACHINE = "sev-snp"` guest image: @@ -133,14 +135,14 @@ platform=amd-sev-snp image_kernel=Linux 6.18.24-dstack with CONFIG_AMD_MEM_ENCRYPT=y, CONFIG_SEV_GUEST=y, CONFIG_TSM_REPORTS=y kms_guest=booted SNP Linux/userspace and started dstack-kms kms_marker=SNP_KMS_CONTAINER_STARTED / KMS runtime ready -kds_proxy=enabled for smoke via DSTACK_SNP_SMOKE_KDS_PROXY_URL=https://cors.litgateway.com/ +kds_base_url=enabled for smoke via DSTACK_SNP_SMOKE_KDS_BASE_URL=https://cors.litgateway.com/https://kdsintf.amd.com/vcek/v1 strict_tcb_probe=denied_as_expected with tcb_status is not allowed success_probe=GetTempCaCert HTTP 200; GetAppKey HTTP 200; SignCert HTTP 200; app container started smoke_result=SNP E2E smoke success no_secret_material_logged=true ``` -This means the PR has live SNP report proof, live golden-vector measurement proof, release-gate unit/integration coverage, and hardware smoke proof through dstack-managed SNP KMS boot, strict TCB denial, app guest key release, and app container startup. The fresh-box smoke now reaches Linux/userspace, `SNP_KMS_CONTAINER_STARTED`, `GetTempCaCert`, `GetAppKey`, `SignCert`, and app container startup when using a coherent **SNP** `meta-dstack` image. During the smoke, AMD KDS throttling was worked around by explicitly routing AMD KDS collateral fetches through the smoke-level `DSTACK_SNP_SMOKE_KDS_PROXY_URL=https://cors.litgateway.com/`; the smoke writes this value to the KMS `[core.sev_snp]` configuration. The proxy is a path-prefix passthrough (`https://cors.litgateway.com/https://kdsintf.amd.com/...`), not a `?url=` wrapper. Host/KMS binaries must match PR #703, guest-side `dstack-util`/`dstack-attest` must include the PR cert-chain/KDS fallback, and the Yocto image must be built with `MACHINE = "sev-snp"` so the guest kernel includes AMD memory-encryption/SNP support. A coherent PR image built with the default `tdx` machine produced a `6.18.24-dstack` kernel with `# CONFIG_AMD_MEM_ENCRYPT is not set`; controlled QEMU tests showed that kernel resets immediately after OVMF loads kernel/initrd, while SNP-capable kernels boot the same QEMU/OVMF path to Linux/SNP markers. +This means the PR has live SNP report proof, live golden-vector measurement proof, release-gate unit/integration coverage, and hardware smoke proof through dstack-managed SNP KMS boot, strict TCB denial, app guest key release, and app container startup. The fresh-box smoke now reaches Linux/userspace, `SNP_KMS_CONTAINER_STARTED`, `GetTempCaCert`, `GetAppKey`, `SignCert`, and app container startup when using a coherent **SNP** `meta-dstack` image. During the smoke, AMD KDS throttling was worked around by explicitly routing AMD KDS collateral fetches through the smoke-level `DSTACK_SNP_SMOKE_KDS_BASE_URL=https://cors.litgateway.com/https://kdsintf.amd.com/vcek/v1`; the smoke writes this value to the KMS `[core.sev_snp]` configuration. This is an AMD-KDS-compatible base URL; requests append relative KDS paths such as `/Milan/cert_chain` or `/Milan/?...`. Host/KMS binaries must match PR #703, guest-side `dstack-util`/`dstack-attest` must include the PR cert-chain/KDS fallback, and the Yocto image must be built with `MACHINE = "sev-snp"` so the guest kernel includes AMD memory-encryption/SNP support. A coherent PR image built with the default `tdx` machine produced a `6.18.24-dstack` kernel with `# CONFIG_AMD_MEM_ENCRYPT is not set`; controlled QEMU tests showed that kernel resets immediately after OVMF loads kernel/initrd, while SNP-capable kernels boot the same QEMU/OVMF path to Linux/SNP markers. ### Fresh SNP host / image requirements @@ -149,13 +151,13 @@ The checked-in smoke is enough to reproduce the current boundary on a compatible - Known-good host for reaching KMS and app `dstack-prepare.sh`: `chris@173.234.27.162` with QEMU 10.0.2, the SNP-capable OVMF above, and a coherent `dstack-dev-0.6.0` guest image built with `MACHINE = "sev-snp"`. - Released images that do not carry PR #703 guest-side `dstack-util`/`dstack-attest` may reject SNP evidence before the newer PR fallback paths can help. - A coherent PR #703 image must be built as an SNP image, not with `meta-dstack`'s default `tdx` machine. The default TDX build can emit a kernel without `CONFIG_AMD_MEM_ENCRYPT`, which fails before Linux serial output under SNP. -- On the same remote host/QEMU/OVMF, a minimal SNP initramfs booted SNP-capable kernels (`6.11.0-rc3-snp-host`, `6.9.0-rc7-snp-host`, and the `MACHINE = "sev-snp"` `6.18.24-dstack` kernel) to Linux/SNP markers, while the default-TDX `6.18.24-dstack` kernel reset immediately after OVMF loaded kernel/initrd. This isolates that failure to the guest kernel config, not PSP firmware, KMS/auth policy, author-key, command line, virtio wiring, or basic host SNP enablement. +- On the same remote host/QEMU/OVMF, a minimal SNP initramfs booted SNP-capable kernels (`6.11.0-rc3-snp-host`, `6.9.0-rc7-snp-host`, and the `MACHINE = "sev-snp"` `6.18.24-dstack` kernel) to Linux/SNP markers, while the default-TDX `6.18.24-dstack` kernel reset immediately after OVMF loaded kernel/initrd. This isolates that failure to the guest kernel config, not PSP firmware, KMS/auth policy, command line, virtio wiring, or basic host SNP enablement. Practical implication for reviewers/testers on a fresh box: 1. Install/use an AMDSEV QEMU 10.x build and the matching SNP-capable OVMF. 2. Build the PR binaries with `cargo build --release -p dstack-vmm -p supervisor -p dstack-kms`. -3. Run `test-scripts/snp-e2e-smoke.sh` unchanged and first confirm it reaches `SNP_KMS_CONTAINER_STARTED`; if AMD KDS throttles the lab host, set `DSTACK_SNP_SMOKE_KDS_PROXY_URL` to a trusted path-prefix AMD-KDS passthrough/cache endpoint such as `https://cors.litgateway.com/` and rerun. The lab success above also used `DSTACK_SNP_SMOKE_ALLOW_OUT_OF_DATE_TCB=1` because the current SNP lab host reports `OutOfDate`; production defaults remain `allowed_tcb_statuses = ["UpToDate"]` with an empty advisory allowlist. +3. Run `test-scripts/snp-e2e-smoke.sh` unchanged and first confirm it reaches `SNP_KMS_CONTAINER_STARTED`; if AMD KDS throttles the lab host, set `DSTACK_SNP_SMOKE_KDS_BASE_URL` to a trusted AMD-KDS-compatible mirror/cache base URL such as `https://mirror.example.com/vcek/v1` (or, for a path-prefix relay, `https://cors.litgateway.com/https://kdsintf.amd.com/vcek/v1`) and rerun. The lab success above also used `DSTACK_SNP_SMOKE_ALLOW_OUT_OF_DATE_TCB=1` because the current SNP lab host reports `OutOfDate`; production defaults remain `allowed_tcb_statuses = ["UpToDate"]` with an empty advisory allowlist. 4. For full `SNP_APP_CONTAINER_STARTED` / `GetAppKey` success, use or publish a coherent `meta-dstack` guest image whose kernel, modules, initramfs, rootfs, verity metadata, and guest userspace include the same PR #703 `dstack-util`/`dstack-attest` SNP cert-chain/KDS fallback code. The reproducible path is to build `meta-dstack` with its `dstack` submodule checked out to this PR branch, for example: ```bash @@ -176,7 +178,7 @@ Practical implication for reviewers/testers on a fresh box: Do not try to inject only a replacement `dstack-util` into the stock image; that experiment changed the initramfs/measurement enough to regress boot. 5. Only after the baseline smoke reaches the app success marker should testers swap the simple app workload for Chipotle. -If the smoke stops after `EFI stub: Loaded initrd ...` with `cpus are not resettable`, use a host/image/kernel that is known to boot dstack under SNP before debugging app-level behavior. If it reaches `Requesting app keys from KMS` and fails with AMD KDS `HTTP 429`, use the smoke proxy hook above; if it fails with missing cert-chain/collateral without KDS proxy evidence, rebuild/use a coherent PR guest image rather than changing KMS release policy. +If the smoke stops after `EFI stub: Loaded initrd ...` with `cpus are not resettable`, use a host/image/kernel that is known to boot dstack under SNP before debugging app-level behavior. If it reaches `Requesting app keys from KMS` and fails with AMD KDS `HTTP 429`, use the smoke KDS base URL hook above; if it fails with missing cert-chain/collateral without KDS base URL evidence, rebuild/use a coherent PR guest image rather than changing KMS release policy. ## Validation commands diff --git a/kms/src/config.rs b/kms/src/config.rs index 934f2a7c4..5a6d7384f 100644 --- a/kms/src/config.rs +++ b/kms/src/config.rs @@ -31,27 +31,17 @@ pub(crate) struct ImageConfig { pub download_timeout: Duration, } -/// Configuration for AMD SEV-SNP measurement/app binding validation. +/// Optional AMD SEV-SNP verifier configuration. #[derive(Debug, Clone, Deserialize)] pub(crate) struct SevSnpMeasureConfig { - /// Path to the AMD SEV-SNP OVMF binary used for this VM image. + /// Optional AMD KDS-compatible base URL used for collateral requests. /// - /// Optional when callers provide OVMF section metadata with the request. - pub ovmf_path: Option, - /// Optional diagnostic/cache proxy used for AMD KDS collateral requests. - /// - /// Empty by default. When set, the KMS process exports the same proxy env - /// used by dstack-attest before any attestation verification happens. + /// Empty by default. When set, the KMS process exports this base URL for + /// dstack-attest before any attestation verification happens. The base URL + /// must expose AMD KDS-compatible paths under `/vcek/v1`, e.g. + /// `https://kdsintf.amd.com/vcek/v1` or a trusted mirror/cache. #[serde(default)] - pub amd_kds_proxy_url: Option, - /// SNP guest features bitmask used at launch. Defaults to SNP with kernel - /// hashes enabled. - #[serde(default = "default_guest_features")] - pub guest_features: u64, -} - -fn default_guest_features() -> u64 { - 0x1 + pub amd_kds_base_url: Option, } #[derive(Debug, Clone, Deserialize)] diff --git a/kms/src/main.rs b/kms/src/main.rs index 7945963d8..ac31a19f7 100644 --- a/kms/src/main.rs +++ b/kms/src/main.rs @@ -105,18 +105,18 @@ fn record_attestation_metrics(req: &rocket::Request<'_>, res: &rocket::Response< .record_attestation_request(res.status().code >= 400); } -fn configure_amd_kds_proxy_from_config(config: &KmsConfig) { - let Some(proxy_url) = config +fn configure_amd_kds_base_from_config(config: &KmsConfig) { + let Some(base_url) = config .sev_snp .as_ref() - .and_then(|sev_snp| sev_snp.amd_kds_proxy_url.as_deref()) + .and_then(|sev_snp| sev_snp.amd_kds_base_url.as_deref()) .map(str::trim) - .filter(|proxy_url| !proxy_url.is_empty()) + .filter(|base_url| !base_url.is_empty()) else { return; }; - std::env::set_var("DSTACK_AMD_KDS_PROXY_URL", proxy_url); - info!("AMD SEV-SNP KDS proxy configured"); + std::env::set_var("DSTACK_AMD_KDS_BASE_URL", base_url); + info!("AMD SEV-SNP KDS base URL configured"); } #[rocket::main] @@ -130,7 +130,7 @@ async fn main() -> Result<()> { let figment = config::load_config_figment(args.config.as_deref()); let config: KmsConfig = figment.focus("core").extract()?; - configure_amd_kds_proxy_from_config(&config); + configure_amd_kds_base_from_config(&config); if config.onboard.enabled && !config.keys_exists() { info!("Onboarding"); @@ -152,10 +152,10 @@ async fn main() -> Result<()> { } let pccs_url = config.pccs_url.clone(); - let amd_kds_proxy_url = config + let amd_kds_base_url = config .sev_snp .as_ref() - .and_then(|sev_snp| sev_snp.amd_kds_proxy_url.clone()); + .and_then(|sev_snp| sev_snp.amd_kds_base_url.clone()); let metrics_enabled = config.metrics.enabled; let state = main_service::KmsState::new(config).context("Failed to initialize KMS state")?; let figment = figment @@ -183,7 +183,7 @@ async fn main() -> Result<()> { .mount("/", rocket::routes![metrics]); } - let verifier = QuoteVerifier::new_with_amd_kds_proxy(pccs_url, amd_kds_proxy_url); + let verifier = QuoteVerifier::new_with_amd_kds_base(pccs_url, amd_kds_base_url); rocket = rocket.manage(verifier); rocket diff --git a/kms/src/main_service.rs b/kms/src/main_service.rs index 6a150da4c..dce8f1a54 100644 --- a/kms/src/main_service.rs +++ b/kms/src/main_service.rs @@ -157,8 +157,16 @@ pub(crate) fn build_boot_info_for_attestation( vm_config_str: &str, ) -> Result { if att.report.amd_snp_report().is_some() { - let config = sev_snp_config - .ok_or_else(|| anyhow::anyhow!("sev_snp config is required for amd sev-snp"))?; + let default_sev_snp_config; + let config = match sev_snp_config { + Some(config) => config, + None => { + default_sev_snp_config = SevSnpMeasureConfig { + amd_kds_base_url: None, + }; + &default_sev_snp_config + } + }; let vm_config_str = if vm_config_str.is_empty() { att.config.as_str() } else { @@ -614,9 +622,7 @@ mod tests { fn sev_snp_config() -> SevSnpMeasureConfig { SevSnpMeasureConfig { - ovmf_path: None, - amd_kds_proxy_url: None, - guest_features: 1, + amd_kds_base_url: None, } } @@ -630,7 +636,6 @@ mod tests { compose_hash: hex_of(0x22, 32), rootfs_hash: hex_of(0x33, 32), base_cmdline: None, - docker_files_hash: Some(hex_of(0x77, 32)), ovmf_hash: hex_of(0x44, 48), kernel_hash: hex_of(0x55, 32), initrd_hash: hex_of(0x66, 32), @@ -638,6 +643,7 @@ mod tests { sev_es_reset_eip: 0xffff_fff0, vcpus: 2, vcpu_type: Some("epyc-v4".to_string()), + guest_features: 1, ovmf_sections: vec![ OvmfSectionParam { gpa: 0x100000, @@ -711,11 +717,11 @@ mod tests { #[test] fn build_boot_info_for_attestation_accepts_snp_vm_config_path() { let input = valid_snp_measurement_input(); - let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); + let measurement = compute_expected_measurement(&input).unwrap(); let mr_config = valid_snp_mr_config(); let attestation = verified_snp_attestation(measurement, [0xab; 64]); let vm_config = serde_json::json!({ - "sev_snp_measurement": input, + "sev_snp_measurement": serde_json::to_string(&input).unwrap(), "mr_config": mr_config.to_canonical_json(), }) .to_string(); @@ -737,10 +743,10 @@ mod tests { #[test] fn build_boot_info_for_attestation_uses_embedded_snp_vm_config_when_external_is_empty() { let input = valid_snp_measurement_input(); - let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); + let measurement = compute_expected_measurement(&input).unwrap(); let mr_config = valid_snp_mr_config(); let embedded_config = serde_json::json!({ - "sev_snp_measurement": input, + "sev_snp_measurement": serde_json::to_string(&input).unwrap(), "mr_config": mr_config.to_canonical_json(), }) .to_string(); @@ -761,32 +767,30 @@ mod tests { } #[test] - fn build_boot_info_for_attestation_requires_snp_config_for_snp() { + fn build_boot_info_for_attestation_accepts_self_contained_snp_input_without_config() { let input = valid_snp_measurement_input(); - let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); + let measurement = compute_expected_measurement(&input).unwrap(); let mr_config = valid_snp_mr_config(); let attestation = verified_snp_attestation(measurement, [0xab; 64]); let vm_config = serde_json::json!({ - "sev_snp_measurement": input, + "sev_snp_measurement": serde_json::to_string(&input).unwrap(), "mr_config": mr_config.to_canonical_json(), }) .to_string(); - let err = build_boot_info_for_attestation(None, &attestation, false, &vm_config) - .expect_err("snp attestation must require sev_snp config"); - assert!( - err.to_string().contains("sev_snp config is required"), - "unexpected error: {err:?}" - ); + let boot_info = build_boot_info_for_attestation(None, &attestation, false, &vm_config) + .expect("self-contained SNP vm_config should not require KMS-local sev_snp config"); + assert_eq!(boot_info.attestation_mode, AttestationMode::DstackAmdSevSnp); + assert_eq!(boot_info.device_id, vec![0xab; 64]); } fn snp_boot_info() -> BootInfo { let input = valid_snp_measurement_input(); - let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); + let measurement = compute_expected_measurement(&input).unwrap(); let mr_config = valid_snp_mr_config(); let attestation = verified_snp_attestation(measurement, [0xab; 64]); let vm_config = serde_json::json!({ - "sev_snp_measurement": input, + "sev_snp_measurement": serde_json::to_string(&input).unwrap(), "mr_config": mr_config.to_canonical_json(), }) .to_string(); diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs index 5993e5383..3b77307a6 100644 --- a/kms/src/main_service/amd_attest.rs +++ b/kms/src/main_service/amd_attest.rs @@ -36,8 +36,7 @@ const MAX_OVMF_METADATA_PAGES: u64 = 16_777_216; // VMSA page GPA: (u64)(-1) page-aligned, bits >51 cleared. const VMSA_GPA: u64 = 0x0000_FFFF_FFFF_F000; -#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)] -#[cfg_attr(test, derive(serde::Serialize))] +#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize, serde::Serialize)] #[serde(deny_unknown_fields)] pub(crate) struct OvmfSectionParam { pub gpa: u64, @@ -48,8 +47,7 @@ pub(crate) struct OvmfSectionParam { pub section_type: u32, } -#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)] -#[cfg_attr(test, derive(serde::Serialize))] +#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize, serde::Serialize)] #[serde(deny_unknown_fields)] pub(crate) struct MeasurementInput { /// Deprecated: app identity is now bound through MrConfigV3/HOST_DATA. @@ -58,15 +56,11 @@ pub(crate) struct MeasurementInput { /// Deprecated: compose identity is now bound through MrConfigV3/HOST_DATA. #[serde(default)] pub compose_hash: String, - /// 32-byte rootfs hash used as the SNP os_image_hash authorization input. + /// 32-byte rootfs hash included in the self-contained SNP measurement input. pub rootfs_hash: String, /// Original image kernel cmdline used for SNP measured launch. pub base_cmdline: Option, - /// Optional 32-byte additional docker files hash included in the measured - /// kernel cmdline when present. - pub docker_files_hash: Option, - /// 48-byte OVMF GCTX launch digest seed. Required when OVMF sections are - /// supplied by the request; optional only when KMS can load ovmf_path. + /// 48-byte OVMF GCTX launch digest seed supplied by the VMM. pub ovmf_hash: String, /// 32-byte kernel SHA-256 hash. pub kernel_hash: String, @@ -79,6 +73,9 @@ pub(crate) struct MeasurementInput { pub sev_es_reset_eip: u32, pub vcpus: u32, pub vcpu_type: Option, + /// SNP guest features bitmask used at launch. QEMU uses 0x1 for SNP with + /// kernel hashes enabled in the current VMM path. + pub guest_features: u64, #[serde(deserialize_with = "deserialize_ovmf_sections_bounded")] pub ovmf_sections: Vec, } @@ -123,14 +120,13 @@ where } pub(crate) fn validate_amd_snp_measurement_binding( - config: Option<&SevSnpMeasureConfig>, + _config: Option<&SevSnpMeasureConfig>, verified_measurement: &[u8; 48], input: &MeasurementInput, ) -> Result<()> { - let config = config.ok_or_else(|| anyhow::anyhow!("sev-snp measurement config is required"))?; - validate_measurement_input(config, input)?; + validate_measurement_input(input)?; - let expected_measurement = compute_expected_measurement(config, input)?; + let expected_measurement = compute_expected_measurement(input)?; if expected_measurement.as_slice() != verified_measurement { bail!("amd sev-snp measurement mismatch"); } @@ -160,6 +156,8 @@ pub(crate) fn build_amd_snp_boot_info( ) -> Result { let mr_config = test_mr_config_from_input(input)?; let mr_config_document = mr_config.to_canonical_json(); + let measurement_document = serde_json::to_string(input) + .context("failed to serialize amd sev-snp measurement input")?; let host_data = MrConfigV3::snp_host_data_from_document(&mr_config_document); build_amd_snp_boot_info_with_tcb_status( config, @@ -169,6 +167,7 @@ pub(crate) fn build_amd_snp_boot_info( "UpToDate", &[], input, + &measurement_document, &mr_config_document, ) } @@ -181,12 +180,13 @@ fn build_amd_snp_boot_info_with_tcb_status( tcb_status: &str, advisory_ids: &[String], input: &MeasurementInput, + measurement_document: &str, mr_config_document: &str, ) -> Result { validate_amd_snp_measurement_binding(Some(config), verified_measurement, input)?; let mr_config = validate_snp_mr_config_binding(verified_host_data, mr_config_document)?; - let rootfs_hash = decode_required_hex("rootfs_hash", &input.rootfs_hash, 32)?; + let os_image_hash = snp_measurement_os_image_hash(measurement_document); let mr_system = Sha256::digest(verified_measurement).to_vec(); let mr_aggregated = snp_mr_aggregated_digest(verified_measurement, verified_host_data); let key_provider_info = mr_config_key_provider_info(&mr_config)?; @@ -194,7 +194,7 @@ fn build_amd_snp_boot_info_with_tcb_status( Ok(BootInfo { attestation_mode: AttestationMode::DstackAmdSevSnp, mr_aggregated, - os_image_hash: rootfs_hash, + os_image_hash, mr_system, app_id: mr_config.app_id.clone(), compose_hash: mr_config.compose_hash.clone(), @@ -217,6 +217,7 @@ pub(crate) fn build_amd_snp_boot_info_from_verified_attestation( config: &SevSnpMeasureConfig, attestation: &VerifiedAttestation, input: &MeasurementInput, + measurement_document: &str, mr_config_document: &str, ) -> Result { let verified = attestation @@ -231,13 +232,14 @@ pub(crate) fn build_amd_snp_boot_info_from_verified_attestation( verified.tcb_info.tcb_status(), &verified.advisory_ids, input, + measurement_document, mr_config_document, ) } #[derive(Debug, serde::Deserialize)] struct SevSnpMeasurementVmConfig { - sev_snp_measurement: Option, + sev_snp_measurement: Option, mr_config: Option, } @@ -251,11 +253,13 @@ pub(crate) fn build_amd_snp_boot_info_from_verified_attestation_and_vm_config( attestation: &VerifiedAttestation, vm_config: &str, ) -> Result { - let (input, mr_config_document) = parse_snp_inputs_from_vm_config(vm_config)?; + let (input, measurement_document, mr_config_document) = + parse_snp_inputs_from_vm_config(vm_config)?; build_amd_snp_boot_info_from_verified_attestation( config, attestation, &input, + &measurement_document, &mr_config_document, ) } @@ -264,7 +268,7 @@ fn parse_measurement_input_from_vm_config(vm_config: &str) -> Result Result<(MeasurementInput, String)> { +fn parse_snp_inputs_from_vm_config(vm_config: &str) -> Result<(MeasurementInput, String, String)> { let value: serde_json::Value = serde_json::from_str(vm_config).context("failed to parse vm_config for amd sev-snp")?; let parsed: SevSnpMeasurementVmConfig = serde_json::from_value(value.clone()) @@ -277,7 +281,7 @@ fn parse_snp_inputs_from_vm_config(vm_config: &str) -> Result<(MeasurementInput, .context("failed to parse nested vm_config for amd sev-snp") }) .transpose()?; - let measurement = parsed + let measurement_document = parsed .sev_snp_measurement .or_else(|| { nested @@ -285,12 +289,14 @@ fn parse_snp_inputs_from_vm_config(vm_config: &str) -> Result<(MeasurementInput, .and_then(|nested| nested.sev_snp_measurement.clone()) }) .ok_or_else(|| anyhow::anyhow!("sev_snp_measurement is required for amd sev-snp"))?; + let measurement: MeasurementInput = serde_json::from_str(&measurement_document) + .context("invalid amd sev-snp measurement document")?; let mr_config = parsed .mr_config .or_else(|| nested.and_then(|nested| nested.mr_config)) .ok_or_else(|| anyhow::anyhow!("mr_config is required for amd sev-snp"))?; MrConfigV3::from_document(&mr_config).context("invalid amd sev-snp mr_config document")?; - Ok((measurement, mr_config)) + Ok((measurement, measurement_document, mr_config)) } /// Explicit helper-only AMD SEV-SNP authorization policy. @@ -414,6 +420,10 @@ fn snp_mr_aggregated_digest(measurement: &[u8; 48], host_data: &[u8; 32]) -> Vec h.finalize().to_vec() } +pub(crate) fn snp_measurement_os_image_hash(measurement_document: &str) -> Vec { + Sha256::digest(measurement_document.as_bytes()).to_vec() +} + fn mr_config_key_provider_info(mr_config: &MrConfigV3) -> Result> { serde_json::to_vec(&KeyProviderInfo::new( mr_config.key_provider_name().to_string(), @@ -461,21 +471,14 @@ fn test_mr_config_from_input(input: &MeasurementInput) -> Result { )) } -fn validate_measurement_input( - config: &SevSnpMeasureConfig, - input: &MeasurementInput, -) -> Result<()> { - if config.guest_features == 0 { +fn validate_measurement_input(input: &MeasurementInput) -> Result<()> { + if input.guest_features == 0 { bail!("guest_features must be non-zero"); } decode_required_hex("rootfs_hash", &input.rootfs_hash, 32)?; decode_required_hex("kernel_hash", &input.kernel_hash, 32)?; decode_optional_hex("initrd_hash", &input.initrd_hash, 32)?; - if let Some(docker_files_hash) = &input.docker_files_hash { - decode_required_hex("docker_files_hash", docker_files_hash, 32)?; - } - if input.vcpus == 0 { bail!("vcpus must be greater than zero"); } @@ -490,19 +493,7 @@ fn validate_measurement_input( } if input.ovmf_sections.is_empty() { - if config - .ovmf_path - .as_deref() - .unwrap_or_default() - .trim() - .is_empty() - { - bail!("ovmf_sections are required when ovmf_path is not configured"); - } - if !input.ovmf_hash.is_empty() { - bail!("ovmf_hash must be empty when ovmf_path is used"); - } - return Ok(()); + bail!("ovmf_sections are required for amd sev-snp"); } decode_required_hex("ovmf_hash", &input.ovmf_hash, 48)?; @@ -935,67 +926,33 @@ fn build_vmsa_page(eip: u32, vcpu_sig: u32, sev_features: u64) -> Box<[u8; 4096] page } -pub(crate) fn compute_expected_measurement( - config: &SevSnpMeasureConfig, - input: &MeasurementInput, -) -> Result<[u8; 48]> { +pub(crate) fn compute_expected_measurement(input: &MeasurementInput) -> Result<[u8; 48]> { let vcpu_type = input .vcpu_type .as_deref() .ok_or_else(|| anyhow::anyhow!("vcpu_type is required"))?; - let mut cmdline = match input.base_cmdline.as_deref() { + let cmdline = match input.base_cmdline.as_deref() { Some(base) if !base.trim().is_empty() => base.trim().to_string(), _ => "console=ttyS0 loglevel=7".to_string(), }; - if let Some(docker_files_hash) = input.docker_files_hash.as_deref() { - cmdline.push_str(&format!( - " docker_additional_files_hash={docker_files_hash}" - )); - } - - let (mut gctx, effective_hashes_gpa, effective_reset_eip, resolved_sections) = - if !input.ovmf_sections.is_empty() { - let sections = input - .ovmf_sections - .iter() - .map(|section| { - let section_type = - SectionType::from_u32(section.section_type).ok_or_else(|| { - anyhow::anyhow!("unknown ovmf section_type {:#x}", section.section_type) - })?; - Ok(MetadataSection { - gpa: section.gpa, - size: section.size, - section_type, - }) - }) - .collect::>>()?; - ( - Gctx::from_ovmf_hash(&input.ovmf_hash)?, - input.sev_hashes_table_gpa, - input.sev_es_reset_eip, - sections, - ) - } else { - let path = config.ovmf_path.as_deref().ok_or_else(|| { - anyhow::anyhow!("ovmf_sections are required when ovmf_path is not configured") + let resolved_sections = input + .ovmf_sections + .iter() + .map(|section| { + let section_type = SectionType::from_u32(section.section_type).ok_or_else(|| { + anyhow::anyhow!("unknown ovmf section_type {:#x}", section.section_type) })?; - let ovmf = OvmfInfo::load(path)?; - let gctx = if input.ovmf_hash.is_empty() { - let mut g = Gctx::new(); - g.update_normal_pages(ovmf.gpa, &ovmf.data); - g - } else { - Gctx::from_ovmf_hash(&input.ovmf_hash)? - }; - ( - gctx, - ovmf.sev_hashes_table_gpa, - ovmf.sev_es_reset_eip, - ovmf.sections, - ) - }; + Ok(MetadataSection { + gpa: section.gpa, + size: section.size, + section_type, + }) + }) + .collect::>>()?; + let mut gctx = Gctx::from_ovmf_hash(&input.ovmf_hash)?; + let effective_hashes_gpa = input.sev_hashes_table_gpa; + let effective_reset_eip = input.sev_es_reset_eip; let mut has_kernel_hashes_section = false; for section in &resolved_sections { @@ -1028,8 +985,8 @@ pub(crate) fn compute_expected_measurement( } let vcpu_sig = vcpu_sig_from_type(vcpu_type)?; - let bsp_vmsa = build_vmsa_page(0xffff_fff0, vcpu_sig, config.guest_features); - let ap_vmsa = build_vmsa_page(effective_reset_eip, vcpu_sig, config.guest_features); + let bsp_vmsa = build_vmsa_page(0xffff_fff0, vcpu_sig, input.guest_features); + let ap_vmsa = build_vmsa_page(effective_reset_eip, vcpu_sig, input.guest_features); for i in 0..input.vcpus as usize { let vmsa_page = if i == 0 { @@ -1049,17 +1006,7 @@ mod tests { fn config() -> SevSnpMeasureConfig { SevSnpMeasureConfig { - ovmf_path: None, - amd_kds_proxy_url: None, - guest_features: 1, - } - } - - fn config_with_path(path: &str) -> SevSnpMeasureConfig { - SevSnpMeasureConfig { - ovmf_path: Some(path.to_string()), - amd_kds_proxy_url: None, - guest_features: 1, + amd_kds_base_url: None, } } @@ -1073,7 +1020,6 @@ mod tests { compose_hash: hex_of(0x22, 32), rootfs_hash: hex_of(0x33, 32), base_cmdline: None, - docker_files_hash: Some(hex_of(0x77, 32)), ovmf_hash: hex_of(0x44, 48), kernel_hash: hex_of(0x55, 32), initrd_hash: hex_of(0x66, 32), @@ -1081,6 +1027,7 @@ mod tests { sev_es_reset_eip: 0xffff_fff0, vcpus: 2, vcpu_type: Some("epyc-v4".to_string()), + guest_features: 1, ovmf_sections: vec![ OvmfSectionParam { gpa: 0x100000, @@ -1110,6 +1057,10 @@ mod tests { test_mr_config_from_input(input) } + fn measurement_document(input: &MeasurementInput) -> String { + serde_json::to_string(input).expect("measurement input should serialize") + } + fn verified_snp_attestation( measurement: [u8; 48], chip_id: [u8; 64], @@ -1151,6 +1102,46 @@ mod tests { ); } + #[test] + fn snp_os_image_hash_covers_all_measurement_input_fields() { + let input = valid_input(); + let baseline = snp_measurement_os_image_hash(&measurement_document(&input)); + + let cases: Vec<(&str, fn(&mut MeasurementInput))> = vec![ + ("app_id", |i| i.app_id = hex_of(0x12, 20)), + ("compose_hash", |i| i.compose_hash = hex_of(0x23, 32)), + ("rootfs_hash", |i| i.rootfs_hash = hex_of(0x34, 32)), + ("base_cmdline", |i| { + i.base_cmdline = Some("console=ttyS0 loglevel=8".to_string()) + }), + ("ovmf_hash", |i| i.ovmf_hash = hex_of(0x45, 48)), + ("kernel_hash", |i| i.kernel_hash = hex_of(0x56, 32)), + ("initrd_hash", |i| i.initrd_hash = hex_of(0x67, 32)), + ("sev_hashes_table_gpa", |i| i.sev_hashes_table_gpa += 0x1000), + ("sev_es_reset_eip", |i| i.sev_es_reset_eip = 0xffff_0000), + ("vcpus", |i| i.vcpus = 3), + ("vcpu_type", |i| { + i.vcpu_type = Some("epyc-milan".to_string()) + }), + ("guest_features", |i| i.guest_features = 3), + ("ovmf_sections.gpa", |i| i.ovmf_sections[0].gpa += 0x1000), + ("ovmf_sections.size", |i| i.ovmf_sections[0].size += 0x1000), + ("ovmf_sections.section_type", |i| { + i.ovmf_sections[0].section_type = 4 + }), + ]; + + for (name, mutate) in cases { + let mut changed = input.clone(); + mutate(&mut changed); + assert_ne!( + baseline, + snp_measurement_os_image_hash(&measurement_document(&changed)), + "{name} must be covered by SNP os_image_hash" + ); + } + } + #[test] fn gctx_update_is_deterministic_and_order_sensitive() { let contents = Gctx::sha384(b"page"); @@ -1199,10 +1190,10 @@ mod tests { #[test] fn accepts_recomputed_matching_measurement_and_rejects_mismatch() { let input = valid_input(); - let expected = compute_expected_measurement(&config(), &input).unwrap(); + let expected = compute_expected_measurement(&input).unwrap(); assert_eq!( hex::encode(expected), - "56a10702f43f18df8a87404fb92637e65d2e069056e58938b3de085d2f33f070aeaa2bac0013e969a2fe283baf40eeaa", + "88a47914470533e33e24befd24ef0ac877658ff82cafc9878bd9566550f100fdf56d62f419e21b959aa228fc98000da4", "synthetic measurement vector should not drift silently" ); validate_amd_snp_measurement_binding(Some(&config()), &expected, &input) @@ -1218,7 +1209,7 @@ mod tests { #[test] fn builds_snp_boot_info_for_matching_measurement_only() { let input = valid_input(); - let verified = compute_expected_measurement(&config(), &input).unwrap(); + let verified = compute_expected_measurement(&input).unwrap(); let chip_id = [0xab; 64]; let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input) @@ -1228,7 +1219,10 @@ mod tests { assert_eq!(boot_info.device_id, chip_id.to_vec()); assert_eq!(boot_info.app_id, vec![0x11; 20]); assert_eq!(boot_info.compose_hash, vec![0x22; 32]); - assert_eq!(boot_info.os_image_hash, vec![0x33; 32]); + assert_eq!( + boot_info.os_image_hash, + snp_measurement_os_image_hash(&measurement_document(&input)) + ); assert_eq!(boot_info.mr_system.len(), 32); assert!(!boot_info.key_provider_info.is_empty()); assert_eq!(boot_info.instance_id.len(), 20); @@ -1246,7 +1240,7 @@ mod tests { #[test] fn builds_snp_boot_info_from_verified_attestation_report() -> Result<()> { let input = valid_input(); - let verified = compute_expected_measurement(&config(), &input).unwrap(); + let verified = compute_expected_measurement(&input).unwrap(); let chip_id = [0xab; 64]; let mr_config = valid_mr_config(&input)?; let mr_config_document = mr_config.to_canonical_json(); @@ -1256,6 +1250,7 @@ mod tests { &config(), &attestation, &input, + &measurement_document(&input), &mr_config_document, ) .expect("verified snp attestation should feed boot info helper"); @@ -1270,7 +1265,7 @@ mod tests { #[test] fn verified_attestation_tcb_status_replaces_snp_placeholder() -> Result<()> { let input = valid_input(); - let verified = compute_expected_measurement(&config(), &input).unwrap(); + let verified = compute_expected_measurement(&input).unwrap(); let chip_id = [0xbc; 64]; let mr_config = valid_mr_config(&input)?; let mr_config_document = mr_config.to_canonical_json(); @@ -1317,6 +1312,7 @@ mod tests { &config(), &attestation, &input, + &measurement_document(&input), &mr_config_document, ) .expect("verified snp attestation should feed boot info helper"); @@ -1339,12 +1335,12 @@ mod tests { #[test] fn builds_snp_boot_info_from_verified_attestation_and_vm_config_json() -> Result<()> { let input = valid_input(); - let verified = compute_expected_measurement(&config(), &input).unwrap(); + let verified = compute_expected_measurement(&input).unwrap(); let chip_id = [0xab; 64]; let mr_config = valid_mr_config(&input)?; let attestation = verified_snp_attestation(verified, chip_id, &mr_config); let vm_config = serde_json::json!({ - "sev_snp_measurement": input, + "sev_snp_measurement": measurement_document(&input), "mr_config": mr_config.to_canonical_json(), }) .to_string(); @@ -1365,7 +1361,7 @@ mod tests { #[test] fn verified_attestation_vm_config_helper_requires_snp_measurement_input() -> Result<()> { let input = valid_input(); - let verified = compute_expected_measurement(&config(), &input).unwrap(); + let verified = compute_expected_measurement(&input).unwrap(); let mr_config = valid_mr_config(&input)?; let attestation = verified_snp_attestation(verified, [0xab; 64], &mr_config); @@ -1387,7 +1383,7 @@ mod tests { let mut measurement = serde_json::to_value(valid_input()).unwrap(); measurement["unexpected"] = serde_json::json!(true); let vm_config = serde_json::json!({ - "sev_snp_measurement": measurement, + "sev_snp_measurement": measurement.to_string(), }) .to_string(); @@ -1414,7 +1410,7 @@ mod tests { .collect(), ); let vm_config = serde_json::json!({ - "sev_snp_measurement": measurement, + "sev_snp_measurement": measurement.to_string(), }) .to_string(); @@ -1458,6 +1454,7 @@ mod tests { &config(), &attestation, &input, + &measurement_document(&input), &mr_config_document, ) .expect_err("non-snp verified attestation must reject"); @@ -1472,13 +1469,13 @@ mod tests { #[test] fn app_id_changes_host_data_and_authorization_binding() -> Result<()> { let input = valid_input(); - let verified = compute_expected_measurement(&config(), &input)?; + let verified = compute_expected_measurement(&input)?; let chip_id = [0xcd; 64]; let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input)?; let mut changed = input.clone(); changed.app_id = hex_of(0x12, 20); - let changed_measurement = compute_expected_measurement(&config(), &changed)?; + let changed_measurement = compute_expected_measurement(&changed)?; assert_eq!( changed_measurement, verified, "app_id must not be added to the SNP measured cmdline" @@ -1487,6 +1484,7 @@ mod tests { assert_ne!(boot_info.app_id, changed_boot_info.app_id); assert_ne!(boot_info.instance_id, changed_boot_info.instance_id); + assert_ne!(boot_info.os_image_hash, changed_boot_info.os_image_hash); assert_ne!(boot_info.mr_aggregated, changed_boot_info.mr_aggregated); assert_eq!(boot_info.mr_system, changed_boot_info.mr_system); Ok(()) @@ -1495,7 +1493,7 @@ mod tests { #[test] fn measured_input_changes_reject_until_measurement_is_recomputed() { let input = valid_input(); - let verified = compute_expected_measurement(&config(), &input).unwrap(); + let verified = compute_expected_measurement(&input).unwrap(); let chip_id = [0xef; 64]; let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input).unwrap(); @@ -1509,19 +1507,20 @@ mod tests { .expect_err("stale verified measurement must reject changed measured input"); assert!(err.to_string().contains("amd sev-snp measurement mismatch")); - let changed_verified = compute_expected_measurement(&config(), &changed).unwrap(); + let changed_verified = compute_expected_measurement(&changed).unwrap(); let changed_boot_info = build_amd_snp_boot_info(&config(), &changed_verified, &chip_id, &changed) .expect("recomputed measurement should build boot info"); assert_ne!(boot_info.mr_aggregated, changed_boot_info.mr_aggregated); assert_ne!(boot_info.mr_system, changed_boot_info.mr_system); + assert_ne!(boot_info.os_image_hash, changed_boot_info.os_image_hash); } } #[test] fn chip_id_maps_to_device_id_and_changes_chip_bound_digests() { let input = valid_input(); - let verified = compute_expected_measurement(&config(), &input).unwrap(); + let verified = compute_expected_measurement(&input).unwrap(); let boot_info = build_amd_snp_boot_info(&config(), &verified, &[0x01; 64], &input).unwrap(); let changed_boot_info = build_amd_snp_boot_info(&config(), &verified, &[0x02; 64], &input).unwrap(); @@ -1559,17 +1558,28 @@ mod tests { let kernel_hash = hex::encode(Sha256::digest(kernel_bytes)); let initrd_hash = hex::encode(Sha256::digest(initrd_bytes)); let mut input = valid_input(); - input.docker_files_hash = None; - input.ovmf_hash.clear(); - input.ovmf_sections.clear(); + let ovmf = OvmfInfo::load(&ovmf_path).expect("ovmf metadata should load"); + let mut gctx = Gctx::new(); + gctx.update_normal_pages(ovmf.gpa, &ovmf.data); + input.ovmf_hash = hex::encode(gctx.ld); + input.sev_hashes_table_gpa = ovmf.sev_hashes_table_gpa; + input.sev_es_reset_eip = ovmf.sev_es_reset_eip; + input.ovmf_sections = ovmf + .sections + .iter() + .map(|section| OvmfSectionParam { + gpa: section.gpa, + size: section.size, + section_type: section.section_type as u32, + }) + .collect(); input.kernel_hash = kernel_hash; input.initrd_hash = initrd_hash; input.vcpus = 2; input.vcpu_type = Some("EPYC-v4".to_string()); - let config = config_with_path(&ovmf_path); - let recomputed = compute_expected_measurement(&config, &input) - .expect("dstack recomputation should succeed"); + let recomputed = + compute_expected_measurement(&input).expect("dstack recomputation should succeed"); let append = "console=ttyS0 loglevel=7"; let output = std::process::Command::new("sev-snp-measure") @@ -1611,7 +1621,7 @@ mod tests { #[test] fn explicit_snp_auth_policy_accepts_only_exact_verified_identity() { let input = valid_input(); - let verified = compute_expected_measurement(&config(), &input).unwrap(); + let verified = compute_expected_measurement(&input).unwrap(); let chip_id = [0x42; 64]; let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input).unwrap(); let policy = AmdSnpAuthPolicy::from_boot_info(&boot_info) @@ -1630,7 +1640,7 @@ mod tests { #[test] fn explicit_snp_auth_policy_rejects_incomplete_or_unsafe_tcb() { let input = valid_input(); - let verified = compute_expected_measurement(&config(), &input).unwrap(); + let verified = compute_expected_measurement(&input).unwrap(); let chip_id = [0x24; 64]; let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input).unwrap(); let policy = AmdSnpAuthPolicy::from_boot_info(&boot_info).unwrap(); @@ -1659,7 +1669,7 @@ mod tests { #[test] fn explicit_snp_auth_policy_rejects_partial_allowlists() { let input = valid_input(); - let verified = compute_expected_measurement(&config(), &input).unwrap(); + let verified = compute_expected_measurement(&input).unwrap(); let chip_id = [0x35; 64]; let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input).unwrap(); @@ -1683,13 +1693,11 @@ mod tests { } #[test] - fn rejects_missing_config() { - let verified = [0xaa; 48]; - let err = validate_amd_snp_measurement_binding(None, &verified, &valid_input()) - .expect_err("missing config must fail closed"); - assert!(err - .to_string() - .contains("sev-snp measurement config is required")); + fn accepts_self_contained_measurement_input_without_sev_snp_config() { + let input = valid_input(); + let expected = compute_expected_measurement(&input).unwrap(); + validate_amd_snp_measurement_binding(None, &expected, &input) + .expect("self-contained SNP launch input should not need KMS-local config"); } #[test] @@ -1712,13 +1720,9 @@ mod tests { let mut input = valid_input(); input.initrd_hash.clear(); - let expected = compute_expected_measurement(&config(), &input).unwrap(); + let expected = compute_expected_measurement(&input).unwrap(); validate_amd_snp_measurement_binding(Some(&config()), &expected, &input) .expect("empty initrd hash should mean empty initrd"); - - let mut input = valid_input(); - input.docker_files_hash = Some(String::new()); - assert_rejects(input, "docker_files_hash must not be empty"); } #[test] @@ -1741,50 +1745,14 @@ mod tests { let mut input = valid_input(); input.ovmf_sections.clear(); - assert_rejects( - input, - "ovmf_sections are required when ovmf_path is not configured", - ); - - let mut input = valid_input(); - input.ovmf_sections.clear(); - input.ovmf_hash.clear(); - let verified = [0xaa; 48]; - let err = validate_amd_snp_measurement_binding( - Some(&config_with_path("/path/that/does/not/exist/ovmf.fd")), - &verified, - &input, - ) - .expect_err("configured ovmf_path should be used for recomputation"); - assert!(err.to_string().contains("cannot read ovmf binary")); - - let mut input = valid_input(); - input.ovmf_sections.clear(); - let err = validate_amd_snp_measurement_binding( - Some(&config_with_path("/opt/amd/ovmf.fd")), - &verified, - &input, - ) - .expect_err("request ovmf_hash must not override configured ovmf_path"); - assert!(err - .to_string() - .contains("ovmf_hash must be empty when ovmf_path is used")); + assert_rejects(input, "ovmf_sections are required for amd sev-snp"); } #[test] fn rejects_unsafe_machine_config() { - let verified = [0xaa; 48]; - let err = validate_amd_snp_measurement_binding( - Some(&SevSnpMeasureConfig { - ovmf_path: None, - amd_kds_proxy_url: None, - guest_features: 0, - }), - &verified, - &valid_input(), - ) - .expect_err("zero guest_features must fail closed"); - assert!(err.to_string().contains("guest_features must be non-zero")); + let mut input = valid_input(); + input.guest_features = 0; + assert_rejects(input, "guest_features must be non-zero"); let mut input = valid_input(); input.ovmf_sections[0].size = 0; @@ -1823,14 +1791,5 @@ mod tests { let mut input = valid_input(); input.sev_es_reset_eip = 0; assert_rejects(input, "sev_es_reset_eip must be non-zero"); - - let mut input = valid_input(); - input.ovmf_sections.clear(); - let err = - validate_amd_snp_measurement_binding(Some(&config_with_path(" ")), &verified, &input) - .expect_err("blank ovmf_path must not bypass section metadata requirement"); - assert!(err - .to_string() - .contains("ovmf_sections are required when ovmf_path is not configured")); } } diff --git a/kms/src/onboard_service.rs b/kms/src/onboard_service.rs index 3ea042abf..617a734ce 100644 --- a/kms/src/onboard_service.rs +++ b/kms/src/onboard_service.rs @@ -197,16 +197,15 @@ mod tests { use crate::{ config::SevSnpMeasureConfig, main_service::amd_attest::{ - compute_expected_measurement, MeasurementInput, OvmfSectionParam, + compute_expected_measurement, snp_measurement_os_image_hash, MeasurementInput, + OvmfSectionParam, }, }; use sha2::Digest; fn sev_snp_config() -> SevSnpMeasureConfig { SevSnpMeasureConfig { - ovmf_path: None, - amd_kds_proxy_url: None, - guest_features: 1, + amd_kds_base_url: None, } } @@ -220,7 +219,6 @@ mod tests { compose_hash: hex_of(0x22, 32), rootfs_hash: hex_of(0x33, 32), base_cmdline: None, - docker_files_hash: Some(hex_of(0x77, 32)), ovmf_hash: hex_of(0x44, 48), kernel_hash: hex_of(0x55, 32), initrd_hash: hex_of(0x66, 32), @@ -228,6 +226,7 @@ mod tests { sev_es_reset_eip: 0xffff_fff0, vcpus: 2, vcpu_type: Some("epyc-v4".to_string()), + guest_features: 1, ovmf_sections: vec![ OvmfSectionParam { gpa: 0x100000, @@ -292,11 +291,11 @@ mod tests { #[test] fn attestation_info_response_uses_snp_boot_info_and_chip_id() { let input = valid_snp_measurement_input(); - let measurement = compute_expected_measurement(&sev_snp_config(), &input).unwrap(); + let measurement = compute_expected_measurement(&input).unwrap(); let mr_config = valid_snp_mr_config(); let attestation = verified_snp_attestation(measurement, [0xab; 64]); let vm_config = serde_json::json!({ - "sev_snp_measurement": input, + "sev_snp_measurement": serde_json::to_string(&input).unwrap(), "mr_config": mr_config.to_canonical_json(), }) .to_string(); @@ -318,7 +317,10 @@ mod tests { ); assert_eq!(response.ppid, vec![0xab; 64]); assert_eq!(response.mr_aggregated.len(), 32); - assert_eq!(response.os_image_hash, vec![0x33; 32]); + assert_eq!( + response.os_image_hash, + snp_measurement_os_image_hash(&serde_json::to_string(&input).unwrap()) + ); assert_eq!(response.attestation_mode, "dstack-amd-sev-snp"); assert_eq!(response.site_name, "test-site"); assert_eq!(response.eth_rpc_url, "https://rpc.example"); diff --git a/ra-rpc/src/rocket_helper.rs b/ra-rpc/src/rocket_helper.rs index 68bb813bc..ede119569 100644 --- a/ra-rpc/src/rocket_helper.rs +++ b/ra-rpc/src/rocket_helper.rs @@ -184,7 +184,7 @@ fn unix_peer_cred(stream: &UnixStream) -> Option { #[derive(Debug, Clone)] pub struct QuoteVerifier { pccs_url: Option, - amd_kds_proxy_url: Option, + amd_kds_base_url: Option, } pub mod deps { @@ -317,24 +317,24 @@ impl<'r> FromRequest<'r> for &'r QuoteVerifier { impl QuoteVerifier { pub fn new(pccs_url: Option) -> Self { - Self::new_with_amd_kds_proxy(pccs_url, None) + Self::new_with_amd_kds_base(pccs_url, None) } - pub fn new_with_amd_kds_proxy( + pub fn new_with_amd_kds_base( pccs_url: Option, - amd_kds_proxy_url: Option, + amd_kds_base_url: Option, ) -> Self { Self { pccs_url, - amd_kds_proxy_url: amd_kds_proxy_url + amd_kds_base_url: amd_kds_base_url .map(|url| url.trim().to_string()) .filter(|url| !url.is_empty()), } } - fn configure_amd_kds_proxy_for_request(&self) { - if let Some(proxy_url) = &self.amd_kds_proxy_url { - std::env::set_var("DSTACK_AMD_KDS_PROXY_URL", proxy_url); + fn configure_amd_kds_base_for_request(&self) { + if let Some(base_url) = &self.amd_kds_base_url { + std::env::set_var("DSTACK_AMD_KDS_BASE_URL", base_url); } } } @@ -460,18 +460,18 @@ mod tests { use std::time::{SystemTime, UNIX_EPOCH}; #[test] - fn quote_verifier_carries_trimmed_amd_kds_proxy_url() { - let verifier = QuoteVerifier::new_with_amd_kds_proxy( + fn quote_verifier_carries_trimmed_amd_kds_base_url() { + let verifier = QuoteVerifier::new_with_amd_kds_base( None, - Some(" https://cors.litgateway.com/ ".to_string()), + Some(" https://mirror.example.com/vcek/v1/ ".to_string()), ); assert_eq!( - verifier.amd_kds_proxy_url.as_deref(), - Some("https://cors.litgateway.com/") + verifier.amd_kds_base_url.as_deref(), + Some("https://mirror.example.com/vcek/v1/") ); - let verifier = QuoteVerifier::new_with_amd_kds_proxy(None, Some(" ".to_string())); - assert!(verifier.amd_kds_proxy_url.is_none()); + let verifier = QuoteVerifier::new_with_amd_kds_base(None, Some(" ".to_string())); + assert!(verifier.amd_kds_base_url.is_none()); } #[test] @@ -567,7 +567,7 @@ pub async fn handle_prpc_impl>( .flatten(); let attestation = match (request.quote_verifier, attestation) { (Some(quote_verifier), Some(attestation)) => { - quote_verifier.configure_amd_kds_proxy_for_request(); + quote_verifier.configure_amd_kds_base_for_request(); let pubkey = request .certificate .context("certificate is missing")? diff --git a/sev-snp-qvl/src/lib.rs b/sev-snp-qvl/src/lib.rs index b860a7e7f..897575a8c 100644 --- a/sev-snp-qvl/src/lib.rs +++ b/sev-snp-qvl/src/lib.rs @@ -24,6 +24,8 @@ const VLEK_CERT_GUID: [u8; 16] = [ 0xa8, 0x07, 0x4b, 0xc2, 0xa2, 0x5a, 0x48, 0x3e, 0xaa, 0xe6, 0x39, 0xc0, 0x45, 0xa0, 0xb8, 0xa1, ]; const CERT_TABLE_ENTRY_SIZE: usize = 24; +const AMD_KDS_BASE_URL_ENV: &str = "DSTACK_AMD_KDS_BASE_URL"; +const AMD_KDS_DEFAULT_BASE_URL: &str = "https://kdsintf.amd.com/vcek/v1"; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum AmdSnpProduct { @@ -369,15 +371,12 @@ fn fetch_amd_kds_collateral_for_product( .context("amd sev-snp chip_id too short")?, ); let vcek_url = amd_kds_vcek_url(product, &chip_id, report.reported_tcb.into())?; - let vcek_request_url = amd_kds_request_url(&vcek_url); let vcek = reqwest::blocking::Client::new() - .get(&vcek_request_url) + .get(&vcek_url) .send() - .with_context(|| format!("failed to request amd sev-snp vcek from {vcek_request_url}"))? + .with_context(|| format!("failed to request amd sev-snp vcek from {vcek_url}"))? .error_for_status() - .with_context(|| { - format!("amd sev-snp vcek request failed for {vcek_url} via {vcek_request_url}") - })? + .with_context(|| format!("amd sev-snp vcek request failed for {vcek_url}"))? .bytes() .context("failed to read amd sev-snp vcek response")? .to_vec(); @@ -392,28 +391,37 @@ fn fetch_amd_kds_collateral_for_product( } fn fetch_amd_kds_ca_chain(product: AmdSnpProduct) -> Result<(CertBytes, CertBytes)> { - let url = format!( - "https://kdsintf.amd.com/vcek/v1/{}/cert_chain", - product.kds_name() - ); - let request_url = amd_kds_request_url(&url); + let url = amd_kds_endpoint(&format!("{}/cert_chain", product.kds_name())); let chain = reqwest::blocking::Client::new() - .get(&request_url) + .get(&url) .send() - .with_context(|| format!("failed to request amd sev-snp cert_chain from {request_url}"))? + .with_context(|| format!("failed to request amd sev-snp cert_chain from {url}"))? .error_for_status() - .with_context(|| format!("amd sev-snp cert_chain request failed for {request_url}"))? + .with_context(|| format!("amd sev-snp cert_chain request failed for {url}"))? .bytes() .context("failed to read amd sev-snp cert_chain response")?; let (_fetched_ark, ask) = extract_ark_ask_from_amd_kds_cert_chain(&chain)?; Ok((product.builtin_ark(), ask)) } -fn amd_kds_request_url(amd_url: &str) -> String { - match std::env::var("DSTACK_AMD_KDS_PROXY_URL") { - Ok(proxy) if !proxy.trim().is_empty() => format!("{}{}", proxy.trim(), amd_url), - _ => amd_url.to_string(), - } +fn amd_kds_base_url() -> String { + std::env::var(AMD_KDS_BASE_URL_ENV) + .ok() + .map(|url| url.trim().trim_end_matches('/').to_string()) + .filter(|url| !url.is_empty()) + .unwrap_or_else(|| AMD_KDS_DEFAULT_BASE_URL.to_string()) +} + +fn amd_kds_endpoint(path: &str) -> String { + join_amd_kds_url(&amd_kds_base_url(), path) +} + +fn join_amd_kds_url(base_url: &str, path: &str) -> String { + format!( + "{}/{}", + base_url.trim().trim_end_matches('/'), + path.trim_start_matches('/') + ) } fn amd_snp_product_candidates_for_report(report: &AttestationReport) -> Result> { @@ -466,31 +474,46 @@ fn amd_kds_vcek_url( product: AmdSnpProduct, chip_id: &[u8; 64], tcb: AmdSnpTcbVersion, +) -> Result { + amd_kds_vcek_url_with_base(&amd_kds_base_url(), product, chip_id, tcb) +} + +fn amd_kds_vcek_url_with_base( + base_url: &str, + product: AmdSnpProduct, + chip_id: &[u8; 64], + tcb: AmdSnpTcbVersion, ) -> Result { let url = match product { AmdSnpProduct::Turin => { let fmc = tcb .fmc .context("amd sev-snp Turin VCEK request requires reported FMC TCB")?; - format!( - "https://kdsintf.amd.com/vcek/v1/{}/{}?fmcSPL={}&blSPL={}&teeSPL={}&snpSPL={}&ucodeSPL={}", + join_amd_kds_url( + base_url, + &format!( + "{}/{}?fmcSPL={}&blSPL={}&teeSPL={}&snpSPL={}&ucodeSPL={}", + product.kds_name(), + hex::encode(&chip_id[..8]), + fmc, + tcb.bootloader, + tcb.tee, + tcb.snp, + tcb.microcode + ), + ) + } + AmdSnpProduct::Milan | AmdSnpProduct::Genoa => join_amd_kds_url( + base_url, + &format!( + "{}/{}?blSPL={}&teeSPL={}&snpSPL={}&ucodeSPL={}", product.kds_name(), - hex::encode(&chip_id[..8]), - fmc, + hex::encode(chip_id), tcb.bootloader, tcb.tee, tcb.snp, tcb.microcode - ) - } - AmdSnpProduct::Milan | AmdSnpProduct::Genoa => format!( - "https://kdsintf.amd.com/vcek/v1/{}/{}?blSPL={}&teeSPL={}&snpSPL={}&ucodeSPL={}", - product.kds_name(), - hex::encode(chip_id), - tcb.bootloader, - tcb.tee, - tcb.snp, - tcb.microcode + ), ), }; Ok(url) @@ -731,7 +754,13 @@ mod tests { microcode: 4, }; - let url = amd_kds_vcek_url(AmdSnpProduct::Genoa, &chip_id, tcb).unwrap(); + let url = amd_kds_vcek_url_with_base( + AMD_KDS_DEFAULT_BASE_URL, + AmdSnpProduct::Genoa, + &chip_id, + tcb, + ) + .unwrap(); assert_eq!( url, @@ -753,7 +782,13 @@ mod tests { microcode: 4, }; - let url = amd_kds_vcek_url(AmdSnpProduct::Turin, &chip_id, tcb).unwrap(); + let url = amd_kds_vcek_url_with_base( + AMD_KDS_DEFAULT_BASE_URL, + AmdSnpProduct::Turin, + &chip_id, + tcb, + ) + .unwrap(); assert_eq!( url, @@ -802,22 +837,11 @@ mod tests { } #[test] - fn amd_kds_proxy_url_wraps_amd_urls_when_configured() { - const ENV_KEY: &str = "DSTACK_AMD_KDS_PROXY_URL"; - let old = std::env::var(ENV_KEY).ok(); - std::env::set_var(ENV_KEY, "https://cors.litgateway.com/"); - - let wrapped = amd_kds_request_url("https://kdsintf.amd.com/vcek/v1/Genoa/cert_chain"); - + fn amd_kds_endpoint_joins_base_url_and_relative_path() { assert_eq!( - wrapped, - "https://cors.litgateway.com/https://kdsintf.amd.com/vcek/v1/Genoa/cert_chain" + join_amd_kds_url("https://mirror.example.com/vcek/v1/", "/Genoa/cert_chain"), + "https://mirror.example.com/vcek/v1/Genoa/cert_chain" ); - if let Some(old) = old { - std::env::set_var(ENV_KEY, old); - } else { - std::env::remove_var(ENV_KEY); - } } #[test] diff --git a/test-scripts/snp-e2e-smoke.sh b/test-scripts/snp-e2e-smoke.sh index 345aa7f59..d9266ad20 100755 --- a/test-scripts/snp-e2e-smoke.sh +++ b/test-scripts/snp-e2e-smoke.sh @@ -40,11 +40,12 @@ # meta-dstack guest image that includes the same PR cert-chain/KDS fallback code. # If AMD KDS throttles VCEK/cert-chain retrieval (for example HTTP 429 from # kdsintf.amd.com), keep verification fail-closed and set -# DSTACK_SNP_SMOKE_KDS_PROXY_URL to a trusted path-prefix AMD-KDS passthrough, -# e.g. https://cors.litgateway.com/ so verifier requests become: -# https://cors.litgateway.com/https://kdsintf.amd.com/... +# DSTACK_SNP_SMOKE_KDS_BASE_URL to a trusted AMD-KDS-compatible mirror/cache +# base, e.g. https://mirror.example.com/vcek/v1. For a path-prefix relay, set +# the full relayed base, e.g.: +# https://cors.litgateway.com/https://kdsintf.amd.com/vcek/v1 # This is an external collateral-fetch boundary, not a guest boot or KMS startup -# failure. Do not use a ?url= wrapper unless the proxy explicitly supports it. +# failure. # One reproducible way is to build meta-dstack with its dstack submodule checked # out to this PR branch, set the Yocto build MACHINE to `sev-snp` (not the # default `tdx`, otherwise the guest kernel can miss AMD memory-encryption @@ -122,8 +123,8 @@ echo "qemu=$QEMU_PATH" echo "qemu_version=$qemu_version_output" echo "ovmf_sha256=$(sha256sum "$OVMF_PATH" | awk '{print $1}')" echo "image=$IMAGE_NAME" -if [[ -n "${DSTACK_SNP_SMOKE_KDS_PROXY_URL:-}" ]]; then - echo "amd_kds_proxy_url=${DSTACK_SNP_SMOKE_KDS_PROXY_URL}" +if [[ -n "${DSTACK_SNP_SMOKE_KDS_BASE_URL:-}" ]]; then + echo "amd_kds_base_url=${DSTACK_SNP_SMOKE_KDS_BASE_URL}" fi cleanup() { @@ -144,7 +145,6 @@ sudo pkill -f "$BIN/supervisor" 2>/dev/null || true sudo rm -rf "$BASE/run"/* "$BASE/tmp"/* cp "$BIN/dstack-kms" "$BASE/http-root/dstack-kms" -cp "$OVMF_PATH" "$BASE/http-root/OVMF.fd" chmod +x "$BASE/http-root/dstack-kms" if [[ ! -d "$BASE/images/$IMAGE_NAME" ]]; then @@ -154,6 +154,9 @@ if [[ ! -d "$BASE/images/$IMAGE_NAME" ]]; then tar -xzf "$BASE/$IMAGE_NAME.tar.gz" -C "$BASE/images/$IMAGE_NAME" --strip-components=1 fi cp "$OVMF_PATH" "$BASE/images/$IMAGE_NAME/ovmf.fd" +tmp_metadata="$(mktemp)" +jq '.bios = "ovmf.fd"' "$BASE/images/$IMAGE_NAME/metadata.json" >"$tmp_metadata" +mv "$tmp_metadata" "$BASE/images/$IMAGE_NAME/metadata.json" jq . "$BASE/images/$IMAGE_NAME/metadata.json" | tee "$ART/image-metadata.json" cat >"$BASE/auth-server.py" <<'PY' @@ -358,9 +361,7 @@ enabled = true auto_bootstrap_domain = "10.0.2.2" [core.sev_snp] -ovmf_path = "/dstack/OVMF.fd" -guest_features = 1 -amd_kds_proxy_url = "${DSTACK_SNP_SMOKE_KDS_PROXY_URL:-}" +amd_kds_base_url = "${DSTACK_SNP_SMOKE_KDS_BASE_URL:-}" [core.sev_snp_key_release] enabled = true @@ -387,7 +388,6 @@ KMS_BASH_SCRIPT=$(cat <<'SH' set -eux mkdir -p /dstack/kms-certs /dstack/kms-images curl -fsS http://10.0.2.2:__DSTACK_HOST_ART_PORT__/dstack-kms -o /dstack/dstack-kms -curl -fsS http://10.0.2.2:__DSTACK_HOST_ART_PORT__/OVMF.fd -o /dstack/OVMF.fd curl -fsS http://10.0.2.2:__DSTACK_HOST_ART_PORT__/kms.toml -o /dstack/kms.toml chmod +x /dstack/dstack-kms echo SNP_KMS_CONTAINER_STARTED diff --git a/vmm/src/app.rs b/vmm/src/app.rs index ee2d7f978..34dd51684 100644 --- a/vmm/src/app.rs +++ b/vmm/src/app.rs @@ -38,6 +38,7 @@ mod id_pool; mod image; mod qemu; pub(crate) mod registry; +mod snp_measure; #[derive(Deserialize, Serialize, Debug, Clone)] pub struct PortMapping { @@ -1219,6 +1220,19 @@ fn file_sha256_hex(path: &Path) -> Result { Ok(hex::encode(sha256_file(path)?)) } +fn amd_sev_snp_ovmf_measurement_info(image: &Image) -> Result { + let bios = image + .bios + .as_deref() + .ok_or_else(|| anyhow::anyhow!("bios/OVMF is required for amd sev-snp measurement"))?; + snp_measure::ovmf_measurement_info(bios).with_context(|| { + format!( + "failed to extract amd sev-snp OVMF measurement metadata from {}", + bios.display() + ) + }) +} + fn image_rootfs_hash(image: &Image) -> Result<&str> { if let Some(rootfs_hash) = image.info.rootfs_hash.as_deref() { return Ok(rootfs_hash); @@ -1278,19 +1292,24 @@ fn make_vm_config( MrConfigV3::from_document(&mr_config).context("Invalid mr_config document")?; config["mr_config"] = serde_json::Value::String(mr_config); } - config["sev_snp_measurement"] = json!({ + let ovmf = amd_sev_snp_ovmf_measurement_info(image)?; + let measurement = json!({ "rootfs_hash": rootfs_hash, "base_cmdline": amd_sev_snp_measurement_base_cmdline(image.info.cmdline.as_deref()), - "docker_files_hash": serde_json::Value::Null, - "ovmf_hash": "", + "ovmf_hash": ovmf.ovmf_hash, "kernel_hash": file_sha256_hex(&image.kernel)?, "initrd_hash": file_sha256_hex(&image.initrd)?, - "sev_hashes_table_gpa": 0, - "sev_es_reset_eip": 0, + "sev_hashes_table_gpa": ovmf.sev_hashes_table_gpa, + "sev_es_reset_eip": ovmf.sev_es_reset_eip, "vcpus": manifest.vcpu, "vcpu_type": "EPYC-v4", - "ovmf_sections": [], + "guest_features": 1, + "ovmf_sections": ovmf.sections, }); + config["sev_snp_measurement"] = serde_json::Value::String( + serde_json::to_string(&measurement) + .context("Failed to serialize amd sev-snp measurement input")?, + ); } Ok(config) } @@ -1306,6 +1325,79 @@ mod tests { hex::encode(vec![byte; len]) } + fn write_u16_le_at(buf: &mut [u8], off: usize, value: u16) { + buf[off..off + 2].copy_from_slice(&value.to_le_bytes()); + } + + fn write_u32_le_at(buf: &mut [u8], off: usize, value: u32) { + buf[off..off + 4].copy_from_slice(&value.to_le_bytes()); + } + + fn ovmf_footer_entry(data: &[u8], guid: &[u8; 16]) -> Vec { + let mut entry = data.to_vec(); + entry.extend_from_slice(&((data.len() + 18) as u16).to_le_bytes()); + entry.extend_from_slice(guid); + entry + } + + fn synthetic_snp_ovmf() -> Vec { + const GUID_FOOTER_TABLE: [u8; 16] = [ + 0xde, 0x82, 0xb5, 0x96, 0xb2, 0x1f, 0xf7, 0x45, 0xba, 0xea, 0xa3, 0x66, 0xc5, 0x5a, + 0x08, 0x2d, + ]; + const GUID_SEV_HASH_TABLE_RV: [u8; 16] = [ + 0x1f, 0x37, 0x55, 0x72, 0x3b, 0x3a, 0x04, 0x4b, 0x92, 0x7b, 0x1d, 0xa6, 0xef, 0xa8, + 0xd4, 0x54, + ]; + const GUID_SEV_ES_RESET_BLK: [u8; 16] = [ + 0xde, 0x71, 0xf7, 0x00, 0x7e, 0x1a, 0xcb, 0x4f, 0x89, 0x0e, 0x68, 0xc7, 0x7e, 0x2f, + 0xb4, 0x4e, + ]; + const GUID_SEV_META_DATA: [u8; 16] = [ + 0x66, 0x65, 0x88, 0xdc, 0x4a, 0x98, 0x98, 0x47, 0xa7, 0x5e, 0x55, 0x85, 0xa7, 0xbf, + 0x67, 0xcc, + ]; + + let mut ovmf = vec![0u8; 4096]; + let meta_start = 512usize; + ovmf[meta_start..meta_start + 4].copy_from_slice(b"ASEV"); + write_u32_le_at(&mut ovmf, meta_start + 8, 1); + write_u32_le_at(&mut ovmf, meta_start + 12, 4); + let sections = [ + (0x1000u32, 0x1000u32, 1u32), + (0x2000u32, 0x1000u32, 2u32), + (0x3000u32, 0x1000u32, 3u32), + (0x4000u32, 0x1000u32, 0x10u32), + ]; + for (i, (gpa, size, section_type)) in sections.into_iter().enumerate() { + let off = meta_start + 16 + i * 12; + write_u32_le_at(&mut ovmf, off, gpa); + write_u32_le_at(&mut ovmf, off + 4, size); + write_u32_le_at(&mut ovmf, off + 8, section_type); + } + + let mut table = Vec::new(); + table.extend(ovmf_footer_entry( + &0x4000u32.to_le_bytes(), + &GUID_SEV_HASH_TABLE_RV, + )); + table.extend(ovmf_footer_entry( + &0xffff_fff0u32.to_le_bytes(), + &GUID_SEV_ES_RESET_BLK, + )); + table.extend(ovmf_footer_entry( + &((ovmf.len() - meta_start) as u32).to_le_bytes(), + &GUID_SEV_META_DATA, + )); + + let footer_off = ovmf.len() - 32 - 18; + let table_start = footer_off - table.len(); + ovmf[table_start..footer_off].copy_from_slice(&table); + write_u16_le_at(&mut ovmf, footer_off, (table.len() + 18) as u16); + ovmf[footer_off + 2..footer_off + 18].copy_from_slice(&GUID_FOOTER_TABLE); + ovmf + } + #[test] fn amd_sev_snp_measurement_base_cmdline_trims_image_cmdline() { assert_eq!( @@ -1327,6 +1419,7 @@ mod tests { fs::write(image_dir.join("kernel"), b"snp-test-kernel")?; fs::write(image_dir.join("initrd"), b"snp-test-initrd")?; fs::write(image_dir.join("rootfs"), b"snp-test-rootfs")?; + fs::write(image_dir.join("ovmf.fd"), synthetic_snp_ovmf())?; fs::write( image_dir.join("metadata.json"), serde_json::json!({ @@ -1334,6 +1427,7 @@ mod tests { "kernel": "kernel", "initrd": "initrd", "rootfs": "rootfs", + "bios": "ovmf.fd", "version": "0.5.11" }) .to_string(), @@ -1378,7 +1472,10 @@ mod tests { .as_str() .context("vm_config must be a string")?, )?; - let measurement = &vm_config["sev_snp_measurement"]; + let measurement_document = vm_config["sev_snp_measurement"] + .as_str() + .context("sev_snp_measurement must be a string")?; + let measurement: serde_json::Value = serde_json::from_str(measurement_document)?; let mr_config_document = sys_config["mr_config"] .as_str() .context("mr_config must be a string")?; @@ -1404,11 +1501,23 @@ mod tests { ); assert_eq!(measurement["vcpus"], 2); assert_eq!(measurement["vcpu_type"], "EPYC-v4"); - assert_eq!(measurement["ovmf_hash"], ""); - assert!(measurement["ovmf_sections"] - .as_array() - .context("ovmf_sections must be an array")? - .is_empty()); + assert_eq!(measurement["guest_features"], 1); + assert_eq!( + measurement["ovmf_hash"] + .as_str() + .context("ovmf_hash must be a string")? + .len(), + 96 + ); + assert_eq!(measurement["sev_hashes_table_gpa"], 0x4000); + assert_eq!(measurement["sev_es_reset_eip"], 0xffff_fff0u32); + assert_eq!( + measurement["ovmf_sections"] + .as_array() + .context("ovmf_sections must be an array")? + .len(), + 4 + ); Ok(()) } } diff --git a/vmm/src/app/qemu.rs b/vmm/src/app/qemu.rs index e8fa6d8b5..19a595af3 100644 --- a/vmm/src/app/qemu.rs +++ b/vmm/src/app/qemu.rs @@ -986,7 +986,7 @@ impl VmConfig { .arg("-object") .arg(amd_sev_snp_memory_backend_arg(mem)); command.arg("-object").arg(format!( - "sev-snp-guest,id=sev0,policy=0x30000,sev-device=/dev/sev,kernel-hashes=on,host-data={host_data},author-key-enabled=on,cbitpos=51,reduced-phys-bits=1" + "sev-snp-guest,id=sev0,policy=0x30000,sev-device=/dev/sev,kernel-hashes=on,host-data={host_data},cbitpos=51,reduced-phys-bits=1" )); command.arg("-machine").arg( "q35,kernel-irqchip=split,confidential-guest-support=sev0,memory-backend=ram1,hpet=off", diff --git a/vmm/src/app/snp_measure.rs b/vmm/src/app/snp_measure.rs new file mode 100644 index 000000000..287de0274 --- /dev/null +++ b/vmm/src/app/snp_measure.rs @@ -0,0 +1,226 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! AMD SEV-SNP launch-measurement metadata extracted by the VMM. +//! +//! The KMS/verifier must not be configured with one local OVMF binary: a VMM can +//! launch many image/OVMF versions. Instead, the VMM records the measured OVMF +//! launch digest seed and OVMF SEV metadata in `.sys-config.json`; the guest then +//! forwards that self-contained launch input to KMS with its attestation. + +use anyhow::{bail, Context, Result}; +use fs_err as fs; +use serde::Serialize; +use sha2::{Digest, Sha384}; +use std::path::Path; + +const LD_BYTES: usize = 48; +const ZEROS_LD: [u8; LD_BYTES] = [0u8; LD_BYTES]; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct OvmfSectionParam { + pub gpa: u64, + pub size: u64, + /// Raw OVMF SEV metadata section type: + /// 1=SNP_SEC_MEMORY, 2=SNP_SECRETS, 3=CPUID, 4=SVSM_CAA, + /// 0x10=SNP_KERNEL_HASHES. + pub section_type: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct OvmfMeasurementInfo { + /// 48-byte GCTX launch digest after measuring the OVMF binary bytes. + pub ovmf_hash: String, + pub sev_hashes_table_gpa: u64, + pub sev_es_reset_eip: u32, + pub sections: Vec, +} + +pub(crate) fn ovmf_measurement_info(path: impl AsRef) -> Result { + let ovmf = OvmfInfo::load(path.as_ref())?; + let mut gctx = Gctx::new(); + gctx.update_normal_pages(ovmf.gpa, &ovmf.data); + Ok(OvmfMeasurementInfo { + ovmf_hash: hex::encode(gctx.ld), + sev_hashes_table_gpa: ovmf.sev_hashes_table_gpa, + sev_es_reset_eip: ovmf.sev_es_reset_eip, + sections: ovmf.sections, + }) +} + +struct Gctx { + ld: [u8; LD_BYTES], +} + +impl Gctx { + fn new() -> Self { + Self { ld: ZEROS_LD } + } + + /// SNP spec §8.17.2 PAGE_INFO layout (112 bytes): current digest, + /// contents digest, length, page type, permissions/reserved, and GPA. + fn update(&mut self, page_type: u8, gpa: u64, contents: &[u8; LD_BYTES]) { + let mut buf = [0u8; 0x70]; + buf[..LD_BYTES].copy_from_slice(&self.ld); + buf[48..96].copy_from_slice(contents); + buf[96..98].copy_from_slice(&0x70u16.to_le_bytes()); + buf[98] = page_type; + buf[104..112].copy_from_slice(&gpa.to_le_bytes()); + let mut digest = [0u8; LD_BYTES]; + digest.copy_from_slice(&Sha384::digest(buf)); + self.ld = digest; + } + + fn sha384(data: &[u8]) -> [u8; LD_BYTES] { + let mut out = [0u8; LD_BYTES]; + out.copy_from_slice(&Sha384::digest(data)); + out + } + + fn update_normal_pages(&mut self, start_gpa: u64, data: &[u8]) { + for (i, chunk) in data.chunks(4096).enumerate() { + self.update(0x01, start_gpa + (i * 4096) as u64, &Self::sha384(chunk)); + } + } +} + +struct OvmfInfo { + data: Vec, + gpa: u64, + sections: Vec, + sev_hashes_table_gpa: u64, + sev_es_reset_eip: u32, +} + +const GUID_FOOTER_TABLE: [u8; 16] = [ + 0xde, 0x82, 0xb5, 0x96, 0xb2, 0x1f, 0xf7, 0x45, 0xba, 0xea, 0xa3, 0x66, 0xc5, 0x5a, 0x08, 0x2d, +]; +const GUID_SEV_HASH_TABLE_RV: [u8; 16] = [ + 0x1f, 0x37, 0x55, 0x72, 0x3b, 0x3a, 0x04, 0x4b, 0x92, 0x7b, 0x1d, 0xa6, 0xef, 0xa8, 0xd4, 0x54, +]; +const GUID_SEV_ES_RESET_BLK: [u8; 16] = [ + 0xde, 0x71, 0xf7, 0x00, 0x7e, 0x1a, 0xcb, 0x4f, 0x89, 0x0e, 0x68, 0xc7, 0x7e, 0x2f, 0xb4, 0x4e, +]; +const GUID_SEV_META_DATA: [u8; 16] = [ + 0x66, 0x65, 0x88, 0xdc, 0x4a, 0x98, 0x98, 0x47, 0xa7, 0x5e, 0x55, 0x85, 0xa7, 0xbf, 0x67, 0xcc, +]; + +fn read_u16_le(buf: &[u8], off: usize) -> u16 { + u16::from_le_bytes([buf[off], buf[off + 1]]) +} + +fn read_u32_le(buf: &[u8], off: usize) -> u32 { + u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]]) +} + +fn validate_section_type(value: u32) -> Result<()> { + match value { + 1 | 2 | 3 | 4 | 0x10 => Ok(()), + _ => bail!("unknown ovmf section_type {value:#x}"), + } +} + +impl OvmfInfo { + fn load(path: &Path) -> Result { + let data = fs::read(path) + .with_context(|| format!("cannot read ovmf binary '{}'", path.display()))?; + let size = data.len(); + let gpa = (0x1_0000_0000u64) + .checked_sub(size as u64) + .context("ovmf binary is larger than 4 gib")?; + + const ENTRY_HDR: usize = 18; + let footer_off = size.saturating_sub(32 + ENTRY_HDR); + if footer_off + ENTRY_HDR > size { + bail!("ovmf binary too small to contain footer table"); + } + if data[footer_off + 2..footer_off + 18] != GUID_FOOTER_TABLE { + bail!("ovmf footer guid not found"); + } + let footer_total_size = read_u16_le(&data, footer_off) as usize; + if footer_total_size < ENTRY_HDR { + bail!("ovmf footer table has invalid total size"); + } + let table_size = footer_total_size - ENTRY_HDR; + if table_size > footer_off { + bail!("ovmf footer table is out of bounds"); + } + let table_start = footer_off - table_size; + let table_bytes = &data[table_start..footer_off]; + + let mut sev_hashes_table_gpa = 0u64; + let mut sev_es_reset_eip = 0u32; + let mut meta_offset_from_end = None; + + let mut pos = table_bytes.len(); + while pos >= ENTRY_HDR { + let entry_off = pos - ENTRY_HDR; + let entry_size = read_u16_le(table_bytes, entry_off) as usize; + if entry_size < ENTRY_HDR || entry_size > pos { + bail!("ovmf footer table has invalid entry size"); + } + let guid = &table_bytes[entry_off + 2..entry_off + 18]; + let data_start = pos - entry_size; + let data_end = pos - ENTRY_HDR; + let entry_data = &table_bytes[data_start..data_end]; + + if guid == GUID_SEV_HASH_TABLE_RV && entry_data.len() >= 4 { + sev_hashes_table_gpa = read_u32_le(entry_data, 0) as u64; + } else if guid == GUID_SEV_ES_RESET_BLK && entry_data.len() >= 4 { + sev_es_reset_eip = read_u32_le(entry_data, 0); + } else if guid == GUID_SEV_META_DATA && entry_data.len() >= 4 { + meta_offset_from_end = Some(read_u32_le(entry_data, 0) as usize); + } + pos -= entry_size; + } + + if sev_hashes_table_gpa == 0 { + bail!("ovmf sev hash table entry not found in footer table"); + } + if sev_es_reset_eip == 0 { + bail!("ovmf sev_es_reset_block entry not found in footer table"); + } + + let mut sections = Vec::new(); + let off_from_end = meta_offset_from_end + .ok_or_else(|| anyhow::anyhow!("ovmf sev metadata entry not found in footer table"))?; + if off_from_end > size { + bail!("ovmf sev metadata offset exceeds file size"); + } + let meta_start = size - off_from_end; + if meta_start + 16 > size { + bail!("ovmf sev metadata header out of bounds"); + } + if &data[meta_start..meta_start + 4] != b"ASEV" { + bail!("ovmf sev metadata has bad signature"); + } + let meta_version = read_u32_le(&data, meta_start + 8); + if meta_version != 1 { + bail!("ovmf sev metadata has unsupported version {meta_version}"); + } + let num_items = read_u32_le(&data, meta_start + 12) as usize; + let items_start = meta_start + 16; + if items_start + num_items * 12 > size { + bail!("ovmf sev metadata sections out of bounds"); + } + for i in 0..num_items { + let off = items_start + i * 12; + let section_type = read_u32_le(&data, off + 8); + validate_section_type(section_type)?; + sections.push(OvmfSectionParam { + gpa: read_u32_le(&data, off) as u64, + size: read_u32_le(&data, off + 4) as u64, + section_type, + }); + } + + Ok(Self { + data, + gpa, + sections, + sev_hashes_table_gpa, + sev_es_reset_eip, + }) + } +} From 48fa21170685e7eb3484bef16d4a21b64f4e4777 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 15 Jun 2026 23:59:27 -0700 Subject: [PATCH 43/67] Detect SEV-SNP C-bit position from CPUID --- vmm/src/app/qemu.rs | 58 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/vmm/src/app/qemu.rs b/vmm/src/app/qemu.rs index 19a595af3..f21821faf 100644 --- a/vmm/src/app/qemu.rs +++ b/vmm/src/app/qemu.rs @@ -75,6 +75,43 @@ fn sanitize_optional>(value: Option) -> Option { value.filter(|value| !value.as_ref().trim().is_empty()) } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct AmdSevSnpCpuid { + cbitpos: u32, + reduced_phys_bits: u32, +} + +fn amd_sev_snp_cpuid_from_ebx(ebx: u32) -> AmdSevSnpCpuid { + AmdSevSnpCpuid { + cbitpos: ebx & 0x3f, + reduced_phys_bits: (ebx >> 6) & 0x3f, + } +} + +#[cfg(target_arch = "x86_64")] +fn detect_amd_sev_snp_cpuid() -> Result { + use std::arch::x86_64::__cpuid_count; + + // AMD CPUID Fn8000_001F reports SME/SEV capabilities. EBX[5:0] is the + // memory-encryption C-bit position and EBX[11:6] is the guest physical + // address size reduction. These are host/platform properties and must match + // the machine QEMU launches on. + let max_extended_leaf = unsafe { __cpuid_count(0x8000_0000, 0).eax }; + if max_extended_leaf < 0x8000_001f { + bail!("host CPUID does not expose AMD memory encryption leaf 0x8000001f"); + } + let leaf = unsafe { __cpuid_count(0x8000_001f, 0) }; + if leaf.eax & (1 << 4) == 0 { + bail!("host CPUID does not report AMD SEV-SNP support"); + } + Ok(amd_sev_snp_cpuid_from_ebx(leaf.ebx)) +} + +#[cfg(not(target_arch = "x86_64"))] +fn detect_amd_sev_snp_cpuid() -> Result { + bail!("AMD SEV-SNP CPUID detection is only supported on x86_64 hosts") +} + #[derive(Debug, Clone, Default, Deserialize, Serialize)] pub struct InstanceInfo { #[serde(default, with = "hex_bytes")] @@ -375,7 +412,10 @@ impl VmState { #[cfg(test)] mod tests { - use super::{amd_sev_snp_memory_backend_arg, sanitize_optional, virtio_pci_device}; + use super::{ + amd_sev_snp_cpuid_from_ebx, amd_sev_snp_memory_backend_arg, sanitize_optional, + virtio_pci_device, + }; #[test] fn sanitize_optional_filters_empty_owned_values() { @@ -405,6 +445,17 @@ mod tests { ); } + #[test] + fn amd_sev_snp_cpuid_from_ebx_extracts_qemu_values() { + let milan = amd_sev_snp_cpuid_from_ebx(51 | (1 << 6)); + assert_eq!(milan.cbitpos, 51); + assert_eq!(milan.reduced_phys_bits, 1); + + let other = amd_sev_snp_cpuid_from_ebx(47 | (5 << 6)); + assert_eq!(other.cbitpos, 47); + assert_eq!(other.reduced_phys_bits, 5); + } + #[test] fn amd_sev_snp_uses_confidential_virtio_pci_options() { assert_eq!( @@ -985,8 +1036,11 @@ impl VmConfig { command .arg("-object") .arg(amd_sev_snp_memory_backend_arg(mem)); + let snp_cpuid = detect_amd_sev_snp_cpuid() + .context("failed to detect AMD SEV-SNP cbitpos/reduced-phys-bits from host CPUID")?; command.arg("-object").arg(format!( - "sev-snp-guest,id=sev0,policy=0x30000,sev-device=/dev/sev,kernel-hashes=on,host-data={host_data},cbitpos=51,reduced-phys-bits=1" + "sev-snp-guest,id=sev0,policy=0x30000,sev-device=/dev/sev,kernel-hashes=on,host-data={host_data},cbitpos={},reduced-phys-bits={}", + snp_cpuid.cbitpos, snp_cpuid.reduced_phys_bits )); command.arg("-machine").arg( "q35,kernel-irqchip=split,confidential-guest-support=sev0,memory-backend=ram1,hpet=off", From 02865d0c2f9d859112026d821bd67aed48d94d89 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 16 Jun 2026 00:31:34 -0700 Subject: [PATCH 44/67] Detect SEV-SNP launch params from QEMU --- vmm/src/app/qemu.rs | 139 ++++++++++++++++++++++++++++++++------------ 1 file changed, 101 insertions(+), 38 deletions(-) diff --git a/vmm/src/app/qemu.rs b/vmm/src/app/qemu.rs index f21821faf..22252de9e 100644 --- a/vmm/src/app/qemu.rs +++ b/vmm/src/app/qemu.rs @@ -12,9 +12,10 @@ use crate::{ use std::{collections::HashMap, os::unix::fs::PermissionsExt}; use std::{ fs::Permissions, + io::Write, ops::Deref, path::{Path, PathBuf}, - process::Command, + process::{Command, Stdio}, time::{Duration, SystemTime}, }; @@ -76,40 +77,101 @@ fn sanitize_optional>(value: Option) -> Option { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct AmdSevSnpCpuid { +struct AmdSevSnpLaunchParams { cbitpos: u32, reduced_phys_bits: u32, } -fn amd_sev_snp_cpuid_from_ebx(ebx: u32) -> AmdSevSnpCpuid { - AmdSevSnpCpuid { - cbitpos: ebx & 0x3f, - reduced_phys_bits: (ebx >> 6) & 0x3f, +fn parse_amd_sev_snp_qmp_capabilities(stdout: &[u8]) -> Result { + let stdout = std::str::from_utf8(stdout).context("QMP output is not valid UTF-8")?; + let mut qmp_error = None; + for line in stdout.lines() { + let Ok(value) = serde_json::from_str::(line) else { + continue; + }; + if let Some(error) = value.get("error") { + qmp_error = Some(error.to_string()); + } + let Some(ret) = value.get("return") else { + continue; + }; + let Some(cbitpos) = ret.get("cbitpos").and_then(|value| value.as_u64()) else { + continue; + }; + let Some(reduced_phys_bits) = ret + .get("reduced-phys-bits") + .and_then(|value| value.as_u64()) + else { + continue; + }; + return Ok(AmdSevSnpLaunchParams { + cbitpos: cbitpos + .try_into() + .context("QMP cbitpos does not fit in u32")?, + reduced_phys_bits: reduced_phys_bits + .try_into() + .context("QMP reduced-phys-bits does not fit in u32")?, + }); } -} -#[cfg(target_arch = "x86_64")] -fn detect_amd_sev_snp_cpuid() -> Result { - use std::arch::x86_64::__cpuid_count; - - // AMD CPUID Fn8000_001F reports SME/SEV capabilities. EBX[5:0] is the - // memory-encryption C-bit position and EBX[11:6] is the guest physical - // address size reduction. These are host/platform properties and must match - // the machine QEMU launches on. - let max_extended_leaf = unsafe { __cpuid_count(0x8000_0000, 0).eax }; - if max_extended_leaf < 0x8000_001f { - bail!("host CPUID does not expose AMD memory encryption leaf 0x8000001f"); - } - let leaf = unsafe { __cpuid_count(0x8000_001f, 0) }; - if leaf.eax & (1 << 4) == 0 { - bail!("host CPUID does not report AMD SEV-SNP support"); + match qmp_error { + Some(error) => bail!("QMP query-sev-capabilities failed: {error}"), + None => bail!("QMP query-sev-capabilities did not return cbitpos/reduced-phys-bits"), } - Ok(amd_sev_snp_cpuid_from_ebx(leaf.ebx)) } -#[cfg(not(target_arch = "x86_64"))] -fn detect_amd_sev_snp_cpuid() -> Result { - bail!("AMD SEV-SNP CPUID detection is only supported on x86_64 hosts") +fn detect_amd_sev_snp_qemu_capabilities(qemu_path: &Path) -> Result { + // QEMU's reduced-phys-bits is not the same value as CPUID Fn8000_001F + // EBX[11:6] on all hosts. Ask the exact QEMU binary that will launch the + // guest for its SEV launch parameters. + let mut child = Command::new(qemu_path) + .args([ + "-machine", + "none,accel=kvm", + "-display", + "none", + "-nodefaults", + "-qmp", + "stdio", + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| { + format!( + "failed to start QEMU to query SEV capabilities: {}", + qemu_path.display() + ) + })?; + + let mut stdin = child + .stdin + .take() + .context("failed to open QEMU QMP stdin")?; + stdin + .write_all( + br#"{"execute":"qmp_capabilities"} +{"execute":"query-sev-capabilities"} +{"execute":"quit"} +"#, + ) + .context("failed to write QMP query-sev-capabilities commands")?; + drop(stdin); + + let output = child + .wait_with_output() + .context("failed to wait for QEMU query-sev-capabilities")?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + bail!( + "QEMU query-sev-capabilities exited with {}: {}", + output.status, + stderr.trim() + ); + } + + parse_amd_sev_snp_qmp_capabilities(&output.stdout) } #[derive(Debug, Clone, Default, Deserialize, Serialize)] @@ -413,7 +475,7 @@ impl VmState { #[cfg(test)] mod tests { use super::{ - amd_sev_snp_cpuid_from_ebx, amd_sev_snp_memory_backend_arg, sanitize_optional, + amd_sev_snp_memory_backend_arg, parse_amd_sev_snp_qmp_capabilities, sanitize_optional, virtio_pci_device, }; @@ -446,14 +508,15 @@ mod tests { } #[test] - fn amd_sev_snp_cpuid_from_ebx_extracts_qemu_values() { - let milan = amd_sev_snp_cpuid_from_ebx(51 | (1 << 6)); - assert_eq!(milan.cbitpos, 51); - assert_eq!(milan.reduced_phys_bits, 1); - - let other = amd_sev_snp_cpuid_from_ebx(47 | (5 << 6)); - assert_eq!(other.cbitpos, 47); - assert_eq!(other.reduced_phys_bits, 5); + fn amd_sev_snp_qmp_capabilities_extracts_launch_params() { + let stdout = br#"{"QMP":{"version":{"qemu":{"major":10,"minor":0,"micro":2}}}} +{"return":{}} +{"return":{"reduced-phys-bits":1,"cbitpos":51,"cert-chain":"ignored","pdh":"ignored","cpu0-id":"ignored"}} +{"return":{}} +"#; + let params = parse_amd_sev_snp_qmp_capabilities(stdout).unwrap(); + assert_eq!(params.cbitpos, 51); + assert_eq!(params.reduced_phys_bits, 1); } #[test] @@ -1036,11 +1099,11 @@ impl VmConfig { command .arg("-object") .arg(amd_sev_snp_memory_backend_arg(mem)); - let snp_cpuid = detect_amd_sev_snp_cpuid() - .context("failed to detect AMD SEV-SNP cbitpos/reduced-phys-bits from host CPUID")?; + let snp_params = detect_amd_sev_snp_qemu_capabilities(&cfg.qemu_path) + .context("failed to detect AMD SEV-SNP cbitpos/reduced-phys-bits from QEMU")?; command.arg("-object").arg(format!( "sev-snp-guest,id=sev0,policy=0x30000,sev-device=/dev/sev,kernel-hashes=on,host-data={host_data},cbitpos={},reduced-phys-bits={}", - snp_cpuid.cbitpos, snp_cpuid.reduced_phys_bits + snp_params.cbitpos, snp_params.reduced_phys_bits )); command.arg("-machine").arg( "q35,kernel-irqchip=split,confidential-guest-support=sev0,memory-backend=ram1,hpet=off", From 74dbd0c978c22a5f01eb4c444df77e0f26d6c9d9 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 16 Jun 2026 18:55:13 -0700 Subject: [PATCH 45/67] vmm: select SEV firmware (bios-sev) for AMD SEV-SNP guests Unified dstack images now ship both the TDX firmware (ovmf.fd) and the AMD SEV firmware (ovmf-sev.fd), the latter referenced by a new "bios-sev" field in metadata.json. Add ImageInfo::bios_sev and an Image::firmware(is_amd_sev_snp) helper that returns bios-sev for SEV-SNP guests (falling back to bios) and bios for TDX. Use it both when launching QEMU (-bios) and when computing the SEV-SNP OVMF launch measurement, so the measured firmware always matches the launched one. TDX behaviour is unchanged; images without bios-sev fall back to bios. --- vmm/src/app.rs | 6 ++++-- vmm/src/app/image.rs | 25 +++++++++++++++++++++++++ vmm/src/app/qemu.rs | 2 +- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/vmm/src/app.rs b/vmm/src/app.rs index 34dd51684..565b4894c 100644 --- a/vmm/src/app.rs +++ b/vmm/src/app.rs @@ -1221,9 +1221,11 @@ fn file_sha256_hex(path: &Path) -> Result { } fn amd_sev_snp_ovmf_measurement_info(image: &Image) -> Result { + // Measure the same firmware the guest launches with: the SEV firmware + // (bios-sev) when present, falling back to the generic bios. let bios = image - .bios - .as_deref() + .firmware(true) + .map(|p| p.as_path()) .ok_or_else(|| anyhow::anyhow!("bios/OVMF is required for amd sev-snp measurement"))?; snp_measure::ovmf_measurement_info(bios).with_context(|| { format!( diff --git a/vmm/src/app/image.rs b/vmm/src/app/image.rs index e863500b3..8e875122f 100644 --- a/vmm/src/app/image.rs +++ b/vmm/src/app/image.rs @@ -17,6 +17,10 @@ pub struct ImageInfo { pub hda: Option, pub rootfs: Option, pub bios: Option, + /// AMD SEV firmware (e.g. ovmf-sev.fd). Present on unified TDX+SEV images; + /// used instead of `bios` when launching as an AMD SEV-SNP guest. + #[serde(default, rename = "bios-sev")] + pub bios_sev: Option, #[serde(default)] pub rootfs_hash: Option, #[serde(default)] @@ -65,9 +69,23 @@ pub struct Image { pub hda: Option, pub rootfs: Option, pub bios: Option, + pub bios_sev: Option, pub digest: Option, } +impl Image { + /// Firmware blob to launch with, given whether this is an AMD SEV-SNP guest. + /// SEV-SNP prefers the dedicated SEV firmware (`bios_sev`) and falls back to + /// the generic `bios`; TDX always uses `bios`. + pub fn firmware(&self, is_amd_sev_snp: bool) -> Option<&PathBuf> { + if is_amd_sev_snp { + self.bios_sev.as_ref().or(self.bios.as_ref()) + } else { + self.bios.as_ref() + } + } +} + impl Image { pub fn load(base_path: impl AsRef) -> Result { let base_path = base_path.as_ref().absolutize()?; @@ -77,6 +95,7 @@ impl Image { let hda = info.hda.as_ref().map(|hda| base_path.join(hda)); let rootfs = info.rootfs.as_ref().map(|rootfs| base_path.join(rootfs)); let bios = info.bios.as_ref().map(|bios| base_path.join(bios)); + let bios_sev = info.bios_sev.as_ref().map(|bios| base_path.join(bios)); let digest = fs::read_to_string(base_path.join("digest.txt")) .ok() .map(|s| s.trim().to_string()); @@ -91,6 +110,7 @@ impl Image { kernel, rootfs, bios, + bios_sev, digest, } .ensure_exists() @@ -118,6 +138,11 @@ impl Image { bail!("Bios does not exist: {}", bios.display()); } } + if let Some(bios_sev) = &self.bios_sev { + if !bios_sev.exists() { + bail!("SEV bios does not exist: {}", bios_sev.display()); + } + } Ok(self) } } diff --git a/vmm/src/app/qemu.rs b/vmm/src/app/qemu.rs index 22252de9e..7da7d8b7f 100644 --- a/vmm/src/app/qemu.rs +++ b/vmm/src/app/qemu.rs @@ -587,7 +587,7 @@ impl VmConfig { workdir.qmp_socket().display() )); } - if let Some(bios) = &self.image.bios { + if let Some(bios) = self.image.firmware(is_amd_sev_snp) { command.arg("-bios").arg(bios); } command.arg("-kernel").arg(&self.image.kernel); From b3b839022735bf9f7534d907546b597c13604ced Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 16 Jun 2026 19:32:10 -0700 Subject: [PATCH 46/67] vmm: auto-detect host TEE platform from /proc/cpuinfo `platform = "auto"` (the default) previously always resolved to TDX, requiring operators to opt into SEV-SNP explicitly. Implement real detection: AMD SEV-SNP hosts advertise the `sev_snp` CPU flag and Intel TDX hosts advertise `tdx_host_platform`; these flags are vendor-exclusive so the flag alone is unambiguous. Unknown hosts still fall back to TDX, and an explicit `platform = "tdx" | "amd-sev-snp"` always overrides detection. Combined with the bios-sev firmware selection, an AMD SEV-SNP host with a default config now auto-launches SEV-SNP guests with the SEV firmware. Verified on real hardware: AMD EPYC SNP host reports `sev_snp`, Intel TDX host reports `tdx_host_platform`. Unit tests cover both plus fallback. --- vmm/src/config.rs | 43 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/vmm/src/config.rs b/vmm/src/config.rs index ce2194d07..7f68b7f5e 100644 --- a/vmm/src/config.rs +++ b/vmm/src/config.rs @@ -125,11 +125,25 @@ impl TeePlatform { } } - pub fn resolve_from_cpuinfo(_cpuinfo: &str) -> Self { - // Keep `auto` conservative while AMD SEV-SNP support is experimental and - // verifier/KMS/app binding are not production-ready. Operators must opt - // into SNP explicitly with `platform = "amd-sev-snp"`. - Self::Tdx + pub fn resolve_from_cpuinfo(cpuinfo: &str) -> Self { + // Detect the host TEE from /proc/cpuinfo CPU flags: + // - AMD SEV-SNP hosts advertise the `sev_snp` flag + // - Intel TDX hosts advertise the `tdx_host_platform` flag + // These flags are vendor-exclusive, so the flag alone is unambiguous. + // Anything else falls back to TDX (the conservative default; the VMM is + // expected to run on a TEE host). Operators can always override the + // detection with an explicit `platform = "tdx" | "amd-sev-snp"`. + let has_flag = |flag: &str| { + cpuinfo + .lines() + .filter(|line| line.starts_with("flags") || line.starts_with("Features")) + .any(|line| line.split_whitespace().any(|f| f == flag)) + }; + if has_flag("sev_snp") { + Self::AmdSevSnp + } else { + Self::Tdx + } } } @@ -170,7 +184,9 @@ impl PortMappingConfig { #[derive(Debug, Clone, Deserialize)] pub struct CvmConfig { - /// TEE platform to use when launching CVMs. + /// TEE platform to use when launching CVMs. Defaults to `auto`, which + /// detects the host TEE from /proc/cpuinfo (AMD SEV-SNP vs Intel TDX); + /// set `tdx` or `amd-sev-snp` to force a platform. #[serde(default)] pub platform: TeePlatform, pub qemu_path: PathBuf, @@ -643,14 +659,23 @@ mod tests { } #[test] - fn tee_platform_auto_stays_tdx_even_with_amd_snp_flag_while_experimental() { + fn tee_platform_auto_detects_amd_sev_snp_from_flag() { let cpuinfo = "flags : fpu svm sev sev_es sev_snp debug_swap"; + assert_eq!( + TeePlatform::resolve_from_cpuinfo(cpuinfo), + TeePlatform::AmdSevSnp + ); + } + + #[test] + fn tee_platform_auto_detects_tdx_host() { + let cpuinfo = "flags : fpu vmx tdx_host_platform"; assert_eq!(TeePlatform::resolve_from_cpuinfo(cpuinfo), TeePlatform::Tdx); } #[test] - fn tee_platform_auto_falls_back_to_tdx_without_amd_snp_flag() { - let cpuinfo = "flags : fpu vmx tdx_guest"; + fn tee_platform_auto_falls_back_to_tdx_without_tee_flag() { + let cpuinfo = "flags : fpu vmx"; assert_eq!(TeePlatform::resolve_from_cpuinfo(cpuinfo), TeePlatform::Tdx); } } From 59a33fda4ea9da1be71f493d78f5aa5b3baba298 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 16 Jun 2026 19:52:52 -0700 Subject: [PATCH 47/67] kms: make SEV-SNP os_image_hash independent of per-deployment params snp_measurement_os_image_hash hashed the entire MeasurementInput document, which includes per-deployment fields (vcpus, vcpu_type, guest_features, app_id, compose_hash). That made the same OS image hash differently for different vCPU counts, breaking per-image on-chain allow-listing. Hash only the image-determined measurement inputs (rootfs_hash, base_cmdline, ovmf_hash, kernel_hash, initrd_hash, sev_hashes_table_gpa, sev_es_reset_eip, ovmf_sections) via a canonical SevOsImageMeasurement projection. The actual SNP launch measurement (compute_expected_measurement) still uses the full input and is unchanged. Test now asserts image fields change the hash while per-deployment fields do not. --- kms/src/main_service/amd_attest.rs | 91 ++++++++++++++++++++++++------ kms/src/onboard_service.rs | 2 +- 2 files changed, 74 insertions(+), 19 deletions(-) diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs index 3b77307a6..80991fd7a 100644 --- a/kms/src/main_service/amd_attest.rs +++ b/kms/src/main_service/amd_attest.rs @@ -186,7 +186,7 @@ fn build_amd_snp_boot_info_with_tcb_status( validate_amd_snp_measurement_binding(Some(config), verified_measurement, input)?; let mr_config = validate_snp_mr_config_binding(verified_host_data, mr_config_document)?; - let os_image_hash = snp_measurement_os_image_hash(measurement_document); + let os_image_hash = snp_measurement_os_image_hash(measurement_document)?; let mr_system = Sha256::digest(verified_measurement).to_vec(); let mr_aggregated = snp_mr_aggregated_digest(verified_measurement, verified_host_data); let key_provider_info = mr_config_key_provider_info(&mr_config)?; @@ -420,8 +420,47 @@ fn snp_mr_aggregated_digest(measurement: &[u8; 48], host_data: &[u8; 32]) -> Vec h.finalize().to_vec() } -pub(crate) fn snp_measurement_os_image_hash(measurement_document: &str) -> Vec { - Sha256::digest(measurement_document.as_bytes()).to_vec() +/// Image-invariant projection of `MeasurementInput`: exactly the fields that are +/// determined by the OS image (firmware, kernel, initrd, cmdline template, +/// rootfs). Serialized canonically (fixed field order) to derive os_image_hash. +#[derive(serde::Serialize)] +struct SevOsImageMeasurement<'a> { + rootfs_hash: &'a str, + base_cmdline: Option<&'a str>, + ovmf_hash: &'a str, + kernel_hash: &'a str, + initrd_hash: &'a str, + sev_hashes_table_gpa: u64, + sev_es_reset_eip: u32, + ovmf_sections: &'a [OvmfSectionParam], +} + +/// Derive the OS image hash from a self-contained SNP measurement document. +/// +/// os_image_hash identifies the OS image only, so it covers exactly the +/// image-determined measurement inputs and EXCLUDES per-deployment values +/// (`vcpus`, `vcpu_type`, `guest_features`, `app_id`, `compose_hash`). Hashing +/// the full `MeasurementInput` made the same image hash differently per vCPU +/// count, which broke per-image on-chain allow-listing. +/// +/// The document is re-projected and canonically serialized here, so the hash is +/// independent of the incoming field order / whitespace. +pub(crate) fn snp_measurement_os_image_hash(measurement_document: &str) -> Result> { + let input: MeasurementInput = serde_json::from_str(measurement_document) + .context("failed to parse sev-snp measurement document for os_image_hash")?; + let image = SevOsImageMeasurement { + rootfs_hash: &input.rootfs_hash, + base_cmdline: input.base_cmdline.as_deref(), + ovmf_hash: &input.ovmf_hash, + kernel_hash: &input.kernel_hash, + initrd_hash: &input.initrd_hash, + sev_hashes_table_gpa: input.sev_hashes_table_gpa, + sev_es_reset_eip: input.sev_es_reset_eip, + ovmf_sections: &input.ovmf_sections, + }; + let canonical = + serde_json::to_vec(&image).context("failed to serialize sev os image measurement")?; + Ok(Sha256::digest(&canonical).to_vec()) } fn mr_config_key_provider_info(mr_config: &MrConfigV3) -> Result> { @@ -1103,13 +1142,14 @@ mod tests { } #[test] - fn snp_os_image_hash_covers_all_measurement_input_fields() { + fn snp_os_image_hash_covers_image_fields_only() { let input = valid_input(); - let baseline = snp_measurement_os_image_hash(&measurement_document(&input)); + let os_image_hash = + |i: &MeasurementInput| snp_measurement_os_image_hash(&measurement_document(i)).unwrap(); + let baseline = os_image_hash(&input); - let cases: Vec<(&str, fn(&mut MeasurementInput))> = vec![ - ("app_id", |i| i.app_id = hex_of(0x12, 20)), - ("compose_hash", |i| i.compose_hash = hex_of(0x23, 32)), + // Image-determined fields MUST change the os_image_hash. + let image_cases: Vec<(&str, fn(&mut MeasurementInput))> = vec![ ("rootfs_hash", |i| i.rootfs_hash = hex_of(0x34, 32)), ("base_cmdline", |i| { i.base_cmdline = Some("console=ttyS0 loglevel=8".to_string()) @@ -1119,25 +1159,40 @@ mod tests { ("initrd_hash", |i| i.initrd_hash = hex_of(0x67, 32)), ("sev_hashes_table_gpa", |i| i.sev_hashes_table_gpa += 0x1000), ("sev_es_reset_eip", |i| i.sev_es_reset_eip = 0xffff_0000), - ("vcpus", |i| i.vcpus = 3), - ("vcpu_type", |i| { - i.vcpu_type = Some("epyc-milan".to_string()) - }), - ("guest_features", |i| i.guest_features = 3), ("ovmf_sections.gpa", |i| i.ovmf_sections[0].gpa += 0x1000), ("ovmf_sections.size", |i| i.ovmf_sections[0].size += 0x1000), ("ovmf_sections.section_type", |i| { i.ovmf_sections[0].section_type = 4 }), ]; - - for (name, mutate) in cases { + for (name, mutate) in image_cases { let mut changed = input.clone(); mutate(&mut changed); assert_ne!( baseline, - snp_measurement_os_image_hash(&measurement_document(&changed)), - "{name} must be covered by SNP os_image_hash" + os_image_hash(&changed), + "{name} must change the SNP os_image_hash" + ); + } + + // Per-deployment fields MUST NOT change the os_image_hash (the same OS + // image must hash identically regardless of vCPU count, app, etc.). + let deployment_cases: Vec<(&str, fn(&mut MeasurementInput))> = vec![ + ("app_id", |i| i.app_id = hex_of(0x12, 20)), + ("compose_hash", |i| i.compose_hash = hex_of(0x23, 32)), + ("vcpus", |i| i.vcpus = 3), + ("vcpu_type", |i| { + i.vcpu_type = Some("epyc-milan".to_string()) + }), + ("guest_features", |i| i.guest_features = 3), + ]; + for (name, mutate) in deployment_cases { + let mut changed = input.clone(); + mutate(&mut changed); + assert_eq!( + baseline, + os_image_hash(&changed), + "{name} must NOT change the SNP os_image_hash" ); } } @@ -1221,7 +1276,7 @@ mod tests { assert_eq!(boot_info.compose_hash, vec![0x22; 32]); assert_eq!( boot_info.os_image_hash, - snp_measurement_os_image_hash(&measurement_document(&input)) + snp_measurement_os_image_hash(&measurement_document(&input)).unwrap() ); assert_eq!(boot_info.mr_system.len(), 32); assert!(!boot_info.key_provider_info.is_empty()); diff --git a/kms/src/onboard_service.rs b/kms/src/onboard_service.rs index 617a734ce..e0b5cbc20 100644 --- a/kms/src/onboard_service.rs +++ b/kms/src/onboard_service.rs @@ -319,7 +319,7 @@ mod tests { assert_eq!(response.mr_aggregated.len(), 32); assert_eq!( response.os_image_hash, - snp_measurement_os_image_hash(&serde_json::to_string(&input).unwrap()) + snp_measurement_os_image_hash(&serde_json::to_string(&input).unwrap()).unwrap() ); assert_eq!(response.attestation_mode, "dstack-amd-sev-snp"); assert_eq!(response.site_name, "test-site"); From 1d907c4ab7cbbce437e1cda76de40c4e9d12bdae Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 16 Jun 2026 20:06:46 -0700 Subject: [PATCH 48/67] vmm/kms: shared SevOsImageMeasurement + sev-os-image-hash tool Factor the SEV-SNP os_image_hash projection into a shared dstack_types::SevOsImageMeasurement (canonical JCS + SHA-256). KMS derives it from a verified launch measurement; add a config-free `dstack-vmm sev-os-image-hash ` subcommand that computes the same value from the OS image artifacts, so the image build can emit digest.sev.txt that matches what the verifier computes. A cross-check test asserts sev_os_image_hash(image) equals the hash derived from the launch measurement document, guarding against field drift between the build and verify paths. --- dstack-types/src/lib.rs | 39 ++++++++++++++++++++ kms/src/main_service/amd_attest.rs | 55 +++++++++++++--------------- vmm/src/app.rs | 59 ++++++++++++++++++++++++++++++ vmm/src/main.rs | 21 +++++++++++ 4 files changed, 144 insertions(+), 30 deletions(-) diff --git a/dstack-types/src/lib.rs b/dstack-types/src/lib.rs index ce10167ff..bf188b695 100644 --- a/dstack-types/src/lib.rs +++ b/dstack-types/src/lib.rs @@ -235,6 +235,45 @@ pub struct VmConfig { pub ovmf_variant: Option, } +/// One OVMF SEV metadata section (gpa/size/type) that affects the SEV-SNP +/// launch measurement. Mirrors the OVMF footer metadata. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OvmfSection { + pub gpa: u64, + pub size: u64, + pub section_type: u32, +} + +/// Image-invariant projection that determines the AMD SEV-SNP OS image identity. +/// +/// `os_image_hash` is the SHA-256 of this projection, canonically serialized +/// (JCS). It is shared by the VMM/KMS (which derive it from a verified launch +/// measurement) and the image build (which precomputes `digest.sev.txt`), so +/// both sides agree. It deliberately EXCLUDES per-deployment values (vcpus, +/// vcpu_type, guest_features, app_id, compose_hash): the same OS image must hash +/// identically regardless of how it is launched. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SevOsImageMeasurement { + pub rootfs_hash: String, + pub base_cmdline: Option, + pub ovmf_hash: String, + pub kernel_hash: String, + pub initrd_hash: String, + pub sev_hashes_table_gpa: u64, + pub sev_es_reset_eip: u32, + pub ovmf_sections: Vec, +} + +impl SevOsImageMeasurement { + /// SHA-256 over the canonical (JCS) serialization of this projection. + pub fn os_image_hash(&self) -> Vec { + use sha2::{Digest, Sha256}; + let canonical = + serde_jcs::to_vec(self).expect("SevOsImageMeasurement is always serializable"); + Sha256::digest(canonical).to_vec() + } +} + #[derive(Serialize, Deserialize, Debug, Clone)] pub struct AppKeys { #[serde(with = "hex_bytes")] diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs index 80991fd7a..a4c5c3d9e 100644 --- a/kms/src/main_service/amd_attest.rs +++ b/kms/src/main_service/amd_attest.rs @@ -420,19 +420,27 @@ fn snp_mr_aggregated_digest(measurement: &[u8; 48], host_data: &[u8; 32]) -> Vec h.finalize().to_vec() } -/// Image-invariant projection of `MeasurementInput`: exactly the fields that are -/// determined by the OS image (firmware, kernel, initrd, cmdline template, -/// rootfs). Serialized canonically (fixed field order) to derive os_image_hash. -#[derive(serde::Serialize)] -struct SevOsImageMeasurement<'a> { - rootfs_hash: &'a str, - base_cmdline: Option<&'a str>, - ovmf_hash: &'a str, - kernel_hash: &'a str, - initrd_hash: &'a str, - sev_hashes_table_gpa: u64, - sev_es_reset_eip: u32, - ovmf_sections: &'a [OvmfSectionParam], +/// Project a verified `MeasurementInput` to the shared image-invariant +/// measurement (excludes per-deployment fields like vcpus/app_id/compose_hash). +fn sev_os_image_measurement(input: &MeasurementInput) -> dstack_types::SevOsImageMeasurement { + dstack_types::SevOsImageMeasurement { + rootfs_hash: input.rootfs_hash.clone(), + base_cmdline: input.base_cmdline.clone(), + ovmf_hash: input.ovmf_hash.clone(), + kernel_hash: input.kernel_hash.clone(), + initrd_hash: input.initrd_hash.clone(), + sev_hashes_table_gpa: input.sev_hashes_table_gpa, + sev_es_reset_eip: input.sev_es_reset_eip, + ovmf_sections: input + .ovmf_sections + .iter() + .map(|s| dstack_types::OvmfSection { + gpa: s.gpa, + size: s.size, + section_type: s.section_type, + }) + .collect(), + } } /// Derive the OS image hash from a self-contained SNP measurement document. @@ -441,26 +449,13 @@ struct SevOsImageMeasurement<'a> { /// image-determined measurement inputs and EXCLUDES per-deployment values /// (`vcpus`, `vcpu_type`, `guest_features`, `app_id`, `compose_hash`). Hashing /// the full `MeasurementInput` made the same image hash differently per vCPU -/// count, which broke per-image on-chain allow-listing. -/// -/// The document is re-projected and canonically serialized here, so the hash is -/// independent of the incoming field order / whitespace. +/// count, which broke per-image on-chain allow-listing. The canonical hashing +/// lives in `dstack_types::SevOsImageMeasurement` so the image build can +/// reproduce the same value as `digest.sev.txt`. pub(crate) fn snp_measurement_os_image_hash(measurement_document: &str) -> Result> { let input: MeasurementInput = serde_json::from_str(measurement_document) .context("failed to parse sev-snp measurement document for os_image_hash")?; - let image = SevOsImageMeasurement { - rootfs_hash: &input.rootfs_hash, - base_cmdline: input.base_cmdline.as_deref(), - ovmf_hash: &input.ovmf_hash, - kernel_hash: &input.kernel_hash, - initrd_hash: &input.initrd_hash, - sev_hashes_table_gpa: input.sev_hashes_table_gpa, - sev_es_reset_eip: input.sev_es_reset_eip, - ovmf_sections: &input.ovmf_sections, - }; - let canonical = - serde_json::to_vec(&image).context("failed to serialize sev os image measurement")?; - Ok(Sha256::digest(&canonical).to_vec()) + Ok(sev_os_image_measurement(&input).os_image_hash()) } fn mr_config_key_provider_info(mr_config: &MrConfigV3) -> Result> { diff --git a/vmm/src/app.rs b/vmm/src/app.rs index 565b4894c..861b4869c 100644 --- a/vmm/src/app.rs +++ b/vmm/src/app.rs @@ -1246,6 +1246,34 @@ fn image_rootfs_hash(image: &Image) -> Result<&str> { .ok_or_else(|| anyhow::anyhow!("rootfs_hash is required for amd sev-snp")) } +/// Compute the AMD SEV-SNP `os_image_hash` for an OS image, from the same +/// image-determined inputs the VMM feeds into the launch measurement. The image +/// build calls this (via the `sev-os-image-hash` subcommand) to emit +/// `digest.sev.txt`; the value matches what KMS derives from a verified launch +/// measurement, since both go through `dstack_types::SevOsImageMeasurement`. +pub fn sev_os_image_hash(image: &Image) -> Result> { + let ovmf = amd_sev_snp_ovmf_measurement_info(image)?; + let measurement = dstack_types::SevOsImageMeasurement { + rootfs_hash: image_rootfs_hash(image)?.to_string(), + base_cmdline: amd_sev_snp_measurement_base_cmdline(image.info.cmdline.as_deref()), + ovmf_hash: ovmf.ovmf_hash, + kernel_hash: file_sha256_hex(&image.kernel)?, + initrd_hash: file_sha256_hex(&image.initrd)?, + sev_hashes_table_gpa: ovmf.sev_hashes_table_gpa, + sev_es_reset_eip: ovmf.sev_es_reset_eip, + ovmf_sections: ovmf + .sections + .into_iter() + .map(|s| dstack_types::OvmfSection { + gpa: s.gpa, + size: s.size, + section_type: s.section_type, + }) + .collect(), + }; + Ok(measurement.os_image_hash()) +} + fn amd_sev_snp_measurement_base_cmdline(base_cmdline: Option<&str>) -> Option { base_cmdline.map(|cmdline| cmdline.trim().to_string()) } @@ -1520,6 +1548,37 @@ mod tests { .len(), 4 ); + + // The build-time os_image_hash (sev_os_image_hash -> digest.sev.txt) + // must equal the os_image_hash a verifier derives from the launch + // measurement document, i.e. the image-invariant projection of it. + let image = Image::load(&image_dir)?; + let build_hash = sev_os_image_hash(&image)?; + let as_str = |v: &serde_json::Value| v.as_str().unwrap().to_string(); + let projected = dstack_types::SevOsImageMeasurement { + rootfs_hash: as_str(&measurement["rootfs_hash"]), + base_cmdline: measurement["base_cmdline"].as_str().map(str::to_string), + ovmf_hash: as_str(&measurement["ovmf_hash"]), + kernel_hash: as_str(&measurement["kernel_hash"]), + initrd_hash: as_str(&measurement["initrd_hash"]), + sev_hashes_table_gpa: measurement["sev_hashes_table_gpa"].as_u64().unwrap(), + sev_es_reset_eip: measurement["sev_es_reset_eip"].as_u64().unwrap() as u32, + ovmf_sections: measurement["ovmf_sections"] + .as_array() + .unwrap() + .iter() + .map(|s| dstack_types::OvmfSection { + gpa: s["gpa"].as_u64().unwrap(), + size: s["size"].as_u64().unwrap(), + section_type: s["section_type"].as_u64().unwrap() as u32, + }) + .collect(), + }; + assert_eq!( + build_hash, + projected.os_image_hash(), + "digest.sev.txt must match the os_image_hash derived from the launch measurement" + ); Ok(()) } } diff --git a/vmm/src/main.rs b/vmm/src/main.rs index 266cc001c..e55814d77 100644 --- a/vmm/src/main.rs +++ b/vmm/src/main.rs @@ -60,6 +60,16 @@ enum Command { Serve, /// One-shot VM execution mode for debugging Run(RunArgs), + /// Compute the AMD SEV-SNP os_image_hash for an OS image and print it as + /// hex. Used by the image build to emit digest.sev.txt; the value matches + /// what KMS derives from a verified launch measurement. + SevOsImageHash(SevOsImageHashArgs), +} + +#[derive(ClapArgs)] +struct SevOsImageHashArgs { + /// Path to the OS image directory (containing metadata.json + artifacts) + image_dir: String, } #[derive(ClapArgs)] @@ -154,6 +164,16 @@ async fn main() -> Result<()> { } let args = Args::parse(); + + // Standalone, config-free subcommand: compute the SEV os_image_hash from an + // OS image directory (used by the image build for digest.sev.txt). + if let Some(Command::SevOsImageHash(a)) = &args.command { + let image = app::Image::load(&a.image_dir)?; + let hash = app::sev_os_image_hash(&image)?; + println!("{}", hex::encode(hash)); + return Ok(()); + } + let figment = config::load_config_figment(args.config.as_deref()); let config = Config::extract_or_default(&figment)?.abs_path()?; @@ -178,6 +198,7 @@ async fn main() -> Result<()> { Command::Serve => { // Default server mode - continue to main server logic } + Command::SevOsImageHash(_) => unreachable!("handled before config load"), } // Register this VMM instance for local discovery From 9830dcee4415232f61dc011169af79548281781b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 16 Jun 2026 21:04:08 -0700 Subject: [PATCH 49/67] dstack-types: return [u8; 32] from SevOsImageMeasurement::os_image_hash sha256 is always 32 bytes; use a fixed-size array instead of Vec for type safety and to avoid the allocation. KMS converts to Vec at the BootInfo boundary; the VMM tool/test use the array directly. --- dstack-types/src/lib.rs | 4 ++-- kms/src/main_service/amd_attest.rs | 2 +- vmm/src/app.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dstack-types/src/lib.rs b/dstack-types/src/lib.rs index bf188b695..5d7357770 100644 --- a/dstack-types/src/lib.rs +++ b/dstack-types/src/lib.rs @@ -266,11 +266,11 @@ pub struct SevOsImageMeasurement { impl SevOsImageMeasurement { /// SHA-256 over the canonical (JCS) serialization of this projection. - pub fn os_image_hash(&self) -> Vec { + pub fn os_image_hash(&self) -> [u8; 32] { use sha2::{Digest, Sha256}; let canonical = serde_jcs::to_vec(self).expect("SevOsImageMeasurement is always serializable"); - Sha256::digest(canonical).to_vec() + Sha256::digest(canonical).into() } } diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs index a4c5c3d9e..65249b039 100644 --- a/kms/src/main_service/amd_attest.rs +++ b/kms/src/main_service/amd_attest.rs @@ -455,7 +455,7 @@ fn sev_os_image_measurement(input: &MeasurementInput) -> dstack_types::SevOsImag pub(crate) fn snp_measurement_os_image_hash(measurement_document: &str) -> Result> { let input: MeasurementInput = serde_json::from_str(measurement_document) .context("failed to parse sev-snp measurement document for os_image_hash")?; - Ok(sev_os_image_measurement(&input).os_image_hash()) + Ok(sev_os_image_measurement(&input).os_image_hash().to_vec()) } fn mr_config_key_provider_info(mr_config: &MrConfigV3) -> Result> { diff --git a/vmm/src/app.rs b/vmm/src/app.rs index 861b4869c..846e22b92 100644 --- a/vmm/src/app.rs +++ b/vmm/src/app.rs @@ -1251,7 +1251,7 @@ fn image_rootfs_hash(image: &Image) -> Result<&str> { /// build calls this (via the `sev-os-image-hash` subcommand) to emit /// `digest.sev.txt`; the value matches what KMS derives from a verified launch /// measurement, since both go through `dstack_types::SevOsImageMeasurement`. -pub fn sev_os_image_hash(image: &Image) -> Result> { +pub fn sev_os_image_hash(image: &Image) -> Result<[u8; 32]> { let ovmf = amd_sev_snp_ovmf_measurement_info(image)?; let measurement = dstack_types::SevOsImageMeasurement { rootfs_hash: image_rootfs_hash(image)?.to_string(), From 4b412d34b409fe4586e49572920207b05e18eefd Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 16 Jun 2026 22:19:04 -0700 Subject: [PATCH 50/67] fix rust-checks CI: drop expect_used, satisfy clippy gates CI runs clippy with -D clippy::expect_used -D clippy::unwrap_used. Replace the two infallible-serialization expect() calls (SevOsImageMeasurement::os_image_hash and MrConfigV3::to_canonical_json) with the repo's or_panic() helper, and add the conventional #[allow(clippy::too_many_arguments)] to the central SNP build_amd_snp_boot_info_with_tcb_status (matching existing usage elsewhere). These were pre-existing rust-checks failures surfaced once expect_used was cleared. Verified: the exact CI clippy command passes clean, fmt --check passes, SNP/types tests pass. --- Cargo.lock | 1 + dstack-types/Cargo.toml | 1 + dstack-types/src/lib.rs | 6 ++++-- dstack-types/src/mr_config.rs | 5 ++++- kms/src/main_service/amd_attest.rs | 1 + 5 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d929b2072..ccc9c1fc8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2656,6 +2656,7 @@ dependencies = [ name = "dstack-types" version = "0.5.11" dependencies = [ + "or-panic", "parity-scale-codec", "serde", "serde-human-bytes", diff --git a/dstack-types/Cargo.toml b/dstack-types/Cargo.toml index 3bc4bfcd5..1bea45ec5 100644 --- a/dstack-types/Cargo.toml +++ b/dstack-types/Cargo.toml @@ -10,6 +10,7 @@ edition.workspace = true license.workspace = true [dependencies] +or-panic.workspace = true scale = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] } serde-human-bytes.workspace = true diff --git a/dstack-types/src/lib.rs b/dstack-types/src/lib.rs index 5d7357770..a149d0f1f 100644 --- a/dstack-types/src/lib.rs +++ b/dstack-types/src/lib.rs @@ -4,6 +4,7 @@ use std::path::Path; +use or_panic::ResultOrPanic; use scale::{Decode, Encode}; use serde::{Deserialize, Serialize}; use serde_human_bytes as hex_bytes; @@ -268,8 +269,9 @@ impl SevOsImageMeasurement { /// SHA-256 over the canonical (JCS) serialization of this projection. pub fn os_image_hash(&self) -> [u8; 32] { use sha2::{Digest, Sha256}; - let canonical = - serde_jcs::to_vec(self).expect("SevOsImageMeasurement is always serializable"); + // JCS serialization of this plain owned struct (strings/ints/array) + // cannot fail; panic loudly if that invariant is ever broken. + let canonical = serde_jcs::to_vec(self).or_panic("SevOsImageMeasurement JCS serialization"); Sha256::digest(canonical).into() } } diff --git a/dstack-types/src/mr_config.rs b/dstack-types/src/mr_config.rs index 7c0a92896..f5f8d09de 100644 --- a/dstack-types/src/mr_config.rs +++ b/dstack-types/src/mr_config.rs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 +use or_panic::ResultOrPanic; use serde::{Deserialize, Serialize}; use serde_human_bytes as hex_bytes; use sha2::Sha256; @@ -142,7 +143,9 @@ impl MrConfigV3 { } pub fn to_canonical_json(&self) -> String { - serde_jcs::to_string(self).expect("MrConfigV3 should serialize to JCS") + // JCS serialization of this owned struct cannot fail; panic loudly if + // that invariant is ever broken. + serde_jcs::to_string(self).or_panic("MrConfigV3 JCS serialization") } pub fn from_document(document: &str) -> Result { diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs index 65249b039..efb4df215 100644 --- a/kms/src/main_service/amd_attest.rs +++ b/kms/src/main_service/amd_attest.rs @@ -172,6 +172,7 @@ pub(crate) fn build_amd_snp_boot_info( ) } +#[allow(clippy::too_many_arguments)] fn build_amd_snp_boot_info_with_tcb_status( config: &SevSnpMeasureConfig, verified_measurement: &[u8; 48], From 9d808d779694861c2fefec15a611937c4eb7cc1a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 16 Jun 2026 22:31:13 -0700 Subject: [PATCH 51/67] kms: fix SNP tests for image-invariant os_image_hash Two tests asserted the old behavior where os_image_hash changed with any MeasurementInput field. Now that os_image_hash is the image-invariant projection, per-deployment fields (app_id, vcpus) must NOT change it: - app_id_changes_host_data_and_authorization_binding: app_id changes the authorization binding but leaves os_image_hash unchanged. - measured_input_changes_reject_until_measurement_is_recomputed: assert os_image_hash changes only for image fields (kernel_hash), not vcpus. (These run under the full test suite; my earlier 'snp'-filtered local run missed them.) --- kms/src/main_service/amd_attest.rs | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs index efb4df215..96f72ba76 100644 --- a/kms/src/main_service/amd_attest.rs +++ b/kms/src/main_service/amd_attest.rs @@ -1535,7 +1535,11 @@ mod tests { assert_ne!(boot_info.app_id, changed_boot_info.app_id); assert_ne!(boot_info.instance_id, changed_boot_info.instance_id); - assert_ne!(boot_info.os_image_hash, changed_boot_info.os_image_hash); + // app_id is an authorization input, not part of the OS image identity. + assert_eq!( + boot_info.os_image_hash, changed_boot_info.os_image_hash, + "app_id must not change the os_image_hash" + ); assert_ne!(boot_info.mr_aggregated, changed_boot_info.mr_aggregated); assert_eq!(boot_info.mr_system, changed_boot_info.mr_system); Ok(()) @@ -1548,10 +1552,13 @@ mod tests { let chip_id = [0xef; 64]; let boot_info = build_amd_snp_boot_info(&config(), &verified, &chip_id, &input).unwrap(); - for mutate in [ - |i: &mut MeasurementInput| i.kernel_hash = hex_of(0x56, 32), - |i: &mut MeasurementInput| i.vcpus = 3, - ] { + // (mutation, is_image_field): both change the SNP measurement (so a stale + // verified measurement rejects), but only image fields change os_image_hash. + let cases: [(fn(&mut MeasurementInput), bool); 2] = [ + (|i| i.kernel_hash = hex_of(0x56, 32), true), + (|i| i.vcpus = 3, false), + ]; + for (mutate, is_image_field) in cases { let mut changed = input.clone(); mutate(&mut changed); let err = build_amd_snp_boot_info(&config(), &verified, &chip_id, &changed) @@ -1564,7 +1571,17 @@ mod tests { .expect("recomputed measurement should build boot info"); assert_ne!(boot_info.mr_aggregated, changed_boot_info.mr_aggregated); assert_ne!(boot_info.mr_system, changed_boot_info.mr_system); - assert_ne!(boot_info.os_image_hash, changed_boot_info.os_image_hash); + if is_image_field { + assert_ne!( + boot_info.os_image_hash, changed_boot_info.os_image_hash, + "image fields must change os_image_hash" + ); + } else { + assert_eq!( + boot_info.os_image_hash, changed_boot_info.os_image_hash, + "per-deployment fields (vcpus) must not change os_image_hash" + ); + } } } From 2553ab95b17657ccaa19fdc2c113ae8667017053 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 17 Jun 2026 17:56:32 -0700 Subject: [PATCH 52/67] dstack-types: single source of truth for host-shared dir & mr_config Add host_shared_dir() honoring DSTACK_HOST_SHARED_DIR, and SysConfig::mr_config_document() (top-level mr_config, falling back to the copy embedded in vm_config). These give every reader one accessor so the guest quote path and the config-id verifier cannot disagree about where host-shared files / the mr_config document live. --- dstack-types/src/lib.rs | 25 +++++++++++++++++++++++++ dstack-types/src/shared_filenames.rs | 15 +++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/dstack-types/src/lib.rs b/dstack-types/src/lib.rs index a149d0f1f..d891eee93 100644 --- a/dstack-types/src/lib.rs +++ b/dstack-types/src/lib.rs @@ -197,6 +197,31 @@ pub struct SysConfig { pub vm_config: String, } +impl SysConfig { + /// Canonical MrConfigV3 document for this VM, if any. + /// + /// The document is carried in the top-level `mr_config` field; older hosts + /// only embedded it inside the serialized `vm_config`, so fall back to that + /// for backward compatibility. This is the single source of truth for all + /// readers (guest quote generation and config-id verification) so they + /// cannot disagree about where `mr_config` lives. + pub fn mr_config_document(&self) -> Option { + if let Some(doc) = self.mr_config.as_deref() { + if !doc.is_empty() { + return Some(doc.to_string()); + } + } + serde_json::from_str::(&self.vm_config) + .ok() + .and_then(|value| { + value + .get("mr_config") + .and_then(|value| value.as_str()) + .map(ToString::to_string) + }) + } +} + #[derive(Deserialize, Serialize, Debug, Clone)] pub struct VmConfig { #[serde(with = "hex_bytes", default)] diff --git a/dstack-types/src/shared_filenames.rs b/dstack-types/src/shared_filenames.rs index 5c3ef8282..a46afbdf9 100644 --- a/dstack-types/src/shared_filenames.rs +++ b/dstack-types/src/shared_filenames.rs @@ -14,6 +14,21 @@ pub const HOST_SHARED_DIR: &str = "/dstack/.host-shared"; pub const HOST_SHARED_DIR_NAME: &str = ".host-shared"; pub const HOST_SHARED_DISK_LABEL: &str = "DSTACKSHR"; +/// Environment variable overriding the host-shared directory location. +pub const HOST_SHARED_DIR_ENV: &str = "DSTACK_HOST_SHARED_DIR"; + +/// Directory the guest reads host-shared files from. +/// +/// `dstack-util setup` runs before `/dstack` is bind-mounted to the work dir, +/// so it exports [`HOST_SHARED_DIR_ENV`] pointing at the real copy directory. +/// Everything that reads host-shared files (including the attestation quote +/// path) honors it, falling back to the canonical [`HOST_SHARED_DIR`]. +pub fn host_shared_dir() -> std::path::PathBuf { + std::env::var_os(HOST_SHARED_DIR_ENV) + .map(std::path::PathBuf::from) + .unwrap_or_else(|| std::path::PathBuf::from(HOST_SHARED_DIR)) +} + pub mod compat_v3 { pub const SYS_CONFIG: &str = "config.json"; pub const ENCRYPTED_ENV: &str = "encrypted-env"; From 84139667caae87fcf05d63111e294e9d30d3fd6d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 17 Jun 2026 17:56:37 -0700 Subject: [PATCH 53/67] guest: read SEV mr_config from the real host-shared copy dir dstack-util setup runs before /dstack/.host-shared is bind-mounted, so the hardcoded path was empty when dstack-attest built the SEV quote, producing 'amd sev-snp mr_config is missing'. setup now exports DSTACK_HOST_SHARED_DIR pointing at its work-dir copy; dstack-attest and the config-id verifier both resolve via host_shared_dir() + SysConfig::mr_config_document(). --- dstack-attest/src/attestation.rs | 30 +++++++++++++++++-- dstack-util/src/system_setup.rs | 8 +++++ .../src/system_setup/config_id_verifier.rs | 17 +++-------- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/dstack-attest/src/attestation.rs b/dstack-attest/src/attestation.rs index f36d8bb17..06d475646 100644 --- a/dstack-attest/src/attestation.rs +++ b/dstack-attest/src/attestation.rs @@ -41,8 +41,15 @@ const DSTACK_AMD_SEV_SNP: &str = "dstack-amd-sev-snp"; const DSTACK_GCP_TDX: &str = "dstack-gcp-tdx"; const DSTACK_NITRO_ENCLAVE: &str = "dstack-nitro-enclave"; +/// Path to sys-config.json in the host-shared dir. +/// +/// Honors `DSTACK_HOST_SHARED_DIR` (exported by `dstack-util setup` because the +/// canonical `/dstack/.host-shared` is only bind-mounted after setup finishes). #[cfg(feature = "quote")] -const SYS_CONFIG_PATH: &str = "/dstack/.host-shared/.sys-config.json"; +fn sys_config_path() -> std::path::PathBuf { + dstack_types::shared_filenames::host_shared_dir() + .join(dstack_types::shared_filenames::SYS_CONFIG) +} /// Global lock for quote generation. The underlying TDX driver does not support concurrent access. #[cfg(feature = "quote")] @@ -51,7 +58,7 @@ static QUOTE_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); /// Read vm_config from sys-config.json #[cfg(feature = "quote")] fn read_vm_config() -> Result { - let content = match fs_err::read_to_string(SYS_CONFIG_PATH) { + let content = match fs_err::read_to_string(sys_config_path()) { Ok(content) => content, Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(String::new()), Err(err) => return Err(err).context("Failed to read sys-config"), @@ -61,6 +68,23 @@ fn read_vm_config() -> Result { Ok(sys_config.vm_config) } +/// Read the canonical mr_config document from sys-config.json. +/// +/// Uses the same accessor as the guest config-id verifier so both agree on +/// where `mr_config` lives (top-level field, falling back to the one embedded +/// in `vm_config`). +#[cfg(feature = "quote")] +fn read_mr_config_document() -> Result> { + let content = match fs_err::read_to_string(sys_config_path()) { + Ok(content) => content, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(err) => return Err(err).context("Failed to read sys-config"), + }; + let sys_config: SysConfig = + serde_json::from_str(&content).context("Failed to parse sys-config")?; + Ok(sys_config.mr_config_document()) +} + fn is_msgpack_map_prefix(byte: u8) -> bool { // fixmap (0x80..=0x8f), map16 (0xde), map32 (0xdf) matches!(byte, 0x80..=0x8f | 0xde | 0xdf) @@ -1661,7 +1685,7 @@ impl Attestation { } }; if let AttestationQuote::DstackAmdSevSnp(quote) = &mut quote { - quote.mr_config = mr_config_document_from_config(&config)? + quote.mr_config = read_mr_config_document()? .context("amd sev-snp mr_config is missing")?; } diff --git a/dstack-util/src/system_setup.rs b/dstack-util/src/system_setup.rs index 73856a1ee..d80f680b7 100644 --- a/dstack-util/src/system_setup.rs +++ b/dstack-util/src/system_setup.rs @@ -852,6 +852,14 @@ impl<'a> Stage0<'a> { } fn load(args: &'a SetupArgs) -> Result { let host_shared_copy_dir = args.work_dir.join(HOST_SHARED_DIR_NAME); + // dstack-attest and the config-id verifier read host-shared files (e.g. + // the SEV mr_config) from this dir. Export it so they don't fall back to + // the canonical /dstack/.host-shared, which is only bind-mounted to the + // work dir after `dstack-util setup` finishes. + std::env::set_var( + dstack_types::shared_filenames::HOST_SHARED_DIR_ENV, + &host_shared_copy_dir, + ); let host_shared = HostShared::copy("/tmp/.host-shared".as_ref(), &host_shared_copy_dir)?; let host_api = HostApi::new( host_shared.sys_config.host_api_url.clone(), diff --git a/dstack-util/src/system_setup/config_id_verifier.rs b/dstack-util/src/system_setup/config_id_verifier.rs index 5d67e8f06..310fb4cd8 100644 --- a/dstack-util/src/system_setup/config_id_verifier.rs +++ b/dstack-util/src/system_setup/config_id_verifier.rs @@ -6,7 +6,7 @@ use anyhow::{bail, Context, Result}; use dstack_attest::attestation::{Attestation, AttestationMode, AttestationQuote}; use dstack_types::{ mr_config::{MrConfig, MrConfigV3}, - shared_filenames::{HOST_SHARED_DIR, SYS_CONFIG}, + shared_filenames::{host_shared_dir, SYS_CONFIG}, KeyProviderKind, SysConfig, }; use tracing::info; @@ -32,21 +32,12 @@ fn read_mr_config_id() -> Result<[u8; 48]> { } fn read_mr_config_document() -> Result { - let path = std::path::Path::new(HOST_SHARED_DIR).join(SYS_CONFIG); + let path = host_shared_dir().join(SYS_CONFIG); let content = fs_err::read_to_string(path).context("Failed to read sys-config")?; let sys_config: SysConfig = serde_json::from_str(&content).context("Failed to parse sys-config")?; - if let Some(mr_config) = sys_config.mr_config { - return Ok(mr_config); - } - serde_json::from_str::(&sys_config.vm_config) - .ok() - .and_then(|value| { - value - .get("mr_config") - .and_then(|value| value.as_str()) - .map(ToString::to_string) - }) + sys_config + .mr_config_document() .context("mr_config is required") } From e0f1fe47297b5519da610d3f63e06d7ccc065461 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 17 Jun 2026 17:56:43 -0700 Subject: [PATCH 54/67] vmm: use launch-measurement os_image_hash for SEV-SNP make_vm_config wrote image.digest (the generic content digest) into vm_config.os_image_hash for every platform. For AMD SEV-SNP the value must be the launch-measurement-derived hash (== sev-os-image-hash subcommand / digest.sev.txt, and what KMS recomputes from the verified measurement). The mismatch left vm_config and the guest app-info reporting a value inconsistent with digest.sev / the KMS-derived one. Compute sev_os_image_hash(image) for SEV, keep image.digest for TDX. --- vmm/src/app.rs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/vmm/src/app.rs b/vmm/src/app.rs index 846e22b92..cc71b2535 100644 --- a/vmm/src/app.rs +++ b/vmm/src/app.rs @@ -1292,11 +1292,23 @@ fn make_vm_config( _compose_hash: &str, mr_config: Option, ) -> Result { - let os_image_hash = image - .digest - .as_ref() - .and_then(|d| hex::decode(d).ok()) - .unwrap_or_default(); + let is_amd_sev_snp = + cfg.cvm.platform.resolve() == crate::config::TeePlatform::AmdSevSnp && !manifest.no_tee; + // AMD SEV-SNP binds the OS image through the launch-measurement-derived + // os_image_hash (the same value produced by the `sev-os-image-hash` + // subcommand / digest.sev.txt and recomputed by KMS from the verified + // measurement), not the generic content digest used for TDX. + let os_image_hash = if is_amd_sev_snp { + sev_os_image_hash(image) + .context("Failed to compute amd sev-snp os_image_hash")? + .to_vec() + } else { + image + .digest + .as_ref() + .and_then(|d| hex::decode(d).ok()) + .unwrap_or_default() + }; let gpus = manifest.gpus.clone().unwrap_or_default(); let mut config = serde_json::to_value(dstack_types::VmConfig { os_image_hash, @@ -1316,7 +1328,7 @@ fn make_vm_config( })?; // For backward compatibility config["spec_version"] = serde_json::Value::from(1); - if cfg.cvm.platform.resolve() == crate::config::TeePlatform::AmdSevSnp && !manifest.no_tee { + if is_amd_sev_snp { let rootfs_hash = image_rootfs_hash(image)?; if let Some(mr_config) = mr_config { MrConfigV3::from_document(&mr_config).context("Invalid mr_config document")?; From b085a529ab4b516aeb5d8892348e5d3b9e486c31 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 17 Jun 2026 17:56:53 -0700 Subject: [PATCH 55/67] dstack-util: report real mr_system/mr_aggregated for SEV in show-mrs show-mrs special-cased AMD SEV-SNP to emit null MRs with a note claiming they were TDX-RTMR-only. The app-info path (Attestation::local()-> decode_app_info) computes mr_system/mr_aggregated for SEV too, so drop the special case and report the real values. --- dstack-util/src/main.rs | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/dstack-util/src/main.rs b/dstack-util/src/main.rs index 3aa60bd82..d2e792b4f 100644 --- a/dstack-util/src/main.rs +++ b/dstack-util/src/main.rs @@ -4,7 +4,7 @@ use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; -use dstack_attest::{attestation::AttestationMode, emit_runtime_event}; +use dstack_attest::emit_runtime_event; use dstack_types::{KeyProvider, KeyProviderKind}; use fs_err as fs; use getrandom::fill as getrandom; @@ -690,21 +690,6 @@ fn cmd_rand(rand_args: RandArgs) -> Result<()> { } fn cmd_show_mrs() -> Result<()> { - if AttestationMode::detect()? == AttestationMode::DstackAmdSevSnp { - serde_json::to_writer_pretty( - io::stdout(), - &serde_json::json!({ - "attestation_mode": AttestationMode::DstackAmdSevSnp.as_str(), - "mr_system": null, - "mr_aggregated": null, - "note": "app-info MRs are TDX RTMR-derived and unavailable for AMD SEV-SNP", - }), - ) - .context("Failed to write app info")?; - println!(); - return Ok(()); - } - let attestation = ra_tls::attestation::Attestation::local().context("Failed to get attestation")?; let app_info = attestation From 3642d0830f97549b612bd2eaab66907011b0856b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 17 Jun 2026 17:56:57 -0700 Subject: [PATCH 56/67] kms: drop SNP key-release self-authorization config gate ensure_snp_key_release_config_safe refused to start the KMS when sev_snp_key_release was enabled without enforce_self_authorization. The self-authorization requirement is not needed for SEV key release, so remove the startup gate, its helper, and the associated test. --- kms/src/main_service.rs | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/kms/src/main_service.rs b/kms/src/main_service.rs index dce8f1a54..67b060fbf 100644 --- a/kms/src/main_service.rs +++ b/kms/src/main_service.rs @@ -117,10 +117,6 @@ impl KmsState { "self-authorization is disabled; trusted RPCs will not be gated by KMS self-attestation - do not use in production TEE deployments" ); } - ensure_snp_key_release_config_safe( - config.enforce_self_authorization, - &config.sev_snp_key_release, - )?; Ok(Self { inner: Arc::new(KmsStateInner { config, @@ -210,16 +206,6 @@ fn ensure_snp_key_release_allowed( Ok(()) } -fn ensure_snp_key_release_config_safe( - enforce_self_authorization: bool, - policy: &SevSnpKeyReleaseConfig, -) -> Result<()> { - if policy.enabled && !enforce_self_authorization { - bail!("self-authorization is required for amd sev-snp key release"); - } - Ok(()) -} - fn ensure_self_key_release_allowed( self_boot_info: Option<&BootInfo>, policy: &SevSnpKeyReleaseConfig, @@ -845,22 +831,6 @@ mod tests { assert!(err.to_string().contains("advisory_id is not allowed")); } - #[test] - fn snp_release_config_requires_self_authorization_when_enabled() { - let policy = SevSnpKeyReleaseConfig { - enabled: true, - ..Default::default() - }; - - let err = ensure_snp_key_release_config_safe(false, &policy) - .expect_err("enabled SNP release must require KMS self-authorization"); - assert!(err - .to_string() - .contains("self-authorization is required for amd sev-snp key release")); - ensure_snp_key_release_config_safe(true, &policy) - .expect("enabled SNP release is safe only with self-authorization enforced"); - } - #[test] fn snp_self_boot_info_uses_same_release_policy_for_temp_ca() { let boot_info = snp_boot_info(); From ae5f561d0985f51fd5f4ebeae1c2b81c4cf5db3e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 17 Jun 2026 18:25:54 -0700 Subject: [PATCH 57/67] dstack-attest: offline SEV-SNP attestation verification test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a real AMD SEV-SNP attestation captured from a live dstack CVM plus its pinned ASK/VCEK, and an integration test that verifies the full chain offline (builtin ARK -> ASK -> VCEK -> report signature) and asserts the report_data marker, launch measurement, and HOST_DATA. Fully deterministic — nothing is fetched from AMD KDS. See sev_snp_fixture.README.md for provenance. --- dstack-attest/tests/sev_snp_ask.pem | 37 ++++++++ dstack-attest/tests/sev_snp_attestation.bin | Bin 0 -> 4130 bytes dstack-attest/tests/sev_snp_fixture.README.md | 52 ++++++++++++ dstack-attest/tests/sev_snp_vcek.pem | 31 +++++++ dstack-attest/tests/sev_snp_verify.rs | 79 ++++++++++++++++++ 5 files changed, 199 insertions(+) create mode 100644 dstack-attest/tests/sev_snp_ask.pem create mode 100644 dstack-attest/tests/sev_snp_attestation.bin create mode 100644 dstack-attest/tests/sev_snp_fixture.README.md create mode 100644 dstack-attest/tests/sev_snp_vcek.pem create mode 100644 dstack-attest/tests/sev_snp_verify.rs diff --git a/dstack-attest/tests/sev_snp_ask.pem b/dstack-attest/tests/sev_snp_ask.pem new file mode 100644 index 000000000..26c059c70 --- /dev/null +++ b/dstack-attest/tests/sev_snp_ask.pem @@ -0,0 +1,37 @@ +-----BEGIN CERTIFICATE----- +MIIGiTCCBDigAwIBAgIDAQABMEYGCSqGSIb3DQEBCjA5oA8wDQYJYIZIAWUDBAIC +BQChHDAaBgkqhkiG9w0BAQgwDQYJYIZIAWUDBAICBQCiAwIBMKMDAgEBMHsxFDAS +BgNVBAsMC0VuZ2luZWVyaW5nMQswCQYDVQQGEwJVUzEUMBIGA1UEBwwLU2FudGEg +Q2xhcmExCzAJBgNVBAgMAkNBMR8wHQYDVQQKDBZBZHZhbmNlZCBNaWNybyBEZXZp +Y2VzMRIwEAYDVQQDDAlBUkstTWlsYW4wHhcNMjAxMDIyMTgyNDIwWhcNNDUxMDIy +MTgyNDIwWjB7MRQwEgYDVQQLDAtFbmdpbmVlcmluZzELMAkGA1UEBhMCVVMxFDAS +BgNVBAcMC1NhbnRhIENsYXJhMQswCQYDVQQIDAJDQTEfMB0GA1UECgwWQWR2YW5j +ZWQgTWljcm8gRGV2aWNlczESMBAGA1UEAwwJU0VWLU1pbGFuMIICIjANBgkqhkiG +9w0BAQEFAAOCAg8AMIICCgKCAgEAnU2drrNTfbhNQIllf+W2y+ROCbSzId1aKZft +2T9zjZQOzjGccl17i1mIKWl7NTcB0VYXt3JxZSzOZjsjLNVAEN2MGj9TiedL+Qew +KZX0JmQEuYjm+WKksLtxgdLp9E7EZNwNDqV1r0qRP5tB8OWkyQbIdLeu4aCz7j/S +l1FkBytev9sbFGzt7cwnjzi9m7noqsk+uRVBp3+In35QPdcj8YflEmnHBNvuUDJh +LCJMW8KOjP6++Phbs3iCitJcANEtW4qTNFoKW3CHlbcSCjTM8KsNbUx3A8ek5EVL +jZWH1pt9E3TfpR6XyfQKnY6kl5aEIPwdW3eFYaqCFPrIo9pQT6WuDSP4JCYJbZne +KKIbZjzXkJt3NQG32EukYImBb9SCkm9+fS5LZFg9ojzubMX3+NkBoSXI7OPvnHMx +jup9mw5se6QUV7GqpCA2TNypolmuQ+cAaxV7JqHE8dl9pWf+Y3arb+9iiFCwFt4l +AlJw5D0CTRTC1Y5YWFDBCrA/vGnmTnqG8C+jjUAS7cjjR8q4OPhyDmJRPnaC/ZG5 +uP0K0z6GoO/3uen9wqshCuHegLTpOeHEJRKrQFr4PVIwVOB0+ebO5FgoyOw43nyF +D5UKBDxEB4BKo/0uAiKHLRvvgLbORbU8KARIs1EoqEjmF8UtrmQWV2hUjwzqwvHF +ei8rPxMCAwEAAaOBozCBoDAdBgNVHQ4EFgQUO8ZuGCrD/T1iZEib47dHLLT8v/gw +HwYDVR0jBBgwFoAUhawa0UP3yKxV1MUdQUir1XhK1FMwEgYDVR0TAQH/BAgwBgEB +/wIBADAOBgNVHQ8BAf8EBAMCAQQwOgYDVR0fBDMwMTAvoC2gK4YpaHR0cHM6Ly9r +ZHNpbnRmLmFtZC5jb20vdmNlay92MS9NaWxhbi9jcmwwRgYJKoZIhvcNAQEKMDmg +DzANBglghkgBZQMEAgIFAKEcMBoGCSqGSIb3DQEBCDANBglghkgBZQMEAgIFAKID +AgEwowMCAQEDggIBAIgeUQScAf3lDYqgWU1VtlDbmIN8S2dC5kmQzsZ/HtAjQnLE +PI1jh3gJbLxL6gf3K8jxctzOWnkYcbdfMOOr28KT35IaAR20rekKRFptTHhe+DFr +3AFzZLDD7cWK29/GpPitPJDKCvI7A4Ug06rk7J0zBe1fz/qe4i2/F12rvfwCGYhc +RxPy7QF3q8fR6GCJdB1UQ5SlwCjFxD4uezURztIlIAjMkt7DFvKRh+2zK+5plVGG +FsjDJtMz2ud9y0pvOE4j3dH5IW9jGxaSGStqNrabnnpF236ETr1/a43b8FFKL5QN +mt8Vr9xnXRpznqCRvqjr+kVrb6dlfuTlliXeQTMlBoRWFJORL8AcBJxGZ4K2mXft +l1jU5TLeh5KXL9NW7a/qAOIUs2FiOhqrtzAhJRg9Ij8QkQ9Pk+cKGzw6El3T3kFr +Eg6zkxmvMuabZOsdKfRkWfhH2ZKcTlDfmH1H0zq0Q2bG3uvaVdiCtFY1LlWyB38J +S2fNsR/Py6t5brEJCFNvzaDky6KeC4ion/cVgUai7zzS3bGQWzKDKU35SqNU2WkP +I8xCZ00WtIiKKFnXWUQxvlKmmgZBIYPe01zD0N8atFxmWiSnfJl690B9rJpNR/fI +ajxCW3Seiws6r1Zm+tCuVbMiNtpS9ThjNX4uve5thyfE2DgoxRFvY1CsoF5M +-----END CERTIFICATE----- diff --git a/dstack-attest/tests/sev_snp_attestation.bin b/dstack-attest/tests/sev_snp_attestation.bin new file mode 100644 index 0000000000000000000000000000000000000000..0c9e9b663175ed3eb1bbda23e432d321e01bcfd1 GIT binary patch literal 4130 zcmeGfZHydc`OdQ-iVzwDJ!^GcX>82y_5GOlW3H?}5kLB)ZBnpUg0`uJKa~_he@HcYend$?B1(yb#H2rrM4DiTH8eHUXJ+quy|#xC zjFH4ic6Z+A^L@Ub?Hky-bf9nUnSYyuEcV+Ei1(JrWk*+@L!|QDW_i8*a^c2&BDHmb z2tS8x(c|VDk8Mu(t^acM=~~~Ot?s5f2KQejlHmil>&tF?YRSWU?%n*_$V;bxd-1U& z`hh!N_}hk!Yx{1!=lxIbx%$Bq?O#4N^@Z(Q{&C~iue|%nLw^@PZe5*tqj zaLqgV*^ciX=st8$_POyN-8C1d6{5ed_{^Qp*Yv*$8Hv7q@}>9uv$EukKRv(ygR7QP zI@Q^~d&d*k=e6IgJ@)#kH(q?^^IPw`^~o(WPmO#^xkneX;?>scD=s;_=H_SC?Aq0u zS!ItNevGd8?1|m$e)#>pU%Ynt?WdIeM)K^M_2!j)*Y<5o{<@f)|HJ>w9(eqssjBL9 znxUx z(AX5NG9G2;0DBTCVm?KfB8~)6l6z8H3Stljh&&4dL`Wd&fyKzfen31XjPV1h6!$!C ze2E;U+&L#CCRQ7+g%BRHHb9oh03*Y=MsRyXq+&tgRHLn;NEjw*V6YQNaNqhGV{RM? zvP3{cT!>=(BaiRA`N0#1?9`@D)aD{ki2K9F znak`IwRP-guU!A@eJ6MR?&H_}00l{s*Q2yDC6J-T_{$zLto zHFDA5`q8zw41Q@Z+WYbqw`|;d{Et6D5GJ^GFHCq~sv4`vo(Wf(e(%xM*HQBO4{@WHSplK}bB}dp;M$=e6pcHkxRj zNx&*bLuiNqdq;=OR?@cIn`J_^YHMN)l5)&u)nTXFnXOghiKsc&nE(Kwi$mPa?h13= zDrgFhB=58*#+r$9XLw3OAukMc13g}kp{zqyM?)Mmt3q`v$?|5_QmJi5i78U3+cE%8 zx{+&kp(s>gR23=&79)ljpt6qD^HyN6HwEKk7mfkp)aYyj9IXzIzU3A;TCI&PXo7Ex zd$BfHN}Vf1Xj`Swx7rFz=Chk&$qnuBJj9&jcW1$s=FH|Km`JCK2-|Kmi#yH8stk7G z$d)+(p-rJ>xvVUl(j8hcWt4SDX{sp4NFvIhas;P=2i;o2aN^Xn0wEVD6+jso7y`~v zWCSL-R30dpd^$4iFt$CZNt{77H1d4%CR7RIFmJ3bzAGIShq{3)vCW4V%5$~gb5G#rF0tDk02b^hGD>N~f zfj@+SIS*YHa>O@=5Uxw1MO;%eFO)gydD>H?a3L%d0^@+1fJ)-ifZ#y+#18^#Q9v~d zG*&|RAlz}EfG+HpLW}$@d}Fg_UW5zUrUE}nSUm{l&}s?-K&CdcdDDiS5=klN5|CZU z1gLW8 +SPDX-License-Identifier: Apache-2.0 +--> + +# AMD SEV-SNP attestation test fixture + +Real AMD SEV-SNP attestation captured from a live dstack CVM, used by +`tests/sev_snp_verify.rs` for an offline end-to-end verification test. + +## Files + +| File | Description | +| --- | --- | +| `sev_snp_attestation.bin` | `VersionedAttestation` (SCALE V0) — the full attestation as produced inside the CVM. Contains the 1184-byte SNP report + the `mr_config` document. | +| `sev_snp_ask.pem` | AMD SEV intermediate cert (ASK, `CN=SEV-Milan`) for the chip that signed the report. | +| `sev_snp_vcek.pem` | Per-chip VCEK (`CN=SEV-VCEK`) for the report's `chip_id` + reported TCB. | + +The AMD root key (ARK) is **not** bundled — `sev-snp-qvl` uses its built-in ARK, +so the test verifies the full chain ARK → ASK → VCEK → report signature with +nothing fetched from AMD KDS (fully offline / deterministic). + +## Provenance + +- Captured 2026-06-17 from a dstack SEV-SNP CVM (app `attest-test`) running the + merged `dstack-nvidia-0.6.0.a2` image on an AMD EPYC Milan host. +- Generated inside the guest with: + ``` + dstack-util quote-report \ + --report-data 6174746573742d746573742d666978747572652d32303236 \ + --output attest.json + ``` + (`report-data` = ASCII `attest-test-fixture-2026`, the marker the test asserts.) +- The `attestation` hex field of that JSON was decoded to `sev_snp_attestation.bin`. +- ASK/VCEK were fetched from AMD KDS (`https://kdsintf.amd.com/vcek/v1/Milan/...`) + for the report's `chip_id` and TCB and pinned here so the test stays offline. + +## Verified values (informational) + +``` +chip_id: 38d174589d2dff97a6d40cb9f9d90b9507c027491219083cef3ce73e + d18f7289142d941ad61eabecd27d25f268c1095d665f6001358e98a4769c82734a6bb877 +measurement 7f51e17f72a04d5422cb2c00998166536019a217376f3aa45a630e59c805a599... +host_data: 783f0057820acb99249af56cc3b07b4e8d80f65183167cba9cf437bb680f742f +tcb_status: OutOfDate (this host's firmware TCB; acceptable per KMS allowed_tcb_statuses) +``` + +## Refreshing + +VCEK/ASK are immutable for a given chip + TCB, so these never expire. If the +report itself is regenerated (e.g. different host or firmware), re-capture all +three files together — the VCEK must match the new report's `chip_id`/TCB. diff --git a/dstack-attest/tests/sev_snp_vcek.pem b/dstack-attest/tests/sev_snp_vcek.pem new file mode 100644 index 000000000..beca88b0e --- /dev/null +++ b/dstack-attest/tests/sev_snp_vcek.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFQzCCAvegAwIBAgIBADBBBgkqhkiG9w0BAQowNKAPMA0GCWCGSAFlAwQCAgUA +oRwwGgYJKoZIhvcNAQEIMA0GCWCGSAFlAwQCAgUAogMCATAwezEUMBIGA1UECwwL +RW5naW5lZXJpbmcxCzAJBgNVBAYTAlVTMRQwEgYDVQQHDAtTYW50YSBDbGFyYTEL +MAkGA1UECAwCQ0ExHzAdBgNVBAoMFkFkdmFuY2VkIE1pY3JvIERldmljZXMxEjAQ +BgNVBAMMCVNFVi1NaWxhbjAeFw0yNjA2MTcwMTA1MDRaFw0zMzA2MTcwMTA1MDRa +MHoxFDASBgNVBAsMC0VuZ2luZWVyaW5nMQswCQYDVQQGEwJVUzEUMBIGA1UEBwwL +U2FudGEgQ2xhcmExCzAJBgNVBAgMAkNBMR8wHQYDVQQKDBZBZHZhbmNlZCBNaWNy +byBEZXZpY2VzMREwDwYDVQQDDAhTRVYtVkNFSzB2MBAGByqGSM49AgEGBSuBBAAi +A2IABEjJ8dwrpAfPmitaXeRU6F3R59/0IU4+7kvjkSmZ970ve2UCVodWScWtL4rM +T4NvH/G/62CohHASNu5yGCjrGVKenpUk0dvCgsIdbuEl6u5onBm+tDIBcraRkRgD +iTU88KOCARcwggETMBAGCSsGAQQBnHgBAQQDAgEAMBcGCSsGAQQBnHgBAgQKFghN +aWxhbi1CMDARBgorBgEEAZx4AQMBBAMCAQQwEQYKKwYBBAGceAEDAgQDAgEAMBEG +CisGAQQBnHgBAwQEAwIBADARBgorBgEEAZx4AQMFBAMCAQAwEQYKKwYBBAGceAED +BgQDAgEAMBEGCisGAQQBnHgBAwcEAwIBADARBgorBgEEAZx4AQMDBAMCARcwEgYK +KwYBBAGceAEDCAQEAgIA1TBNBgkrBgEEAZx4AQQEQDjRdFidLf+XptQMufnZC5UH +wCdJEhkIPO885z7Rj3KJFC2UGtYeq+zSfSXyaMEJXWZfYAE1jpikdpyCc0pruHcw +QQYJKoZIhvcNAQEKMDSgDzANBglghkgBZQMEAgIFAKEcMBoGCSqGSIb3DQEBCDAN +BglghkgBZQMEAgIFAKIDAgEwA4ICAQBvaS6IR9JySxWPvBjCixAReaCzpS34Rf7q +nV1HIUEXK72H6XPyET8zjZgYhkGzr99B5jOf+bZj2XqeT6t8vAG8+VwrZEnxRz14 +wepI0V1RIMu9mb1hFQlKqKrrVy9jRA9Nd2sjtav8zy4xv+neAV+6HjWH3W2RiSne +SLbOwUkYKvLZ0ZmlxvFUz4Z0E5o0ofQDXf/XRYnTMJTI3nkNGC05IRY02seKFRKp +f649cmpy8sXj+GS4FqOjeymW9WBgxsxeyV9+DhJ0u6N7Tx+QHHJuc4AGSnVq2KJy +ndrknp6bk/yISY13DuUkeF71Q/FGk3sQe5PsK7kSLcsoGaDURuA3wrrstpO/ooIa +OnyqBW6AKL6vluwzPcCuMtxJ8iV/NdXIsSUolPni8hZQ7MYh474bl68NnlD+v8CP +yC6cgHevQmKFtWAtWXlxapzUUlBIlMIZAKUe+Hel6MhUF8vLYUfrERoCnaclZokp +BYVvgj4QLqugzYVyHBsRlnyepMuUT6KxZf01LW2RqvUwMF00xR7mqQVDZoidAwF2 +0RZ4+GoL8yKSR6nlCBTCfIOlDoBQPRDGY3RMGWBjhcc1VnzxxGT++/uauKKRiUKE +64qaVQp0eYTD1CZGq4I6YY/Sb8b2U5SS2yvoF9GJN873wqGnxXipNxZAWTI+pB77 +sAs/3DdQrA== +-----END CERTIFICATE----- diff --git a/dstack-attest/tests/sev_snp_verify.rs b/dstack-attest/tests/sev_snp_verify.rs new file mode 100644 index 000000000..3352031e1 --- /dev/null +++ b/dstack-attest/tests/sev_snp_verify.rs @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Integration test: verify a real AMD SEV-SNP attestation end-to-end, offline. +//! +//! The fixtures were captured from a live dstack SEV-SNP CVM (see +//! `sev_snp_fixture.README.md`). Verification is fully offline: the VCEK and ASK +//! are bundled so the test never reaches AMD KDS, and the AMD root (ARK) is the +//! one built into `sev-snp-qvl`. + +use dstack_attest::attestation::{AttestationQuote, VersionedAttestation}; +use sev_snp_qvl::{verify_amd_snp_attestation, AmdSnpAttestationInput}; + +/// Real SEV-SNP attestation captured from a dstack CVM (VersionedAttestation, SCALE V0). +const SEV_ATTESTATION_BIN: &[u8] = include_bytes!("sev_snp_attestation.bin"); +/// AMD SEV intermediate (ASK / CN=SEV-Milan) for the chip that produced the report. +const SEV_ASK_PEM: &[u8] = include_bytes!("sev_snp_ask.pem"); +/// Per-chip VCEK (CN=SEV-VCEK) for the report's chip_id + reported TCB. +const SEV_VCEK_PEM: &[u8] = include_bytes!("sev_snp_vcek.pem"); + +/// report_data marker passed to `dstack-util quote-report` when capturing the fixture. +const REPORT_DATA_MARKER: &[u8] = b"attest-test-fixture-2026"; + +fn expected_report_data() -> [u8; 64] { + let mut rd = [0u8; 64]; + rd[..REPORT_DATA_MARKER.len()].copy_from_slice(REPORT_DATA_MARKER); + rd +} + +#[test] +fn verify_sev_snp_attestation_bin() { + // Decode the VersionedAttestation captured from the CVM. + let versioned = + VersionedAttestation::from_scale(SEV_ATTESTATION_BIN).expect("decode VersionedAttestation"); + let VersionedAttestation::V0 { attestation } = versioned else { + panic!("expected V0 attestation"); + }; + + // The outer report_data carries our capture marker. + assert_eq!( + attestation.report_data, + expected_report_data(), + "outer attestation report_data marker" + ); + + let AttestationQuote::DstackAmdSevSnp(quote) = &attestation.quote else { + panic!("expected an AMD SEV-SNP quote"); + }; + assert_eq!(quote.report.len(), 1184, "raw SNP report length"); + assert!( + !quote.mr_config.is_empty(), + "SEV-SNP quote must carry the mr_config document" + ); + + // Offline hardware verification: ARK (builtin) -> ASK -> VCEK -> report signature. + let verified = verify_amd_snp_attestation(&AmdSnpAttestationInput { + report: "e.report, + ask_pem: SEV_ASK_PEM, + vcek_pem: SEV_VCEK_PEM, + }) + .expect("verify SEV-SNP attestation offline"); + + // The signed report_data matches the marker we requested. + assert_eq!( + verified.report_data, + expected_report_data(), + "signed report_data marker" + ); + // A real launch measurement is present. + assert_ne!(verified.measurement, [0u8; 48], "measurement must be set"); + // HOST_DATA binds the mr_config document; it must be non-zero for a dstack CVM. + assert_ne!(verified.host_data, [0u8; 32], "host_data must be set"); + + println!("measurement: {}", hex::encode(verified.measurement)); + println!("host_data: {}", hex::encode(verified.host_data)); + println!("chip_id: {}", hex::encode(verified.chip_id)); + println!("tcb_status: {}", verified.tcb_info.tcb_status()); +} From 2e3aca572687d5884d8085081b83502ca22494df Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 17 Jun 2026 19:01:42 -0700 Subject: [PATCH 58/67] dstack-mr: add shared AMD SEV-SNP launch-measurement module Move the SEV-SNP launch-measurement recomputation and os_image_hash derivation into a new dstack-mr::sev module so the KMS (key release) and the verifier (attestation verification) compute identical values from a single source of truth, instead of the verifier lacking it entirely. Primitive-typed API (measurement/host_data byte arrays) keeps the module free of attestation/RA-TLS types, avoiding a dependency cycle. Includes a real-fixture regression test that recomputes the captured CVM's launch measurement (7f51e17f...) and os_image_hash (32b47673...). --- dstack-mr/src/lib.rs | 1 + dstack-mr/src/sev.rs | 1017 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1018 insertions(+) create mode 100644 dstack-mr/src/sev.rs diff --git a/dstack-mr/src/lib.rs b/dstack-mr/src/lib.rs index d4d41638c..ad71c0aee 100644 --- a/dstack-mr/src/lib.rs +++ b/dstack-mr/src/lib.rs @@ -18,6 +18,7 @@ mod acpi; mod kernel; mod machine; mod num; +pub mod sev; mod tdvf; mod uefi_var; mod util; diff --git a/dstack-mr/src/sev.rs b/dstack-mr/src/sev.rs new file mode 100644 index 000000000..40048697c --- /dev/null +++ b/dstack-mr/src/sev.rs @@ -0,0 +1,1017 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! AMD SEV-SNP launch-measurement recomputation and `os_image_hash` derivation. +//! +//! This is the single source of truth shared by `dstack-kms` (key release) and +//! `dstack-verifier` (attestation verification). It recomputes the expected SNP +//! launch `MEASUREMENT` from self-contained launch inputs (the +//! `sev_snp_measurement` document a VMM embeds in `vm_config`) and derives the +//! image-invariant `os_image_hash`. +//! +//! It deals only in primitive, hardware-verified values (`measurement`, +//! `host_data`) so it can stay free of attestation/RA-TLS types and be reused by +//! both the KMS and the verifier without a dependency cycle. Verifying the report +//! signature/collateral is the caller's job; this module recomputes the launch +//! measurement and checks it against the already-verified one. + +use anyhow::{bail, Context, Result}; +use dstack_types::mr_config::MrConfigV3; +use sha2::{Digest, Sha256, Sha384}; +use std::fs; + +const LD_BYTES: usize = 48; +const ZEROS_LD: [u8; LD_BYTES] = [0u8; LD_BYTES]; +/// Maximum number of vCPUs accepted in a measurement input. +pub const MAX_VCPUS: u32 = 512; +/// Maximum number of OVMF metadata sections accepted in a measurement input. +pub const MAX_OVMF_SECTIONS: usize = 64; +/// 64 GiB worth of 4 KiB pages — upper bound on measured OVMF metadata pages. +pub const MAX_OVMF_METADATA_PAGES: u64 = 16_777_216; +// VMSA page GPA: (u64)(-1) page-aligned, bits >51 cleared. +const VMSA_GPA: u64 = 0x0000_FFFF_FFFF_F000; + +#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +#[serde(deny_unknown_fields)] +pub struct OvmfSectionParam { + pub gpa: u64, + pub size: u64, + /// Raw OVMF SEV metadata section type: + /// 1=SNP_SEC_MEMORY, 2=SNP_SECRETS, 3=CPUID, 4=SVSM_CAA, + /// 0x10=SNP_KERNEL_HASHES. + pub section_type: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +#[serde(deny_unknown_fields)] +pub struct MeasurementInput { + /// Deprecated: app identity is now bound through MrConfigV3/HOST_DATA. + #[serde(default)] + pub app_id: String, + /// Deprecated: compose identity is now bound through MrConfigV3/HOST_DATA. + #[serde(default)] + pub compose_hash: String, + /// 32-byte rootfs hash included in the self-contained SNP measurement input. + pub rootfs_hash: String, + /// Original image kernel cmdline used for SNP measured launch. + pub base_cmdline: Option, + /// 48-byte OVMF GCTX launch digest seed supplied by the VMM. + pub ovmf_hash: String, + /// 32-byte kernel SHA-256 hash. + pub kernel_hash: String, + /// 32-byte initrd SHA-256 hash. An empty string is treated as the SHA-256 of + /// an empty initrd, matching QEMU/sev-snp-measure behavior. + pub initrd_hash: String, + /// GPA of the SevHashTable, from OVMF footer metadata. + pub sev_hashes_table_gpa: u64, + /// AP reset EIP, from OVMF footer metadata. + pub sev_es_reset_eip: u32, + pub vcpus: u32, + pub vcpu_type: Option, + /// SNP guest features bitmask used at launch. QEMU uses 0x1 for SNP with + /// kernel hashes enabled in the current VMM path. + pub guest_features: u64, + #[serde(deserialize_with = "deserialize_ovmf_sections_bounded")] + pub ovmf_sections: Vec, +} + +fn deserialize_ovmf_sections_bounded<'de, D>( + deserializer: D, +) -> std::result::Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + struct BoundedOvmfSections; + + impl<'de> serde::de::Visitor<'de> for BoundedOvmfSections { + type Value = Vec; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "at most {MAX_OVMF_SECTIONS} OVMF metadata sections" + ) + } + + fn visit_seq(self, mut seq: A) -> std::result::Result, A::Error> + where + A: serde::de::SeqAccess<'de>, + { + let mut sections = + Vec::with_capacity(seq.size_hint().unwrap_or(0).min(MAX_OVMF_SECTIONS)); + while let Some(section) = seq.next_element()? { + if sections.len() >= MAX_OVMF_SECTIONS { + return Err(serde::de::Error::custom(format!( + "ovmf section count must not exceed {MAX_OVMF_SECTIONS}" + ))); + } + sections.push(section); + } + Ok(sections) + } + } + + deserializer.deserialize_seq(BoundedOvmfSections) +} + +/// Validate a `MeasurementInput` for shape/bounds before recomputation. +pub fn validate_measurement_input(input: &MeasurementInput) -> Result<()> { + if input.guest_features == 0 { + bail!("guest_features must be non-zero"); + } + + decode_required_hex("rootfs_hash", &input.rootfs_hash, 32)?; + decode_required_hex("kernel_hash", &input.kernel_hash, 32)?; + decode_optional_hex("initrd_hash", &input.initrd_hash, 32)?; + if input.vcpus == 0 { + bail!("vcpus must be greater than zero"); + } + if input.vcpus > MAX_VCPUS { + bail!("vcpus must not exceed {MAX_VCPUS}"); + } + match input.vcpu_type.as_deref() { + Some(vcpu_type) if !vcpu_type.trim().is_empty() => { + vcpu_sig_from_type(vcpu_type)?; + } + _ => bail!("vcpu_type is required"), + } + + if input.ovmf_sections.is_empty() { + bail!("ovmf_sections are required for amd sev-snp"); + } + + decode_required_hex("ovmf_hash", &input.ovmf_hash, 48)?; + if input.ovmf_sections.len() > MAX_OVMF_SECTIONS { + bail!("ovmf section count must not exceed {MAX_OVMF_SECTIONS}"); + } + if input.sev_hashes_table_gpa == 0 { + bail!("sev_hashes_table_gpa must be non-zero"); + } + if input.sev_es_reset_eip == 0 { + bail!("sev_es_reset_eip must be non-zero"); + } + + let mut has_kernel_hashes_section = false; + let mut measured_pages = 0u64; + for section in &input.ovmf_sections { + if section.size == 0 { + bail!("ovmf section size must be greater than zero"); + } + let pages = section.size.div_ceil(4096); + measured_pages = measured_pages + .checked_add(pages) + .ok_or_else(|| anyhow::anyhow!("ovmf metadata page count overflow"))?; + if measured_pages > MAX_OVMF_METADATA_PAGES { + bail!("ovmf metadata page count must not exceed {MAX_OVMF_METADATA_PAGES}"); + } + let section_type = SectionType::from_u32(section.section_type).ok_or_else(|| { + anyhow::anyhow!("unknown ovmf section_type {:#x}", section.section_type) + })?; + has_kernel_hashes_section |= section_type == SectionType::SnpKernelHashes; + } + if !has_kernel_hashes_section { + bail!("ovmf metadata does not include a snp_kernel_hashes section"); + } + + Ok(()) +} + +pub fn decode_required_hex(name: &str, value: &str, expected_len: usize) -> Result> { + if value.is_empty() { + bail!("{name} must not be empty"); + } + decode_optional_hex(name, value, expected_len) +} + +pub fn decode_optional_hex(name: &str, value: &str, expected_len: usize) -> Result> { + if value.is_empty() { + return Ok(Vec::new()); + } + let bytes = hex::decode(value).map_err(|_| anyhow::anyhow!("{name} must be valid hex"))?; + if bytes.len() != expected_len { + bail!("{name} must be {expected_len} bytes"); + } + Ok(bytes) +} + +struct Gctx { + ld: [u8; LD_BYTES], +} + +impl Gctx { + #[cfg(test)] + fn new() -> Self { + Self { ld: ZEROS_LD } + } + + fn from_ovmf_hash(hex_value: &str) -> Result { + let raw = hex::decode(hex_value).context("ovmf_hash must be valid hex")?; + let ld: [u8; LD_BYTES] = raw + .try_into() + .map_err(|_| anyhow::anyhow!("ovmf_hash must be 48 bytes"))?; + Ok(Self { ld }) + } + + /// SNP spec §8.17.2 PAGE_INFO layout (112 bytes): current digest, + /// contents digest, length, page type, permissions/reserved, and GPA. + fn update(&mut self, page_type: u8, gpa: u64, contents: &[u8; LD_BYTES]) { + let mut buf = [0u8; 0x70]; + buf[..LD_BYTES].copy_from_slice(&self.ld); + buf[48..96].copy_from_slice(contents); + buf[96..98].copy_from_slice(&0x70u16.to_le_bytes()); + buf[98] = page_type; + buf[104..112].copy_from_slice(&gpa.to_le_bytes()); + let mut digest = [0u8; LD_BYTES]; + digest.copy_from_slice(&Sha384::digest(buf)); + self.ld = digest; + } + + fn sha384(data: &[u8]) -> [u8; LD_BYTES] { + let mut out = [0u8; LD_BYTES]; + out.copy_from_slice(&Sha384::digest(data)); + out + } + + fn update_normal_pages(&mut self, start_gpa: u64, data: &[u8]) { + for (i, chunk) in data.chunks(4096).enumerate() { + self.update(0x01, start_gpa + (i * 4096) as u64, &Self::sha384(chunk)); + } + } + + fn update_zero_pages(&mut self, gpa: u64, len: usize) { + for i in (0..len).step_by(4096) { + self.update(0x03, gpa + i as u64, &ZEROS_LD); + } + } + + fn update_secrets_page(&mut self, gpa: u64) { + self.update(0x05, gpa, &ZEROS_LD); + } + + fn update_cpuid_page(&mut self, gpa: u64) { + self.update(0x06, gpa, &ZEROS_LD); + } + + fn update_vmsa_page(&mut self, page: &[u8]) { + self.update(0x02, VMSA_GPA, &Self::sha384(page)); + } +} + +const GUID_LE_HASH_TABLE_HEADER: [u8; 16] = [ + 0x06, 0xd6, 0x38, 0x94, 0x22, 0x4f, 0xc9, 0x4c, 0xb4, 0x79, 0xa7, 0x93, 0xd4, 0x11, 0xfd, 0x21, +]; +const GUID_LE_KERNEL_ENTRY: [u8; 16] = [ + 0x37, 0x94, 0xe7, 0x4d, 0xd2, 0xab, 0x7f, 0x42, 0xb8, 0x35, 0xd5, 0xb1, 0x72, 0xd2, 0x04, 0x5b, +]; +const GUID_LE_INITRD_ENTRY: [u8; 16] = [ + 0x31, 0xf7, 0xba, 0x44, 0x2f, 0x3a, 0xd7, 0x4b, 0x9a, 0xf1, 0x41, 0xe2, 0x91, 0x69, 0x78, 0x1d, +]; +const GUID_LE_CMDLINE_ENTRY: [u8; 16] = [ + 0xd8, 0x2d, 0xd0, 0x97, 0x20, 0xbd, 0x94, 0x4c, 0xaa, 0x78, 0xe7, 0x71, 0x4d, 0x36, 0xab, 0x2a, +]; + +fn sev_entry(guid: &[u8; 16], hash: &[u8; 32]) -> [u8; 50] { + let mut entry = [0u8; 50]; + entry[..16].copy_from_slice(guid); + entry[16..18].copy_from_slice(&50u16.to_le_bytes()); + entry[18..].copy_from_slice(hash); + entry +} + +fn build_sev_hashes_page( + kernel_hash_hex: &str, + initrd_hash_hex: &str, + append: &str, + page_offset: usize, +) -> Result<[u8; 4096]> { + let kernel_hash: [u8; 32] = hex::decode(kernel_hash_hex) + .context("kernel_hash must be valid hex")? + .try_into() + .map_err(|_| anyhow::anyhow!("kernel_hash must be 32 bytes"))?; + + let initrd_hash: [u8; 32] = if initrd_hash_hex.is_empty() { + let mut h = [0u8; 32]; + h.copy_from_slice(&Sha256::digest(b"")); + h + } else { + hex::decode(initrd_hash_hex) + .context("initrd_hash must be valid hex")? + .try_into() + .map_err(|_| anyhow::anyhow!("initrd_hash must be 32 bytes"))? + }; + + let mut cmdline_bytes = append.as_bytes().to_vec(); + cmdline_bytes.push(0); + let mut cmdline_hash = [0u8; 32]; + cmdline_hash.copy_from_slice(&Sha256::digest(&cmdline_bytes)); + + let cmdline_entry = sev_entry(&GUID_LE_CMDLINE_ENTRY, &cmdline_hash); + let initrd_entry = sev_entry(&GUID_LE_INITRD_ENTRY, &initrd_hash); + let kernel_entry = sev_entry(&GUID_LE_KERNEL_ENTRY, &kernel_hash); + + const TABLE_SIZE: usize = 16 + 2 + 50 + 50 + 50; + let mut table = [0u8; TABLE_SIZE]; + table[..16].copy_from_slice(&GUID_LE_HASH_TABLE_HEADER); + table[16..18].copy_from_slice(&(TABLE_SIZE as u16).to_le_bytes()); + table[18..68].copy_from_slice(&cmdline_entry); + table[68..118].copy_from_slice(&initrd_entry); + table[118..168].copy_from_slice(&kernel_entry); + + const PADDED: usize = (TABLE_SIZE + 15) & !(15usize); + if page_offset + PADDED > 4096 { + bail!("sev hash table overflows 4096-byte page"); + } + let mut page = [0u8; 4096]; + page[page_offset..page_offset + TABLE_SIZE].copy_from_slice(&table); + Ok(page) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SectionType { + SnpSecMemory = 1, + SnpSecrets = 2, + Cpuid = 3, + SvsmCaa = 4, + SnpKernelHashes = 0x10, +} + +impl SectionType { + pub fn from_u32(value: u32) -> Option { + match value { + 1 => Some(Self::SnpSecMemory), + 2 => Some(Self::SnpSecrets), + 3 => Some(Self::Cpuid), + 4 => Some(Self::SvsmCaa), + 0x10 => Some(Self::SnpKernelHashes), + _ => None, + } + } +} + +pub struct MetadataSection { + pub gpa: u64, + pub size: u64, + pub section_type: SectionType, +} + +pub struct OvmfInfo { + pub data: Vec, + pub gpa: u64, + pub sections: Vec, + pub sev_hashes_table_gpa: u64, + pub sev_es_reset_eip: u32, +} + +const GUID_FOOTER_TABLE: [u8; 16] = [ + 0xde, 0x82, 0xb5, 0x96, 0xb2, 0x1f, 0xf7, 0x45, 0xba, 0xea, 0xa3, 0x66, 0xc5, 0x5a, 0x08, 0x2d, +]; +const GUID_SEV_HASH_TABLE_RV: [u8; 16] = [ + 0x1f, 0x37, 0x55, 0x72, 0x3b, 0x3a, 0x04, 0x4b, 0x92, 0x7b, 0x1d, 0xa6, 0xef, 0xa8, 0xd4, 0x54, +]; +const GUID_SEV_ES_RESET_BLK: [u8; 16] = [ + 0xde, 0x71, 0xf7, 0x00, 0x7e, 0x1a, 0xcb, 0x4f, 0x89, 0x0e, 0x68, 0xc7, 0x7e, 0x2f, 0xb4, 0x4e, +]; +const GUID_SEV_META_DATA: [u8; 16] = [ + 0x66, 0x65, 0x88, 0xdc, 0x4a, 0x98, 0x98, 0x47, 0xa7, 0x5e, 0x55, 0x85, 0xa7, 0xbf, 0x67, 0xcc, +]; + +fn read_u16_le(buf: &[u8], off: usize) -> u16 { + u16::from_le_bytes([buf[off], buf[off + 1]]) +} + +fn read_u32_le(buf: &[u8], off: usize) -> u32 { + u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]]) +} + +impl OvmfInfo { + pub fn load(path: &str) -> Result { + let data = fs::read(path).with_context(|| format!("cannot read ovmf binary '{path}'"))?; + let size = data.len(); + let gpa = (0x1_0000_0000u64) + .checked_sub(size as u64) + .context("ovmf binary is larger than 4 gib")?; + + const ENTRY_HDR: usize = 18; + let footer_off = size.saturating_sub(32 + ENTRY_HDR); + if footer_off + ENTRY_HDR > size { + bail!("ovmf binary too small to contain footer table"); + } + if data[footer_off + 2..footer_off + 18] != GUID_FOOTER_TABLE { + bail!("ovmf footer guid not found"); + } + let footer_total_size = read_u16_le(&data, footer_off) as usize; + if footer_total_size < ENTRY_HDR { + bail!("ovmf footer table has invalid total size"); + } + let table_size = footer_total_size - ENTRY_HDR; + if table_size > footer_off { + bail!("ovmf footer table is out of bounds"); + } + let table_start = footer_off - table_size; + let table_bytes = &data[table_start..footer_off]; + + let mut sev_hashes_table_gpa = 0u64; + let mut sev_es_reset_eip = 0u32; + let mut meta_offset_from_end = None; + + let mut pos = table_bytes.len(); + while pos >= ENTRY_HDR { + let entry_off = pos - ENTRY_HDR; + let entry_size = read_u16_le(table_bytes, entry_off) as usize; + if entry_size < ENTRY_HDR || entry_size > pos { + bail!("ovmf footer table has invalid entry size"); + } + let guid = &table_bytes[entry_off + 2..entry_off + 18]; + let data_start = pos - entry_size; + let data_end = pos - ENTRY_HDR; + let entry_data = &table_bytes[data_start..data_end]; + + if guid == GUID_SEV_HASH_TABLE_RV && entry_data.len() >= 4 { + sev_hashes_table_gpa = read_u32_le(entry_data, 0) as u64; + } else if guid == GUID_SEV_ES_RESET_BLK && entry_data.len() >= 4 { + sev_es_reset_eip = read_u32_le(entry_data, 0); + } else if guid == GUID_SEV_META_DATA && entry_data.len() >= 4 { + meta_offset_from_end = Some(read_u32_le(entry_data, 0) as usize); + } + pos -= entry_size; + } + + if sev_hashes_table_gpa == 0 { + bail!("ovmf sev hash table entry not found in footer table"); + } + if sev_es_reset_eip == 0 { + bail!("ovmf sev_es_reset_block entry not found in footer table"); + } + + let mut sections = Vec::new(); + let off_from_end = meta_offset_from_end + .ok_or_else(|| anyhow::anyhow!("ovmf sev metadata entry not found in footer table"))?; + if off_from_end > size { + bail!("ovmf sev metadata offset exceeds file size"); + } + let meta_start = size - off_from_end; + if meta_start + 16 > size { + bail!("ovmf sev metadata header out of bounds"); + } + if &data[meta_start..meta_start + 4] != b"ASEV" { + bail!("ovmf sev metadata has bad signature"); + } + let meta_version = read_u32_le(&data, meta_start + 8); + if meta_version != 1 { + bail!("ovmf sev metadata has unsupported version {meta_version}"); + } + let num_items = read_u32_le(&data, meta_start + 12) as usize; + let items_start = meta_start + 16; + if items_start + num_items * 12 > size { + bail!("ovmf sev metadata sections out of bounds"); + } + for i in 0..num_items { + let off = items_start + i * 12; + let section_type_value = read_u32_le(&data, off + 8); + let section_type = SectionType::from_u32(section_type_value).ok_or_else(|| { + anyhow::anyhow!("unknown ovmf section_type {section_type_value:#x}") + })?; + sections.push(MetadataSection { + gpa: read_u32_le(&data, off) as u64, + size: read_u32_le(&data, off + 4) as u64, + section_type, + }); + } + + Ok(Self { + data, + gpa, + sections, + sev_hashes_table_gpa, + sev_es_reset_eip, + }) + } +} + +fn write_u16_le_at(buf: &mut [u8], off: usize, value: u16) { + buf[off..off + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn write_u32_le_at(buf: &mut [u8], off: usize, value: u32) { + buf[off..off + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn write_u64_le_at(buf: &mut [u8], off: usize, value: u64) { + buf[off..off + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn write_vmcb_seg(buf: &mut [u8], off: usize, selector: u16, attrib: u16, limit: u32, base: u64) { + write_u16_le_at(buf, off, selector); + write_u16_le_at(buf, off + 2, attrib); + write_u32_le_at(buf, off + 4, limit); + write_u64_le_at(buf, off + 8, base); +} + +fn amd_cpu_sig(family: u32, model: u32, stepping: u32) -> u32 { + let (family_low, family_high) = if family > 0xf { + (0xf, (family - 0xf) & 0xff) + } else { + (family, 0) + }; + let model_low = model & 0xf; + let model_high = (model >> 4) & 0xf; + (family_high << 20) + | (model_high << 16) + | (family_low << 8) + | (model_low << 4) + | (stepping & 0xf) +} + +fn vcpu_sig_from_type(vcpu_type: &str) -> Result { + match vcpu_type.trim().to_lowercase().as_str() { + "epyc" | "epyc-v1" | "epyc-v2" | "epyc-ibpb" | "epyc-v3" | "epyc-v4" => { + Ok(amd_cpu_sig(23, 1, 2)) + } + "epyc-rome" | "epyc-rome-v1" | "epyc-rome-v2" | "epyc-rome-v3" => { + Ok(amd_cpu_sig(23, 49, 0)) + } + "epyc-milan" | "epyc-milan-v1" | "epyc-milan-v2" => Ok(amd_cpu_sig(25, 1, 1)), + "epyc-genoa" | "epyc-genoa-v1" => Ok(amd_cpu_sig(25, 17, 0)), + other => bail!("unknown vcpu_type {other:?}"), + } +} + +fn build_vmsa_page(eip: u32, vcpu_sig: u32, sev_features: u64) -> Box<[u8; 4096]> { + let mut page = Box::new([0u8; 4096]); + let p = page.as_mut_slice(); + + let cs_base = (eip as u64) & 0xffff_0000; + let rip = (eip as u64) & 0x0000_ffff; + + write_vmcb_seg(p, 0x000, 0, 0x0093, 0xffff, 0); + write_vmcb_seg(p, 0x010, 0xf000, 0x009b, 0xffff, cs_base); + write_vmcb_seg(p, 0x020, 0, 0x0093, 0xffff, 0); + write_vmcb_seg(p, 0x030, 0, 0x0093, 0xffff, 0); + write_vmcb_seg(p, 0x040, 0, 0x0093, 0xffff, 0); + write_vmcb_seg(p, 0x050, 0, 0x0093, 0xffff, 0); + write_vmcb_seg(p, 0x060, 0, 0x0000, 0xffff, 0); + write_vmcb_seg(p, 0x070, 0, 0x0082, 0xffff, 0); + write_vmcb_seg(p, 0x080, 0, 0x0000, 0xffff, 0); + write_vmcb_seg(p, 0x090, 0, 0x008b, 0xffff, 0); + + write_u64_le_at(p, 0x0D0, 0x1000); + write_u64_le_at(p, 0x148, 0x40); + write_u64_le_at(p, 0x158, 0x10); + write_u64_le_at(p, 0x160, 0x400); + write_u64_le_at(p, 0x168, 0xffff_0ff0); + write_u64_le_at(p, 0x170, 0x2); + write_u64_le_at(p, 0x178, rip); + write_u64_le_at(p, 0x268, 0x0007_0406_0007_0406); + write_u64_le_at(p, 0x310, vcpu_sig as u64); + write_u64_le_at(p, 0x3B0, sev_features); + write_u64_le_at(p, 0x3E8, 0x1); + write_u32_le_at(p, 0x408, 0x1f80); + write_u16_le_at(p, 0x410, 0x037f); + + page +} + +/// Recompute the AMD SEV-SNP launch `MEASUREMENT` from self-contained inputs. +pub fn compute_expected_measurement(input: &MeasurementInput) -> Result<[u8; 48]> { + let vcpu_type = input + .vcpu_type + .as_deref() + .ok_or_else(|| anyhow::anyhow!("vcpu_type is required"))?; + + let cmdline = match input.base_cmdline.as_deref() { + Some(base) if !base.trim().is_empty() => base.trim().to_string(), + _ => "console=ttyS0 loglevel=7".to_string(), + }; + let resolved_sections = input + .ovmf_sections + .iter() + .map(|section| { + let section_type = SectionType::from_u32(section.section_type).ok_or_else(|| { + anyhow::anyhow!("unknown ovmf section_type {:#x}", section.section_type) + })?; + Ok(MetadataSection { + gpa: section.gpa, + size: section.size, + section_type, + }) + }) + .collect::>>()?; + let mut gctx = Gctx::from_ovmf_hash(&input.ovmf_hash)?; + let effective_hashes_gpa = input.sev_hashes_table_gpa; + let effective_reset_eip = input.sev_es_reset_eip; + + let mut has_kernel_hashes_section = false; + for section in &resolved_sections { + let gpa = section.gpa; + let size = usize::try_from(section.size) + .map_err(|_| anyhow::anyhow!("ovmf section size is too large"))?; + match section.section_type { + SectionType::SnpSecMemory => gctx.update_zero_pages(gpa, size), + SectionType::SnpSecrets => gctx.update_secrets_page(gpa), + SectionType::Cpuid => gctx.update_cpuid_page(gpa), + SectionType::SvsmCaa => gctx.update_zero_pages(gpa, size), + SectionType::SnpKernelHashes => { + has_kernel_hashes_section = true; + if effective_hashes_gpa == 0 { + bail!("snp_kernel_hashes section present but sev_hashes_table_gpa is 0"); + } + let page_offset = (effective_hashes_gpa & 0xfff) as usize; + let page = build_sev_hashes_page( + &input.kernel_hash, + &input.initrd_hash, + &cmdline, + page_offset, + )?; + gctx.update_normal_pages(gpa, &page); + } + } + } + if !has_kernel_hashes_section { + bail!("ovmf metadata does not include a snp_kernel_hashes section"); + } + + let vcpu_sig = vcpu_sig_from_type(vcpu_type)?; + let bsp_vmsa = build_vmsa_page(0xffff_fff0, vcpu_sig, input.guest_features); + let ap_vmsa = build_vmsa_page(effective_reset_eip, vcpu_sig, input.guest_features); + + for i in 0..input.vcpus as usize { + let vmsa_page = if i == 0 { + bsp_vmsa.as_ref() + } else { + ap_vmsa.as_ref() + }; + gctx.update_vmsa_page(vmsa_page); + } + + Ok(gctx.ld) +} + +/// Project a verified `MeasurementInput` to the shared image-invariant +/// measurement (excludes per-deployment fields like vcpus/app_id/compose_hash). +fn sev_os_image_measurement(input: &MeasurementInput) -> dstack_types::SevOsImageMeasurement { + dstack_types::SevOsImageMeasurement { + rootfs_hash: input.rootfs_hash.clone(), + base_cmdline: input.base_cmdline.clone(), + ovmf_hash: input.ovmf_hash.clone(), + kernel_hash: input.kernel_hash.clone(), + initrd_hash: input.initrd_hash.clone(), + sev_hashes_table_gpa: input.sev_hashes_table_gpa, + sev_es_reset_eip: input.sev_es_reset_eip, + ovmf_sections: input + .ovmf_sections + .iter() + .map(|s| dstack_types::OvmfSection { + gpa: s.gpa, + size: s.size, + section_type: s.section_type, + }) + .collect(), + } +} + +/// Derive the OS image hash from a self-contained SNP measurement document. +/// +/// os_image_hash identifies the OS image only, so it covers exactly the +/// image-determined measurement inputs and EXCLUDES per-deployment values +/// (`vcpus`, `vcpu_type`, `guest_features`, `app_id`, `compose_hash`). Hashing +/// the full `MeasurementInput` made the same image hash differently per vCPU +/// count, which broke per-image on-chain allow-listing. The canonical hashing +/// lives in `dstack_types::SevOsImageMeasurement` so the image build can +/// reproduce the same value as `digest.sev.txt`. +pub fn snp_measurement_os_image_hash(measurement_document: &str) -> Result> { + let input: MeasurementInput = serde_json::from_str(measurement_document) + .context("failed to parse sev-snp measurement document for os_image_hash")?; + Ok(sev_os_image_measurement(&input).os_image_hash().to_vec()) +} + +/// `sha256(MEASUREMENT || HOST_DATA)` — the SNP aggregated identity digest. +pub fn snp_mr_aggregated_digest(measurement: &[u8; 48], host_data: &[u8; 32]) -> Vec { + let mut h = Sha256::new(); + h.update(measurement); + h.update(host_data); + h.finalize().to_vec() +} + +/// Validate the shape of an MrConfigV3 document carried by HOST_DATA. +pub fn validate_mr_config(mr_config: &MrConfigV3) -> Result<()> { + if mr_config.version != 3 { + bail!("mr_config version must be 3"); + } + ensure_len("mr_config.app_id", &mr_config.app_id, 20)?; + ensure_len("mr_config.compose_hash", &mr_config.compose_hash, 32)?; + if !mr_config.instance_id.is_empty() { + ensure_len("mr_config.instance_id", &mr_config.instance_id, 20)?; + } + Ok(()) +} + +fn ensure_len(name: &str, value: &[u8], expected_len: usize) -> Result<()> { + if value.len() != expected_len { + bail!("{name} must be {expected_len} bytes"); + } + Ok(()) +} + +/// Check that the hardware-verified `HOST_DATA` equals the hash of the supplied +/// MrConfigV3 document, binding app/config identity to the report. +pub fn validate_snp_mr_config_binding( + host_data: &[u8; 32], + mr_config_document: &str, +) -> Result { + let mr_config = MrConfigV3::from_document(mr_config_document) + .context("invalid amd sev-snp mr_config document")?; + let expected = MrConfigV3::snp_host_data_from_document(mr_config_document); + if expected != *host_data { + bail!("amd sev-snp host_data mismatch"); + } + validate_mr_config(&mr_config)?; + Ok(mr_config) +} + +#[derive(Debug, serde::Deserialize)] +struct SevSnpMeasurementVmConfig { + sev_snp_measurement: Option, + mr_config: Option, +} + +/// Launch inputs extracted from a VMM-produced `vm_config` string. +pub struct SnpLaunchInputs { + pub input: MeasurementInput, + /// Raw `sev_snp_measurement` document used for os_image_hash derivation. + pub measurement_document: String, + /// Raw MrConfigV3 document bound by HOST_DATA. + pub mr_config_document: String, +} + +/// Parse the SNP launch-measurement inputs (`sev_snp_measurement`) and the +/// `mr_config` document out of a VMM `vm_config` JSON string. +/// +/// The fields are intentionally explicit so missing SNP launch inputs fail +/// closed instead of falling back to TDX event-log decoding. Both the top-level +/// shape and the legacy nested `vm_config` string shape are accepted. +pub fn parse_snp_inputs_from_vm_config(vm_config: &str) -> Result { + let value: serde_json::Value = + serde_json::from_str(vm_config).context("failed to parse vm_config for amd sev-snp")?; + let parsed: SevSnpMeasurementVmConfig = serde_json::from_value(value.clone()) + .context("failed to parse vm_config for amd sev-snp")?; + let nested = value + .get("vm_config") + .and_then(|value| value.as_str()) + .map(|vm_config| { + serde_json::from_str::(vm_config) + .context("failed to parse nested vm_config for amd sev-snp") + }) + .transpose()?; + let measurement_document = parsed + .sev_snp_measurement + .or_else(|| { + nested + .as_ref() + .and_then(|nested| nested.sev_snp_measurement.clone()) + }) + .ok_or_else(|| anyhow::anyhow!("sev_snp_measurement is required for amd sev-snp"))?; + let input: MeasurementInput = serde_json::from_str(&measurement_document) + .context("invalid amd sev-snp measurement document")?; + let mr_config_document = parsed + .mr_config + .or_else(|| nested.and_then(|nested| nested.mr_config)) + .ok_or_else(|| anyhow::anyhow!("mr_config is required for amd sev-snp"))?; + MrConfigV3::from_document(&mr_config_document) + .context("invalid amd sev-snp mr_config document")?; + Ok(SnpLaunchInputs { + input, + measurement_document, + mr_config_document, + }) +} + +/// The verified SNP image binding produced by [`verify_sev_launch`]. +pub struct SevImageBinding { + /// Image-invariant os_image_hash derived from the (now measurement-bound) + /// launch inputs. + pub os_image_hash: Vec, + /// App/config identity bound by HOST_DATA. + pub mr_config: MrConfigV3, +} + +/// End-to-end SNP launch verification against an already hardware-verified +/// report. +/// +/// Given the verified `MEASUREMENT` and `HOST_DATA` from a report whose +/// signature/collateral have already been checked, this: +/// 1. parses `sev_snp_measurement` + `mr_config` from `vm_config`, +/// 2. recomputes the launch measurement and checks it equals `measurement` +/// (this is what makes the otherwise-untrusted launch inputs trustworthy), +/// 3. checks `HOST_DATA` binds the `mr_config` document, and +/// 4. derives the image-invariant `os_image_hash`. +pub fn verify_sev_launch( + verified_measurement: &[u8; 48], + verified_host_data: &[u8; 32], + vm_config: &str, +) -> Result { + let inputs = parse_snp_inputs_from_vm_config(vm_config)?; + validate_measurement_input(&inputs.input)?; + let expected = compute_expected_measurement(&inputs.input)?; + if &expected != verified_measurement { + bail!("amd sev-snp measurement mismatch"); + } + let mr_config = validate_snp_mr_config_binding(verified_host_data, &inputs.mr_config_document)?; + let os_image_hash = snp_measurement_os_image_hash(&inputs.measurement_document)?; + Ok(SevImageBinding { + os_image_hash, + mr_config, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn hex_of(byte: u8, len: usize) -> String { + hex::encode(vec![byte; len]) + } + + fn valid_input() -> MeasurementInput { + MeasurementInput { + app_id: hex_of(0x11, 20), + compose_hash: hex_of(0x22, 32), + rootfs_hash: hex_of(0x33, 32), + base_cmdline: None, + ovmf_hash: hex_of(0x44, 48), + kernel_hash: hex_of(0x55, 32), + initrd_hash: hex_of(0x66, 32), + sev_hashes_table_gpa: 0x80_1000, + sev_es_reset_eip: 0xffff_fff0, + vcpus: 2, + vcpu_type: Some("epyc-v4".to_string()), + guest_features: 1, + ovmf_sections: vec![ + OvmfSectionParam { + gpa: 0x100000, + size: 0x2000, + section_type: 1, + }, + OvmfSectionParam { + gpa: 0x80_0000, + size: 0x1000, + section_type: 0x10, + }, + OvmfSectionParam { + gpa: 0x81_0000, + size: 0x1000, + section_type: 2, + }, + OvmfSectionParam { + gpa: 0x82_0000, + size: 0x1000, + section_type: 3, + }, + ], + } + } + + fn measurement_document(input: &MeasurementInput) -> String { + serde_json::to_string(input).expect("measurement input should serialize") + } + + #[test] + fn snp_os_image_hash_covers_image_fields_only() { + let input = valid_input(); + let os_image_hash = + |i: &MeasurementInput| snp_measurement_os_image_hash(&measurement_document(i)).unwrap(); + let baseline = os_image_hash(&input); + + // Image-determined fields MUST change the os_image_hash. + let image_cases: Vec<(&str, fn(&mut MeasurementInput))> = vec![ + ("rootfs_hash", |i| i.rootfs_hash = hex_of(0x34, 32)), + ("base_cmdline", |i| { + i.base_cmdline = Some("console=ttyS0 loglevel=8".to_string()) + }), + ("ovmf_hash", |i| i.ovmf_hash = hex_of(0x45, 48)), + ("kernel_hash", |i| i.kernel_hash = hex_of(0x56, 32)), + ("initrd_hash", |i| i.initrd_hash = hex_of(0x67, 32)), + ("sev_hashes_table_gpa", |i| i.sev_hashes_table_gpa += 0x1000), + ("sev_es_reset_eip", |i| i.sev_es_reset_eip = 0xffff_0000), + ("ovmf_sections.gpa", |i| i.ovmf_sections[0].gpa += 0x1000), + ("ovmf_sections.size", |i| i.ovmf_sections[0].size += 0x1000), + ("ovmf_sections.section_type", |i| { + i.ovmf_sections[0].section_type = 4 + }), + ]; + for (name, mutate) in image_cases { + let mut changed = input.clone(); + mutate(&mut changed); + assert_ne!( + baseline, + os_image_hash(&changed), + "{name} must change the SNP os_image_hash" + ); + } + + // Per-deployment fields MUST NOT change the os_image_hash (the same OS + // image must hash identically regardless of vCPU count, app, etc.). + let deployment_cases: Vec<(&str, fn(&mut MeasurementInput))> = vec![ + ("app_id", |i| i.app_id = hex_of(0x12, 20)), + ("compose_hash", |i| i.compose_hash = hex_of(0x23, 32)), + ("vcpus", |i| i.vcpus = 3), + ("vcpu_type", |i| { + i.vcpu_type = Some("epyc-milan".to_string()) + }), + ("guest_features", |i| i.guest_features = 3), + ]; + for (name, mutate) in deployment_cases { + let mut changed = input.clone(); + mutate(&mut changed); + assert_eq!( + baseline, + os_image_hash(&changed), + "{name} must NOT change the SNP os_image_hash" + ); + } + } + + #[test] + fn gctx_update_is_deterministic_and_order_sensitive() { + let contents = Gctx::sha384(b"page"); + let mut first = Gctx::new(); + first.update(0x01, 0x1000, &contents); + assert_eq!( + hex::encode(first.ld), + "3ebc1a70acc0bae5ae2788fae29a0371f983b19a68faf9843064f36040f58571ce5bb6bcdc9c361087073f8cffd92635" + ); + + let mut second = Gctx::new(); + second.update(0x01, 0x2000, &contents); + assert_ne!(first.ld, second.ld); + } + + #[test] + fn builds_sev_hashes_page_at_requested_offset() { + let page = build_sev_hashes_page(&hex_of(0x55, 32), "", "console=ttyS0", 0x80) + .expect("sev hashes page should build"); + assert_eq!(&page[..0x80], &[0u8; 0x80]); + assert_eq!(&page[0x80..0x90], &GUID_LE_HASH_TABLE_HEADER); + assert_eq!(u16::from_le_bytes([page[0x90], page[0x91]]), 168); + assert_eq!( + &page[0x92..0xa2], + &GUID_LE_CMDLINE_ENTRY, + "cmdline entry must be first" + ); + let empty_hash = Sha256::digest(b""); + assert_eq!(&page[0x80 + 68 + 18..0x80 + 68 + 50], empty_hash.as_slice()); + } + + #[test] + fn vcpu_type_mapping_is_strict() { + assert_eq!( + vcpu_sig_from_type("EPYC-v4").unwrap(), + amd_cpu_sig(23, 1, 2) + ); + assert_eq!( + vcpu_sig_from_type("epyc-genoa-v1").unwrap(), + amd_cpu_sig(25, 17, 0) + ); + let err = vcpu_sig_from_type("not-a-cpu").expect_err("unknown vcpu should reject"); + assert!(err.to_string().contains("unknown vcpu_type")); + } + + #[test] + fn measurement_vector_does_not_drift() { + let input = valid_input(); + let expected = compute_expected_measurement(&input).unwrap(); + assert_eq!( + hex::encode(expected), + "88a47914470533e33e24befd24ef0ac877658ff82cafc9878bd9566550f100fdf56d62f419e21b959aa228fc98000da4", + "synthetic measurement vector should not drift silently" + ); + } + + /// Real `sev_snp_measurement` document captured from a live dstack SEV-SNP + /// CVM (the same fixture used by `dstack-attest/tests/sev_snp_verify.rs`). + const REAL_MEASUREMENT_DOC: &str = r#"{"rootfs_hash":"ca5adaef0ac3a36108035925763b48a5818f634e700fbaab561d419fd30d7121","base_cmdline":"console=ttyS0 init=/init panic=1 net.ifnames=0 biosdevname=0 mce=off oops=panic pci=noearly pci=nommconf random.trust_cpu=y random.trust_bootloader=n tsc=reliable no-kvmclock dstack.rootfs_hash=ca5adaef0ac3a36108035925763b48a5818f634e700fbaab561d419fd30d7121 dstack.rootfs_size=490713088","ovmf_hash":"ffb57e393469a497c0e3b07bd1c97d8611e555f464d14491837665893ac642b263a71f9507ff100a847897fe0c3f8c6f","kernel_hash":"dd9ea274ce9a07090b22e8284b0c841b65c021c2d15ca57d0f16731089dd226c","initrd_hash":"5f844c4a2ca5a3d0711b3db38293b21ba929bb8e0b3c5bc1a779a57f69221c19","sev_hashes_table_gpa":8457216,"sev_es_reset_eip":8433668,"vcpus":2,"vcpu_type":"EPYC-v4","guest_features":1,"ovmf_sections":[{"gpa":8388608,"size":36864,"section_type":1},{"gpa":8429568,"size":12288,"section_type":1},{"gpa":8441856,"size":4096,"section_type":2},{"gpa":8445952,"size":4096,"section_type":3},{"gpa":8450048,"size":4096,"section_type":4},{"gpa":8458240,"size":61440,"section_type":1},{"gpa":8454144,"size":4096,"section_type":16}]}"#; + + #[test] + fn real_fixture_recomputes_measurement_and_os_image_hash() { + let input: MeasurementInput = + serde_json::from_str(REAL_MEASUREMENT_DOC).expect("real measurement doc parses"); + validate_measurement_input(&input).expect("real measurement input is valid"); + + // Recomputed launch measurement must equal the hardware-signed value + // from the captured report (see sev_snp_fixture.README.md). + let measurement = compute_expected_measurement(&input).expect("recompute measurement"); + assert_eq!( + hex::encode(measurement), + "7f51e17f72a04d5422cb2c00998166536019a217376f3aa45a630e59c805a599847ff250dbffcd07e1ba639771d6f05d", + ); + + // os_image_hash derived from the same document must match the value the + // CVM advertised in its vm_config (and digest.sev.txt). + let os_image_hash = + snp_measurement_os_image_hash(REAL_MEASUREMENT_DOC).expect("derive os_image_hash"); + assert_eq!( + hex::encode(os_image_hash), + "32b4767373ad7fa0f9c418925006194d5c3f5619529f309fe81156789fecd8bc", + ); + } +} From 81379afaca98f250c0623debfdabe5472af19300 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 17 Jun 2026 19:01:42 -0700 Subject: [PATCH 59/67] kms: source SEV-SNP measurement from dstack-mr::sev Replace the in-tree launch-measurement recomputation, os_image_hash derivation, OVMF parsing and mr_config binding with re-exports from dstack-mr::sev. The KMS keeps its authorization BootInfo/policy layer on top. Behaviour is unchanged: all 28 KMS tests (incl. the pinned 88a479... measurement vector) pass against the shared implementation. --- kms/src/main_service/amd_attest.rs | 939 +---------------------------- 1 file changed, 24 insertions(+), 915 deletions(-) diff --git a/kms/src/main_service/amd_attest.rs b/kms/src/main_service/amd_attest.rs index 96f72ba76..0139a0888 100644 --- a/kms/src/main_service/amd_attest.rs +++ b/kms/src/main_service/amd_attest.rs @@ -14,110 +14,35 @@ //! boot inputs; app identity is bound by checking that the verified report //! `HOST_DATA` equals the attached MrConfigV3 document hash. Do not use this //! helper by itself to release app keys. +//! +//! The launch-measurement recomputation and `os_image_hash` derivation live in +//! `dstack_mr::sev` so the KMS (key release) and the verifier (attestation +//! verification) compute identical values from a single source of truth. The +//! pieces below add the KMS-specific authorization `BootInfo`/policy layer on +//! top of those shared primitives. #![allow(dead_code)] use anyhow::{bail, Context, Result}; use dstack_types::{mr_config::MrConfigV3, KeyProviderInfo}; use ra_tls::attestation::{AttestationMode, VerifiedAttestation}; -use sha2::{Digest, Sha256, Sha384}; -use std::fs; +use sha2::{Digest, Sha256}; use crate::config::SevSnpMeasureConfig; use super::upgrade_authority::BootInfo; -const LD_BYTES: usize = 48; -const ZEROS_LD: [u8; LD_BYTES] = [0u8; LD_BYTES]; -const MAX_VCPUS: u32 = 512; -const MAX_OVMF_SECTIONS: usize = 64; -/// 64 GiB worth of 4 KiB pages. -const MAX_OVMF_METADATA_PAGES: u64 = 16_777_216; -// VMSA page GPA: (u64)(-1) page-aligned, bits >51 cleared. -const VMSA_GPA: u64 = 0x0000_FFFF_FFFF_F000; - -#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize, serde::Serialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct OvmfSectionParam { - pub gpa: u64, - pub size: u64, - /// Raw OVMF SEV metadata section type: - /// 1=SNP_SEC_MEMORY, 2=SNP_SECRETS, 3=CPUID, 4=SVSM_CAA, - /// 0x10=SNP_KERNEL_HASHES. - pub section_type: u32, -} - -#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize, serde::Serialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct MeasurementInput { - /// Deprecated: app identity is now bound through MrConfigV3/HOST_DATA. - #[serde(default)] - pub app_id: String, - /// Deprecated: compose identity is now bound through MrConfigV3/HOST_DATA. - #[serde(default)] - pub compose_hash: String, - /// 32-byte rootfs hash included in the self-contained SNP measurement input. - pub rootfs_hash: String, - /// Original image kernel cmdline used for SNP measured launch. - pub base_cmdline: Option, - /// 48-byte OVMF GCTX launch digest seed supplied by the VMM. - pub ovmf_hash: String, - /// 32-byte kernel SHA-256 hash. - pub kernel_hash: String, - /// 32-byte initrd SHA-256 hash. An empty string is treated as the SHA-256 of - /// an empty initrd, matching QEMU/sev-snp-measure behavior. - pub initrd_hash: String, - /// GPA of the SevHashTable, from OVMF footer metadata. - pub sev_hashes_table_gpa: u64, - /// AP reset EIP, from OVMF footer metadata. - pub sev_es_reset_eip: u32, - pub vcpus: u32, - pub vcpu_type: Option, - /// SNP guest features bitmask used at launch. QEMU uses 0x1 for SNP with - /// kernel hashes enabled in the current VMM path. - pub guest_features: u64, - #[serde(deserialize_with = "deserialize_ovmf_sections_bounded")] - pub ovmf_sections: Vec, -} - -fn deserialize_ovmf_sections_bounded<'de, D>( - deserializer: D, -) -> std::result::Result, D::Error> -where - D: serde::Deserializer<'de>, -{ - struct BoundedOvmfSections; - - impl<'de> serde::de::Visitor<'de> for BoundedOvmfSections { - type Value = Vec; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - formatter, - "at most {MAX_OVMF_SECTIONS} OVMF metadata sections" - ) - } - - fn visit_seq(self, mut seq: A) -> std::result::Result, A::Error> - where - A: serde::de::SeqAccess<'de>, - { - let mut sections = - Vec::with_capacity(seq.size_hint().unwrap_or(0).min(MAX_OVMF_SECTIONS)); - while let Some(section) = seq.next_element()? { - if sections.len() >= MAX_OVMF_SECTIONS { - return Err(serde::de::Error::custom(format!( - "ovmf section count must not exceed {MAX_OVMF_SECTIONS}" - ))); - } - sections.push(section); - } - Ok(sections) - } - } - - deserializer.deserialize_seq(BoundedOvmfSections) -} +// Shared SEV-SNP launch-measurement primitives now live in `dstack-mr::sev` +// (single source of truth shared with `dstack-verifier`). Re-export the symbols +// the rest of the KMS and its tests reference so existing call sites keep +// working. `allow(unused_imports)` because some are consumed only by tests. +#[allow(unused_imports)] +pub(crate) use dstack_mr::sev::{ + compute_expected_measurement, decode_required_hex, parse_snp_inputs_from_vm_config, + snp_measurement_os_image_hash, snp_mr_aggregated_digest, validate_measurement_input, + validate_snp_mr_config_binding, MeasurementInput, OvmfSectionParam, SnpLaunchInputs, + MAX_OVMF_METADATA_PAGES, MAX_OVMF_SECTIONS, MAX_VCPUS, +}; pub(crate) fn validate_amd_snp_measurement_binding( _config: Option<&SevSnpMeasureConfig>, @@ -238,12 +163,6 @@ pub(crate) fn build_amd_snp_boot_info_from_verified_attestation( ) } -#[derive(Debug, serde::Deserialize)] -struct SevSnpMeasurementVmConfig { - sev_snp_measurement: Option, - mr_config: Option, -} - /// Parses SNP launch-measurement inputs from the KMS request `vm_config` and /// builds helper-only SNP `BootInfo` from an already verified attestation. /// @@ -254,8 +173,11 @@ pub(crate) fn build_amd_snp_boot_info_from_verified_attestation_and_vm_config( attestation: &VerifiedAttestation, vm_config: &str, ) -> Result { - let (input, measurement_document, mr_config_document) = - parse_snp_inputs_from_vm_config(vm_config)?; + let SnpLaunchInputs { + input, + measurement_document, + mr_config_document, + } = parse_snp_inputs_from_vm_config(vm_config)?; build_amd_snp_boot_info_from_verified_attestation( config, attestation, @@ -266,38 +188,7 @@ pub(crate) fn build_amd_snp_boot_info_from_verified_attestation_and_vm_config( } fn parse_measurement_input_from_vm_config(vm_config: &str) -> Result { - Ok(parse_snp_inputs_from_vm_config(vm_config)?.0) -} - -fn parse_snp_inputs_from_vm_config(vm_config: &str) -> Result<(MeasurementInput, String, String)> { - let value: serde_json::Value = - serde_json::from_str(vm_config).context("failed to parse vm_config for amd sev-snp")?; - let parsed: SevSnpMeasurementVmConfig = serde_json::from_value(value.clone()) - .context("failed to parse vm_config for amd sev-snp")?; - let nested = value - .get("vm_config") - .and_then(|value| value.as_str()) - .map(|vm_config| { - serde_json::from_str::(vm_config) - .context("failed to parse nested vm_config for amd sev-snp") - }) - .transpose()?; - let measurement_document = parsed - .sev_snp_measurement - .or_else(|| { - nested - .as_ref() - .and_then(|nested| nested.sev_snp_measurement.clone()) - }) - .ok_or_else(|| anyhow::anyhow!("sev_snp_measurement is required for amd sev-snp"))?; - let measurement: MeasurementInput = serde_json::from_str(&measurement_document) - .context("invalid amd sev-snp measurement document")?; - let mr_config = parsed - .mr_config - .or_else(|| nested.and_then(|nested| nested.mr_config)) - .ok_or_else(|| anyhow::anyhow!("mr_config is required for amd sev-snp"))?; - MrConfigV3::from_document(&mr_config).context("invalid amd sev-snp mr_config document")?; - Ok((measurement, measurement_document, mr_config)) + Ok(parse_snp_inputs_from_vm_config(vm_config)?.input) } /// Explicit helper-only AMD SEV-SNP authorization policy. @@ -414,51 +305,6 @@ fn ensure_allowed_string(name: &str, value: &str, allowed: &[String]) -> Result< bail!("{name} is not allowed") } -fn snp_mr_aggregated_digest(measurement: &[u8; 48], host_data: &[u8; 32]) -> Vec { - let mut h = Sha256::new(); - h.update(measurement); - h.update(host_data); - h.finalize().to_vec() -} - -/// Project a verified `MeasurementInput` to the shared image-invariant -/// measurement (excludes per-deployment fields like vcpus/app_id/compose_hash). -fn sev_os_image_measurement(input: &MeasurementInput) -> dstack_types::SevOsImageMeasurement { - dstack_types::SevOsImageMeasurement { - rootfs_hash: input.rootfs_hash.clone(), - base_cmdline: input.base_cmdline.clone(), - ovmf_hash: input.ovmf_hash.clone(), - kernel_hash: input.kernel_hash.clone(), - initrd_hash: input.initrd_hash.clone(), - sev_hashes_table_gpa: input.sev_hashes_table_gpa, - sev_es_reset_eip: input.sev_es_reset_eip, - ovmf_sections: input - .ovmf_sections - .iter() - .map(|s| dstack_types::OvmfSection { - gpa: s.gpa, - size: s.size, - section_type: s.section_type, - }) - .collect(), - } -} - -/// Derive the OS image hash from a self-contained SNP measurement document. -/// -/// os_image_hash identifies the OS image only, so it covers exactly the -/// image-determined measurement inputs and EXCLUDES per-deployment values -/// (`vcpus`, `vcpu_type`, `guest_features`, `app_id`, `compose_hash`). Hashing -/// the full `MeasurementInput` made the same image hash differently per vCPU -/// count, which broke per-image on-chain allow-listing. The canonical hashing -/// lives in `dstack_types::SevOsImageMeasurement` so the image build can -/// reproduce the same value as `digest.sev.txt`. -pub(crate) fn snp_measurement_os_image_hash(measurement_document: &str) -> Result> { - let input: MeasurementInput = serde_json::from_str(measurement_document) - .context("failed to parse sev-snp measurement document for os_image_hash")?; - Ok(sev_os_image_measurement(&input).os_image_hash().to_vec()) -} - fn mr_config_key_provider_info(mr_config: &MrConfigV3) -> Result> { serde_json::to_vec(&KeyProviderInfo::new( mr_config.key_provider_name().to_string(), @@ -467,32 +313,6 @@ fn mr_config_key_provider_info(mr_config: &MrConfigV3) -> Result> { .context("failed to serialize key provider info") } -fn validate_snp_mr_config_binding( - host_data: &[u8; 32], - mr_config_document: &str, -) -> Result { - let mr_config = MrConfigV3::from_document(mr_config_document) - .context("invalid amd sev-snp mr_config document")?; - let expected = MrConfigV3::snp_host_data_from_document(mr_config_document); - if expected != *host_data { - bail!("amd sev-snp host_data mismatch"); - } - validate_mr_config(&mr_config)?; - Ok(mr_config) -} - -fn validate_mr_config(mr_config: &MrConfigV3) -> Result<()> { - if mr_config.version != 3 { - bail!("mr_config version must be 3"); - } - ensure_len("mr_config.app_id", &mr_config.app_id, 20)?; - ensure_len("mr_config.compose_hash", &mr_config.compose_hash, 32)?; - if !mr_config.instance_id.is_empty() { - ensure_len("mr_config.instance_id", &mr_config.instance_id, 20)?; - } - Ok(()) -} - #[cfg(test)] fn test_mr_config_from_input(input: &MeasurementInput) -> Result { let app_id = decode_required_hex("app_id", &input.app_id, 20)?; @@ -506,535 +326,6 @@ fn test_mr_config_from_input(input: &MeasurementInput) -> Result { )) } -fn validate_measurement_input(input: &MeasurementInput) -> Result<()> { - if input.guest_features == 0 { - bail!("guest_features must be non-zero"); - } - - decode_required_hex("rootfs_hash", &input.rootfs_hash, 32)?; - decode_required_hex("kernel_hash", &input.kernel_hash, 32)?; - decode_optional_hex("initrd_hash", &input.initrd_hash, 32)?; - if input.vcpus == 0 { - bail!("vcpus must be greater than zero"); - } - if input.vcpus > MAX_VCPUS { - bail!("vcpus must not exceed {MAX_VCPUS}"); - } - match input.vcpu_type.as_deref() { - Some(vcpu_type) if !vcpu_type.trim().is_empty() => { - vcpu_sig_from_type(vcpu_type)?; - } - _ => bail!("vcpu_type is required"), - } - - if input.ovmf_sections.is_empty() { - bail!("ovmf_sections are required for amd sev-snp"); - } - - decode_required_hex("ovmf_hash", &input.ovmf_hash, 48)?; - if input.ovmf_sections.len() > MAX_OVMF_SECTIONS { - bail!("ovmf section count must not exceed {MAX_OVMF_SECTIONS}"); - } - if input.sev_hashes_table_gpa == 0 { - bail!("sev_hashes_table_gpa must be non-zero"); - } - if input.sev_es_reset_eip == 0 { - bail!("sev_es_reset_eip must be non-zero"); - } - - let mut has_kernel_hashes_section = false; - let mut measured_pages = 0u64; - for section in &input.ovmf_sections { - if section.size == 0 { - bail!("ovmf section size must be greater than zero"); - } - let pages = section.size.div_ceil(4096); - measured_pages = measured_pages - .checked_add(pages) - .ok_or_else(|| anyhow::anyhow!("ovmf metadata page count overflow"))?; - if measured_pages > MAX_OVMF_METADATA_PAGES { - bail!("ovmf metadata page count must not exceed {MAX_OVMF_METADATA_PAGES}"); - } - let section_type = SectionType::from_u32(section.section_type).ok_or_else(|| { - anyhow::anyhow!("unknown ovmf section_type {:#x}", section.section_type) - })?; - has_kernel_hashes_section |= section_type == SectionType::SnpKernelHashes; - } - if !has_kernel_hashes_section { - bail!("ovmf metadata does not include a snp_kernel_hashes section"); - } - - Ok(()) -} - -fn decode_required_hex(name: &str, value: &str, expected_len: usize) -> Result> { - if value.is_empty() { - bail!("{name} must not be empty"); - } - decode_optional_hex(name, value, expected_len) -} - -fn decode_optional_hex(name: &str, value: &str, expected_len: usize) -> Result> { - if value.is_empty() { - return Ok(Vec::new()); - } - let bytes = hex::decode(value).map_err(|_| anyhow::anyhow!("{name} must be valid hex"))?; - if bytes.len() != expected_len { - bail!("{name} must be {expected_len} bytes"); - } - Ok(bytes) -} - -struct Gctx { - ld: [u8; LD_BYTES], -} - -impl Gctx { - fn new() -> Self { - Self { ld: ZEROS_LD } - } - - fn from_ovmf_hash(hex_value: &str) -> Result { - let raw = hex::decode(hex_value).context("ovmf_hash must be valid hex")?; - let ld: [u8; LD_BYTES] = raw - .try_into() - .map_err(|_| anyhow::anyhow!("ovmf_hash must be 48 bytes"))?; - Ok(Self { ld }) - } - - /// SNP spec §8.17.2 PAGE_INFO layout (112 bytes): current digest, - /// contents digest, length, page type, permissions/reserved, and GPA. - fn update(&mut self, page_type: u8, gpa: u64, contents: &[u8; LD_BYTES]) { - let mut buf = [0u8; 0x70]; - buf[..LD_BYTES].copy_from_slice(&self.ld); - buf[48..96].copy_from_slice(contents); - buf[96..98].copy_from_slice(&0x70u16.to_le_bytes()); - buf[98] = page_type; - buf[104..112].copy_from_slice(&gpa.to_le_bytes()); - let mut digest = [0u8; LD_BYTES]; - digest.copy_from_slice(&Sha384::digest(buf)); - self.ld = digest; - } - - fn sha384(data: &[u8]) -> [u8; LD_BYTES] { - let mut out = [0u8; LD_BYTES]; - out.copy_from_slice(&Sha384::digest(data)); - out - } - - fn update_normal_pages(&mut self, start_gpa: u64, data: &[u8]) { - for (i, chunk) in data.chunks(4096).enumerate() { - self.update(0x01, start_gpa + (i * 4096) as u64, &Self::sha384(chunk)); - } - } - - fn update_zero_pages(&mut self, gpa: u64, len: usize) { - for i in (0..len).step_by(4096) { - self.update(0x03, gpa + i as u64, &ZEROS_LD); - } - } - - fn update_secrets_page(&mut self, gpa: u64) { - self.update(0x05, gpa, &ZEROS_LD); - } - - fn update_cpuid_page(&mut self, gpa: u64) { - self.update(0x06, gpa, &ZEROS_LD); - } - - fn update_vmsa_page(&mut self, page: &[u8]) { - self.update(0x02, VMSA_GPA, &Self::sha384(page)); - } -} - -const GUID_LE_HASH_TABLE_HEADER: [u8; 16] = [ - 0x06, 0xd6, 0x38, 0x94, 0x22, 0x4f, 0xc9, 0x4c, 0xb4, 0x79, 0xa7, 0x93, 0xd4, 0x11, 0xfd, 0x21, -]; -const GUID_LE_KERNEL_ENTRY: [u8; 16] = [ - 0x37, 0x94, 0xe7, 0x4d, 0xd2, 0xab, 0x7f, 0x42, 0xb8, 0x35, 0xd5, 0xb1, 0x72, 0xd2, 0x04, 0x5b, -]; -const GUID_LE_INITRD_ENTRY: [u8; 16] = [ - 0x31, 0xf7, 0xba, 0x44, 0x2f, 0x3a, 0xd7, 0x4b, 0x9a, 0xf1, 0x41, 0xe2, 0x91, 0x69, 0x78, 0x1d, -]; -const GUID_LE_CMDLINE_ENTRY: [u8; 16] = [ - 0xd8, 0x2d, 0xd0, 0x97, 0x20, 0xbd, 0x94, 0x4c, 0xaa, 0x78, 0xe7, 0x71, 0x4d, 0x36, 0xab, 0x2a, -]; - -fn sev_entry(guid: &[u8; 16], hash: &[u8; 32]) -> [u8; 50] { - let mut entry = [0u8; 50]; - entry[..16].copy_from_slice(guid); - entry[16..18].copy_from_slice(&50u16.to_le_bytes()); - entry[18..].copy_from_slice(hash); - entry -} - -fn build_sev_hashes_page( - kernel_hash_hex: &str, - initrd_hash_hex: &str, - append: &str, - page_offset: usize, -) -> Result<[u8; 4096]> { - let kernel_hash: [u8; 32] = hex::decode(kernel_hash_hex) - .context("kernel_hash must be valid hex")? - .try_into() - .map_err(|_| anyhow::anyhow!("kernel_hash must be 32 bytes"))?; - - let initrd_hash: [u8; 32] = if initrd_hash_hex.is_empty() { - let mut h = [0u8; 32]; - h.copy_from_slice(&Sha256::digest(b"")); - h - } else { - hex::decode(initrd_hash_hex) - .context("initrd_hash must be valid hex")? - .try_into() - .map_err(|_| anyhow::anyhow!("initrd_hash must be 32 bytes"))? - }; - - let mut cmdline_bytes = append.as_bytes().to_vec(); - cmdline_bytes.push(0); - let mut cmdline_hash = [0u8; 32]; - cmdline_hash.copy_from_slice(&Sha256::digest(&cmdline_bytes)); - - let cmdline_entry = sev_entry(&GUID_LE_CMDLINE_ENTRY, &cmdline_hash); - let initrd_entry = sev_entry(&GUID_LE_INITRD_ENTRY, &initrd_hash); - let kernel_entry = sev_entry(&GUID_LE_KERNEL_ENTRY, &kernel_hash); - - const TABLE_SIZE: usize = 16 + 2 + 50 + 50 + 50; - let mut table = [0u8; TABLE_SIZE]; - table[..16].copy_from_slice(&GUID_LE_HASH_TABLE_HEADER); - table[16..18].copy_from_slice(&(TABLE_SIZE as u16).to_le_bytes()); - table[18..68].copy_from_slice(&cmdline_entry); - table[68..118].copy_from_slice(&initrd_entry); - table[118..168].copy_from_slice(&kernel_entry); - - const PADDED: usize = (TABLE_SIZE + 15) & !(15usize); - if page_offset + PADDED > 4096 { - bail!("sev hash table overflows 4096-byte page"); - } - let mut page = [0u8; 4096]; - page[page_offset..page_offset + TABLE_SIZE].copy_from_slice(&table); - Ok(page) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum SectionType { - SnpSecMemory = 1, - SnpSecrets = 2, - Cpuid = 3, - SvsmCaa = 4, - SnpKernelHashes = 0x10, -} - -impl SectionType { - fn from_u32(value: u32) -> Option { - match value { - 1 => Some(Self::SnpSecMemory), - 2 => Some(Self::SnpSecrets), - 3 => Some(Self::Cpuid), - 4 => Some(Self::SvsmCaa), - 0x10 => Some(Self::SnpKernelHashes), - _ => None, - } - } -} - -struct MetadataSection { - gpa: u64, - size: u64, - section_type: SectionType, -} - -struct OvmfInfo { - data: Vec, - gpa: u64, - sections: Vec, - sev_hashes_table_gpa: u64, - sev_es_reset_eip: u32, -} - -const GUID_FOOTER_TABLE: [u8; 16] = [ - 0xde, 0x82, 0xb5, 0x96, 0xb2, 0x1f, 0xf7, 0x45, 0xba, 0xea, 0xa3, 0x66, 0xc5, 0x5a, 0x08, 0x2d, -]; -const GUID_SEV_HASH_TABLE_RV: [u8; 16] = [ - 0x1f, 0x37, 0x55, 0x72, 0x3b, 0x3a, 0x04, 0x4b, 0x92, 0x7b, 0x1d, 0xa6, 0xef, 0xa8, 0xd4, 0x54, -]; -const GUID_SEV_ES_RESET_BLK: [u8; 16] = [ - 0xde, 0x71, 0xf7, 0x00, 0x7e, 0x1a, 0xcb, 0x4f, 0x89, 0x0e, 0x68, 0xc7, 0x7e, 0x2f, 0xb4, 0x4e, -]; -const GUID_SEV_META_DATA: [u8; 16] = [ - 0x66, 0x65, 0x88, 0xdc, 0x4a, 0x98, 0x98, 0x47, 0xa7, 0x5e, 0x55, 0x85, 0xa7, 0xbf, 0x67, 0xcc, -]; - -fn read_u16_le(buf: &[u8], off: usize) -> u16 { - u16::from_le_bytes([buf[off], buf[off + 1]]) -} - -fn read_u32_le(buf: &[u8], off: usize) -> u32 { - u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]]) -} - -impl OvmfInfo { - fn load(path: &str) -> Result { - let data = fs::read(path).with_context(|| format!("cannot read ovmf binary '{path}'"))?; - let size = data.len(); - let gpa = (0x1_0000_0000u64) - .checked_sub(size as u64) - .context("ovmf binary is larger than 4 gib")?; - - const ENTRY_HDR: usize = 18; - let footer_off = size.saturating_sub(32 + ENTRY_HDR); - if footer_off + ENTRY_HDR > size { - bail!("ovmf binary too small to contain footer table"); - } - if data[footer_off + 2..footer_off + 18] != GUID_FOOTER_TABLE { - bail!("ovmf footer guid not found"); - } - let footer_total_size = read_u16_le(&data, footer_off) as usize; - if footer_total_size < ENTRY_HDR { - bail!("ovmf footer table has invalid total size"); - } - let table_size = footer_total_size - ENTRY_HDR; - if table_size > footer_off { - bail!("ovmf footer table is out of bounds"); - } - let table_start = footer_off - table_size; - let table_bytes = &data[table_start..footer_off]; - - let mut sev_hashes_table_gpa = 0u64; - let mut sev_es_reset_eip = 0u32; - let mut meta_offset_from_end = None; - - let mut pos = table_bytes.len(); - while pos >= ENTRY_HDR { - let entry_off = pos - ENTRY_HDR; - let entry_size = read_u16_le(table_bytes, entry_off) as usize; - if entry_size < ENTRY_HDR || entry_size > pos { - bail!("ovmf footer table has invalid entry size"); - } - let guid = &table_bytes[entry_off + 2..entry_off + 18]; - let data_start = pos - entry_size; - let data_end = pos - ENTRY_HDR; - let entry_data = &table_bytes[data_start..data_end]; - - if guid == GUID_SEV_HASH_TABLE_RV && entry_data.len() >= 4 { - sev_hashes_table_gpa = read_u32_le(entry_data, 0) as u64; - } else if guid == GUID_SEV_ES_RESET_BLK && entry_data.len() >= 4 { - sev_es_reset_eip = read_u32_le(entry_data, 0); - } else if guid == GUID_SEV_META_DATA && entry_data.len() >= 4 { - meta_offset_from_end = Some(read_u32_le(entry_data, 0) as usize); - } - pos -= entry_size; - } - - if sev_hashes_table_gpa == 0 { - bail!("ovmf sev hash table entry not found in footer table"); - } - if sev_es_reset_eip == 0 { - bail!("ovmf sev_es_reset_block entry not found in footer table"); - } - - let mut sections = Vec::new(); - let off_from_end = meta_offset_from_end - .ok_or_else(|| anyhow::anyhow!("ovmf sev metadata entry not found in footer table"))?; - if off_from_end > size { - bail!("ovmf sev metadata offset exceeds file size"); - } - let meta_start = size - off_from_end; - if meta_start + 16 > size { - bail!("ovmf sev metadata header out of bounds"); - } - if &data[meta_start..meta_start + 4] != b"ASEV" { - bail!("ovmf sev metadata has bad signature"); - } - let meta_version = read_u32_le(&data, meta_start + 8); - if meta_version != 1 { - bail!("ovmf sev metadata has unsupported version {meta_version}"); - } - let num_items = read_u32_le(&data, meta_start + 12) as usize; - let items_start = meta_start + 16; - if items_start + num_items * 12 > size { - bail!("ovmf sev metadata sections out of bounds"); - } - for i in 0..num_items { - let off = items_start + i * 12; - let section_type_value = read_u32_le(&data, off + 8); - let section_type = SectionType::from_u32(section_type_value).ok_or_else(|| { - anyhow::anyhow!("unknown ovmf section_type {section_type_value:#x}") - })?; - sections.push(MetadataSection { - gpa: read_u32_le(&data, off) as u64, - size: read_u32_le(&data, off + 4) as u64, - section_type, - }); - } - - Ok(Self { - data, - gpa, - sections, - sev_hashes_table_gpa, - sev_es_reset_eip, - }) - } -} - -fn write_u16_le_at(buf: &mut [u8], off: usize, value: u16) { - buf[off..off + 2].copy_from_slice(&value.to_le_bytes()); -} - -fn write_u32_le_at(buf: &mut [u8], off: usize, value: u32) { - buf[off..off + 4].copy_from_slice(&value.to_le_bytes()); -} - -fn write_u64_le_at(buf: &mut [u8], off: usize, value: u64) { - buf[off..off + 8].copy_from_slice(&value.to_le_bytes()); -} - -fn write_vmcb_seg(buf: &mut [u8], off: usize, selector: u16, attrib: u16, limit: u32, base: u64) { - write_u16_le_at(buf, off, selector); - write_u16_le_at(buf, off + 2, attrib); - write_u32_le_at(buf, off + 4, limit); - write_u64_le_at(buf, off + 8, base); -} - -fn amd_cpu_sig(family: u32, model: u32, stepping: u32) -> u32 { - let (family_low, family_high) = if family > 0xf { - (0xf, (family - 0xf) & 0xff) - } else { - (family, 0) - }; - let model_low = model & 0xf; - let model_high = (model >> 4) & 0xf; - (family_high << 20) - | (model_high << 16) - | (family_low << 8) - | (model_low << 4) - | (stepping & 0xf) -} - -fn vcpu_sig_from_type(vcpu_type: &str) -> Result { - match vcpu_type.trim().to_lowercase().as_str() { - "epyc" | "epyc-v1" | "epyc-v2" | "epyc-ibpb" | "epyc-v3" | "epyc-v4" => { - Ok(amd_cpu_sig(23, 1, 2)) - } - "epyc-rome" | "epyc-rome-v1" | "epyc-rome-v2" | "epyc-rome-v3" => { - Ok(amd_cpu_sig(23, 49, 0)) - } - "epyc-milan" | "epyc-milan-v1" | "epyc-milan-v2" => Ok(amd_cpu_sig(25, 1, 1)), - "epyc-genoa" | "epyc-genoa-v1" => Ok(amd_cpu_sig(25, 17, 0)), - other => bail!("unknown vcpu_type {other:?}"), - } -} - -fn build_vmsa_page(eip: u32, vcpu_sig: u32, sev_features: u64) -> Box<[u8; 4096]> { - let mut page = Box::new([0u8; 4096]); - let p = page.as_mut_slice(); - - let cs_base = (eip as u64) & 0xffff_0000; - let rip = (eip as u64) & 0x0000_ffff; - - write_vmcb_seg(p, 0x000, 0, 0x0093, 0xffff, 0); - write_vmcb_seg(p, 0x010, 0xf000, 0x009b, 0xffff, cs_base); - write_vmcb_seg(p, 0x020, 0, 0x0093, 0xffff, 0); - write_vmcb_seg(p, 0x030, 0, 0x0093, 0xffff, 0); - write_vmcb_seg(p, 0x040, 0, 0x0093, 0xffff, 0); - write_vmcb_seg(p, 0x050, 0, 0x0093, 0xffff, 0); - write_vmcb_seg(p, 0x060, 0, 0x0000, 0xffff, 0); - write_vmcb_seg(p, 0x070, 0, 0x0082, 0xffff, 0); - write_vmcb_seg(p, 0x080, 0, 0x0000, 0xffff, 0); - write_vmcb_seg(p, 0x090, 0, 0x008b, 0xffff, 0); - - write_u64_le_at(p, 0x0D0, 0x1000); - write_u64_le_at(p, 0x148, 0x40); - write_u64_le_at(p, 0x158, 0x10); - write_u64_le_at(p, 0x160, 0x400); - write_u64_le_at(p, 0x168, 0xffff_0ff0); - write_u64_le_at(p, 0x170, 0x2); - write_u64_le_at(p, 0x178, rip); - write_u64_le_at(p, 0x268, 0x0007_0406_0007_0406); - write_u64_le_at(p, 0x310, vcpu_sig as u64); - write_u64_le_at(p, 0x3B0, sev_features); - write_u64_le_at(p, 0x3E8, 0x1); - write_u32_le_at(p, 0x408, 0x1f80); - write_u16_le_at(p, 0x410, 0x037f); - - page -} - -pub(crate) fn compute_expected_measurement(input: &MeasurementInput) -> Result<[u8; 48]> { - let vcpu_type = input - .vcpu_type - .as_deref() - .ok_or_else(|| anyhow::anyhow!("vcpu_type is required"))?; - - let cmdline = match input.base_cmdline.as_deref() { - Some(base) if !base.trim().is_empty() => base.trim().to_string(), - _ => "console=ttyS0 loglevel=7".to_string(), - }; - let resolved_sections = input - .ovmf_sections - .iter() - .map(|section| { - let section_type = SectionType::from_u32(section.section_type).ok_or_else(|| { - anyhow::anyhow!("unknown ovmf section_type {:#x}", section.section_type) - })?; - Ok(MetadataSection { - gpa: section.gpa, - size: section.size, - section_type, - }) - }) - .collect::>>()?; - let mut gctx = Gctx::from_ovmf_hash(&input.ovmf_hash)?; - let effective_hashes_gpa = input.sev_hashes_table_gpa; - let effective_reset_eip = input.sev_es_reset_eip; - - let mut has_kernel_hashes_section = false; - for section in &resolved_sections { - let gpa = section.gpa; - let size = usize::try_from(section.size) - .map_err(|_| anyhow::anyhow!("ovmf section size is too large"))?; - match section.section_type { - SectionType::SnpSecMemory => gctx.update_zero_pages(gpa, size), - SectionType::SnpSecrets => gctx.update_secrets_page(gpa), - SectionType::Cpuid => gctx.update_cpuid_page(gpa), - SectionType::SvsmCaa => gctx.update_zero_pages(gpa, size), - SectionType::SnpKernelHashes => { - has_kernel_hashes_section = true; - if effective_hashes_gpa == 0 { - bail!("snp_kernel_hashes section present but sev_hashes_table_gpa is 0"); - } - let page_offset = (effective_hashes_gpa & 0xfff) as usize; - let page = build_sev_hashes_page( - &input.kernel_hash, - &input.initrd_hash, - &cmdline, - page_offset, - )?; - gctx.update_normal_pages(gpa, &page); - } - } - } - if !has_kernel_hashes_section { - bail!("ovmf metadata does not include a snp_kernel_hashes section"); - } - - let vcpu_sig = vcpu_sig_from_type(vcpu_type)?; - let bsp_vmsa = build_vmsa_page(0xffff_fff0, vcpu_sig, input.guest_features); - let ap_vmsa = build_vmsa_page(effective_reset_eip, vcpu_sig, input.guest_features); - - for i in 0..input.vcpus as usize { - let vmsa_page = if i == 0 { - bsp_vmsa.as_ref() - } else { - ap_vmsa.as_ref() - }; - gctx.update_vmsa_page(vmsa_page); - } - - Ok(gctx.ld) -} - #[cfg(test)] mod tests { use super::*; @@ -1137,107 +428,6 @@ mod tests { ); } - #[test] - fn snp_os_image_hash_covers_image_fields_only() { - let input = valid_input(); - let os_image_hash = - |i: &MeasurementInput| snp_measurement_os_image_hash(&measurement_document(i)).unwrap(); - let baseline = os_image_hash(&input); - - // Image-determined fields MUST change the os_image_hash. - let image_cases: Vec<(&str, fn(&mut MeasurementInput))> = vec![ - ("rootfs_hash", |i| i.rootfs_hash = hex_of(0x34, 32)), - ("base_cmdline", |i| { - i.base_cmdline = Some("console=ttyS0 loglevel=8".to_string()) - }), - ("ovmf_hash", |i| i.ovmf_hash = hex_of(0x45, 48)), - ("kernel_hash", |i| i.kernel_hash = hex_of(0x56, 32)), - ("initrd_hash", |i| i.initrd_hash = hex_of(0x67, 32)), - ("sev_hashes_table_gpa", |i| i.sev_hashes_table_gpa += 0x1000), - ("sev_es_reset_eip", |i| i.sev_es_reset_eip = 0xffff_0000), - ("ovmf_sections.gpa", |i| i.ovmf_sections[0].gpa += 0x1000), - ("ovmf_sections.size", |i| i.ovmf_sections[0].size += 0x1000), - ("ovmf_sections.section_type", |i| { - i.ovmf_sections[0].section_type = 4 - }), - ]; - for (name, mutate) in image_cases { - let mut changed = input.clone(); - mutate(&mut changed); - assert_ne!( - baseline, - os_image_hash(&changed), - "{name} must change the SNP os_image_hash" - ); - } - - // Per-deployment fields MUST NOT change the os_image_hash (the same OS - // image must hash identically regardless of vCPU count, app, etc.). - let deployment_cases: Vec<(&str, fn(&mut MeasurementInput))> = vec![ - ("app_id", |i| i.app_id = hex_of(0x12, 20)), - ("compose_hash", |i| i.compose_hash = hex_of(0x23, 32)), - ("vcpus", |i| i.vcpus = 3), - ("vcpu_type", |i| { - i.vcpu_type = Some("epyc-milan".to_string()) - }), - ("guest_features", |i| i.guest_features = 3), - ]; - for (name, mutate) in deployment_cases { - let mut changed = input.clone(); - mutate(&mut changed); - assert_eq!( - baseline, - os_image_hash(&changed), - "{name} must NOT change the SNP os_image_hash" - ); - } - } - - #[test] - fn gctx_update_is_deterministic_and_order_sensitive() { - let contents = Gctx::sha384(b"page"); - let mut first = Gctx::new(); - first.update(0x01, 0x1000, &contents); - assert_eq!( - hex::encode(first.ld), - "3ebc1a70acc0bae5ae2788fae29a0371f983b19a68faf9843064f36040f58571ce5bb6bcdc9c361087073f8cffd92635" - ); - - let mut second = Gctx::new(); - second.update(0x01, 0x2000, &contents); - assert_ne!(first.ld, second.ld); - } - - #[test] - fn builds_sev_hashes_page_at_requested_offset() { - let page = build_sev_hashes_page(&hex_of(0x55, 32), "", "console=ttyS0", 0x80) - .expect("sev hashes page should build"); - assert_eq!(&page[..0x80], &[0u8; 0x80]); - assert_eq!(&page[0x80..0x90], &GUID_LE_HASH_TABLE_HEADER); - assert_eq!(u16::from_le_bytes([page[0x90], page[0x91]]), 168); - assert_eq!( - &page[0x92..0xa2], - &GUID_LE_CMDLINE_ENTRY, - "cmdline entry must be first" - ); - let empty_hash = Sha256::digest(b""); - assert_eq!(&page[0x80 + 68 + 18..0x80 + 68 + 50], empty_hash.as_slice()); - } - - #[test] - fn vcpu_type_mapping_is_strict() { - assert_eq!( - vcpu_sig_from_type("EPYC-v4").unwrap(), - amd_cpu_sig(23, 1, 2) - ); - assert_eq!( - vcpu_sig_from_type("epyc-genoa-v1").unwrap(), - amd_cpu_sig(25, 17, 0) - ); - let err = vcpu_sig_from_type("not-a-cpu").expect_err("unknown vcpu should reject"); - assert!(err.to_string().contains("unknown vcpu_type")); - } - #[test] fn accepts_recomputed_matching_measurement_and_rejects_mismatch() { let input = valid_input(); @@ -1605,87 +795,6 @@ mod tests { assert_eq!(boot_info.mr_system, changed_boot_info.mr_system); } - #[test] - #[ignore = "requires sev-snp-measure and an SNP-capable OVMF binary"] - fn recomputation_matches_sev_snp_measure_live_golden_vector() { - let ovmf_path = std::env::var("DSTACK_SEV_SNP_GOLDEN_OVMF") - .unwrap_or_else(|_| "/opt/AMDSEV/usr/local/share/qemu/OVMF.fd".to_string()); - assert!( - std::path::Path::new(&ovmf_path).exists(), - "set DSTACK_SEV_SNP_GOLDEN_OVMF to an SNP-capable OVMF binary" - ); - - let dir = tempfile::tempdir().expect("tempdir should be available"); - let kernel_path = dir.path().join("kernel.bin"); - let initrd_path = dir.path().join("initrd.bin"); - let kernel_bytes = b"golden-kernel-for-dstack-sev-snp-measure\n"; - let initrd_bytes = b"golden-initrd-for-dstack-sev-snp-measure\n"; - std::fs::write(&kernel_path, kernel_bytes).expect("kernel fixture should be written"); - std::fs::write(&initrd_path, initrd_bytes).expect("initrd fixture should be written"); - - let kernel_hash = hex::encode(Sha256::digest(kernel_bytes)); - let initrd_hash = hex::encode(Sha256::digest(initrd_bytes)); - let mut input = valid_input(); - let ovmf = OvmfInfo::load(&ovmf_path).expect("ovmf metadata should load"); - let mut gctx = Gctx::new(); - gctx.update_normal_pages(ovmf.gpa, &ovmf.data); - input.ovmf_hash = hex::encode(gctx.ld); - input.sev_hashes_table_gpa = ovmf.sev_hashes_table_gpa; - input.sev_es_reset_eip = ovmf.sev_es_reset_eip; - input.ovmf_sections = ovmf - .sections - .iter() - .map(|section| OvmfSectionParam { - gpa: section.gpa, - size: section.size, - section_type: section.section_type as u32, - }) - .collect(); - input.kernel_hash = kernel_hash; - input.initrd_hash = initrd_hash; - input.vcpus = 2; - input.vcpu_type = Some("EPYC-v4".to_string()); - - let recomputed = - compute_expected_measurement(&input).expect("dstack recomputation should succeed"); - - let append = "console=ttyS0 loglevel=7"; - let output = std::process::Command::new("sev-snp-measure") - .args([ - "--mode", - "snp", - "--vcpus", - "2", - "--vcpu-type", - "EPYC-v4", - "--ovmf", - &ovmf_path, - "--kernel", - kernel_path.to_str().unwrap(), - "--initrd", - initrd_path.to_str().unwrap(), - "--append", - append, - "--guest-features", - "0x1", - "--output-format", - "hex", - ]) - .output() - .expect("sev-snp-measure should be installed"); - assert!( - output.status.success(), - "sev-snp-measure failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - let tool_measurement = String::from_utf8(output.stdout) - .expect("sev-snp-measure output should be utf8") - .trim() - .to_string(); - - assert_eq!(hex::encode(recomputed), tool_measurement); - } - #[test] fn explicit_snp_auth_policy_accepts_only_exact_verified_identity() { let input = valid_input(); From 678bb8a5d5c5643197dde318bdf4c6cd8533250e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 17 Jun 2026 19:01:42 -0700 Subject: [PATCH 60/67] verifier: verify AMD SEV-SNP os_image_hash verify_os_image_hash previously bailed "Unsupported attestation quote" for DstackAmdSevSnp, so SEV-SNP attestations always returned is_valid=false. Add verify_os_image_hash_for_dstack_sev: recompute the launch measurement from the self-contained sev_snp_measurement inputs carried in the attestation config, require it to equal the hardware-signed MEASUREMENT, require HOST_DATA to bind the MrConfigV3 document, then derive and surface the image-invariant os_image_hash. Also fills tcb_status/advisory_ids for SEV. Same dstack-mr::sev code path the KMS uses for key release, so a quote the KMS would release keys for now verifies here too (is_valid=true). --- verifier/src/verification.rs | 55 ++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/verifier/src/verification.rs b/verifier/src/verification.rs index bb5638c2a..49326d30c 100644 --- a/verifier/src/verification.rs +++ b/verifier/src/verification.rs @@ -509,7 +509,15 @@ impl CvmVerifier { debug: bool, details: &mut VerificationDetails, ) -> Result { - let vm_config = attestation + // The raw config string used for platform-specific binding: the explicit + // request `vm_config` when supplied, otherwise the one embedded in the + // attestation (mirroring `decode_vm_config`'s own fallback). + let raw_config = if vm_config.is_empty() { + attestation.config.clone() + } else { + vm_config.clone() + }; + let mut vm_config = attestation .decode_vm_config(&vm_config) .context("Failed to decode VM config")?; match &attestation.quote { @@ -528,15 +536,52 @@ impl CvmVerifier { self.verify_os_image_hash_for_nitro_enclave(&vm_config, &report.pcrs)?; } AttestationQuote::DstackAmdSevSnp(_) => { - bail!( - "Unsupported attestation quote: {:?}", - attestation.quote.mode() - ); + self.verify_os_image_hash_for_dstack_sev( + attestation, + &raw_config, + &mut vm_config, + details, + )?; } } Ok(vm_config) } + /// Verify the AMD SEV-SNP OS image binding. + /// + /// Unlike TDX (which replays RTMRs against a downloaded image), the SNP boot + /// is summarised by the launch `MEASUREMENT`. The CVM advertises the + /// self-contained launch inputs (`sev_snp_measurement`) and the MrConfigV3 + /// document in its `vm_config`; we recompute the launch measurement from + /// those inputs and require it to equal the hardware-signed `MEASUREMENT` + /// (which is what makes the otherwise-untrusted inputs trustworthy), require + /// `HOST_DATA` to bind the MrConfigV3 document, and then derive the + /// image-invariant `os_image_hash`. The shared recomputation in + /// `dstack_mr::sev` is the same code path the KMS uses for key release, so a + /// quote that the KMS would release keys for verifies here too. + fn verify_os_image_hash_for_dstack_sev( + &self, + attestation: &VerifiedAttestation, + raw_config: &str, + vm_config: &mut VmConfig, + details: &mut VerificationDetails, + ) -> Result<()> { + let report = attestation + .report + .amd_snp_report() + .context("internal error: sev-snp quote without a verified sev-snp report")?; + let binding = + dstack_mr::sev::verify_sev_launch(&report.measurement, &report.host_data, raw_config) + .context("amd sev-snp launch verification failed")?; + // The os_image_hash derived from the measurement-bound launch inputs is + // the authoritative one; surface it (overriding any guest-advertised + // value, which is not independently trusted). + vm_config.os_image_hash = binding.os_image_hash; + details.tcb_status = Some(report.tcb_info.tcb_status().to_string()); + details.advisory_ids = report.advisory_ids.clone(); + Ok(()) + } + async fn verify_os_image_hash_for_dstack_tdx( &self, vm_config: &VmConfig, From 59b722096337082e308dc136cc8a5a5272f20ee8 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 17 Jun 2026 19:01:54 -0700 Subject: [PATCH 61/67] guest, dstack-util: platform-adaptive quote (incl. AMD SEV-SNP) dstack-util quote was TDX-only (read the Intel configfs directly and failed on SEV hosts); make it detect the running TEE via Attestation::quote and emit the platform's raw hardware quote (TDX DCAP quote or SNP report). GetQuoteResponse gains an 'attestation' field carrying the platform- adaptive versioned attestation, populated on every platform. On non-TDX (SEV-SNP) the legacy quote/event_log fields are empty, so this is the verifier-ready payload to send to dstack-verifier's /verify 'attestation' field. Populated in the real, simulator and test backends; exposed in the Rust SDK GetQuoteResponse with a decode_attestation helper. --- dstack-attest/src/attestation.rs | 4 ++-- dstack-util/src/main.rs | 15 +++++++++++++-- guest-agent-simulator/src/simulator.rs | 5 +++++ guest-agent/rpc/proto/agent_rpc.proto | 9 +++++++-- guest-agent/src/backend.rs | 7 +++++++ guest-agent/src/rpc_service.rs | 5 +++++ sdk/rust/types/src/dstack.rs | 10 ++++++++++ 7 files changed, 49 insertions(+), 6 deletions(-) diff --git a/dstack-attest/src/attestation.rs b/dstack-attest/src/attestation.rs index 06d475646..64b158f5a 100644 --- a/dstack-attest/src/attestation.rs +++ b/dstack-attest/src/attestation.rs @@ -1685,8 +1685,8 @@ impl Attestation { } }; if let AttestationQuote::DstackAmdSevSnp(quote) = &mut quote { - quote.mr_config = read_mr_config_document()? - .context("amd sev-snp mr_config is missing")?; + quote.mr_config = + read_mr_config_document()?.context("amd sev-snp mr_config is missing")?; } Ok(Self { diff --git a/dstack-util/src/main.rs b/dstack-util/src/main.rs index d2e792b4f..c9ec7ef70 100644 --- a/dstack-util/src/main.rs +++ b/dstack-util/src/main.rs @@ -12,7 +12,7 @@ use host_api::HostApi; use k256::schnorr::SigningKey; use ra_rpc::Attestation; use ra_tls::{ - attestation::{QuoteContentType, VersionedAttestation}, + attestation::{AttestationQuote, QuoteContentType, VersionedAttestation}, cert::{generate_ra_cert, generate_ra_cert_with_app_id}, kdf::{derive_key, derive_p256_key_pair_from_bytes}, rcgen::KeyPair, @@ -654,7 +654,18 @@ fn cmd_quote() -> Result<()> { io::stdin() .read_exact(&mut report_data) .context("Failed to read report data")?; - let quote = att::get_quote(&report_data).context("Failed to get quote")?; + // Platform-adaptive: detect the running TEE and emit its raw hardware quote + // (the TDX DCAP quote, or the AMD SEV-SNP report). For a verifier-ready, + // platform-agnostic payload (with event log / mr_config), use `quote-report`. + let attestation = Attestation::quote(&report_data).context("Failed to get quote")?; + let quote = match &attestation.quote { + AttestationQuote::DstackTdx(tdx) => tdx.quote.clone(), + AttestationQuote::DstackGcpTdx(gcp) => gcp.tdx_quote.quote.clone(), + AttestationQuote::DstackAmdSevSnp(snp) => snp.report.clone(), + AttestationQuote::DstackNitroEnclave(_) => { + anyhow::bail!("nitro enclave has no raw quote; use `quote-report` instead"); + } + }; io::stdout() .write_all("e) .context("Failed to write quote")?; diff --git a/guest-agent-simulator/src/simulator.rs b/guest-agent-simulator/src/simulator.rs index 902dfe87e..ed148429b 100644 --- a/guest-agent-simulator/src/simulator.rs +++ b/guest-agent-simulator/src/simulator.rs @@ -34,12 +34,17 @@ pub fn simulated_quote_response( let Some(quote) = attestation.tdx_quote_bytes() else { return Err(anyhow!("Quote not found")); }; + let versioned = VersionedAttestation::V1 { + attestation: attestation.clone(), + } + .to_bytes()?; Ok(GetQuoteResponse { quote, event_log: attestation.tdx_event_log_string().unwrap_or_default(), report_data: report_data.to_vec(), vm_config: vm_config.to_string(), + attestation: versioned, }) } diff --git a/guest-agent/rpc/proto/agent_rpc.proto b/guest-agent/rpc/proto/agent_rpc.proto index 241b543bf..3226d2ef3 100644 --- a/guest-agent/rpc/proto/agent_rpc.proto +++ b/guest-agent/rpc/proto/agent_rpc.proto @@ -191,14 +191,19 @@ message AttestResponse { } message GetQuoteResponse { - // TDX quote + // TDX quote (empty on non-TDX platforms such as AMD SEV-SNP) bytes quote = 1; - // Event log + // Event log (empty on non-TDX platforms) string event_log = 2; // Report data bytes report_data = 3; // Hw config string vm_config = 4; + // Platform-adaptive versioned attestation (SCALE/msgpack encoded). Populated + // for every TEE platform (TDX, AMD SEV-SNP, ...) and is the verifier-ready + // payload to send to dstack-verifier's `/verify` `attestation` field. Use + // this instead of `quote`/`event_log` for platform-agnostic verification. + bytes attestation = 5; } message EmitEventArgs { diff --git a/guest-agent/src/backend.rs b/guest-agent/src/backend.rs index 3344ff909..a8e06cd8e 100644 --- a/guest-agent/src/backend.rs +++ b/guest-agent/src/backend.rs @@ -37,11 +37,18 @@ impl PlatformBackend for RealPlatform { let attestation = Attestation::quote(&report_data).context("Failed to get quote")?; let tdx_quote = attestation.get_tdx_quote_bytes(); let tdx_event_log = attestation.get_tdx_event_log_string(); + // Always carry the platform-adaptive versioned attestation so callers on + // non-TDX platforms (AMD SEV-SNP) still get a verifier-ready payload. + let versioned = attestation + .into_versioned() + .to_bytes() + .context("Failed to encode versioned attestation")?; Ok(GetQuoteResponse { quote: tdx_quote.unwrap_or_default(), event_log: tdx_event_log.unwrap_or_default(), report_data: report_data.to_vec(), vm_config: vm_config.to_string(), + attestation: versioned, }) } diff --git a/guest-agent/src/rpc_service.rs b/guest-agent/src/rpc_service.rs index f9e01ca9e..984da23b7 100644 --- a/guest-agent/src/rpc_service.rs +++ b/guest-agent/src/rpc_service.rs @@ -839,6 +839,10 @@ pNs85uhOZE8z2jr8Pg== let Some(quote) = attestation.platform.tdx_quote().map(ToOwned::to_owned) else { return Err(anyhow::anyhow!("Quote not found")); }; + let versioned = VersionedAttestation::V1 { + attestation: attestation.clone(), + } + .to_bytes()?; Ok(GetQuoteResponse { quote, event_log: serde_json::to_string( @@ -847,6 +851,7 @@ pNs85uhOZE8z2jr8Pg== .unwrap_or_default(), report_data: report_data.to_vec(), vm_config: vm_config.to_string(), + attestation: versioned, }) } diff --git a/sdk/rust/types/src/dstack.rs b/sdk/rust/types/src/dstack.rs index 3ae8603b9..aa6263fe0 100644 --- a/sdk/rust/types/src/dstack.rs +++ b/sdk/rust/types/src/dstack.rs @@ -111,6 +111,11 @@ pub struct GetQuoteResponse { /// VM configuration #[serde(default)] pub vm_config: String, + /// Platform-adaptive versioned attestation in hexadecimal format. Populated + /// for every TEE platform (TDX, AMD SEV-SNP, ...); this is the payload to + /// send to dstack-verifier for platform-agnostic verification. + #[serde(default)] + pub attestation: String, } /// Response containing a versioned attestation @@ -133,6 +138,11 @@ impl GetQuoteResponse { hex::decode(&self.quote) } + /// Decode the platform-adaptive versioned attestation bytes, if present. + pub fn decode_attestation(&self) -> Result, FromHexError> { + hex::decode(&self.attestation) + } + pub fn decode_event_log(&self) -> Result, serde_json::Error> { serde_json::from_str(&self.event_log) } From b2dc95ad36c3e3a71da7cf3fcef39d94f9a2a2d0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 17 Jun 2026 19:01:54 -0700 Subject: [PATCH 62/67] dstack-attest: offline end-to-end SEV-SNP os_image_hash test Extend the offline SEV-SNP fixture test to also run the verifier's full binding path with no network: after the hardware report verifies, recompute the launch measurement from the attestation's embedded sev_snp_measurement, confirm HOST_DATA binds the mr_config, and assert the derived os_image_hash (32b47673...) and HOST_DATA-bound app_id. Adds dstack-mr as a dev-dep. --- Cargo.lock | 1 + dstack-attest/Cargo.toml | 1 + dstack-attest/tests/sev_snp_verify.rs | 27 +++++++++++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index ccc9c1fc8..c8a55c7bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2326,6 +2326,7 @@ dependencies = [ "anyhow", "cc-eventlog", "dcap-qvl", + "dstack-mr", "dstack-types", "errify", "ez-hash", diff --git a/dstack-attest/Cargo.toml b/dstack-attest/Cargo.toml index 8b78ad840..f538f344d 100644 --- a/dstack-attest/Cargo.toml +++ b/dstack-attest/Cargo.toml @@ -44,3 +44,4 @@ quote = [] [dev-dependencies] futures = { workspace = true } tokio = { workspace = true, features = ["full"] } +dstack-mr = { workspace = true } diff --git a/dstack-attest/tests/sev_snp_verify.rs b/dstack-attest/tests/sev_snp_verify.rs index 3352031e1..410f91afc 100644 --- a/dstack-attest/tests/sev_snp_verify.rs +++ b/dstack-attest/tests/sev_snp_verify.rs @@ -76,4 +76,31 @@ fn verify_sev_snp_attestation_bin() { println!("host_data: {}", hex::encode(verified.host_data)); println!("chip_id: {}", hex::encode(verified.chip_id)); println!("tcb_status: {}", verified.tcb_info.tcb_status()); + + // End-to-end OS image binding, fully offline — exactly what dstack-verifier + // does after the hardware report verifies. Recompute the launch measurement + // from the self-contained `sev_snp_measurement` document embedded in the + // attestation config, require it to equal the hardware MEASUREMENT, require + // HOST_DATA to bind the MrConfigV3 document, and derive the os_image_hash. + let binding = dstack_mr::sev::verify_sev_launch( + &verified.measurement, + &verified.host_data, + &attestation.config, + ) + .expect("recompute SEV launch + derive os_image_hash from the attestation config"); + + // The os_image_hash matches the value advertised in the CVM config and the + // image build's digest.sev.txt. + assert_eq!( + hex::encode(&binding.os_image_hash), + "32b4767373ad7fa0f9c418925006194d5c3f5619529f309fe81156789fecd8bc", + "derived os_image_hash" + ); + // The HOST_DATA-bound app identity is recovered from the mr_config document. + assert_eq!( + hex::encode(&binding.mr_config.app_id), + "86e59625be93207bc2351c4d1bba20037cec8e16", + "mr_config app_id bound by HOST_DATA" + ); + println!("os_image_hash: {}", hex::encode(&binding.os_image_hash)); } From d80daeb5245ccdd74b52ea91be3771cac53e18ae Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 17 Jun 2026 19:10:20 -0700 Subject: [PATCH 63/67] reuse: license SEV-SNP test fixtures (CC0-1.0) The binary/PEM SEV-SNP fixtures can't carry inline SPDX headers; annotate them in REUSE.toml as CC0-1.0 alongside the existing nitro fixtures so the REUSE compliance check passes. --- REUSE.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/REUSE.toml b/REUSE.toml index d5b23f3f4..20dc81df6 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -81,6 +81,9 @@ path = [ "tpm-qvl/certs/gcp-root-ca.pem", "dstack-attest/tests/nitro_attestation.bin", "dstack-attest/tests/nitro_attestation_dbg.bin", + "dstack-attest/tests/sev_snp_attestation.bin", + "dstack-attest/tests/sev_snp_ask.pem", + "dstack-attest/tests/sev_snp_vcek.pem", "nsm-attest/tests/nitro_attestation.bin", "nsm-qvl/tests/nitro_attestation.bin", "nsm-qvl/certs/AWS_NitroEnclaves_Root-G1.pem", From 719370a26d8c35e44ac9a923803b3c66493173fb Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 17 Jun 2026 19:34:25 -0700 Subject: [PATCH 64/67] test: AMD SEV-SNP forged-quote / tampered-input coverage Adversarial negative tests for the SEV-SNP verification path: dstack-mr::sev (synthetic, deterministic): - forged hardware MEASUREMENT and HOST_DATA are rejected - every measured launch field (ovmf/kernel/initrd hashes, cmdline, hash-table offset, reset eip, section gpa, vcpus, vcpu_type, guest_features) is caught by the measurement-equality check - substituting a different MrConfigV3 (app/compose/instance id) breaks the HOST_DATA binding - an advertised top-level os_image_hash is ignored (derived value wins) - booting a different image cannot present an allow-listed image's inputs - missing sev_snp_measurement / mr_config fail closed - documents that rootfs_hash is os_image_hash-only (bound via the measured cmdline), so tampering it changes the derived os_image_hash rather than failing the measurement check dstack-attest (real fixture, offline): - flipping any signed report field (report_data/measurement/host_data) or the signature invalidates VCEK verification; zeroed/truncated reports rejected - wrong collateral (ASK-as-VCEK, malformed VCEK) rejected - forged measurement/host_data, tampered launch inputs, substituted mr_config and bogus advertised os_image_hash all handled correctly against real data Derive Debug on SevImageBinding for test ergonomics. --- dstack-attest/tests/sev_snp_verify.rs | 226 +++++++++++++++++++++- dstack-mr/src/sev.rs | 257 ++++++++++++++++++++++++++ 2 files changed, 482 insertions(+), 1 deletion(-) diff --git a/dstack-attest/tests/sev_snp_verify.rs b/dstack-attest/tests/sev_snp_verify.rs index 410f91afc..933311264 100644 --- a/dstack-attest/tests/sev_snp_verify.rs +++ b/dstack-attest/tests/sev_snp_verify.rs @@ -10,7 +10,9 @@ //! one built into `sev-snp-qvl`. use dstack_attest::attestation::{AttestationQuote, VersionedAttestation}; -use sev_snp_qvl::{verify_amd_snp_attestation, AmdSnpAttestationInput}; +use dstack_mr::sev::verify_sev_launch; +use dstack_types::{mr_config::MrConfigV3, KeyProviderKind}; +use sev_snp_qvl::{verify_amd_snp_attestation, AmdSnpAttestationInput, VerifiedAmdSnpReport}; /// Real SEV-SNP attestation captured from a dstack CVM (VersionedAttestation, SCALE V0). const SEV_ATTESTATION_BIN: &[u8] = include_bytes!("sev_snp_attestation.bin"); @@ -104,3 +106,225 @@ fn verify_sev_snp_attestation_bin() { ); println!("os_image_hash: {}", hex::encode(&binding.os_image_hash)); } + +// --------------------------------------------------------------------------- +// Forged / tampered quote coverage (all offline, using the real fixture). +// --------------------------------------------------------------------------- + +const OS_IMAGE_HASH: &str = "32b4767373ad7fa0f9c418925006194d5c3f5619529f309fe81156789fecd8bc"; + +fn decoded_attestation() -> dstack_attest::attestation::Attestation { + let versioned = + VersionedAttestation::from_scale(SEV_ATTESTATION_BIN).expect("decode VersionedAttestation"); + let VersionedAttestation::V0 { attestation } = versioned else { + panic!("expected V0 attestation"); + }; + attestation +} + +fn fixture_report() -> Vec { + let attestation = decoded_attestation(); + let AttestationQuote::DstackAmdSevSnp(quote) = &attestation.quote else { + panic!("expected an AMD SEV-SNP quote"); + }; + quote.report.clone() +} + +fn fixture_config() -> String { + decoded_attestation().config +} + +fn verified_fixture_report() -> VerifiedAmdSnpReport { + let report = fixture_report(); + verify_amd_snp_attestation(&AmdSnpAttestationInput { + report: &report, + ask_pem: SEV_ASK_PEM, + vcek_pem: SEV_VCEK_PEM, + }) + .expect("verify SEV-SNP attestation offline") +} + +/// Rewrite one field inside the embedded `sev_snp_measurement` document. +fn with_measurement_field(config: &str, f: impl FnOnce(&mut serde_json::Value)) -> String { + let mut value: serde_json::Value = serde_json::from_str(config).expect("config json"); + let measurement_doc = value["sev_snp_measurement"] + .as_str() + .expect("sev_snp_measurement string") + .to_string(); + let mut measurement: serde_json::Value = + serde_json::from_str(&measurement_doc).expect("measurement json"); + f(&mut measurement); + value["sev_snp_measurement"] = + serde_json::Value::String(serde_json::to_string(&measurement).expect("reserialize")); + value.to_string() +} + +/// Replace the embedded MrConfigV3 document with a different one. +fn set_mr_config(config: &str, mr_config_doc: &str) -> String { + let mut value: serde_json::Value = serde_json::from_str(config).expect("config json"); + value["mr_config"] = serde_json::Value::String(mr_config_doc.to_string()); + value.to_string() +} + +#[test] +fn forged_report_bytes_fail_signature_verification() { + let report = fixture_report(); + // Flip a byte in each signed field (and the signature itself); the VCEK + // signature over the report must no longer verify. + // SNP ATTESTATION_REPORT offsets: report_data 0x50, measurement 0x90, + // host_data 0xC0, signature 0x2A0. + for (name, offset) in [ + ("report_data", 0x50usize), + ("measurement", 0x90), + ("host_data", 0xC0), + ("signature", 0x2A0), + ] { + let mut tampered = report.clone(); + tampered[offset] ^= 0xff; + let result = verify_amd_snp_attestation(&AmdSnpAttestationInput { + report: &tampered, + ask_pem: SEV_ASK_PEM, + vcek_pem: SEV_VCEK_PEM, + }); + assert!( + result.is_err(), + "tampering the {name} field must invalidate the report signature" + ); + } + + // A well-formed-length but zeroed report has no valid signature. + let zeroed = vec![0u8; 1184]; + assert!( + verify_amd_snp_attestation(&AmdSnpAttestationInput { + report: &zeroed, + ask_pem: SEV_ASK_PEM, + vcek_pem: SEV_VCEK_PEM, + }) + .is_err(), + "a zeroed report must not verify" + ); + + // A truncated report must be rejected, not parsed. + assert!( + verify_amd_snp_attestation(&AmdSnpAttestationInput { + report: &report[..200], + ask_pem: SEV_ASK_PEM, + vcek_pem: SEV_VCEK_PEM, + }) + .is_err(), + "a truncated report must be rejected" + ); +} + +#[test] +fn wrong_collateral_is_rejected() { + let report = fixture_report(); + // The ASK presented as the VCEK leaf: the report signature won't verify + // against the intermediate key. + assert!( + verify_amd_snp_attestation(&AmdSnpAttestationInput { + report: &report, + ask_pem: SEV_ASK_PEM, + vcek_pem: SEV_ASK_PEM, + }) + .is_err(), + "using the ASK as the VCEK must be rejected" + ); + + // Garbage VCEK PEM. + let junk = b"-----BEGIN CERTIFICATE-----\nbm90IGEgY2VydA==\n-----END CERTIFICATE-----\n"; + assert!( + verify_amd_snp_attestation(&AmdSnpAttestationInput { + report: &report, + ask_pem: SEV_ASK_PEM, + vcek_pem: junk, + }) + .is_err(), + "a malformed VCEK must be rejected" + ); +} + +#[test] +fn forged_launch_measurement_is_rejected() { + let verified = verified_fixture_report(); + let config = fixture_config(); + let mut forged = verified.measurement; + forged[0] ^= 0xff; + let err = verify_sev_launch(&forged, &verified.host_data, &config) + .expect_err("a measurement that disagrees with the launch inputs must reject"); + assert!( + err.to_string().contains("amd sev-snp measurement mismatch"), + "unexpected error: {err:?}" + ); +} + +#[test] +fn forged_host_data_is_rejected() { + let verified = verified_fixture_report(); + let config = fixture_config(); + let mut forged = verified.host_data; + forged[0] ^= 0xff; + let err = verify_sev_launch(&verified.measurement, &forged, &config) + .expect_err("host_data that does not bind the mr_config must reject"); + assert!( + err.to_string().contains("amd sev-snp host_data mismatch"), + "unexpected error: {err:?}" + ); +} + +#[test] +fn tampered_launch_inputs_break_os_image_binding() { + // Swap in a different kernel hash in the advertised launch inputs: the + // recomputed measurement no longer equals the hardware MEASUREMENT, so the + // forged (allow-listed-looking) os_image_hash is never trusted. + let verified = verified_fixture_report(); + let tampered = with_measurement_field(&fixture_config(), |m| { + m["kernel_hash"] = serde_json::Value::String("00".repeat(32)); + }); + let err = verify_sev_launch(&verified.measurement, &verified.host_data, &tampered) + .expect_err("tampered launch inputs must reject"); + assert!( + err.to_string().contains("amd sev-snp measurement mismatch"), + "unexpected error: {err:?}" + ); +} + +#[test] +fn substituted_mr_config_breaks_host_data_binding() { + // Present a well-formed but different-identity MrConfigV3 document. The + // hardware HOST_DATA still binds the original document, so this is rejected. + let verified = verified_fixture_report(); + let evil = MrConfigV3::new( + vec![0xab; 20], + vec![0xcd; 32], + KeyProviderKind::None, + Vec::new(), + vec![0xef; 20], + ); + let tampered = set_mr_config(&fixture_config(), &evil.to_canonical_json()); + let err = verify_sev_launch(&verified.measurement, &verified.host_data, &tampered) + .expect_err("a substituted mr_config must reject"); + assert!( + err.to_string().contains("amd sev-snp host_data mismatch"), + "unexpected error: {err:?}" + ); +} + +#[test] +fn advertised_os_image_hash_is_ignored() { + // A forged top-level os_image_hash is ignored; the authoritative value is + // derived from the measurement-bound launch inputs. + let verified = verified_fixture_report(); + let mut value: serde_json::Value = + serde_json::from_str(&fixture_config()).expect("config json"); + value["os_image_hash"] = serde_json::Value::String("de".repeat(32)); + let tampered = value.to_string(); + + let binding = verify_sev_launch(&verified.measurement, &verified.host_data, &tampered) + .expect("a bogus advertised os_image_hash is ignored, not fatal"); + assert_eq!( + hex::encode(&binding.os_image_hash), + OS_IMAGE_HASH, + "derived os_image_hash must win over the advertised one" + ); +} diff --git a/dstack-mr/src/sev.rs b/dstack-mr/src/sev.rs index 40048697c..cef25aa0e 100644 --- a/dstack-mr/src/sev.rs +++ b/dstack-mr/src/sev.rs @@ -787,6 +787,7 @@ pub fn parse_snp_inputs_from_vm_config(vm_config: &str) -> Result MrConfigV3 { + MrConfigV3::new( + vec![0x11; 20], + vec![0x22; 32], + dstack_types::KeyProviderKind::None, + Vec::new(), + vec![0x33; 20], + ) + } + + fn synthetic_vm_config(input: &MeasurementInput, mr_config: &MrConfigV3) -> String { + serde_json::json!({ + "sev_snp_measurement": serde_json::to_string(input).expect("serialize input"), + "mr_config": mr_config.to_canonical_json(), + }) + .to_string() + } + + /// Returns `(input, mr_config, verified_measurement, verified_host_data, vm_config)` + /// for an honest, internally-consistent SNP launch. + fn honest_case() -> (MeasurementInput, MrConfigV3, [u8; 48], [u8; 32], String) { + let input = valid_input(); + let mr_config = synthetic_mr_config(); + let host_data = MrConfigV3::snp_host_data_from_document(&mr_config.to_canonical_json()); + let measurement = compute_expected_measurement(&input).expect("measurement"); + let vm_config = synthetic_vm_config(&input, &mr_config); + (input, mr_config, measurement, host_data, vm_config) + } + + #[test] + fn verify_sev_launch_accepts_consistent_inputs() { + let (input, mr_config, measurement, host_data, vm_config) = honest_case(); + let binding = verify_sev_launch(&measurement, &host_data, &vm_config) + .expect("honest launch verifies"); + assert_eq!( + binding.os_image_hash, + snp_measurement_os_image_hash(&serde_json::to_string(&input).unwrap()).unwrap() + ); + assert_eq!(binding.mr_config.app_id, mr_config.app_id); + } + + #[test] + fn verify_sev_launch_rejects_forged_measurement() { + let (_input, _mr, measurement, host_data, vm_config) = honest_case(); + let mut forged = measurement; + forged[0] ^= 0xff; + let err = verify_sev_launch(&forged, &host_data, &vm_config) + .expect_err("forged hardware measurement must reject"); + assert!( + err.to_string().contains("amd sev-snp measurement mismatch"), + "unexpected error: {err:?}" + ); + } + + #[test] + fn verify_sev_launch_rejects_forged_host_data() { + let (_input, _mr, measurement, host_data, vm_config) = honest_case(); + let mut forged = host_data; + forged[0] ^= 0xff; + let err = verify_sev_launch(&measurement, &forged, &vm_config) + .expect_err("forged hardware host_data must reject"); + assert!( + err.to_string().contains("amd sev-snp host_data mismatch"), + "unexpected error: {err:?}" + ); + } + + #[test] + fn verify_sev_launch_rejects_tampered_measured_inputs() { + // Fields that feed the launch MEASUREMENT: tampering the advertised + // inputs while keeping the honest hardware MEASUREMENT is caught by the + // measurement-equality check, so the (would-be different) os_image_hash + // never gets a chance to be trusted. + let (input, mr_config, measurement, host_data, _vm_config) = honest_case(); + let cases: Vec<(&str, fn(&mut MeasurementInput))> = vec![ + ("base_cmdline", |i| { + i.base_cmdline = Some("console=ttyS0 evil=1".to_string()) + }), + ("ovmf_hash", |i| i.ovmf_hash = hex_of(0x99, 48)), + ("kernel_hash", |i| i.kernel_hash = hex_of(0x99, 32)), + ("initrd_hash", |i| i.initrd_hash = hex_of(0x99, 32)), + // Only the in-page offset (& 0xfff) of the hash table is measured, so + // tamper the low bits to actually move the measured table position. + ("sev_hashes_table_gpa", |i| i.sev_hashes_table_gpa += 0x40), + ("sev_es_reset_eip", |i| i.sev_es_reset_eip = 0xffff_0000), + ("ovmf_sections.gpa", |i| i.ovmf_sections[0].gpa += 0x1000), + ("vcpus", |i| i.vcpus = 4), + ("vcpu_type", |i| { + i.vcpu_type = Some("epyc-milan".to_string()) + }), + ("guest_features", |i| i.guest_features = 3), + ]; + for (name, mutate) in cases { + let mut tampered = input.clone(); + mutate(&mut tampered); + let vm_config = synthetic_vm_config(&tampered, &mr_config); + let err = match verify_sev_launch(&measurement, &host_data, &vm_config) { + Ok(binding) => panic!( + "{name} tampering was accepted; derived os_image_hash {}", + hex::encode(binding.os_image_hash) + ), + Err(e) => e.to_string(), + }; + assert!( + err.contains("amd sev-snp measurement mismatch"), + "{name}: unexpected error: {err}" + ); + } + } + + #[test] + fn tampering_rootfs_hash_changes_os_image_hash() { + // `rootfs_hash` is bound into the launch MEASUREMENT indirectly, through + // the measured kernel cmdline (`dstack.rootfs_hash=...`), but the + // standalone field also feeds the os_image_hash projection. Tampering the + // standalone field is therefore not a measurement mismatch — it yields a + // DIFFERENT os_image_hash that cannot match an allow-listed image, so the + // forgery is caught downstream by the os_image_hash allowlist instead. + let (input, mr_config, measurement, host_data, vm_config) = honest_case(); + let honest = verify_sev_launch(&measurement, &host_data, &vm_config) + .expect("honest launch verifies"); + + let mut tampered = input.clone(); + tampered.rootfs_hash = hex_of(0x99, 32); + let tampered_vm = synthetic_vm_config(&tampered, &mr_config); + let forged = verify_sev_launch(&measurement, &host_data, &tampered_vm) + .expect("rootfs_hash is not in the measurement, so the launch still verifies"); + assert_ne!( + honest.os_image_hash, forged.os_image_hash, + "a tampered rootfs_hash must change the derived os_image_hash" + ); + } + + #[test] + fn verify_sev_launch_rejects_tampered_mr_config() { + // Changing app/compose/instance identity changes the MrConfigV3 document, + // so the honest HOST_DATA no longer binds it. + let (input, _mr, measurement, host_data, _vm) = honest_case(); + let evil_mr_configs = [ + MrConfigV3::new( + vec![0xee; 20], + vec![0x22; 32], + dstack_types::KeyProviderKind::None, + Vec::new(), + vec![0x33; 20], + ), + MrConfigV3::new( + vec![0x11; 20], + vec![0xee; 32], + dstack_types::KeyProviderKind::None, + Vec::new(), + vec![0x33; 20], + ), + MrConfigV3::new( + vec![0x11; 20], + vec![0x22; 32], + dstack_types::KeyProviderKind::None, + Vec::new(), + vec![0xee; 20], + ), + ]; + for evil in evil_mr_configs { + let vm_config = synthetic_vm_config(&input, &evil); + let err = verify_sev_launch(&measurement, &host_data, &vm_config) + .expect_err("substituted mr_config must reject"); + assert!( + err.to_string().contains("amd sev-snp host_data mismatch"), + "unexpected error: {err:?}" + ); + } + } + + #[test] + fn verify_sev_launch_ignores_advertised_os_image_hash() { + // The os_image_hash is derived from the measurement-bound inputs; a + // top-level attacker-advertised os_image_hash is ignored entirely. + let (input, mr_config, measurement, host_data, _vm) = honest_case(); + let bogus = vec![0xde; 32]; + let vm_config = serde_json::json!({ + "os_image_hash": hex::encode(&bogus), + "sev_snp_measurement": serde_json::to_string(&input).unwrap(), + "mr_config": mr_config.to_canonical_json(), + }) + .to_string(); + let binding = verify_sev_launch(&measurement, &host_data, &vm_config) + .expect("bogus advertised os_image_hash is ignored, not fatal"); + let expected = + snp_measurement_os_image_hash(&serde_json::to_string(&input).unwrap()).unwrap(); + assert_eq!(binding.os_image_hash, expected); + assert_ne!(binding.os_image_hash, bogus); + } + + #[test] + fn swapping_os_image_changes_hash_and_is_rejected() { + // An attacker booting a different OS image cannot present an allowed + // image's inputs: the booted image's MEASUREMENT differs from the + // advertised inputs' recomputed measurement. + let honest = valid_input(); + let honest_hash = + snp_measurement_os_image_hash(&serde_json::to_string(&honest).unwrap()).unwrap(); + + let mut malicious = honest.clone(); + malicious.kernel_hash = hex_of(0xab, 32); // different kernel == different image + let malicious_measurement = compute_expected_measurement(&malicious).unwrap(); + let malicious_hash = + snp_measurement_os_image_hash(&serde_json::to_string(&malicious).unwrap()).unwrap(); + assert_ne!( + honest_hash, malicious_hash, + "different image must hash differently" + ); + + let mr_config = synthetic_mr_config(); + let host_data = MrConfigV3::snp_host_data_from_document(&mr_config.to_canonical_json()); + // Hardware measured the malicious image, but the quote advertises the + // honest (allowed) inputs. + let vm_config = synthetic_vm_config(&honest, &mr_config); + let err = verify_sev_launch(&malicious_measurement, &host_data, &vm_config) + .expect_err("advertised honest inputs must not pass for a different booted image"); + assert!( + err.to_string().contains("amd sev-snp measurement mismatch"), + "unexpected error: {err:?}" + ); + } + + #[test] + fn verify_sev_launch_requires_measurement_and_mr_config() { + let (input, mr_config, measurement, host_data, _vm) = honest_case(); + + let no_measurement = + serde_json::json!({ "mr_config": mr_config.to_canonical_json() }).to_string(); + let err = verify_sev_launch(&measurement, &host_data, &no_measurement) + .expect_err("missing sev_snp_measurement must fail closed"); + assert!( + err.to_string().contains("sev_snp_measurement is required"), + "unexpected error: {err:?}" + ); + + let no_mr_config = + serde_json::json!({ "sev_snp_measurement": serde_json::to_string(&input).unwrap() }) + .to_string(); + let err = verify_sev_launch(&measurement, &host_data, &no_mr_config) + .expect_err("missing mr_config must fail closed"); + assert!( + err.to_string().contains("mr_config is required"), + "unexpected error: {err:?}" + ); + } } From 776f2abb8a706cb394c1784e7f18daf1cc4a4222 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 17 Jun 2026 19:58:57 -0700 Subject: [PATCH 65/67] dstack-mr: own SEV os_image_hash CLI; vmm reads digest.sev.txt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the AMD SEV-SNP os_image_hash computation out of dstack-vmm into the dstack-mr crate, and add a `dstack-mr sev-os-image-hash ` command that emits the value (digest.sev.txt). dstack-mr now parses metadata.json, measures the SEV firmware (GCTX over ovmf-sev.fd), hashes kernel/initrd and projects them through dstack_types::SevOsImageMeasurement — the single hashing path already shared with KMS/verifier. dstack-vmm no longer recomputes the SEV os_image_hash at deploy: Image::load reads digest.sev.txt and make_vm_config uses it directly (failing closed if the file is absent), mirroring how TDX uses digest.txt. The vmm `sev-os-image-hash` subcommand is removed. Verified the new CLI reproduces the existing digest.sev.txt byte-for-byte (32b47673...) on the nvidia-0.6.0.a2 image, matching the value the verifier and CVM report. --- Cargo.lock | 1 + dstack-mr/Cargo.toml | 10 +++- dstack-mr/src/main.rs | 32 +++++++++++ dstack-mr/src/sev.rs | 121 +++++++++++++++++++++++++++++++++++++++++- vmm/Cargo.toml | 1 + vmm/src/app.rs | 67 ++++++++++------------- vmm/src/app/image.rs | 9 ++++ vmm/src/main.rs | 20 ------- 8 files changed, 200 insertions(+), 61 deletions(-) create mode 100644 dstack-mr/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index c8a55c7bb..05d8e5cdc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2763,6 +2763,7 @@ dependencies = [ "clap", "dirs 6.0.0", "dstack-kms-rpc", + "dstack-mr", "dstack-port-forward", "dstack-types", "dstack-vmm-rpc", diff --git a/dstack-mr/Cargo.toml b/dstack-mr/Cargo.toml index 83bfa31b2..4968409a0 100644 --- a/dstack-mr/Cargo.toml +++ b/dstack-mr/Cargo.toml @@ -9,7 +9,15 @@ version.workspace = true authors.workspace = true edition.workspace = true license.workspace = true -description = "A CLI tool for calculating TDX measurements for dstack images" +description = "A CLI tool for calculating TDX/SEV measurements for dstack images" + +[lib] +name = "dstack_mr" +path = "src/lib.rs" + +[[bin]] +name = "dstack-mr" +path = "src/main.rs" [dependencies] serde = { workspace = true, features = ["derive"] } diff --git a/dstack-mr/src/main.rs b/dstack-mr/src/main.rs new file mode 100644 index 000000000..2dca7574f --- /dev/null +++ b/dstack-mr/src/main.rs @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! `dstack-mr` CLI. +//! +//! Currently exposes the AMD SEV-SNP `os_image_hash` computation used by the +//! image build to emit `digest.sev.txt`. + +use anyhow::{bail, Context, Result}; +use std::path::Path; + +const USAGE: &str = "usage: dstack-mr sev-os-image-hash "; + +fn main() -> Result<()> { + let mut args = std::env::args().skip(1); + match args.next().as_deref() { + Some("sev-os-image-hash") => { + let image_dir = args.next().context(USAGE)?; + let hash = dstack_mr::sev::sev_os_image_hash_for_image_dir(Path::new(&image_dir)) + .context("failed to compute amd sev-snp os_image_hash")?; + println!("{}", hex::encode(hash)); + Ok(()) + } + Some("-h") | Some("--help") => { + println!("{USAGE}"); + Ok(()) + } + Some(other) => bail!("unknown subcommand {other:?}\n{USAGE}"), + None => bail!("{USAGE}"), + } +} diff --git a/dstack-mr/src/sev.rs b/dstack-mr/src/sev.rs index cef25aa0e..aeaae172a 100644 --- a/dstack-mr/src/sev.rs +++ b/dstack-mr/src/sev.rs @@ -20,6 +20,7 @@ use anyhow::{bail, Context, Result}; use dstack_types::mr_config::MrConfigV3; use sha2::{Digest, Sha256, Sha384}; use std::fs; +use std::path::Path; const LD_BYTES: usize = 48; const ZEROS_LD: [u8; LD_BYTES] = [0u8; LD_BYTES]; @@ -200,7 +201,6 @@ struct Gctx { } impl Gctx { - #[cfg(test)] fn new() -> Self { Self { ld: ZEROS_LD } } @@ -685,6 +685,125 @@ pub fn snp_measurement_os_image_hash(measurement_document: &str) -> Result, +} + +/// Parse an OVMF (SEV firmware) binary and compute its launch-measurement +/// metadata: the GCTX digest over the firmware bytes plus the SEV footer fields. +pub fn ovmf_measurement_info(path: &Path) -> Result { + let path_str = path.to_str().context("ovmf path must be valid utf-8")?; + let ovmf = OvmfInfo::load(path_str)?; + let mut gctx = Gctx::new(); + gctx.update_normal_pages(ovmf.gpa, &ovmf.data); + Ok(OvmfMeasurementInfo { + ovmf_hash: hex::encode(gctx.ld), + sev_hashes_table_gpa: ovmf.sev_hashes_table_gpa, + sev_es_reset_eip: ovmf.sev_es_reset_eip, + sections: ovmf + .sections + .into_iter() + .map(|s| OvmfSectionParam { + gpa: s.gpa, + size: s.size, + section_type: s.section_type as u32, + }) + .collect(), + }) +} + +/// The subset of an image's `metadata.json` needed to compute the SEV +/// os_image_hash. Kept local (rather than depending on the VMM `ImageInfo`) so +/// `dstack-mr` stays self-contained. +#[derive(Debug, serde::Deserialize)] +struct ImageMetadata { + #[serde(default)] + cmdline: Option, + kernel: String, + initrd: String, + #[serde(default)] + bios: Option, + #[serde(default, rename = "bios-sev")] + bios_sev: Option, + #[serde(default)] + rootfs_hash: Option, +} + +fn file_sha256_hex(path: &Path) -> Result { + let data = fs::read(path).with_context(|| format!("cannot read {}", path.display()))?; + Ok(hex::encode(Sha256::digest(data))) +} + +fn resolve_rootfs_hash(meta: &ImageMetadata) -> Result { + if let Some(hash) = meta.rootfs_hash.as_deref() { + if !hash.is_empty() { + return Ok(hash.to_string()); + } + } + // Fall back to the `dstack.rootfs_hash=` kernel cmdline parameter. + meta.cmdline + .as_deref() + .unwrap_or_default() + .split_whitespace() + .find_map(|param| param.strip_prefix("dstack.rootfs_hash=")) + .map(ToString::to_string) + .context( + "rootfs_hash is required for amd sev-snp \ + (set metadata.rootfs_hash or dstack.rootfs_hash= in the cmdline)", + ) +} + +/// Compute the AMD SEV-SNP `os_image_hash` from an OS image directory containing +/// `metadata.json` plus the SEV firmware, kernel and initrd. +/// +/// This is the canonical producer of `digest.sev.txt`. The value equals the +/// `os_image_hash` the KMS and verifier derive from a hardware-verified launch +/// measurement, because both go through [`snp_measurement_os_image_hash`] / +/// `dstack_types::SevOsImageMeasurement`. +pub fn sev_os_image_hash_for_image_dir(image_dir: &Path) -> Result<[u8; 32]> { + let meta_path = image_dir.join("metadata.json"); + let meta_str = fs::read_to_string(&meta_path) + .with_context(|| format!("cannot read {}", meta_path.display()))?; + let meta: ImageMetadata = + serde_json::from_str(&meta_str).context("failed to parse image metadata.json")?; + + // Measure the firmware the guest actually launches with: prefer the SEV + // firmware (bios-sev), fall back to the generic bios. + let bios = meta + .bios_sev + .as_deref() + .or(meta.bios.as_deref()) + .context("bios-sev/bios is required for amd sev-snp os_image_hash")?; + let ovmf = ovmf_measurement_info(&image_dir.join(bios))?; + + let measurement = dstack_types::SevOsImageMeasurement { + rootfs_hash: resolve_rootfs_hash(&meta)?, + base_cmdline: meta.cmdline.as_deref().map(|c| c.trim().to_string()), + ovmf_hash: ovmf.ovmf_hash, + kernel_hash: file_sha256_hex(&image_dir.join(&meta.kernel))?, + initrd_hash: file_sha256_hex(&image_dir.join(&meta.initrd))?, + sev_hashes_table_gpa: ovmf.sev_hashes_table_gpa, + sev_es_reset_eip: ovmf.sev_es_reset_eip, + ovmf_sections: ovmf + .sections + .into_iter() + .map(|s| dstack_types::OvmfSection { + gpa: s.gpa, + size: s.size, + section_type: s.section_type, + }) + .collect(), + }; + Ok(measurement.os_image_hash()) +} + /// `sha256(MEASUREMENT || HOST_DATA)` — the SNP aggregated identity digest. pub fn snp_mr_aggregated_digest(measurement: &[u8; 48], host_data: &[u8; 32]) -> Vec { let mut h = Sha256::new(); diff --git a/vmm/Cargo.toml b/vmm/Cargo.toml index 31150243c..b78dc3554 100644 --- a/vmm/Cargo.toml +++ b/vmm/Cargo.toml @@ -63,6 +63,7 @@ tar.workspace = true [dev-dependencies] insta.workspace = true +dstack-mr.workspace = true [build-dependencies] or-panic.workspace = true diff --git a/vmm/src/app.rs b/vmm/src/app.rs index cc71b2535..49f6aed33 100644 --- a/vmm/src/app.rs +++ b/vmm/src/app.rs @@ -1246,34 +1246,6 @@ fn image_rootfs_hash(image: &Image) -> Result<&str> { .ok_or_else(|| anyhow::anyhow!("rootfs_hash is required for amd sev-snp")) } -/// Compute the AMD SEV-SNP `os_image_hash` for an OS image, from the same -/// image-determined inputs the VMM feeds into the launch measurement. The image -/// build calls this (via the `sev-os-image-hash` subcommand) to emit -/// `digest.sev.txt`; the value matches what KMS derives from a verified launch -/// measurement, since both go through `dstack_types::SevOsImageMeasurement`. -pub fn sev_os_image_hash(image: &Image) -> Result<[u8; 32]> { - let ovmf = amd_sev_snp_ovmf_measurement_info(image)?; - let measurement = dstack_types::SevOsImageMeasurement { - rootfs_hash: image_rootfs_hash(image)?.to_string(), - base_cmdline: amd_sev_snp_measurement_base_cmdline(image.info.cmdline.as_deref()), - ovmf_hash: ovmf.ovmf_hash, - kernel_hash: file_sha256_hex(&image.kernel)?, - initrd_hash: file_sha256_hex(&image.initrd)?, - sev_hashes_table_gpa: ovmf.sev_hashes_table_gpa, - sev_es_reset_eip: ovmf.sev_es_reset_eip, - ovmf_sections: ovmf - .sections - .into_iter() - .map(|s| dstack_types::OvmfSection { - gpa: s.gpa, - size: s.size, - section_type: s.section_type, - }) - .collect(), - }; - Ok(measurement.os_image_hash()) -} - fn amd_sev_snp_measurement_base_cmdline(base_cmdline: Option<&str>) -> Option { base_cmdline.map(|cmdline| cmdline.trim().to_string()) } @@ -1295,13 +1267,16 @@ fn make_vm_config( let is_amd_sev_snp = cfg.cvm.platform.resolve() == crate::config::TeePlatform::AmdSevSnp && !manifest.no_tee; // AMD SEV-SNP binds the OS image through the launch-measurement-derived - // os_image_hash (the same value produced by the `sev-os-image-hash` - // subcommand / digest.sev.txt and recomputed by KMS from the verified - // measurement), not the generic content digest used for TDX. + // os_image_hash, computed at image build time by `dstack-mr sev-os-image-hash` + // and shipped as `digest.sev.txt` (the same value KMS/verifier derive from a + // verified launch measurement). The VMM reads it from the image rather than + // recomputing it; TDX still uses the generic content digest. let os_image_hash = if is_amd_sev_snp { - sev_os_image_hash(image) - .context("Failed to compute amd sev-snp os_image_hash")? - .to_vec() + let digest = image.sev_digest.as_deref().context( + "amd sev-snp image is missing digest.sev.txt; \ + rebuild the image so `dstack-mr sev-os-image-hash` emits it", + )?; + hex::decode(digest).context("digest.sev.txt is not valid hex")? } else { image .digest @@ -1506,6 +1481,13 @@ mod tests { vec![0x44; 20], ) .to_canonical_json(); + + // digest.sev.txt is produced at build time by the `dstack-mr + // sev-os-image-hash` command; the VMM reads it instead of recomputing. + // Emit it here so the deploy path (make_vm_config) can read it back. + let build_hash = dstack_mr::sev::sev_os_image_hash_for_image_dir(&image_dir)?; + fs::write(image_dir.join("digest.sev.txt"), hex::encode(build_hash))?; + let sys_config_document = make_sys_config(&config, &manifest, &compose_hash, Some(mr_config))?; let sys_config: serde_json::Value = serde_json::from_str(&sys_config_document)?; @@ -1526,6 +1508,15 @@ mod tests { assert_eq!(parsed_mr_config.app_id, vec![0x11; 20]); assert_eq!(parsed_mr_config.compose_hash, vec![0x22; 32]); assert_eq!(vm_config["mr_config"], sys_config["mr_config"]); + // The deploy path must surface the os_image_hash straight from + // digest.sev.txt (not recompute it). + assert_eq!( + vm_config["os_image_hash"] + .as_str() + .context("os_image_hash must be a string")?, + hex::encode(build_hash), + "vm_config os_image_hash must come from digest.sev.txt" + ); assert!(measurement.get("app_id").is_none()); assert!(measurement.get("compose_hash").is_none()); assert_eq!(measurement["rootfs_hash"], hex_of(0x33, 32)); @@ -1561,11 +1552,9 @@ mod tests { 4 ); - // The build-time os_image_hash (sev_os_image_hash -> digest.sev.txt) - // must equal the os_image_hash a verifier derives from the launch - // measurement document, i.e. the image-invariant projection of it. - let image = Image::load(&image_dir)?; - let build_hash = sev_os_image_hash(&image)?; + // The build-time os_image_hash (dstack-mr sev-os-image-hash -> + // digest.sev.txt) must equal the os_image_hash a verifier derives from + // the launch measurement document, i.e. the image-invariant projection. let as_str = |v: &serde_json::Value| v.as_str().unwrap().to_string(); let projected = dstack_types::SevOsImageMeasurement { rootfs_hash: as_str(&measurement["rootfs_hash"]), diff --git a/vmm/src/app/image.rs b/vmm/src/app/image.rs index 8e875122f..c8e7d255d 100644 --- a/vmm/src/app/image.rs +++ b/vmm/src/app/image.rs @@ -71,6 +71,10 @@ pub struct Image { pub bios: Option, pub bios_sev: Option, pub digest: Option, + /// AMD SEV-SNP os_image_hash, read from `digest.sev.txt` (produced at image + /// build time by `dstack-mr sev-os-image-hash`). The VMM does not recompute + /// it; the deploy path reads this value directly. + pub sev_digest: Option, } impl Image { @@ -99,6 +103,10 @@ impl Image { let digest = fs::read_to_string(base_path.join("digest.txt")) .ok() .map(|s| s.trim().to_string()); + let sev_digest = fs::read_to_string(base_path.join("digest.sev.txt")) + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()); if info.version.is_empty() { // Older images does not have version field. Fallback to the version of the image folder name info.version = guess_version(&base_path).unwrap_or_default(); @@ -112,6 +120,7 @@ impl Image { bios, bios_sev, digest, + sev_digest, } .ensure_exists() } diff --git a/vmm/src/main.rs b/vmm/src/main.rs index e55814d77..8ab3be383 100644 --- a/vmm/src/main.rs +++ b/vmm/src/main.rs @@ -60,16 +60,6 @@ enum Command { Serve, /// One-shot VM execution mode for debugging Run(RunArgs), - /// Compute the AMD SEV-SNP os_image_hash for an OS image and print it as - /// hex. Used by the image build to emit digest.sev.txt; the value matches - /// what KMS derives from a verified launch measurement. - SevOsImageHash(SevOsImageHashArgs), -} - -#[derive(ClapArgs)] -struct SevOsImageHashArgs { - /// Path to the OS image directory (containing metadata.json + artifacts) - image_dir: String, } #[derive(ClapArgs)] @@ -165,15 +155,6 @@ async fn main() -> Result<()> { let args = Args::parse(); - // Standalone, config-free subcommand: compute the SEV os_image_hash from an - // OS image directory (used by the image build for digest.sev.txt). - if let Some(Command::SevOsImageHash(a)) = &args.command { - let image = app::Image::load(&a.image_dir)?; - let hash = app::sev_os_image_hash(&image)?; - println!("{}", hex::encode(hash)); - return Ok(()); - } - let figment = config::load_config_figment(args.config.as_deref()); let config = Config::extract_or_default(&figment)?.abs_path()?; @@ -198,7 +179,6 @@ async fn main() -> Result<()> { Command::Serve => { // Default server mode - continue to main server logic } - Command::SevOsImageHash(_) => unreachable!("handled before config load"), } // Register this VMM instance for local discovery From a9f4b360f6501ea2b88fd7fff4c6276079d2cb8f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 17 Jun 2026 20:03:20 -0700 Subject: [PATCH 66/67] vmm: reuse dstack-mr OVMF measurement for sev_snp_measurement The sev_snp_measurement launch-input document built at deploy time used vmm's own snp_measure.rs (OVMF footer parse + GCTX). That logic is byte-for-byte the same as dstack_mr::sev::ovmf_measurement_info (added for the os_image_hash CLI), so delegate to it and delete the duplicate module. dstack-mr becomes a normal vmm dependency. Output is unchanged: the measurement-doc test and its os_image_hash projection cross-check still pass. --- vmm/Cargo.toml | 2 +- vmm/src/app.rs | 8 +- vmm/src/app/snp_measure.rs | 226 ------------------------------------- 3 files changed, 5 insertions(+), 231 deletions(-) delete mode 100644 vmm/src/app/snp_measure.rs diff --git a/vmm/Cargo.toml b/vmm/Cargo.toml index b78dc3554..9dbc6399c 100644 --- a/vmm/Cargo.toml +++ b/vmm/Cargo.toml @@ -48,6 +48,7 @@ load_config.workspace = true key-provider-client.workspace = true dstack-port-forward.workspace = true dstack-types.workspace = true +dstack-mr.workspace = true hex_fmt.workspace = true lspci.workspace = true base64.workspace = true @@ -63,7 +64,6 @@ tar.workspace = true [dev-dependencies] insta.workspace = true -dstack-mr.workspace = true [build-dependencies] or-panic.workspace = true diff --git a/vmm/src/app.rs b/vmm/src/app.rs index 49f6aed33..91741fce6 100644 --- a/vmm/src/app.rs +++ b/vmm/src/app.rs @@ -38,7 +38,6 @@ mod id_pool; mod image; mod qemu; pub(crate) mod registry; -mod snp_measure; #[derive(Deserialize, Serialize, Debug, Clone)] pub struct PortMapping { @@ -1220,14 +1219,15 @@ fn file_sha256_hex(path: &Path) -> Result { Ok(hex::encode(sha256_file(path)?)) } -fn amd_sev_snp_ovmf_measurement_info(image: &Image) -> Result { +fn amd_sev_snp_ovmf_measurement_info(image: &Image) -> Result { // Measure the same firmware the guest launches with: the SEV firmware - // (bios-sev) when present, falling back to the generic bios. + // (bios-sev) when present, falling back to the generic bios. The OVMF + // parsing/GCTX logic is shared with `dstack-mr sev-os-image-hash`. let bios = image .firmware(true) .map(|p| p.as_path()) .ok_or_else(|| anyhow::anyhow!("bios/OVMF is required for amd sev-snp measurement"))?; - snp_measure::ovmf_measurement_info(bios).with_context(|| { + dstack_mr::sev::ovmf_measurement_info(bios).with_context(|| { format!( "failed to extract amd sev-snp OVMF measurement metadata from {}", bios.display() diff --git a/vmm/src/app/snp_measure.rs b/vmm/src/app/snp_measure.rs deleted file mode 100644 index 287de0274..000000000 --- a/vmm/src/app/snp_measure.rs +++ /dev/null @@ -1,226 +0,0 @@ -// SPDX-FileCopyrightText: © 2024-2025 Phala Network -// -// SPDX-License-Identifier: Apache-2.0 - -//! AMD SEV-SNP launch-measurement metadata extracted by the VMM. -//! -//! The KMS/verifier must not be configured with one local OVMF binary: a VMM can -//! launch many image/OVMF versions. Instead, the VMM records the measured OVMF -//! launch digest seed and OVMF SEV metadata in `.sys-config.json`; the guest then -//! forwards that self-contained launch input to KMS with its attestation. - -use anyhow::{bail, Context, Result}; -use fs_err as fs; -use serde::Serialize; -use sha2::{Digest, Sha384}; -use std::path::Path; - -const LD_BYTES: usize = 48; -const ZEROS_LD: [u8; LD_BYTES] = [0u8; LD_BYTES]; - -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub(crate) struct OvmfSectionParam { - pub gpa: u64, - pub size: u64, - /// Raw OVMF SEV metadata section type: - /// 1=SNP_SEC_MEMORY, 2=SNP_SECRETS, 3=CPUID, 4=SVSM_CAA, - /// 0x10=SNP_KERNEL_HASHES. - pub section_type: u32, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct OvmfMeasurementInfo { - /// 48-byte GCTX launch digest after measuring the OVMF binary bytes. - pub ovmf_hash: String, - pub sev_hashes_table_gpa: u64, - pub sev_es_reset_eip: u32, - pub sections: Vec, -} - -pub(crate) fn ovmf_measurement_info(path: impl AsRef) -> Result { - let ovmf = OvmfInfo::load(path.as_ref())?; - let mut gctx = Gctx::new(); - gctx.update_normal_pages(ovmf.gpa, &ovmf.data); - Ok(OvmfMeasurementInfo { - ovmf_hash: hex::encode(gctx.ld), - sev_hashes_table_gpa: ovmf.sev_hashes_table_gpa, - sev_es_reset_eip: ovmf.sev_es_reset_eip, - sections: ovmf.sections, - }) -} - -struct Gctx { - ld: [u8; LD_BYTES], -} - -impl Gctx { - fn new() -> Self { - Self { ld: ZEROS_LD } - } - - /// SNP spec §8.17.2 PAGE_INFO layout (112 bytes): current digest, - /// contents digest, length, page type, permissions/reserved, and GPA. - fn update(&mut self, page_type: u8, gpa: u64, contents: &[u8; LD_BYTES]) { - let mut buf = [0u8; 0x70]; - buf[..LD_BYTES].copy_from_slice(&self.ld); - buf[48..96].copy_from_slice(contents); - buf[96..98].copy_from_slice(&0x70u16.to_le_bytes()); - buf[98] = page_type; - buf[104..112].copy_from_slice(&gpa.to_le_bytes()); - let mut digest = [0u8; LD_BYTES]; - digest.copy_from_slice(&Sha384::digest(buf)); - self.ld = digest; - } - - fn sha384(data: &[u8]) -> [u8; LD_BYTES] { - let mut out = [0u8; LD_BYTES]; - out.copy_from_slice(&Sha384::digest(data)); - out - } - - fn update_normal_pages(&mut self, start_gpa: u64, data: &[u8]) { - for (i, chunk) in data.chunks(4096).enumerate() { - self.update(0x01, start_gpa + (i * 4096) as u64, &Self::sha384(chunk)); - } - } -} - -struct OvmfInfo { - data: Vec, - gpa: u64, - sections: Vec, - sev_hashes_table_gpa: u64, - sev_es_reset_eip: u32, -} - -const GUID_FOOTER_TABLE: [u8; 16] = [ - 0xde, 0x82, 0xb5, 0x96, 0xb2, 0x1f, 0xf7, 0x45, 0xba, 0xea, 0xa3, 0x66, 0xc5, 0x5a, 0x08, 0x2d, -]; -const GUID_SEV_HASH_TABLE_RV: [u8; 16] = [ - 0x1f, 0x37, 0x55, 0x72, 0x3b, 0x3a, 0x04, 0x4b, 0x92, 0x7b, 0x1d, 0xa6, 0xef, 0xa8, 0xd4, 0x54, -]; -const GUID_SEV_ES_RESET_BLK: [u8; 16] = [ - 0xde, 0x71, 0xf7, 0x00, 0x7e, 0x1a, 0xcb, 0x4f, 0x89, 0x0e, 0x68, 0xc7, 0x7e, 0x2f, 0xb4, 0x4e, -]; -const GUID_SEV_META_DATA: [u8; 16] = [ - 0x66, 0x65, 0x88, 0xdc, 0x4a, 0x98, 0x98, 0x47, 0xa7, 0x5e, 0x55, 0x85, 0xa7, 0xbf, 0x67, 0xcc, -]; - -fn read_u16_le(buf: &[u8], off: usize) -> u16 { - u16::from_le_bytes([buf[off], buf[off + 1]]) -} - -fn read_u32_le(buf: &[u8], off: usize) -> u32 { - u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]]) -} - -fn validate_section_type(value: u32) -> Result<()> { - match value { - 1 | 2 | 3 | 4 | 0x10 => Ok(()), - _ => bail!("unknown ovmf section_type {value:#x}"), - } -} - -impl OvmfInfo { - fn load(path: &Path) -> Result { - let data = fs::read(path) - .with_context(|| format!("cannot read ovmf binary '{}'", path.display()))?; - let size = data.len(); - let gpa = (0x1_0000_0000u64) - .checked_sub(size as u64) - .context("ovmf binary is larger than 4 gib")?; - - const ENTRY_HDR: usize = 18; - let footer_off = size.saturating_sub(32 + ENTRY_HDR); - if footer_off + ENTRY_HDR > size { - bail!("ovmf binary too small to contain footer table"); - } - if data[footer_off + 2..footer_off + 18] != GUID_FOOTER_TABLE { - bail!("ovmf footer guid not found"); - } - let footer_total_size = read_u16_le(&data, footer_off) as usize; - if footer_total_size < ENTRY_HDR { - bail!("ovmf footer table has invalid total size"); - } - let table_size = footer_total_size - ENTRY_HDR; - if table_size > footer_off { - bail!("ovmf footer table is out of bounds"); - } - let table_start = footer_off - table_size; - let table_bytes = &data[table_start..footer_off]; - - let mut sev_hashes_table_gpa = 0u64; - let mut sev_es_reset_eip = 0u32; - let mut meta_offset_from_end = None; - - let mut pos = table_bytes.len(); - while pos >= ENTRY_HDR { - let entry_off = pos - ENTRY_HDR; - let entry_size = read_u16_le(table_bytes, entry_off) as usize; - if entry_size < ENTRY_HDR || entry_size > pos { - bail!("ovmf footer table has invalid entry size"); - } - let guid = &table_bytes[entry_off + 2..entry_off + 18]; - let data_start = pos - entry_size; - let data_end = pos - ENTRY_HDR; - let entry_data = &table_bytes[data_start..data_end]; - - if guid == GUID_SEV_HASH_TABLE_RV && entry_data.len() >= 4 { - sev_hashes_table_gpa = read_u32_le(entry_data, 0) as u64; - } else if guid == GUID_SEV_ES_RESET_BLK && entry_data.len() >= 4 { - sev_es_reset_eip = read_u32_le(entry_data, 0); - } else if guid == GUID_SEV_META_DATA && entry_data.len() >= 4 { - meta_offset_from_end = Some(read_u32_le(entry_data, 0) as usize); - } - pos -= entry_size; - } - - if sev_hashes_table_gpa == 0 { - bail!("ovmf sev hash table entry not found in footer table"); - } - if sev_es_reset_eip == 0 { - bail!("ovmf sev_es_reset_block entry not found in footer table"); - } - - let mut sections = Vec::new(); - let off_from_end = meta_offset_from_end - .ok_or_else(|| anyhow::anyhow!("ovmf sev metadata entry not found in footer table"))?; - if off_from_end > size { - bail!("ovmf sev metadata offset exceeds file size"); - } - let meta_start = size - off_from_end; - if meta_start + 16 > size { - bail!("ovmf sev metadata header out of bounds"); - } - if &data[meta_start..meta_start + 4] != b"ASEV" { - bail!("ovmf sev metadata has bad signature"); - } - let meta_version = read_u32_le(&data, meta_start + 8); - if meta_version != 1 { - bail!("ovmf sev metadata has unsupported version {meta_version}"); - } - let num_items = read_u32_le(&data, meta_start + 12) as usize; - let items_start = meta_start + 16; - if items_start + num_items * 12 > size { - bail!("ovmf sev metadata sections out of bounds"); - } - for i in 0..num_items { - let off = items_start + i * 12; - let section_type = read_u32_le(&data, off + 8); - validate_section_type(section_type)?; - sections.push(OvmfSectionParam { - gpa: read_u32_le(&data, off) as u64, - size: read_u32_le(&data, off + 4) as u64, - section_type, - }); - } - - Ok(Self { - data, - gpa, - sections, - sev_hashes_table_gpa, - sev_es_reset_eip, - }) - } -} From 52f277f62efa32d0cc71dc87f3ccd520f38624d0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 18 Jun 2026 02:32:00 -0700 Subject: [PATCH 67/67] vmm: drop TeePlatform::Auto, model auto as Option TeePlatform::resolve() folded an 'Auto' variant into the resolved type, so every match on a resolved platform carried a dead Auto arm (e.g. `Tdx | Auto` in the -machine selection). Remove the Auto variant: the config field becomes `Option` (None = auto-detect), and CvmConfig::resolved_platform() returns the pinned platform or TeePlatform::detect(). Matches on the resolved platform are now exhaustive over {Tdx, AmdSevSnp} with no unreachable arm. A back-compat deserializer still accepts the literal `platform = "auto"` (mapped to None) so existing vmm.toml configs keep working. --- vmm/src/app.rs | 6 ++-- vmm/src/app/qemu.rs | 6 ++-- vmm/src/config.rs | 75 ++++++++++++++++++++++++++++++++++++--------- vmm/src/one_shot.rs | 2 +- 4 files changed, 67 insertions(+), 22 deletions(-) diff --git a/vmm/src/app.rs b/vmm/src/app.rs index 91741fce6..8a8773003 100644 --- a/vmm/src/app.rs +++ b/vmm/src/app.rs @@ -1001,7 +1001,7 @@ impl App { let manifest = work_dir.manifest().context("Failed to read manifest")?; let cfg = &self.config; let compose_hash = sha256_file(shared_dir.join(APP_COMPOSE))?; - let platform = cfg.cvm.platform.resolve(); + let platform = cfg.cvm.resolved_platform(); let app_compose = work_dir .app_compose() .context("Failed to get app compose")?; @@ -1265,7 +1265,7 @@ fn make_vm_config( mr_config: Option, ) -> Result { let is_amd_sev_snp = - cfg.cvm.platform.resolve() == crate::config::TeePlatform::AmdSevSnp && !manifest.no_tee; + cfg.cvm.resolved_platform() == crate::config::TeePlatform::AmdSevSnp && !manifest.no_tee; // AMD SEV-SNP binds the OS image through the launch-measurement-derived // os_image_hash, computed at image build time by `dstack-mr sev-os-image-hash` // and shipped as `digest.sev.txt` (the same value KMS/verifier derive from a @@ -1452,7 +1452,7 @@ mod tests { let mut config: Config = Figment::from(load_config_figment(None)).extract()?; config.image.path = image_root; - config.cvm.platform = TeePlatform::AmdSevSnp; + config.cvm.platform = Some(TeePlatform::AmdSevSnp); let compose_hash = hex_of(0x22, 32); let manifest = Manifest { id: "snp-test".to_string(), diff --git a/vmm/src/app/qemu.rs b/vmm/src/app/qemu.rs index 7da7d8b7f..71159b3d7 100644 --- a/vmm/src/app/qemu.rs +++ b/vmm/src/app/qemu.rs @@ -566,7 +566,7 @@ impl VmConfig { let app_compose = workdir.app_compose().context("Failed to get app compose")?; let qemu = &cfg.qemu_path; let is_amd_sev_snp = - cfg.platform.resolve() == TeePlatform::AmdSevSnp && !self.manifest.no_tee; + cfg.resolved_platform() == TeePlatform::AmdSevSnp && !self.manifest.no_tee; let mut smp = self.manifest.vcpu.max(1); let mut mem = self.manifest.memory; let mut command = Command::new(qemu); @@ -969,8 +969,8 @@ impl VmConfig { return Ok(()); } - match cfg.platform.resolve() { - TeePlatform::Tdx | TeePlatform::Auto => { + match cfg.resolved_platform() { + TeePlatform::Tdx => { command .arg("-machine") .arg("q35,kernel-irqchip=split,confidential-guest-support=tdx,hpet=off"); diff --git a/vmm/src/config.rs b/vmm/src/config.rs index 7f68b7f5e..b0b234a29 100644 --- a/vmm/src/config.rs +++ b/vmm/src/config.rs @@ -106,23 +106,18 @@ impl Protocol { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize, Serialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] pub enum TeePlatform { - #[default] - Auto, Tdx, AmdSevSnp, } impl TeePlatform { - pub fn resolve(self) -> Self { - match self { - Self::Auto => Self::resolve_from_cpuinfo( - &fs_err::read_to_string("/proc/cpuinfo").unwrap_or_default(), - ), - platform => platform, - } + /// Detect the host TEE platform from /proc/cpuinfo. Used when the operator + /// did not pin a platform in the config (`platform` omitted, or `auto`). + pub fn detect() -> Self { + Self::resolve_from_cpuinfo(&fs_err::read_to_string("/proc/cpuinfo").unwrap_or_default()) } pub fn resolve_from_cpuinfo(cpuinfo: &str) -> Self { @@ -182,13 +177,44 @@ impl PortMappingConfig { } } +/// Deserialize the optional `platform` config field. `None` (field omitted, or +/// the legacy literal `auto`) means "detect the host TEE"; `tdx` / `amd-sev-snp` +/// pin a platform. Keeping `auto` accepted preserves existing vmm.toml configs. +fn deserialize_platform<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(rename_all = "kebab-case")] + enum PlatformSetting { + Auto, + Tdx, + AmdSevSnp, + } + Ok( + match Option::::deserialize(deserializer)? { + None | Some(PlatformSetting::Auto) => None, + Some(PlatformSetting::Tdx) => Some(TeePlatform::Tdx), + Some(PlatformSetting::AmdSevSnp) => Some(TeePlatform::AmdSevSnp), + }, + ) +} + +impl CvmConfig { + /// The effective TEE platform: the configured one, or host auto-detection + /// when left unset (`platform` omitted / `auto`). + pub fn resolved_platform(&self) -> TeePlatform { + self.platform.unwrap_or_else(TeePlatform::detect) + } +} + #[derive(Debug, Clone, Deserialize)] pub struct CvmConfig { - /// TEE platform to use when launching CVMs. Defaults to `auto`, which - /// detects the host TEE from /proc/cpuinfo (AMD SEV-SNP vs Intel TDX); - /// set `tdx` or `amd-sev-snp` to force a platform. - #[serde(default)] - pub platform: TeePlatform, + /// TEE platform to use when launching CVMs. Omit (or set `auto`) to detect + /// the host TEE from /proc/cpuinfo (AMD SEV-SNP vs Intel TDX); set `tdx` or + /// `amd-sev-snp` to force a platform. + #[serde(default, deserialize_with = "deserialize_platform")] + pub platform: Option, pub qemu_path: PathBuf, /// The URL of the KMS server pub kms_urls: Vec, @@ -658,6 +684,25 @@ mod tests { assert_eq!(platform, TeePlatform::AmdSevSnp); } + #[test] + fn platform_config_maps_auto_and_omitted_to_none() { + #[derive(Deserialize)] + struct Wrap { + #[serde(default, deserialize_with = "deserialize_platform")] + platform: Option, + } + let parse = |s: &str| serde_json::from_str::(s).unwrap().platform; + // Omitted and the legacy `auto` literal both mean "auto-detect" (None). + assert_eq!(parse("{}"), None); + assert_eq!(parse(r#"{"platform":"auto"}"#), None); + // Explicit platforms are pinned. + assert_eq!(parse(r#"{"platform":"tdx"}"#), Some(TeePlatform::Tdx)); + assert_eq!( + parse(r#"{"platform":"amd-sev-snp"}"#), + Some(TeePlatform::AmdSevSnp) + ); + } + #[test] fn tee_platform_auto_detects_amd_sev_snp_from_flag() { let cpuinfo = "flags : fpu svm sev sev_es sev_snp debug_swap"; diff --git a/vmm/src/one_shot.rs b/vmm/src/one_shot.rs index 36f46d9e7..2b9b49f8f 100644 --- a/vmm/src/one_shot.rs +++ b/vmm/src/one_shot.rs @@ -238,7 +238,7 @@ Compose file content (first 200 chars): let app_compose = vm_work_dir .app_compose() .context("Failed to get app compose")?; - let platform = config.cvm.platform.resolve(); + let platform = config.cvm.resolved_platform(); let use_mr_config_v3 = !manifest.no_tee && (platform == crate::config::TeePlatform::AmdSevSnp || (platform == crate::config::TeePlatform::Tdx