From 62265bbbd700ab5da8677b9ec2cc40f7d95cf7ca Mon Sep 17 00:00:00 2001 From: Hang Yin Date: Wed, 17 Jun 2026 00:18:08 +0000 Subject: [PATCH 1/8] docs: add onboarding-redesign design doc Captures the problem (the ~22-step path to a first app), the hardware-validated findings, and the locked design for hands-off single-node onboarding: dstackup (host setup) + dstack (client), single-node KMS bootstrap, a Rust auth webhook, and the crates/ layout. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/onboarding-redesign.md | 181 ++++++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 docs/onboarding-redesign.md diff --git a/docs/onboarding-redesign.md b/docs/onboarding-redesign.md new file mode 100644 index 00000000..14f68e5a --- /dev/null +++ b/docs/onboarding-redesign.md @@ -0,0 +1,181 @@ +# dstack onboarding redesign + +**Status:** design / planning (no implementation yet) +**Tracking issue:** [#699](https://github.com/Dstack-TEE/dstack/issues/699) +**Scope:** cut "time to first dstack app" from ~22 manual steps to a single command, for a self-hoster on their own TDX+SGX host. + +This document is the canonical design. The issue tracks task status; this file explains the *why* and the *shape*. + +--- + +## 1. Problem + +Following the deployment guide end-to-end today takes roughly **22 ordered steps across two repos** before a `docker-compose.yaml` is reachable in a browser. The sharp edges: + +- A chicken-and-egg dance with the KMS `mrAggregated` allowlist on first bootstrap. +- Three deploy scripts that use the "exit 1, edit the `.env`, run me again" pattern. +- "Copy-paste this hash, then type `y`" gates that the script could do itself. +- Three foreground processes, no systemd, manual restarts after config edits. +- A domain + DNS + ACME token required even to see the dashboard. +- `vmm.toml` URLs read only at startup, so bring-up means edit → restart → deploy → edit → restart. + +## 2. Goals & non-goals + +**Goals** +- One command to stand up the host stack; one command to deploy an app. +- No domain, no DNS, no ACME for the first app. +- The full security model still applies: real TDX attestation, KMS in a CVM, real per-app key derivation. +- Smooth on a server that already has another instance/workload running. + +**Non-goals (for the quickstart path)** +- Running on hosts without SGX (we fail fast — see §6). +- Multi-node KMS replication (single-node only; replication stays an advanced topic). +- On-chain governance for the first run (`auth-eth` is the documented upgrade). + +## 3. Principles + +- If it can be done in one script run, don't ask for two. +- If config can be generated, don't ask the user to write it. +- Pull prebuilt, reproducible artifacts; don't compile from source on first boot. +- Default to the simplest, most secure access path; make exposure opt-in. + +## 4. Two-tier onboarding + +**Tier 1 — first app (default).** `vmm` + single-node KMS (in a CVM) + your app, reachable via a **direct host:port** mapping. No gateway, no domain, no DNS, no ACME. This is the headline quickstart. + +**Tier 2 — managed HTTPS + routing (opt-in, separate).** Set up the gateway when you want `https://-.` URLs, automatic Let's Encrypt certs, and load-balanced routing. All the domain/DNS/ACME complexity lives here, so Tier 1 never blocks on it. + +The single decision that unblocks "deploy without a domain" is making the gateway opt-in: the domain complexity is entirely gateway-side. + +## 5. What we validated (on real TDX+SGX hardware) + +1. **Hands-off single-node KMS bootstrap works.** With `enforce_self_authorization = false` + a set `auto_bootstrap_domain`, a KMS CVM bootstraps unattended (~30 s): generates its CA + k256 root keys *inside the TEE* and serves `KMS.GetMeta` — no browser, no measurement pre-registration, no manual step. +2. **Necessary but not sufficient — bootstrap needs a genesis quote.** `Keys::generate` → `attest_keys` requires a TDX quote from the guest agent, so a *completed* bootstrap only happens **inside a CVM** (a bare host fails at "Failed to get quote" regardless of the flag). The flag removes the *authorization* round-trip; it does not remove the *attestation* requirement. +3. **The flag does not weaken app security.** App boot still goes through `bootAuth/app` (compose-hash allowlist), and `GetAppKey`/`SignCert` still verify each app's own TDX quote. The `mrAggregated` allowlist only does real work for KMS→KMS replication, which single-node never hits. +4. **Coexistence on a busy server is fine** — a new KMS CVM came up alongside an existing VM without disturbing it (CID auto-allocated by scanning live VMs). Friction surfaced: manual host-port selection, undefined image source on a clean box, and two-VMM contention (see §7.3 — the answer is to attach to the one VMM, not spawn a second). +5. **Browser secure-context constraint.** The web deploy UI uses `crypto.subtle` (AES-256-GCM for env encryption, SHA-256 for the compose hash), which is gated to secure contexts. Verified in a real browser: plain HTTP on a non-loopback IP **breaks** it; **`http://localhost` (loopback) works** (secure context); self-signed HTTPS after clicking through the warning **works**. The Rust `dstack run` path encrypts natively and never touches `crypto.subtle`. +6. **Prebuilt, reproducible images already exist.** `dstacktee/dstack-kms` (+ `-gateway`, `-verifier`) on Docker Hub, versioned tags, built with `SOURCE_DATE_EPOCH` + pinned `DSTACK_REV` and a build-provenance attestation pushed to the registry — so the published digest is independently rebuildable from source. + +## 6. Decisions (locked) + +| # | Decision | +|---|----------| +| Two-tier | KMS-first (mandatory for the real experience); gateway is an opt-in Tier-2 step. | +| Hardware | **SGX required.** `dstackup install` refuses on non-SGX hosts with a clear message; no silent degrade to host-mode KMS. | +| TLS / access | **Dashboard binds `localhost` by default** (secure context over plain HTTP via SSH tunnel / on-box → no cert needed). `--expose ` is the opt-in that binds the IP and mints a self-signed cert (SAN = that IP). Real-domain certs only in Tier 2. | +| Process / packaging | **systemd-native**, installed as an OS package (deb first, rpm next). The package owns install/uninstall (`apt remove`); `dstackup` does deployment bring-up/teardown. | +| CLI shape | **Two binaries** (see §7.1): `dstackup` (host setup, local + privileged) and `dstack` (client, local or remote). `dstack init` = scaffold an app project. | +| Image source | **Pull the prebuilt, pinned, reproducible images** from Docker Hub — not the dev build-from-source compose. Keep the pin current (the in-repo `deploy-to-vmm.sh` digest is stale at 0.5.5 vs latest 0.5.11). | +| Auth backend (Tier 1) | **Reimplement the simple JSON-allowlist webhook in Rust** (drop the `bun`-based `auth-simple`). Webhook → compose-hash allowlist + `enforce_self_authorization = false`. `auth-eth` (on-chain) stays the documented upgrade. | +| Teardown | `dstackup destroy`/`reset` keeps KMS keys by default (re-init against the same identity); `--purge` wipes everything. | +| vmm-cli.py | Wrap-then-deprecate — keep it working during a deprecation window while `dstack` supersedes it. | +| meta-dstack | Keep the two repos separate for now; may merge host artifacts into the package later. | + +## 7. Architecture + +### 7.1 Binaries & command taxonomy + +Precedent: kubeadm/kubectl, rustup/cargo — a privileged setup tool vs an everyday client. + +**`dstackup`** — host setup & lifecycle. **Local + privileged only** (touches `/dev/sgx`, systemd, local files, the local `vmm.sock`). +- `dstackup install` — the bring-up pipeline (§7.3). Idempotent. +- `dstackup status` — health of the host stack. +- `dstackup destroy [--purge]` — teardown. +- `--expose ` — opt-in network exposure of the dashboard (mints a self-signed cert). + +**`dstack`** — the client. **Local or remote** (defaults to the local `vmm.sock`; `--host --token ` for a remote VMM over TLS). +- `dstack run ` — deploy an app (§7.4). +- `dstack ls` / `logs` / `info` / `upgrade` — day-to-day ops. +- `dstack init` — scaffold a new app project (`app-compose.yaml` + `.env` template). The conventional meaning of `init`. + +### 7.2 Local vs remote + +- **Setup commands** (`dstackup *`) are local-only and privileged. +- **Client commands** (`dstack *`) work locally (unix socket) or against a remote VMM (TLS + token, using the VMM's existing `[auth] tokens`). + +### 7.3 `dstackup install` pipeline + +Talks to the VMM's existing `Vmm` prpc service over `vmm.sock` (reusing the `ra-rpc` client + the `vmm-rpc` proto crate — not a rewrite). Phases: + +0. **Preflight** — SGX gate (refuse on non-SGX); detect the primary routable host IP; **detect an existing VMM and attach to it** (the fix for two-VMM contention) rather than spawning a second. CID allocation is already VMM-managed. +1. **Render configs** — `vmm.toml`, `kms.toml`, `auth-allowlist.json`. Dashboard bound to `localhost` by default; `--expose ` mints a self-signed cert (`ra-tls` `CertRequest`, SAN = the IP) and enables `[tls]` on the dashboard listener. +2. **systemd units** — `dstack-vmm`, `dstack-auth` (the Rust webhook), `dstack-key-provider` (Gramine), with `After=`/`Requires=` ordering. +3. **Gramine key-provider** — write `sgx_default_qcnl.conf`, bind `0.0.0.0:3443`, `docker compose up -d`, poll `:3443` until healthy. (Automates the three manual gates in the current tutorial.) +4. **KMS-in-CVM bootstrap** — deploy the KMS CVM from the **pinned published image** with `enforce_self_authorization = false` + `auto_bootstrap_domain = `; **auto-pick free host ports** (the VMM does not auto-allocate host ports — `dstackup` bind-tests and assigns them); poll `KMS.GetMeta` for readiness; wire `kms_urls` **per-deploy via the RPC (no VMM restart)**. +5. **Report** — print the dashboard URL and KMS-ready confirmation. + +Each phase checks state first and resumes; re-running converges. + +### 7.4 `dstack run` + +Wraps compose + register + deploy into one step: compute the compose hash (`Vmm.GetComposeHash`), add it to the auth allowlist, **encrypt env vars natively in Rust** (no browser `crypto.subtle`), `Vmm.CreateVm` with an auto-allocated host port, and report `http://:`. This retires the "edit `.env`, re-run" and "copy the hash, type `y`" dances; the deploy scripts fold into subcommands. + +### 7.5 Auth webhook (Rust) + +A small host service (`dstack-auth.service`) reimplementing the `auth-simple` JSON-allowlist webhook in Rust: `bootAuth/app` checks the app's compose hash against an allowlist; `bootAuth/kms` is unused for single-node (KMS self-bootstrap is hands-off via `enforce_self_authorization = false`). Reachable from CVMs at `10.0.2.2:` under user-mode networking. Removes the `bun` runtime dependency. + +### 7.6 Dashboard access model + +- **Default:** `http://localhost:9080` via SSH tunnel or on-box — a secure context, so the browser deploy UI's `crypto.subtle` works with no TLS and no cert. +- **Opt-in:** `dstackup install --expose ` binds the IP and serves self-signed HTTPS (click-through); the cert SAN matches the IP exactly, so the optional "install the CA to silence the warning" path also works. +- The `dstack run` CLI path is unaffected either way (native crypto). + +## 8. Directory / crate structure + +The workspace has ~40 crates, mostly at the repo root. **Decision: introduce a single `crates/` directory and move all Rust crates into it** (libraries *and* binaries together), preserving each component's nesting. + +We deliberately do **not** split libraries vs binaries into separate top-level dirs (`crates/` vs `bin/`): the repo already keeps each component's binary and its library together and splits them at the *sub-crate* level — `kms` (bin) + `kms/rpc` (lib), `vmm` + `vmm/rpc`, `gateway` + `gateway/rpc`, `supervisor` + `supervisor/client`, `dstack-mr` (lib) + `dstack-mr/cli` (bin). A top-level lib/bin split would scatter those pairs across two trees and has no clean home for crates that are both. Binaries are identified by their `[[bin]]` targets / `cargo run -p`, not by directory. + +Target layout (existing components keep their shape, just relocated): + +``` +crates/ + vmm/ vmm/rpc/ # (and the rest of the current root crates, moved as-is) + kms/ kms/rpc/ + gateway/ gateway/rpc/ + ... + dstack/ # NEW: client binary -> `dstack` + dstackup/ # NEW: setup binary -> `dstackup` + dstack-core/ # NEW: shared lib (vmm prpc client, config render, port alloc) + dstack-auth/ # NEW: Rust JSON-allowlist webhook (binary: `dstack-auth`) +``` + +`sdk/` keeps its own grouping (it spans Rust + JS + Python + Go); only its Rust members relocate if/when convenient. + +**Migration is cheap mechanically, expensive to coordinate.** In-repo the fixups are concentrated: the root `Cargo.toml` (`members` + the 28 `path = "..."` entries in `[workspace.dependencies]`), exactly **one** inter-crate `../` path dep (survives a together-move), and **4** CI workflow files (`kms-release`, `gateway-release`, `docker-build-check`, `vmm-ui`). `include_str!`/fixtures are all within-crate and unaffected. `git mv` preserves history. The real cost is coordination — the reproducible image build contexts (`kms/dstack-app/builder`, `gateway/dstack-app/builder`), the separate **meta-dstack** repo, and conflicts with every open PR. + +**Sequencing:** the new onboarding crates go into `crates/` from day one (so the root never grows). The bulk move of the existing 40 crates is done **atomically in its own dedicated PR** (not in the onboarding branch), timed for a low-PR window and coordinated with CI / reproducible-builds / meta-dstack. + +Changes to existing crates (no new root crates): +- `vmm/` — bind the dashboard to `localhost` by default; add an opt-in `[tls]` path for `--expose` (self-signed cert via `ra-tls`). + +## 9. KMS modes — quickstart target + +Three independent axes get conflated in the docs: + +- **Boot mode:** Non-KMS (ephemeral) / Local-Key-Provider (SGX-sealed, how the KMS itself runs) / **KMS Mode** (deterministic per-app keys, upgradeable — what apps should run in). +- **Auth backend:** auth-mock (demo) / **auth-simple-style allowlist** (single-operator — our Rust webhook) / auth-eth (on-chain upgrade). +- **Where the KMS runs:** host (no real attestation) / **in a CVM with Gramine** (real TDX attestation). + +**Quickstart target:** KMS Mode (apps) + KMS-in-CVM-with-Gramine + the Rust allowlist webhook. Full security story minus the blockchain. + +## 10. Implementation roadmap + +Dependency-ordered; the critical path to Tier 1 is 1–6. Status tracked in [#699](https://github.com/Dstack-TEE/dstack/issues/699). + +1. **Hands-off single-node KMS bootstrap** — ✅ validated on hardware (§5). +2. **systemd units** (`dstack-vmm`, `dstack-auth`, `dstack-key-provider`). +3. **Gramine key-provider bring-up automation.** +4. **CLI crates + prpc client** — `dstack` (client) + `dstackup` (setup); ship read-only `dstack ls`/`logs` first. +5. **`dstackup install`** — the §7.3 pipeline. +6. **`dstack run`** — §7.4. +7. **`dstackup destroy`/`reset`** — teardown (keep keys + `--purge`). +8. **OS package** (deb first, rpm next). +9. **Tier-2 `dstack gateway` (opt-in).** +10. **Docs: two-tier quickstart rewrite.** + +## 11. Open questions + +- Cross-CVM KMS reachability under user-mode networking: `kms_urls` must use the guest-visible host address (`10.0.2.2:`), not `127.0.0.1` — nail down during #5. +- Behavior when a *foreign* (non-dstack) VMM already owns `vmm.sock`/ports: adopt vs warn-and-abort. +- Auth webhook home (`kms/auth-rs` crate vs `dstackup` subcommand) and `cli/core` timing (see §8). From 17bb05b1a5abd9ce0002981eed82b719b790055d Mon Sep 17 00:00:00 2001 From: Hang Yin Date: Wed, 17 Jun 2026 00:18:08 +0000 Subject: [PATCH 2/8] feat(cli): dstack + dstackup for hands-off single-node onboarding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds four crates under crates/ (no changes to existing crates): - dstack-core: shared library — typed VMM prpc client, config rendering (vmm.toml / kms.toml / auth-allowlist.json), app-compose build, host/SGX detection, free-port + --port spec helpers. - dstack: client CLI — run / ls / logs (info / upgrade / init are scaffolds); talks to a local VMM over its unix socket or a remote one over http(s). - dstackup: host setup CLI — install / status / destroy. SGX preflight, renders configs, manages dstack-vmm + dstack-auth as systemd units, deploys and bootstraps a single-node KMS-in-CVM. Idempotent install, deterministic cgroup teardown on destroy. - dstack-auth: Rust reimplementation of the single-operator KMS auth webhook (compose-hash allowlist, re-read per request, fails closed). Validated end-to-end on a TDX host against the official meta-dstack v0.5.11 release image: dstackup install -> KMS bootstrap -> dstack run -> app serves HTTP 200 -> dstackup destroy, all at the default 1 GB. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 44 ++ Cargo.toml | 5 + crates/dstack-auth/Cargo.toml | 20 + crates/dstack-auth/src/main.rs | 272 ++++++++++++ crates/dstack-core/Cargo.toml | 18 + crates/dstack-core/src/compose.rs | 28 ++ crates/dstack-core/src/config.rs | 419 ++++++++++++++++++ crates/dstack-core/src/host.rs | 66 +++ crates/dstack-core/src/lib.rs | 24 + crates/dstack-core/src/ports.rs | 55 +++ crates/dstack-core/src/vmm.rs | 123 ++++++ crates/dstack/Cargo.toml | 19 + crates/dstack/src/main.rs | 272 ++++++++++++ crates/dstackup/Cargo.toml | 21 + crates/dstackup/src/main.rs | 713 ++++++++++++++++++++++++++++++ 15 files changed, 2099 insertions(+) create mode 100644 crates/dstack-auth/Cargo.toml create mode 100644 crates/dstack-auth/src/main.rs create mode 100644 crates/dstack-core/Cargo.toml create mode 100644 crates/dstack-core/src/compose.rs create mode 100644 crates/dstack-core/src/config.rs create mode 100644 crates/dstack-core/src/host.rs create mode 100644 crates/dstack-core/src/lib.rs create mode 100644 crates/dstack-core/src/ports.rs create mode 100644 crates/dstack-core/src/vmm.rs create mode 100644 crates/dstack/Cargo.toml create mode 100644 crates/dstack/src/main.rs create mode 100644 crates/dstackup/Cargo.toml create mode 100644 crates/dstackup/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 3c8510b4..2cf5d79b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2286,6 +2286,16 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dstack" +version = "0.5.11" +dependencies = [ + "anyhow", + "clap", + "dstack-core", + "tokio", +] + [[package]] name = "dstack-attest" version = "0.5.11" @@ -2319,6 +2329,28 @@ dependencies = [ "tracing", ] +[[package]] +name = "dstack-auth" +version = "0.5.11" +dependencies = [ + "anyhow", + "clap", + "rocket", + "serde", + "serde_json", +] + +[[package]] +name = "dstack-core" +version = "0.5.11" +dependencies = [ + "anyhow", + "dstack-vmm-rpc", + "http-client", + "serde_json", + "toml", +] + [[package]] name = "dstack-gateway" version = "0.5.11" @@ -2780,6 +2812,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "dstackup" +version = "0.5.11" +dependencies = [ + "anyhow", + "clap", + "dstack-core", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "dunce" version = "1.0.5" diff --git a/Cargo.toml b/Cargo.toml index 1677b9a7..ed832f30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,6 +60,10 @@ members = [ "sdk/rust", "sdk/rust/types", "no_std_check", + "crates/dstack-core", + "crates/dstack", + "crates/dstackup", + "crates/dstack-auth", ] resolver = "2" @@ -72,6 +76,7 @@ dstack-gateway-rpc = { path = "gateway/rpc" } dstack-kms-rpc = { path = "kms/rpc" } dstack-guest-agent-rpc = { path = "guest-agent/rpc" } dstack-vmm-rpc = { path = "vmm/rpc" } +dstack-core = { path = "crates/dstack-core" } dstack-port-forward = { path = "port-forward" } cc-eventlog = { path = "cc-eventlog" } supervisor = { path = "supervisor" } diff --git a/crates/dstack-auth/Cargo.toml b/crates/dstack-auth/Cargo.toml new file mode 100644 index 00000000..de892fc6 --- /dev/null +++ b/crates/dstack-auth/Cargo.toml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: © 2025 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "dstack-auth" +version.workspace = true +edition.workspace = true +license.workspace = true + +[[bin]] +name = "dstack-auth" +path = "src/main.rs" + +[dependencies] +anyhow.workspace = true +clap.workspace = true +rocket = { workspace = true, features = ["json"] } +serde.workspace = true +serde_json.workspace = true diff --git a/crates/dstack-auth/src/main.rs b/crates/dstack-auth/src/main.rs new file mode 100644 index 00000000..03e2a357 --- /dev/null +++ b/crates/dstack-auth/src/main.rs @@ -0,0 +1,272 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! `dstack-auth` — the single-operator KMS auth webhook (Rust reimplementation +//! of `auth-simple`). +//! +//! Runs on the host as `dstack-auth.service`; the KMS-in-CVM reaches it at +//! `http://10.0.2.2:` under user-mode networking and POSTs `BootInfo` to +//! `/bootAuth/app` (compose-hash allowlist) and `/bootAuth/kms` (mrAggregated +//! allowlist). The allowlist JSON is re-read on every request, so `dstack run` +//! can add an app without a restart. Fails closed: a missing/invalid allowlist +//! denies everything. + +use anyhow::Result; +use clap::Parser; +use rocket::serde::json::Json; +use rocket::{get, post, routes, State}; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use std::collections::HashMap; +use std::path::PathBuf; + +#[derive(Parser, Clone)] +#[command( + name = "dstack-auth", + version, + about = "single-operator KMS auth webhook" +)] +struct Cli { + /// path to the allowlist JSON (re-read on every request). + #[arg(long, default_value = "/var/lib/dstack/auth-allowlist.json")] + config: PathBuf, + /// bind address. Defaults to loopback (reachable from CVMs at 10.0.2.2 via + /// user-mode networking, and not exposed externally). + #[arg(long, default_value = "127.0.0.1")] + address: String, + /// bind port. + #[arg(long, default_value_t = 8001)] + port: u16, +} + +/// boot info the KMS sends (camelCase; byte fields are hex strings). Only the +/// fields the allowlist checks are captured; the rest are ignored. +#[derive(Deserialize, Default)] +#[serde(rename_all = "camelCase", default)] +struct BootInfo { + mr_aggregated: String, + os_image_hash: String, + app_id: String, + compose_hash: String, + device_id: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct BootResponse { + is_allowed: bool, + gateway_app_id: String, + reason: String, +} + +#[derive(Deserialize, Default)] +#[serde(rename_all = "camelCase", default)] +struct Allowlist { + os_images: Vec, + gateway_app_id: String, + kms: KmsRules, + apps: HashMap, +} + +#[derive(Deserialize, Default)] +#[serde(rename_all = "camelCase", default)] +struct KmsRules { + mr_aggregated: Vec, + devices: Vec, + allow_any_device: bool, +} + +#[derive(Deserialize, Default)] +#[serde(rename_all = "camelCase", default)] +struct AppRules { + compose_hashes: Vec, + devices: Vec, + allow_any_device: bool, +} + +/// normalize a hex string for comparison: trim, drop a `0x` prefix, lowercase. +fn norm(s: &str) -> String { + let s = s.trim(); + let s = s + .strip_prefix("0x") + .or_else(|| s.strip_prefix("0X")) + .unwrap_or(s); + s.to_lowercase() +} + +fn contains(list: &[String], value: &str) -> bool { + let v = norm(value); + list.iter().any(|x| norm(x) == v) +} + +fn device_ok(allow_any: bool, devices: &[String], device_id: &str) -> bool { + allow_any || contains(devices, device_id) +} + +fn deny(al: &Allowlist, reason: &str) -> BootResponse { + BootResponse { + is_allowed: false, + gateway_app_id: al.gateway_app_id.clone(), + reason: reason.to_string(), + } +} + +fn allow(al: &Allowlist) -> BootResponse { + BootResponse { + is_allowed: true, + gateway_app_id: al.gateway_app_id.clone(), + reason: "ok".to_string(), + } +} + +fn check_app(info: &BootInfo, al: &Allowlist) -> BootResponse { + if !al.os_images.is_empty() && !contains(&al.os_images, &info.os_image_hash) { + return deny(al, "os image not allowed"); + } + let app_id = norm(&info.app_id); + let Some(app) = al + .apps + .iter() + .find(|(k, _)| norm(k) == app_id) + .map(|(_, v)| v) + else { + return deny(al, "app not registered"); + }; + if !contains(&app.compose_hashes, &info.compose_hash) { + return deny(al, "compose hash not allowed"); + } + if !device_ok(app.allow_any_device, &app.devices, &info.device_id) { + return deny(al, "device not allowed"); + } + allow(al) +} + +fn check_kms(info: &BootInfo, al: &Allowlist) -> BootResponse { + if !contains(&al.kms.mr_aggregated, &info.mr_aggregated) { + return deny(al, "kms mrAggregated not allowed"); + } + if !device_ok(al.kms.allow_any_device, &al.kms.devices, &info.device_id) { + return deny(al, "device not allowed"); + } + allow(al) +} + +/// load the allowlist, failing closed (deny-all) if it's missing or invalid. +fn load(path: &PathBuf) -> Allowlist { + match std::fs::read_to_string(path) { + Ok(body) => serde_json::from_str(&body).unwrap_or_else(|e| { + rocket::warn!("allowlist {} is invalid: {e}; denying all", path.display()); + Allowlist::default() + }), + Err(e) => { + rocket::warn!("allowlist {} unreadable: {e}; denying all", path.display()); + Allowlist::default() + } + } +} + +#[post("/bootAuth/app", data = "")] +fn boot_app(info: Json, cli: &State) -> Json { + let r = check_app(&info, &load(&cli.config)); + rocket::info!( + "bootAuth/app app={} compose={} -> allowed={} ({})", + norm(&info.app_id), + norm(&info.compose_hash), + r.is_allowed, + r.reason + ); + Json(r) +} + +#[post("/bootAuth/kms", data = "")] +fn boot_kms(info: Json, cli: &State) -> Json { + let r = check_kms(&info, &load(&cli.config)); + rocket::info!( + "bootAuth/kms mr={} -> allowed={} ({})", + norm(&info.mr_aggregated), + r.is_allowed, + r.reason + ); + Json(r) +} + +/// info endpoint the KMS GETs to populate its metadata. Single-node: no chain. +#[get("/")] +fn info() -> Json { + Json(json!({ + "status": "ok", + "kmsContractAddr": "", + "ethRpcUrl": "", + "gatewayAppId": "", + "chainId": 0, + "appImplementation": "" + })) +} + +#[rocket::main] +async fn main() -> Result<()> { + let cli = Cli::parse(); + let figment = rocket::Config::figment() + .merge(("address", cli.address.clone())) + .merge(("port", cli.port)); + rocket::custom(figment) + .manage(cli) + .mount("/", routes![info, boot_app, boot_kms]) + .launch() + .await + .map_err(|e| anyhow::anyhow!("auth webhook failed: {e}"))?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn allowlist() -> Allowlist { + serde_json::from_str( + r#"{ + "osImages": ["0xIMG"], + "kms": { "mrAggregated": ["0xMR"], "allowAnyDevice": true }, + "apps": { "0xApp1": { "composeHashes": ["0xHASH"], "allowAnyDevice": true } } + }"#, + ) + .unwrap() + } + + fn boot(app: &str, hash: &str, img: &str) -> BootInfo { + BootInfo { + app_id: app.into(), + compose_hash: hash.into(), + os_image_hash: img.into(), + ..Default::default() + } + } + + #[test] + fn app_allowed_with_normalized_hex() { + // differing 0x/case must still match. + let r = check_app(&boot("APP1", "hash", "img"), &allowlist()); + assert!(r.is_allowed, "{}", r.reason); + } + + #[test] + fn app_denied_unknown_app_hash_or_image() { + let al = allowlist(); + assert!(!check_app(&boot("0xnope", "0xHASH", "0xIMG"), &al).is_allowed); + assert!(!check_app(&boot("0xApp1", "0xnope", "0xIMG"), &al).is_allowed); + assert!(!check_app(&boot("0xApp1", "0xHASH", "0xnope"), &al).is_allowed); + } + + #[test] + fn kms_allowlist_and_empty_default() { + let al = allowlist(); + let info = BootInfo { + mr_aggregated: "0xMR".into(), + ..Default::default() + }; + assert!(check_kms(&info, &al).is_allowed); + // fail closed: empty allowlist denies (the single-node case never calls this). + assert!(!check_kms(&info, &Allowlist::default()).is_allowed); + } +} diff --git a/crates/dstack-core/Cargo.toml b/crates/dstack-core/Cargo.toml new file mode 100644 index 00000000..a4d20799 --- /dev/null +++ b/crates/dstack-core/Cargo.toml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: © 2025 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "dstack-core" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +anyhow.workspace = true +http-client = { workspace = true, features = ["prpc"] } +dstack-vmm-rpc.workspace = true +serde_json.workspace = true + +[dev-dependencies] +toml.workspace = true diff --git a/crates/dstack-core/src/compose.rs b/crates/dstack-core/src/compose.rs new file mode 100644 index 00000000..caf14b2c --- /dev/null +++ b/crates/dstack-core/src/compose.rs @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! build the app-compose manifest — the JSON document the VMM hashes (to derive +//! the app id) and deploys. The raw docker-compose YAML is embedded as a string. + +use serde_json::json; + +/// build a minimal Tier-1 app-compose manifest from a docker-compose YAML body. +/// +/// `kms_enabled` selects KMS mode (deterministic, upgradeable per-app keys); +/// gateway and local-key-provider are off for the direct-port Tier-1 flow. +pub fn build_app_compose(name: &str, docker_compose_yaml: &str, kms_enabled: bool) -> String { + let manifest = json!({ + "manifest_version": 2, + "name": name, + "runner": "docker-compose", + "docker_compose_file": docker_compose_yaml, + "kms_enabled": kms_enabled, + "gateway_enabled": false, + "local_key_provider_enabled": false, + "public_logs": true, + "public_sysinfo": true, + "no_instance_id": false, + }); + serde_json::to_string_pretty(&manifest).expect("app-compose manifest is always serializable") +} diff --git a/crates/dstack-core/src/config.rs b/crates/dstack-core/src/config.rs new file mode 100644 index 00000000..eeabe255 --- /dev/null +++ b/crates/dstack-core/src/config.rs @@ -0,0 +1,419 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! render the config files `dstackup install` writes. +//! +//! Two are produced here: +//! * `kms.toml` — embedded into the KMS-in-CVM app-compose; this is the +//! single-node Tier-1 config (webhook auth + `enforce_self_authorization = +//! false` + a set `auto_bootstrap_domain`, the combination validated to make +//! bootstrap hands-off — see docs/onboarding-redesign.md §5). +//! * `auth-allowlist.json` — read by the host-side Rust auth webhook. +//! +//! `vmm.toml` rendering is a follow-up. + +use anyhow::{Context, Result}; +use serde_json::json; +use std::path::Path; + +/// register an app (id + compose hash) in the auth webhook's allowlist file, +/// so the KMS will issue keys to it. Read-modify-write; idempotent. +pub fn register_app_in_allowlist(path: &Path, app_id: &str, compose_hash: &str) -> Result<()> { + let body = std::fs::read_to_string(path) + .with_context(|| format!("reading allowlist {}", path.display()))?; + let mut v: serde_json::Value = serde_json::from_str(&body).context("parsing allowlist json")?; + let apps = v + .get_mut("apps") + .and_then(|a| a.as_object_mut()) + .context("allowlist has no `apps` object")?; + let entry = apps + .entry(app_id.to_string()) + .or_insert_with(|| json!({ "composeHashes": [], "devices": [], "allowAnyDevice": true })); + let hashes = entry + .get_mut("composeHashes") + .and_then(|h| h.as_array_mut()) + .context("app entry missing `composeHashes`")?; + let norm = compose_hash.trim().trim_start_matches("0x").to_lowercase(); + let present = hashes.iter().any(|h| { + h.as_str() + .map(|s| s.trim_start_matches("0x").to_lowercase() == norm) + .unwrap_or(false) + }); + if !present { + hashes.push(serde_json::Value::String(compose_hash.to_string())); + } + std::fs::write(path, serde_json::to_string_pretty(&v)?) + .with_context(|| format!("writing allowlist {}", path.display()))?; + Ok(()) +} + +/// public OS-image download URL template used by the KMS image-hash verifier. +pub const DEFAULT_IMAGE_DOWNLOAD_URL: &str = + "https://download.dstack.org/os-images/mr_{OS_IMAGE_HASH}.tar.gz"; + +/// inputs that parameterize the rendered configs. +#[derive(Debug, Clone)] +pub struct HostConfig { + /// URL the KMS-in-CVM uses to reach the host auth webhook + /// (the host as seen from the CVM under user-mode networking, e.g. + /// `http://10.0.2.2:8001`). + pub auth_webhook_url: String, + /// KMS bootstrap domain — the host address as seen from the CVM + /// (e.g. `10.0.2.2`); the bootstrapped RPC cert is issued for this. + pub kms_bootstrap_domain: String, + /// OS image hash to allow apps to boot from (the measured guest image). + pub os_image_hash: String, + /// OS image download URL template (must contain `{OS_IMAGE_HASH}`). + pub image_download_url: String, + /// whether the KMS verifies the OS image hash on app key requests. + pub verify_os_image: bool, +} + +impl Default for HostConfig { + fn default() -> Self { + Self { + auth_webhook_url: "http://10.0.2.2:8001".to_string(), + kms_bootstrap_domain: "10.0.2.2".to_string(), + os_image_hash: String::new(), + image_download_url: DEFAULT_IMAGE_DOWNLOAD_URL.to_string(), + verify_os_image: true, + } + } +} + +/// render the single-node KMS config (lives at `/kms/kms.toml` inside the CVM). +pub fn kms_toml(cfg: &HostConfig) -> String { + format!( + r#"# generated by `dstackup install` — single-node Tier-1 KMS + +[rpc] +address = "0.0.0.0" +port = 8000 + +[rpc.tls] +key = "/kms/certs/rpc.key" +certs = "/kms/certs/rpc.crt" + +[rpc.tls.mutual] +ca_certs = "/kms/certs/tmp-ca.crt" +mandatory = false + +[core] +cert_dir = "/kms/certs" +admin_token_hash = "" +# single-node: the KMS does not self-attest to its own auth API before +# bootstrap (it still attests the genesis keys via the guest agent, and app +# auth + per-app quote checks are unaffected). See onboarding-redesign.md §5. +enforce_self_authorization = false + +[core.image] +verify = {verify} +cache_dir = "/kms/images" +download_url = "{download_url}" +download_timeout = "2m" + +[core.metrics] +enabled = false + +[core.auth_api] +type = "webhook" + +[core.auth_api.webhook] +url = "{webhook_url}" + +[core.onboard] +enabled = true +auto_bootstrap_domain = "{bootstrap_domain}" +address = "0.0.0.0" +port = 8000 +"#, + verify = cfg.verify_os_image, + download_url = cfg.image_download_url, + webhook_url = cfg.auth_webhook_url, + bootstrap_domain = cfg.kms_bootstrap_domain, + ) +} + +/// render the host-side auth webhook allowlist. +/// +/// Tier-1 single-node: the OS image is allowed, the KMS `mrAggregated` allowlist +/// is empty (no replication; self-bootstrap is hands-off), and per-app compose +/// hashes are added by `dstack run`. +pub fn auth_allowlist_json(cfg: &HostConfig) -> String { + let allowlist = json!({ + "osImages": if cfg.os_image_hash.is_empty() { + Vec::::new() + } else { + vec![cfg.os_image_hash.clone()] + }, + "kms": { + "mrAggregated": [], + "devices": [], + "allowAnyDevice": true + }, + "apps": {} + }); + serde_json::to_string_pretty(&allowlist).expect("allowlist is always serializable") +} + +/// default pinned, reproducibly-built KMS image (Docker Hub). +pub const DEFAULT_KMS_IMAGE: &str = "dstacktee/dstack-kms:0.5.11"; + +/// build the KMS-in-CVM app-compose manifest. The CVM runs in +/// local-key-provider mode; an init script writes the rendered `kms.toml` into +/// the guest, and the KMS container mounts it. +pub fn kms_app_compose(kms_toml: &str, kms_image: &str) -> String { + let docker_compose = format!( + r#"services: + kms: + image: {kms_image} + volumes: + - kms-volume:/kms + - /var/run/dstack.sock:/var/run/dstack.sock + - /dstack/kms-config/kms.toml:/kms/kms.toml:ro + ports: + - "8000:8000" + restart: unless-stopped + command: sh -c 'mkdir -p /kms/certs /kms/images && exec dstack-kms -c /kms/kms.toml' +volumes: + kms-volume: +"# + ); + let init_script = format!( + "mkdir -p /dstack/kms-config\ncat > /dstack/kms-config/kms.toml <<'KMSTOML'\n{kms_toml}\nKMSTOML\ntrue\n" + ); + let manifest = json!({ + "manifest_version": 2, + "name": "dstack-kms", + "runner": "docker-compose", + "docker_compose_file": docker_compose, + "init_script": init_script, + "kms_enabled": false, + "gateway_enabled": false, + "local_key_provider_enabled": true, + "public_logs": true, + "public_sysinfo": true, + "public_tcbinfo": true, + "no_instance_id": false, + "secure_time": false, + "allowed_envs": [] + }); + serde_json::to_string_pretty(&manifest).expect("kms app-compose is always serializable") +} + +/// inputs for rendering `vmm.toml`. Defaults target a localhost dashboard and +/// reuse of an existing local key provider; the isolation knobs (ports, cid +/// range, prefix) let a fresh instance coexist with an existing VMM. +#[derive(Debug, Clone)] +pub struct VmmRender { + /// Rocket endpoint for the dashboard + management API + /// (e.g. `tcp:127.0.0.1:9080`, or `unix:`). + pub dashboard_addr: String, + /// guest image directory. + pub image_path: String, + /// qemu binary path. + pub qemu_path: String, + /// run directory for the supervisor socket/pid/log. + pub run_dir: String, + /// VM storage directory (isolated per install; default `~/.dstack-vmm/vm`). + pub vm_path: String, + /// supervisor binary path. + pub supervisor_exe: String, + /// CID pool start (raise to coexist with an existing VMM). + pub cid_start: u32, + /// CID pool size. + pub cid_pool_size: u32, + /// host-api vsock port (raise to coexist with an existing VMM on 10000). + pub host_api_port: u32, + /// local key-provider address (reuse the running one). + pub key_provider_addr: String, + /// local key-provider port. + pub key_provider_port: u32, + /// KMS URLs injected into app CVMs (the guest-visible KMS address). + pub kms_urls: Vec, +} + +impl Default for VmmRender { + fn default() -> Self { + Self { + dashboard_addr: "tcp:127.0.0.1:9080".to_string(), + image_path: "/var/lib/dstack/images".to_string(), + qemu_path: "/usr/bin/qemu-system-x86_64".to_string(), + run_dir: "/var/lib/dstack/run".to_string(), + vm_path: "/var/lib/dstack/vm".to_string(), + supervisor_exe: "/usr/bin/dstack-supervisor".to_string(), + cid_start: 1000, + cid_pool_size: 1000, + host_api_port: 10000, + key_provider_addr: "127.0.0.1".to_string(), + key_provider_port: 3443, + kms_urls: Vec::new(), + } + } +} + +/// render the host `vmm.toml`. Gateway and auth-token gating are off (Tier-1 +/// direct-port access); CVMs use user-mode networking with host port mapping. +pub fn vmm_toml(r: &VmmRender) -> String { + format!( + r#"# generated by `dstackup install` + +workers = 8 +max_blocking = 64 +ident = "dstack VMM" +temp_dir = "/tmp" +keep_alive = 10 +log_level = "info" +address = "{dashboard_addr}" +reuse = true +kms_url = "" +event_buffer_size = 20 +node_name = "" +run_path = "{vm_path}" + +[image] +path = "{image_path}" +registry = "" + +[cvm] +qemu_path = "{qemu_path}" +kms_urls = [{kms_urls}] +gateway_urls = [] +pccs_url = "" +docker_registry = "" +cid_start = {cid_start} +cid_pool_size = {cid_pool_size} +max_allocable_vcpu = 20 +max_allocable_memory_in_mb = 100_000 +qmp_socket = false +user = "" +use_mrconfigid = false +qemu_pci_hole64_size = 0 +qemu_hotplug_off = false +host_share_mode = "9p" +qgs_port = 4050 + +[cvm.product] +sys_vendor = "dstack" +product_name = "dstack" + +[cvm.networking] +mode = "user" +net = "10.0.2.0/24" +dhcp_start = "10.0.2.10" +restrict = false +forward_service_enabled = false + +[cvm.port_mapping] +enabled = true +address = "127.0.0.1" +range = [ + {{ protocol = "tcp", from = 1, to = 20000 }}, +] + +[cvm.auto_restart] +enabled = true +interval = 20 + +[cvm.gpu] +enabled = false +listing = [] +exclude = [] +include = [] +allow_attach_all = false + +[gateway] +base_domain = "localhost" +port = 8082 +agent_port = 8090 + +[auth] +enabled = false +tokens = [] + +[supervisor] +exe = "{supervisor_exe}" +sock = "{run_dir}/supervisor.sock" +pid_file = "{run_dir}/supervisor.pid" +log_file = "{run_dir}/supervisor.log" +detached = true +auto_start = true + +[host_api] +ident = "dstack VMM" +address = "vsock:2" +port = {host_api_port} + +[key_provider] +enabled = true +address = "{kp_addr}" +port = {kp_port} +"#, + dashboard_addr = r.dashboard_addr, + image_path = r.image_path, + vm_path = r.vm_path, + qemu_path = r.qemu_path, + kms_urls = r + .kms_urls + .iter() + .map(|u| format!("\"{u}\"")) + .collect::>() + .join(", "), + cid_start = r.cid_start, + cid_pool_size = r.cid_pool_size, + supervisor_exe = r.supervisor_exe, + run_dir = r.run_dir, + host_api_port = r.host_api_port, + kp_addr = r.key_provider_addr, + kp_port = r.key_provider_port, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn vmm_toml_is_valid_and_parameterized() { + let r = VmmRender { + dashboard_addr: "tcp:127.0.0.1:19080".into(), + cid_start: 2000, + host_api_port: 10001, + ..Default::default() + }; + let rendered = vmm_toml(&r); + assert!(rendered.contains(r#"address = "tcp:127.0.0.1:19080""#)); + assert!(rendered.contains("cid_start = 2000")); + assert!(rendered.contains("port = 10001")); + toml::from_str::(&rendered).expect("vmm.toml must be valid TOML"); + } + + #[test] + fn kms_toml_has_tier1_invariants() { + let cfg = HostConfig { + auth_webhook_url: "http://10.0.2.2:8001".into(), + kms_bootstrap_domain: "10.0.2.2".into(), + ..Default::default() + }; + let toml = kms_toml(&cfg); + assert!(toml.contains("enforce_self_authorization = false")); + assert!(toml.contains(r#"auto_bootstrap_domain = "10.0.2.2""#)); + assert!(toml.contains(r#"type = "webhook""#)); + assert!(toml.contains(r#"url = "http://10.0.2.2:8001""#)); + // sanity: it parses as TOML. + toml::from_str::(&toml).expect("kms.toml must be valid TOML"); + } + + #[test] + fn allowlist_shape() { + let cfg = HostConfig { + os_image_hash: "0xabc".into(), + ..Default::default() + }; + let v: serde_json::Value = serde_json::from_str(&auth_allowlist_json(&cfg)).unwrap(); + assert_eq!(v["osImages"][0], "0xabc"); + assert_eq!(v["kms"]["mrAggregated"].as_array().unwrap().len(), 0); + assert!(v["apps"].as_object().unwrap().is_empty()); + } +} diff --git a/crates/dstack-core/src/host.rs b/crates/dstack-core/src/host.rs new file mode 100644 index 00000000..cbc49b59 --- /dev/null +++ b/crates/dstack-core/src/host.rs @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! host environment checks used by `dstackup` — SGX presence and the primary IP. + +use anyhow::{bail, Result}; +use std::net::{IpAddr, UdpSocket}; +use std::path::Path; + +/// presence of the SGX device nodes the local key provider needs. +#[derive(Debug, Clone, Copy)] +pub struct Sgx { + pub enclave: bool, + pub provision: bool, +} + +impl Sgx { + pub fn ok(&self) -> bool { + self.enclave && self.provision + } +} + +/// check for `/dev/sgx_enclave` and `/dev/sgx_provision`. +pub fn check_sgx() -> Sgx { + Sgx { + enclave: Path::new("/dev/sgx_enclave").exists(), + provision: Path::new("/dev/sgx_provision").exists(), + } +} + +/// require SGX, with a clear message if it is missing (design decision: fail fast +/// rather than silently degrade to a host-mode KMS with no real attestation). +pub fn require_sgx() -> Result<()> { + let sgx = check_sgx(); + if !sgx.ok() { + let mut missing = Vec::new(); + if !sgx.enclave { + missing.push("/dev/sgx_enclave"); + } + if !sgx.provision { + missing.push("/dev/sgx_provision"); + } + bail!( + "sgx not available (missing {}); dstack requires Intel SGX for the local key provider — enable SGX in BIOS, or run on a TDX+SGX host", + missing.join(", ") + ); + } + Ok(()) +} + +/// best-effort primary routable IPv4 of this host. +/// +/// uses the standard UDP-connect trick: connecting a datagram socket sends no +/// packets but makes the kernel pick the source address it would route from. +pub fn detect_host_ip() -> Result { + let socket = UdpSocket::bind("0.0.0.0:0")?; + socket.connect("8.8.8.8:80")?; + Ok(socket.local_addr()?.ip()) +} + +/// whether `ip` is a link-local address (169.254/16) — usable, but a poor +/// default for a dashboard SAN or KMS bootstrap domain. +pub fn is_link_local(ip: &IpAddr) -> bool { + matches!(ip, IpAddr::V4(v4) if v4.is_link_local()) +} diff --git a/crates/dstack-core/src/lib.rs b/crates/dstack-core/src/lib.rs new file mode 100644 index 00000000..8b6fb01e --- /dev/null +++ b/crates/dstack-core/src/lib.rs @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! shared internals for the `dstack` (client) and `dstackup` (host setup) binaries. +//! +//! `vmm` is a thin typed client over the VMM `Vmm` prpc service; `compose` builds +//! the app-compose manifest; `ports` does host-port allocation. `config` (config +//! rendering for `dstackup install`) is still a stub — see +//! docs/onboarding-redesign.md §7. + +/// re-export the generated VMM rpc types (VmConfiguration, PortMapping, …). +pub use dstack_vmm_rpc as rpc; + +/// identifier string attached to outbound RPC calls. +pub fn user_agent() -> String { + format!("dstack-cli/{}", env!("CARGO_PKG_VERSION")) +} + +pub mod compose; +pub mod config; +pub mod host; +pub mod ports; +pub mod vmm; diff --git a/crates/dstack-core/src/ports.rs b/crates/dstack-core/src/ports.rs new file mode 100644 index 00000000..a3919877 --- /dev/null +++ b/crates/dstack-core/src/ports.rs @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! host-port helpers. The VMM does not auto-allocate host ports, so the client +//! picks a free one and passes it explicitly in the VM configuration. + +use anyhow::{bail, Context, Result}; +use dstack_vmm_rpc::PortMapping; +use std::net::TcpListener; + +/// pick a currently-free TCP port on loopback by binding to port 0. +/// +/// inherently racy (the port could be taken before the VMM binds it), but fine +/// for a single interactive deploy; the VMM will surface a bind conflict. +pub fn free_local_port() -> Result { + let listener = TcpListener::bind("127.0.0.1:0").context("failed to find a free host port")?; + let port = listener.local_addr()?.port(); + Ok(port) +} + +/// parse a `--port` spec into a [`PortMapping`], auto-allocating the host port +/// when it is omitted, `0`, or `auto`. Accepted forms: +/// +/// * `` — auto host port, tcp, 127.0.0.1 +/// * `:` — tcp, 127.0.0.1 +/// * `::` +/// * `:::` +pub fn parse_port(spec: &str) -> Result { + let parts: Vec<&str> = spec.split(':').collect(); + let (proto, addr, host, vm) = match parts.as_slice() { + [vm] => ("tcp", "127.0.0.1", "auto", *vm), + [host, vm] => ("tcp", "127.0.0.1", *host, *vm), + [proto, host, vm] => (*proto, "127.0.0.1", *host, *vm), + [proto, addr, host, vm] => (*proto, *addr, *host, *vm), + _ => bail!( + "invalid --port '{spec}': expected vm | host:vm | proto:host:vm | proto:addr:host:vm" + ), + }; + let vm_port: u32 = vm + .parse() + .with_context(|| format!("invalid vm port in '{spec}'"))?; + let host_port: u32 = if host.is_empty() || host == "auto" || host == "0" { + free_local_port()? as u32 + } else { + host.parse() + .with_context(|| format!("invalid host port in '{spec}'"))? + }; + Ok(PortMapping { + protocol: proto.to_string(), + host_address: addr.to_string(), + host_port, + vm_port, + }) +} diff --git a/crates/dstack-core/src/vmm.rs b/crates/dstack-core/src/vmm.rs new file mode 100644 index 00000000..27693f0f --- /dev/null +++ b/crates/dstack-core/src/vmm.rs @@ -0,0 +1,123 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! thin typed client over the VMM `Vmm` prpc service. +//! +//! talks to a local VMM over its unix control socket, or a remote VMM over an +//! http(s) endpoint. prpc calls go to `/prpc/?json`; a few endpoints +//! (e.g. `/logs`) are plain HTTP and are reached with [`http_client::http_request`]. + +use anyhow::{anyhow, bail, Result}; +use dstack_vmm_rpc::vmm_client::VmmClient; +use dstack_vmm_rpc::{Id, StatusRequest, StatusResponse, VmConfiguration}; +use http_client::http_request; +use http_client::prpc::PrpcClient; + +/// default local VMM control socket (created by `dstackup install`). +pub const DEFAULT_HOST: &str = "unix:/var/run/dstack/vmm.sock"; + +/// a connection to a VMM — local unix socket or remote http endpoint. +pub struct Vmm { + rpc: VmmClient, + /// base string usable with [`http_request`] for non-prpc endpoints. + base: String, +} + +impl Vmm { + /// connect to a VMM addressed by `host`: + /// `unix:/path/to/vmm.sock` (local) or `http(s)://host:port` (remote). + pub fn connect(host: &str) -> Result { + let host = host.trim(); + if let Some(sock) = host.strip_prefix("unix:") { + let rpc = VmmClient::new(PrpcClient::new_unix(sock.to_string(), "/prpc".to_string())); + Ok(Self { + rpc, + base: format!("unix:{sock}"), + }) + } else if host.starts_with("http://") || host.starts_with("https://") { + let base = host.trim_end_matches('/').to_string(); + let rpc = VmmClient::new(PrpcClient::new(format!("{base}/prpc"))); + Ok(Self { rpc, base }) + } else { + bail!( + "unsupported host '{host}': expected unix:/path/to/vmm.sock or http(s)://host:port" + ); + } + } + + /// whether this connection targets a local unix socket. + pub fn is_local(&self) -> bool { + self.base.starts_with("unix:") + } + + /// list deployed VMs (brief: no full configuration). + pub async fn status(&self) -> Result { + self.rpc + .status(StatusRequest { + brief: true, + ..Default::default() + }) + .await + .map_err(|e| anyhow!("vmm Status rpc failed: {e}")) + } + + /// compute the compose hash for a VM configuration (no side effects). + /// the app id is the first 40 hex chars of this hash. + pub async fn get_compose_hash(&self, cfg: &VmConfiguration) -> Result { + self.rpc + .get_compose_hash(cfg.clone()) + .await + .map(|c| c.hash) + .map_err(|e| anyhow!("vmm GetComposeHash rpc failed: {e}")) + } + + /// create (and, unless `cfg.stopped`, start) a VM; returns the new VM id. + pub async fn create_vm(&self, cfg: VmConfiguration) -> Result { + self.rpc + .create_vm(cfg) + .await + .map(|id| id.id) + .map_err(|e| anyhow!("vmm CreateVm rpc failed: {e}")) + } + + /// stop a VM by id, keeping its disk (so its keys survive a re-install). + pub async fn stop_vm(&self, id: &str) -> Result<()> { + self.rpc + .stop_vm(Id { id: id.to_string() }) + .await + .map_err(|e| anyhow!("vmm StopVm rpc failed: {e}")) + } + + /// remove (and stop) a VM by id. + pub async fn remove_vm(&self, id: &str) -> Result<()> { + self.rpc + .remove_vm(Id { id: id.to_string() }) + .await + .map_err(|e| anyhow!("vmm RemoveVm rpc failed: {e}")) + } + + /// whether a VM with the given id currently exists. + pub async fn has_vm(&self, id: &str) -> bool { + match self.status().await { + Ok(s) => s.vms.iter().any(|v| v.id == id), + Err(_) => false, + } + } + + /// fetch the last `lines` log lines for a VM (non-following). + /// + /// `/logs` is a plain-HTTP endpoint; only the local unix-socket transport is + /// supported today (the shared http helper posts on the remote http path). + pub async fn logs(&self, id: &str, lines: u32) -> Result { + if !self.is_local() { + bail!("`dstack logs` currently supports a local VMM (unix socket) only"); + } + let path = format!("/logs?id={id}&follow=false&ansi=false&lines={lines}"); + let (status, body) = http_request("GET", &self.base, &path, b"").await?; + if status != 200 { + bail!("vmm /logs returned status {status}"); + } + Ok(String::from_utf8_lossy(&body).into_owned()) + } +} diff --git a/crates/dstack/Cargo.toml b/crates/dstack/Cargo.toml new file mode 100644 index 00000000..4929c272 --- /dev/null +++ b/crates/dstack/Cargo.toml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: © 2025 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "dstack" +version.workspace = true +edition.workspace = true +license.workspace = true + +[[bin]] +name = "dstack" +path = "src/main.rs" + +[dependencies] +anyhow.workspace = true +clap.workspace = true +dstack-core.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/dstack/src/main.rs b/crates/dstack/src/main.rs new file mode 100644 index 00000000..1eb3acf7 --- /dev/null +++ b/crates/dstack/src/main.rs @@ -0,0 +1,272 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! `dstack` — client for deploying and managing apps on a dstack host. +//! +//! Works against a local VMM (unix socket) or a remote one (`--host` + `--token`). +//! Setup/host tasks live in the separate `dstackup` binary. + +use anyhow::{bail, Context, Result}; +use clap::{Parser, Subcommand}; +use dstack_core::vmm::{Vmm, DEFAULT_HOST}; +use dstack_core::{compose, ports, rpc}; + +#[derive(Parser)] +#[command( + name = "dstack", + version, + about = "client for deploying and managing dstack apps" +)] +struct Cli { + /// VMM endpoint: `unix:/path/to/vmm.sock` (local) or `http(s)://host:port` (remote). + /// Defaults to the local control socket. + #[arg(long, global = true)] + host: Option, + + /// auth token for a remote VMM. + #[arg(long, global = true)] + token: Option, + + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Deploy an app from a docker-compose file. + Run { + /// path to the docker-compose file. + compose: String, + /// app name. + #[arg(long, default_value = "app")] + name: String, + /// guest OS image version (required to actually deploy). + #[arg(long)] + image: Option, + /// vCPUs. + #[arg(long, default_value_t = 2)] + vcpu: u32, + /// memory in MB (matches vmm-cli's default; images with a large + /// initramfs-rootfs may need more — raise it if the guest fails early). + #[arg(long, default_value_t = 1024)] + memory: u32, + /// disk size in GB. + #[arg(long, default_value_t = 20)] + disk: u32, + /// expose a port: `vm` | `host:vm` | `proto:host:vm` | `proto:addr:host:vm` + /// (host omitted/`auto`/`0` ⇒ a free host port is picked). Repeatable. + #[arg(long = "port", value_name = "SPEC")] + ports: Vec, + /// deploy in non-KMS mode (ephemeral keys; no KMS required). + #[arg(long)] + no_kms: bool, + /// register the app's compose hash in this auth-allowlist.json (local, + /// KMS mode) so the KMS will issue it keys. Usually the path printed by + /// `dstackup install`. + #[arg(long, value_name = "PATH")] + allowlist: Option, + /// build + hash the compose and print it, without deploying. + #[arg(long)] + dry_run: bool, + }, + /// List deployed apps. + Ls, + /// Show recent logs for an app. + Logs { + /// app, instance, or VM id. + id: String, + /// number of trailing log lines to fetch. + #[arg(long, default_value_t = 200)] + lines: u32, + }, + /// Show details for an app. + Info { + /// app or instance id. + id: String, + }, + /// Upgrade an app to a new compose. + Upgrade { + /// app or instance id. + id: String, + /// path to the new docker-compose file. + compose: String, + }, + /// Scaffold a new app project in the current directory. + Init, +} + +#[tokio::main] +async fn main() -> Result<()> { + let cli = Cli::parse(); + // remote-auth wiring lands with the TLS+token transport. + let _ = &cli.token; + let host = cli.host.as_deref().unwrap_or(DEFAULT_HOST); + + match cli.command { + Command::Ls => cmd_ls(host).await, + Command::Logs { id, lines } => cmd_logs(host, &id, lines).await, + Command::Run { + compose, + name, + image, + vcpu, + memory, + disk, + ports, + no_kms, + allowlist, + dry_run, + } => { + cmd_run( + host, + &compose, + &name, + image.as_deref(), + vcpu, + memory, + disk, + &ports, + no_kms, + allowlist.as_deref(), + dry_run, + ) + .await + } + Command::Info { .. } => stub("info"), + Command::Upgrade { .. } => stub("upgrade"), + Command::Init => stub("init"), + } +} + +#[allow(clippy::too_many_arguments)] +async fn cmd_run( + host: &str, + compose_path: &str, + name: &str, + image: Option<&str>, + vcpu: u32, + memory: u32, + disk: u32, + port_specs: &[String], + no_kms: bool, + allowlist: Option<&str>, + dry_run: bool, +) -> Result<()> { + let yaml = std::fs::read_to_string(compose_path) + .with_context(|| format!("reading compose file '{compose_path}'"))?; + let app_compose = compose::build_app_compose(name, &yaml, !no_kms); + + let mut port_maps = Vec::new(); + for spec in port_specs { + port_maps.push(ports::parse_port(spec)?); + } + + let mut cfg = rpc::VmConfiguration { + name: name.to_string(), + image: image.unwrap_or_default().to_string(), + compose_file: app_compose.clone(), + vcpu, + memory, + disk_size: disk, + ports: port_maps.clone(), + ..Default::default() + }; + + let vmm = Vmm::connect(host)?; + let hash = vmm.get_compose_hash(&cfg).await?; + let app_id = short(&hash, 40); + cfg.app_id = Some(app_id.clone()); + println!("compose hash: {hash}"); + println!("app id: {app_id}"); + + if dry_run { + println!("--- app-compose ---\n{app_compose}"); + println!("(dry run — not deploying)"); + return Ok(()); + } + if cfg.image.is_empty() { + bail!("an image is required to deploy (pass --image )"); + } + + // register the compose hash so the KMS will issue keys (KMS mode, local). + if let Some(path) = allowlist { + dstack_core::config::register_app_in_allowlist(std::path::Path::new(path), &app_id, &hash) + .with_context(|| { + format!("registering app in {path} (it is usually root-owned — run with sudo, or make it writable)") + })?; + println!("registered in allowlist: {path}"); + } else if !no_kms { + println!("note: no --allowlist given; a KMS-mode app needs its compose hash registered to get keys"); + } + + let id = vmm.create_vm(cfg).await?; + println!("deployed: vm {id}"); + if port_maps.is_empty() { + println!("(no ports mapped — add --port to expose the app)"); + } + for p in &port_maps { + let addr = if p.host_address.is_empty() { + "127.0.0.1" + } else { + &p.host_address + }; + println!(" app :{} -> http://{}:{}/", p.vm_port, addr, p.host_port); + } + Ok(()) +} + +fn stub(name: &str) -> Result<()> { + eprintln!( + "dstack {name}: not yet implemented (scaffold) — {}", + dstack_core::user_agent() + ); + Ok(()) +} + +async fn cmd_ls(host: &str) -> Result<()> { + let vmm = Vmm::connect(host)?; + let resp = vmm.status().await?; + if resp.vms.is_empty() { + println!("no apps deployed"); + return Ok(()); + } + println!( + "{:<14} {:<22} {:<10} {:<14} APP ID", + "ID", "NAME", "STATUS", "UPTIME" + ); + for vm in resp.vms { + println!( + "{:<14} {:<22} {:<10} {:<14} {}", + short(&vm.id, 12), + trunc(&vm.name, 22), + trunc(&vm.status, 10), + trunc(&vm.uptime, 14), + short(&vm.app_id, 40), + ); + } + Ok(()) +} + +async fn cmd_logs(host: &str, id: &str, lines: u32) -> Result<()> { + let vmm = Vmm::connect(host)?; + let logs = vmm.logs(id, lines).await?; + print!("{logs}"); + Ok(()) +} + +/// first `n` chars of an id-like string. +fn short(s: &str, n: usize) -> String { + s.chars().take(n).collect() +} + +/// truncate to `n` chars with an ellipsis if longer. +fn trunc(s: &str, n: usize) -> String { + if s.chars().count() <= n { + s.to_string() + } else { + let mut out: String = s.chars().take(n.saturating_sub(1)).collect(); + out.push('…'); + out + } +} diff --git a/crates/dstackup/Cargo.toml b/crates/dstackup/Cargo.toml new file mode 100644 index 00000000..b26bfc51 --- /dev/null +++ b/crates/dstackup/Cargo.toml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: © 2025 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "dstackup" +version.workspace = true +edition.workspace = true +license.workspace = true + +[[bin]] +name = "dstackup" +path = "src/main.rs" + +[dependencies] +anyhow.workspace = true +clap.workspace = true +dstack-core.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] } diff --git a/crates/dstackup/src/main.rs b/crates/dstackup/src/main.rs new file mode 100644 index 00000000..df84c978 --- /dev/null +++ b/crates/dstackup/src/main.rs @@ -0,0 +1,713 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! `dstackup` — host setup and lifecycle for a dstack host. +//! +//! Local + privileged only (touches `/dev/sgx`, systemd, local files, the local +//! VMM socket). Day-to-day app operations live in the separate `dstack` binary. + +use anyhow::{bail, Context, Result}; +use clap::{Parser, Subcommand}; +use dstack_core::config::{self, HostConfig, VmmRender}; +use dstack_core::vmm::{Vmm, DEFAULT_HOST}; +use dstack_core::{host, ports, rpc}; +use serde::{Deserialize, Serialize}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command as PCommand; +use std::time::Duration; + +/// what an install put in place, recorded so re-runs are idempotent and +/// `destroy` can reverse it cleanly. +#[derive(Serialize, Deserialize, Default)] +struct State { + prefix: String, + client_url: String, + auth_port: u16, + /// systemd unit names (without the `.service` suffix). + #[serde(default)] + vmm_unit: String, + #[serde(default)] + auth_unit: String, + #[serde(default)] + kms_vm_id: Option, + #[serde(default)] + kms_url: String, + /// docker-compose project for a key provider we started ourselves. + #[serde(default)] + kp_own_project: Option, +} + +fn state_path(prefix: &Path) -> PathBuf { + prefix.join("dstackup-state.json") +} + +fn read_state(prefix: &Path) -> Option { + let body = fs::read_to_string(state_path(prefix)).ok()?; + serde_json::from_str(&body).ok() +} + +fn write_state(prefix: &Path, st: &State) -> Result<()> { + write(&state_path(prefix), &serde_json::to_string_pretty(st)?) +} + +/// systemd unit name (no `.service` suffix): `dstack-` or, with an +/// instance, `dstack--` (so a fresh install coexists with an +/// existing `dstack-vmm.service`). +fn unit_name(base: &str, instance: &Option) -> String { + match instance { + Some(i) if !i.is_empty() => format!("dstack-{base}-{i}"), + _ => format!("dstack-{base}"), + } +} + +fn systemctl(args: &[&str]) -> bool { + PCommand::new("systemctl") + .args(args) + .status() + .map(|s| s.success()) + .unwrap_or(false) +} + +fn unit_active(unit: &str) -> bool { + systemctl(&["is-active", "--quiet", &format!("{unit}.service")]) +} + +/// write a unit file, reload systemd, and enable+start it (idempotent). +fn install_unit(unit: &str, contents: &str) -> Result<()> { + let path = format!("/etc/systemd/system/{unit}.service"); + fs::write(&path, contents).with_context(|| format!("writing {path}"))?; + systemctl(&["daemon-reload"]); + if !systemctl(&["enable", "--now", &format!("{unit}.service")]) { + bail!("failed to enable+start {unit}.service"); + } + Ok(()) +} + +/// stop, disable, and remove a unit (idempotent — missing unit is fine). +fn remove_unit(unit: &str) { + let svc = format!("{unit}.service"); + let _ = systemctl(&["disable", "--now", &svc]); + let _ = fs::remove_file(format!("/etc/systemd/system/{svc}")); +} + +fn auth_unit_file(bin: &str, allowlist: &Path, port: u16, prefix: &Path) -> String { + format!( + "[Unit]\nDescription=dstack auth webhook\nAfter=network.target\n\n[Service]\n\ + ExecStart={bin} --config {cfg} --address 127.0.0.1 --port {port}\n\ + Restart=always\nRestartSec=2\nWorkingDirectory={wd}\n\n\ + [Install]\nWantedBy=multi-user.target\n", + cfg = allowlist.display(), + wd = prefix.display(), + ) +} + +fn vmm_unit_file(bin: &str, config: &Path, prefix: &Path, auth_unit: &str) -> String { + // KillMode defaults to control-group, so `systemctl stop` tears down the + // VMM + supervisor + CVM qemus together (deterministic teardown). + format!( + "[Unit]\nDescription=dstack VMM\nAfter=network.target docker.service {auth}.service\nWants={auth}.service\n\n\ + [Service]\nExecStart={bin} -c {cfg}\nRestart=always\nRestartSec=2\n\ + TimeoutStopSec=120\nWorkingDirectory={wd}\n\n\ + [Install]\nWantedBy=multi-user.target\n", + auth = auth_unit, + cfg = config.display(), + wd = prefix.display(), + ) +} + +#[derive(Parser)] +#[command(name = "dstackup", version, about = "set up and manage a dstack host")] +struct Cli { + /// VMM control socket / endpoint to talk to (for status and attach). + #[arg(long, global = true)] + host: Option, + + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +// `Install` carries all the host-setup flags; the size gap to `Status`/`Destroy` +// is irrelevant for a CLI enum constructed once at startup. +#[allow(clippy::large_enum_variant)] +enum Command { + /// Bring up the host stack: SGX preflight, render configs, and start the + /// VMM + auth webhook. (Gramine bring-up and KMS-in-CVM bootstrap follow.) + Install { + /// expose the dashboard on this IP (default: bind localhost only — + /// reach it via an SSH tunnel). + #[arg(long, value_name = "IP")] + expose: Option, + + /// guest OS image version to deploy. + #[arg(long, value_name = "VERSION")] + image: Option, + + /// install prefix for configs, certs, run state. + #[arg(long, default_value = "/var/lib/dstack")] + prefix: String, + + /// systemd instance suffix: units become `dstack-vmm-` etc., + /// so a fresh install coexists with an existing `dstack-vmm.service`. + #[arg(long)] + instance: Option, + + /// guest image directory (default: /images). + #[arg(long)] + image_path: Option, + + /// dstack-vmm binary. + #[arg(long, default_value = "dstack-vmm")] + vmm_bin: String, + + /// dstack-auth binary. + #[arg(long, default_value = "dstack-auth")] + auth_bin: String, + + /// dstack-supervisor binary. + #[arg(long, default_value = "dstack-supervisor")] + supervisor_bin: String, + + /// qemu binary. + #[arg(long, default_value = "/usr/bin/qemu-system-x86_64")] + qemu: String, + + /// dashboard TCP port. + #[arg(long, default_value_t = 9080)] + dashboard_port: u16, + + /// auth webhook port. + #[arg(long, default_value_t = 8001)] + auth_port: u16, + + /// host-api vsock port (raise to coexist with an existing VMM on 10000). + #[arg(long, default_value_t = 10000)] + host_api_port: u32, + + /// CID pool start (raise to coexist with an existing VMM). + #[arg(long, default_value_t = 1000)] + cid_start: u32, + + /// use an existing key provider at ADDR:PORT instead of running our own. + #[arg(long, value_name = "ADDR:PORT")] + use_existing_key_provider: Option, + + /// port for our own key provider (when not using an existing one). + #[arg(long, default_value_t = 3443)] + key_provider_port: u16, + + /// key-provider build/compose directory (to start our own). + #[arg(long)] + key_provider_src: Option, + + /// KMS container image. + #[arg(long, default_value = "dstacktee/dstack-kms:0.5.11")] + kms_image: String, + + /// host port for the KMS RPC (default: an auto-picked free port). + #[arg(long)] + kms_port: Option, + + /// skip the KMS-in-CVM deploy (bring up VMM + auth only). + #[arg(long)] + no_kms: bool, + + /// render + write configs only; do not start any process. + #[arg(long)] + no_start: bool, + }, + /// Show the health of the host stack. + Status, + /// Tear down the deployment (keeps configs + KMS keys unless --purge). + Destroy { + /// install prefix to tear down. + #[arg(long, default_value = "/var/lib/dstack")] + prefix: String, + /// also wipe the prefix (configs + KMS keys). + #[arg(long)] + purge: bool, + }, +} + +#[tokio::main] +async fn main() -> Result<()> { + let cli = Cli::parse(); + let host = cli.host.as_deref().unwrap_or(DEFAULT_HOST); + match cli.command { + Command::Status => cmd_status(host).await, + Command::Install { + expose, + image, + prefix, + instance, + image_path, + vmm_bin, + auth_bin, + supervisor_bin, + qemu, + dashboard_port, + auth_port, + host_api_port, + cid_start, + use_existing_key_provider, + key_provider_port, + key_provider_src, + kms_image, + kms_port, + no_kms, + no_start, + } => { + let opts = InstallOpts { + expose, + image, + prefix, + instance, + image_path, + vmm_bin, + auth_bin, + supervisor_bin, + qemu, + dashboard_port, + auth_port, + host_api_port, + cid_start, + use_existing_key_provider, + key_provider_port, + key_provider_src, + kms_image, + kms_port, + no_kms, + no_start, + }; + let _ = host; // install uses its own prefix-derived endpoint + cmd_install(opts).await + } + Command::Destroy { prefix, purge } => cmd_destroy(&prefix, purge).await, + } +} + +async fn cmd_status(host: &str) -> Result<()> { + let sgx = host::check_sgx(); + println!( + "sgx: enclave={} provision={} => {}", + sgx.enclave, + sgx.provision, + if sgx.ok() { "ok" } else { "MISSING" } + ); + match host::detect_host_ip() { + Ok(ip) => { + let note = if host::is_link_local(&ip) { + " (link-local)" + } else { + "" + }; + println!("host ip: {ip}{note}"); + } + Err(e) => println!("host ip: (undetected: {e})"), + } + print!("vmm: {host} => "); + match Vmm::connect(host) { + Ok(vmm) => match vmm.status().await { + Ok(s) => println!("reachable ({} vms)", s.vms.len()), + Err(e) => println!("unreachable ({e})"), + }, + Err(e) => println!("invalid endpoint ({e})"), + } + Ok(()) +} + +struct InstallOpts { + expose: Option, + image: Option, + prefix: String, + instance: Option, + image_path: Option, + vmm_bin: String, + auth_bin: String, + supervisor_bin: String, + qemu: String, + dashboard_port: u16, + auth_port: u16, + host_api_port: u32, + cid_start: u32, + use_existing_key_provider: Option, + key_provider_port: u16, + key_provider_src: Option, + kms_image: String, + kms_port: Option, + no_kms: bool, + no_start: bool, +} + +async fn cmd_install(o: InstallOpts) -> Result<()> { + println!("dstackup install — preflight"); + + // 1. hardware gate (fail fast on non-SGX hosts). + host::require_sgx()?; + println!(" [ok] sgx present"); + + // 2. host IP (informational; used as the bind/SAN when --expose is set). + match host::detect_host_ip() { + Ok(ip) if host::is_link_local(&ip) => { + println!(" [!] host ip {ip} is link-local") + } + Ok(ip) => println!(" [ok] host ip: {ip}"), + Err(e) => println!(" [!] could not detect host ip: {e}"), + } + + // 3. lay out the prefix. + let prefix = Path::new(&o.prefix); + let run_dir = prefix.join("run"); + let images = o + .image_path + .clone() + .unwrap_or_else(|| prefix.join("images").display().to_string()); + for dir in [prefix.to_path_buf(), prefix.join("certs"), run_dir.clone()] { + fs::create_dir_all(&dir).with_context(|| format!("creating {}", dir.display()))?; + } + + // 4. resolve the key provider — run our own unless told to use an existing one. + let (kp_addr, kp_port, kp_own_project) = resolve_key_provider(&o)?; + + // read prior state up front so re-runs are idempotent. + let mut st = read_state(prefix).unwrap_or_default(); + + // KMS host port + URL, known up front so the VMM can inject kms_urls into + // app CVMs (the guest reaches the KMS at 10.0.2.2: via user-net). + let kms_port: u16 = if o.no_kms { + 0 + } else if let Some(p) = o.kms_port { + p + } else if let Some(p) = st.kms_url.rsplit(':').next().and_then(|s| s.parse().ok()) { + p // reuse the port a prior install assigned + } else { + ports::free_local_port()? + }; + let kms_urls = if o.no_kms { + vec![] + } else { + vec![format!("https://10.0.2.2:{kms_port}")] + }; + + // 5. render configs. + let bind = o.expose.clone().unwrap_or_else(|| "127.0.0.1".to_string()); + let dashboard_addr = format!("tcp:{bind}:{}", o.dashboard_port); + let client_url = format!("http://{bind}:{}", o.dashboard_port); + + let vmm = config::vmm_toml(&VmmRender { + dashboard_addr: dashboard_addr.clone(), + image_path: images.clone(), + qemu_path: o.qemu.clone(), + run_dir: run_dir.display().to_string(), + vm_path: prefix.join("vm").display().to_string(), + supervisor_exe: o.supervisor_bin.clone(), + cid_start: o.cid_start, + host_api_port: o.host_api_port, + key_provider_addr: kp_addr, + key_provider_port: kp_port as u32, + kms_urls: kms_urls.clone(), + ..Default::default() + }); + // the KMS-in-CVM reaches the host auth webhook at 10.0.2.2:. + // os-image re-verify is off for the single-node flow (the operator controls + // the image at install time; turning it on needs a published image source). + let host_cfg = HostConfig { + auth_webhook_url: format!("http://10.0.2.2:{}", o.auth_port), + verify_os_image: false, + ..Default::default() + }; + let kms = config::kms_toml(&host_cfg); + let allowlist = config::auth_allowlist_json(&host_cfg); + + let vmm_path = prefix.join("vmm.toml"); + let kms_path = prefix.join("kms.toml"); + let allow_path = prefix.join("auth-allowlist.json"); + write(&vmm_path, &vmm)?; + write(&kms_path, &kms)?; + write(&allow_path, &allowlist)?; + println!( + " [ok] wrote {}, {}, {}", + vmm_path.display(), + kms_path.display(), + allow_path.display() + ); + + if o.no_start { + println!(" (--no-start: configs written; not starting any process)"); + return Ok(()); + } + + st.prefix = prefix.display().to_string(); + st.client_url = client_url.clone(); + st.auth_port = o.auth_port; + let auth_unit = unit_name("auth", &o.instance); + let vmm_unit = unit_name("vmm", &o.instance); + + // 5. auth webhook systemd unit (idempotent). + if unit_active(&auth_unit) { + println!(" [ok] {auth_unit}.service already active"); + } else { + install_unit( + &auth_unit, + &auth_unit_file(&o.auth_bin, &allow_path, o.auth_port, prefix), + ) + .context("installing the auth webhook unit")?; + println!( + " [ok] started {auth_unit}.service on 127.0.0.1:{}", + o.auth_port + ); + } + st.auth_unit = auth_unit.clone(); + + // 6. VMM systemd unit (idempotent). + if vmm_reachable(&client_url).await { + println!(" [ok] VMM already serving at {client_url}"); + } else { + install_unit( + &vmm_unit, + &vmm_unit_file(&o.vmm_bin, &vmm_path, prefix, &auth_unit), + ) + .context("installing the VMM unit")?; + println!(" [ok] started {vmm_unit}.service"); + print!(" [..] waiting for VMM at {client_url} "); + if wait_ready(&client_url, Duration::from_secs(25)).await { + println!("=> ready"); + } else { + println!("=> not ready within timeout (journalctl -u {vmm_unit})"); + } + } + st.vmm_unit = vmm_unit.clone(); + + // persist what we have so far (so a later step / destroy can see it). + st.kp_own_project = kp_own_project; + write_state(prefix, &st)?; + + // 7. deploy + bootstrap the KMS-in-CVM (idempotent). + if o.no_kms { + println!(" (--no-kms: skipping KMS deploy)"); + } else { + let vmm = Vmm::connect(&client_url)?; + let existing = match &st.kms_vm_id { + Some(id) if vmm.has_vm(id).await => Some(id.clone()), + _ => None, + }; + if let Some(id) = existing { + println!(" [ok] KMS CVM already deployed (vm {id})"); + } else { + let img = o + .image + .clone() + .context("KMS deploy needs --image (or pass --no-kms)")?; + let compose = config::kms_app_compose(&kms, &o.kms_image); + let cfg = rpc::VmConfiguration { + name: "dstack-kms".into(), + image: img.clone(), + compose_file: compose, + vcpu: 4, + memory: 8192, + disk_size: 20, + ports: vec![rpc::PortMapping { + protocol: "tcp".into(), + host_address: "127.0.0.1".into(), + host_port: kms_port as u32, + vm_port: 8000, + }], + ..Default::default() + }; + println!(" [..] deploying KMS CVM (os {img}, kms {})", o.kms_image); + let vm_id = vmm + .create_vm(cfg) + .await + .context("CreateVm for KMS failed")?; + print!(" [..] waiting for KMS bootstrap on :{kms_port} "); + if wait_kms_ready(kms_port, Duration::from_secs(240)).await { + println!("=> bootstrapped"); + } else { + println!("=> not ready in time (check `dstack logs {vm_id}` / VMM log)"); + } + st.kms_vm_id = Some(vm_id); + st.kms_url = format!("https://10.0.2.2:{kms_port}"); + write_state(prefix, &st)?; + } + } + + println!(); + println!("dashboard: {client_url} (localhost — reach it via an SSH tunnel)"); + if !st.kms_url.is_empty() { + println!( + "kms: {} (apps reach it via this address)", + st.kms_url + ); + } + println!( + "deploy an app with: dstack --host {client_url} run --image {} --port --allowlist {}", + o.image.as_deref().unwrap_or(""), + allow_path.display() + ); + Ok(()) +} + +/// resolve the key provider for this install. Returns (addr, port, own_project). +fn resolve_key_provider(o: &InstallOpts) -> Result<(String, u16, Option)> { + if let Some(ep) = &o.use_existing_key_provider { + let (addr, port) = split_addr_port(ep)?; + println!(" [ok] using existing key provider at {addr}:{port}"); + return Ok((addr, port, None)); + } + // run our own (default). + let src = o.key_provider_src.as_deref().context( + "no key provider: pass --use-existing-key-provider ADDR:PORT, or --key-provider-src DIR to run our own", + )?; + let project = format!("dstack-kp-{}", o.key_provider_port); + let status = PCommand::new("docker") + .args([ + "compose", + "-p", + &project, + "-f", + &format!("{src}/docker-compose.yaml"), + "up", + "-d", + ]) + .status() + .context("running docker compose for the key provider")?; + if !status.success() { + bail!("failed to start our own key provider (docker compose up)"); + } + println!( + " [ok] started our own key provider (project {project}, :{})", + o.key_provider_port + ); + Ok(("127.0.0.1".to_string(), o.key_provider_port, Some(project))) +} + +fn split_addr_port(ep: &str) -> Result<(String, u16)> { + let (addr, port) = ep + .rsplit_once(':') + .with_context(|| format!("expected ADDR:PORT, got '{ep}'"))?; + Ok(( + addr.to_string(), + port.parse() + .with_context(|| format!("bad port in '{ep}'"))?, + )) +} + +/// poll the KMS `GetMeta` RPC (self-signed TLS) via curl until it bootstraps. +async fn wait_kms_ready(port: u16, timeout: Duration) -> bool { + let deadline = tokio::time::Instant::now() + timeout; + loop { + if kms_get_meta_ok(port) { + return true; + } + if tokio::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(Duration::from_secs(2)).await; + } +} + +fn kms_get_meta_ok(port: u16) -> bool { + let out = PCommand::new("curl") + .args([ + "-sk", + "--max-time", + "4", + "-X", + "POST", + &format!("https://127.0.0.1:{port}/prpc/KMS.GetMeta?json"), + "-d", + "{}", + ]) + .output(); + matches!(out, Ok(o) if String::from_utf8_lossy(&o.stdout).contains("ca_cert")) +} + +/// tear down what `install` started; idempotent. Keeps the prefix (configs + +/// KMS keys) unless `--purge`. +async fn cmd_destroy(prefix: &str, purge: bool) -> Result<()> { + let prefix = Path::new(prefix); + println!("dstackup destroy ({})", prefix.display()); + match read_state(prefix) { + Some(st) => { + // gracefully stop the KMS CVM first so its keys flush to disk + // (unless we purge). Stopping the VMM unit below reaps the supervisor + // and the CVM qemu via the unit's cgroup. + if let Some(id) = &st.kms_vm_id { + if let Ok(vmm) = Vmm::connect(&st.client_url) { + if vmm.has_vm(id).await { + let _ = vmm.stop_vm(id).await; + println!(" stopping KMS CVM (vm {id})"); + } + } + } + // stop + remove the units. `systemctl stop` is synchronous and tears + // down the whole unit cgroup (VMM + supervisor + CVM qemu), so the + // host is back to baseline when this returns. + if !st.vmm_unit.is_empty() { + remove_unit(&st.vmm_unit); + println!(" stopped {}.service", st.vmm_unit); + } + if !st.auth_unit.is_empty() { + remove_unit(&st.auth_unit); + println!(" stopped {}.service", st.auth_unit); + } + systemctl(&["daemon-reload"]); + // stop our own key provider, if we started one. + if let Some(project) = &st.kp_own_project { + let _ = PCommand::new("docker") + .args(["compose", "-p", project, "down"]) + .status(); + println!(" stopped key provider (project {project})"); + } + // remove the runtime-state marker so a later install starts fresh. + let _ = fs::remove_file(state_path(prefix)); + } + None => println!( + " no install state at {} (nothing running to stop)", + prefix.display() + ), + } + + if purge { + if prefix.exists() { + fs::remove_dir_all(prefix).with_context(|| format!("purging {}", prefix.display()))?; + println!(" purged {} (configs + KMS keys wiped)", prefix.display()); + } + } else { + println!( + " configs + KMS keys kept at {} (use --purge to wipe)", + prefix.display() + ); + } + Ok(()) +} + +fn write(path: &Path, body: &str) -> Result<()> { + fs::write(path, body).with_context(|| format!("writing {}", path.display())) +} + +/// one-shot liveness probe of the VMM. +async fn vmm_reachable(client_url: &str) -> bool { + match Vmm::connect(client_url) { + Ok(vmm) => vmm.status().await.is_ok(), + Err(_) => false, + } +} + +/// poll the VMM `Status` RPC until it succeeds or the deadline passes. +async fn wait_ready(client_url: &str, timeout: Duration) -> bool { + let deadline = tokio::time::Instant::now() + timeout; + loop { + if let Ok(vmm) = Vmm::connect(client_url) { + if vmm.status().await.is_ok() { + return true; + } + } + if tokio::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(Duration::from_millis(500)).await; + } +} From 01e84541d90aae764a76162535c62398e03c70b9 Mon Sep 17 00:00:00 2001 From: Hang Yin Date: Thu, 18 Jun 2026 09:34:49 +0000 Subject: [PATCH 3/8] fix(cli): harden allowlist/state writes, destroy, and --expose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses code-review blockers: - allowlist + state files are now written atomically (temp + rename), and the allowlist read-modify-write holds an exclusive flock — so a concurrent `dstack run` or a crash can no longer corrupt it. A torn allowlist matters: the webhook fails closed on invalid JSON, i.e. denies keys to every app on the host. New `dstack-core::fsutil` (write_atomic, lock_exclusive), tested. - `dstackup destroy` now finds the KMS CVM by recorded id OR by name, so an install that died before persisting kms_vm_id can't orphan the CVM. - `--expose` fails fast with guidance (use an SSH tunnel): it would otherwise bind the VM-control plane with neither TLS nor an auth token. Minors: align hex normalization between `dstack run` and the webhook and store the normalized hash; command stubs exit non-zero; dedupe the KMS image default against `config::DEFAULT_KMS_IMAGE`; fix a stale doc comment and duplicate step numbering. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 1 + crates/dstack-auth/src/main.rs | 5 +- crates/dstack-core/Cargo.toml | 3 + crates/dstack-core/src/config.rs | 36 +++++++++--- crates/dstack-core/src/fsutil.rs | 95 ++++++++++++++++++++++++++++++++ crates/dstack-core/src/lib.rs | 7 ++- crates/dstack/src/main.rs | 8 +-- crates/dstackup/src/main.rs | 49 ++++++++++++---- 8 files changed, 177 insertions(+), 27 deletions(-) create mode 100644 crates/dstack-core/src/fsutil.rs diff --git a/Cargo.lock b/Cargo.lock index 2cf5d79b..f9a42d23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2347,6 +2347,7 @@ dependencies = [ "anyhow", "dstack-vmm-rpc", "http-client", + "rustix 0.38.44", "serde_json", "toml", ] diff --git a/crates/dstack-auth/src/main.rs b/crates/dstack-auth/src/main.rs index 03e2a357..d976d69a 100644 --- a/crates/dstack-auth/src/main.rs +++ b/crates/dstack-auth/src/main.rs @@ -85,7 +85,10 @@ struct AppRules { allow_any_device: bool, } -/// normalize a hex string for comparison: trim, drop a `0x` prefix, lowercase. +/// normalize a hex string for comparison: trim, drop a `0x`/`0X` prefix, +/// lowercase. MUST stay in sync with `dstack-core::config::norm_hex` — both +/// `dstack run` (writing the allowlist) and this webhook (reading it) must +/// agree on the canonical form, or apps are silently denied. fn norm(s: &str) -> String { let s = s.trim(); let s = s diff --git a/crates/dstack-core/Cargo.toml b/crates/dstack-core/Cargo.toml index a4d20799..71c31130 100644 --- a/crates/dstack-core/Cargo.toml +++ b/crates/dstack-core/Cargo.toml @@ -13,6 +13,9 @@ anyhow.workspace = true http-client = { workspace = true, features = ["prpc"] } dstack-vmm-rpc.workspace = true serde_json.workspace = true +# advisory file locking (flock) for the allowlist/state read-modify-write; +# already in the dependency tree transitively, so no extra compile cost. +rustix = { version = "0.38", features = ["fs"] } [dev-dependencies] toml.workspace = true diff --git a/crates/dstack-core/src/config.rs b/crates/dstack-core/src/config.rs index eeabe255..0c2e31f8 100644 --- a/crates/dstack-core/src/config.rs +++ b/crates/dstack-core/src/config.rs @@ -17,9 +17,29 @@ use anyhow::{Context, Result}; use serde_json::json; use std::path::Path; +/// normalize a hex string for comparison: trim, drop a single `0x`/`0X` +/// prefix, lowercase. MUST stay in sync with `dstack-auth`'s `norm()` — the +/// webhook compares allowlist entries against KMS-supplied hashes with the same +/// rule, so a divergence here silently denies (or wrongly allows) apps. +pub fn norm_hex(s: &str) -> String { + let s = s.trim(); + let s = s + .strip_prefix("0x") + .or_else(|| s.strip_prefix("0X")) + .unwrap_or(s); + s.to_lowercase() +} + /// register an app (id + compose hash) in the auth webhook's allowlist file, /// so the KMS will issue keys to it. Read-modify-write; idempotent. +/// +/// Holds an exclusive lock for the whole read-modify-write (so two concurrent +/// `dstack run`s can't clobber each other) and writes atomically (so a crash or +/// partial write can't leave torn JSON — which the webhook would read as +/// deny-all). The stored hash is normalized so the on-disk file can't +/// accumulate visually-distinct-but-equal entries. pub fn register_app_in_allowlist(path: &Path, app_id: &str, compose_hash: &str) -> Result<()> { + let _lock = crate::fsutil::lock_exclusive(path)?; let body = std::fs::read_to_string(path) .with_context(|| format!("reading allowlist {}", path.display()))?; let mut v: serde_json::Value = serde_json::from_str(&body).context("parsing allowlist json")?; @@ -28,22 +48,20 @@ pub fn register_app_in_allowlist(path: &Path, app_id: &str, compose_hash: &str) .and_then(|a| a.as_object_mut()) .context("allowlist has no `apps` object")?; let entry = apps - .entry(app_id.to_string()) + .entry(norm_hex(app_id)) .or_insert_with(|| json!({ "composeHashes": [], "devices": [], "allowAnyDevice": true })); let hashes = entry .get_mut("composeHashes") .and_then(|h| h.as_array_mut()) .context("app entry missing `composeHashes`")?; - let norm = compose_hash.trim().trim_start_matches("0x").to_lowercase(); - let present = hashes.iter().any(|h| { - h.as_str() - .map(|s| s.trim_start_matches("0x").to_lowercase() == norm) - .unwrap_or(false) - }); + let norm = norm_hex(compose_hash); + let present = hashes + .iter() + .any(|h| h.as_str().map(|s| norm_hex(s) == norm).unwrap_or(false)); if !present { - hashes.push(serde_json::Value::String(compose_hash.to_string())); + hashes.push(serde_json::Value::String(norm)); } - std::fs::write(path, serde_json::to_string_pretty(&v)?) + crate::fsutil::write_atomic(path, &serde_json::to_string_pretty(&v)?) .with_context(|| format!("writing allowlist {}", path.display()))?; Ok(()) } diff --git a/crates/dstack-core/src/fsutil.rs b/crates/dstack-core/src/fsutil.rs new file mode 100644 index 00000000..0d955a5e --- /dev/null +++ b/crates/dstack-core/src/fsutil.rs @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! small filesystem helpers: atomic file replace + advisory locking. +//! +//! The allowlist and the install state file are read-modify-written from more +//! than one process (`dstack run` adds an app while the webhook reads; a second +//! `dstack run` can race the first). A torn write there is not cosmetic: the +//! auth webhook fails *closed* on invalid JSON, so a half-written allowlist +//! denies keys to every app on the host. These helpers make the write atomic +//! and serialize concurrent writers. + +use anyhow::{Context, Result}; +use std::ffi::OsString; +use std::fs::{File, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; + +/// `path` with `suffix` appended to its full name (not replacing the extension, +/// so `a/b.json` + `.tmp` → `a/b.json.tmp`, a sibling in the same directory). +fn sibling(path: &Path, suffix: &str) -> PathBuf { + let mut s: OsString = path.as_os_str().to_os_string(); + s.push(suffix); + PathBuf::from(s) +} + +/// atomically replace `path`'s contents: write a sibling temp file, fsync it, +/// then rename it over the target. A crash or partial write never leaves the +/// target torn — a reader sees either the old file or the new one, never a +/// fragment. `tmp` and `path` are in the same directory so the rename is atomic. +pub fn write_atomic(path: &Path, contents: &str) -> Result<()> { + let tmp = sibling(path, ".tmp"); + let mut f = + File::create(&tmp).with_context(|| format!("creating temp file {}", tmp.display()))?; + f.write_all(contents.as_bytes()) + .with_context(|| format!("writing {}", tmp.display()))?; + f.sync_all() + .with_context(|| format!("syncing {}", tmp.display()))?; + drop(f); + std::fs::rename(&tmp, path) + .with_context(|| format!("renaming {} -> {}", tmp.display(), path.display()))?; + Ok(()) +} + +/// acquire an exclusive advisory lock tied to `path` (held on a sibling +/// `.lock` file). The lock releases when the returned guard is dropped — +/// including on process exit, so a crash never leaves a stale lock. Hold it +/// around a read-modify-write of `path` to serialize concurrent processes. +#[must_use = "the lock is released when the returned guard is dropped"] +pub fn lock_exclusive(path: &Path) -> Result { + let lock_path = sibling(path, ".lock"); + let f = OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .open(&lock_path) + .with_context(|| format!("opening lock {}", lock_path.display()))?; + rustix::fs::flock(&f, rustix::fs::FlockOperation::LockExclusive) + .with_context(|| format!("locking {}", lock_path.display()))?; + Ok(f) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn atomic_write_replaces_contents() { + let dir = std::env::temp_dir().join(format!("dstack-fsutil-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let p = dir.join("x.json"); + write_atomic(&p, "one").unwrap(); + assert_eq!(std::fs::read_to_string(&p).unwrap(), "one"); + write_atomic(&p, "two").unwrap(); + assert_eq!(std::fs::read_to_string(&p).unwrap(), "two"); + // no temp file left behind. + assert!(!sibling(&p, ".tmp").exists()); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn lock_is_reentrant_within_process_after_drop() { + let dir = std::env::temp_dir().join(format!("dstack-fslock-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let p = dir.join("y.json"); + std::fs::write(&p, "{}").unwrap(); + { + let _g = lock_exclusive(&p).unwrap(); + } + // re-acquire after the first guard dropped. + let _g2 = lock_exclusive(&p).unwrap(); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/dstack-core/src/lib.rs b/crates/dstack-core/src/lib.rs index 8b6fb01e..d4158564 100644 --- a/crates/dstack-core/src/lib.rs +++ b/crates/dstack-core/src/lib.rs @@ -5,9 +5,9 @@ //! shared internals for the `dstack` (client) and `dstackup` (host setup) binaries. //! //! `vmm` is a thin typed client over the VMM `Vmm` prpc service; `compose` builds -//! the app-compose manifest; `ports` does host-port allocation. `config` (config -//! rendering for `dstackup install`) is still a stub — see -//! docs/onboarding-redesign.md §7. +//! the app-compose manifest; `ports` does host-port allocation; `config` renders +//! the config files `dstackup install` writes; `fsutil` provides the atomic +//! write + advisory lock the allowlist/state files need. /// re-export the generated VMM rpc types (VmConfiguration, PortMapping, …). pub use dstack_vmm_rpc as rpc; @@ -19,6 +19,7 @@ pub fn user_agent() -> String { pub mod compose; pub mod config; +pub mod fsutil; pub mod host; pub mod ports; pub mod vmm; diff --git a/crates/dstack/src/main.rs b/crates/dstack/src/main.rs index 1eb3acf7..1ce162a4 100644 --- a/crates/dstack/src/main.rs +++ b/crates/dstack/src/main.rs @@ -217,11 +217,11 @@ async fn cmd_run( } fn stub(name: &str) -> Result<()> { - eprintln!( - "dstack {name}: not yet implemented (scaffold) — {}", + // exit non-zero so `dstack && next` doesn't proceed as if it worked. + bail!( + "dstack {name}: not yet implemented ({})", dstack_core::user_agent() - ); - Ok(()) + ) } async fn cmd_ls(host: &str) -> Result<()> { diff --git a/crates/dstackup/src/main.rs b/crates/dstackup/src/main.rs index df84c978..b6febeca 100644 --- a/crates/dstackup/src/main.rs +++ b/crates/dstackup/src/main.rs @@ -203,7 +203,7 @@ enum Command { key_provider_src: Option, /// KMS container image. - #[arg(long, default_value = "dstacktee/dstack-kms:0.5.11")] + #[arg(long, default_value = config::DEFAULT_KMS_IMAGE)] kms_image: String, /// host port for the KMS RPC (default: an auto-picked free port). @@ -342,6 +342,20 @@ struct InstallOpts { } async fn cmd_install(o: InstallOpts) -> Result<()> { + // --expose is not safe yet: the rendered vmm.toml binds the VM-control + // plane with neither TLS nor an auth token (the management RPCs are not + // behind an auth guard), so exposing it would hand deploy/destroy to anyone + // who can reach the IP. Refuse until the TLS+token transport lands; the + // supported path is localhost + an SSH tunnel. + if let Some(ip) = &o.expose { + bail!( + "--expose {ip} is not yet safe: it would bind the VM-control plane on \ + {ip}:{port} with no TLS and no auth. reach the dashboard over an SSH \ + tunnel instead: ssh -L {port}:127.0.0.1:{port} ", + port = o.dashboard_port + ); + } + println!("dstackup install — preflight"); // 1. hardware gate (fail fast on non-SGX hosts). @@ -445,7 +459,7 @@ async fn cmd_install(o: InstallOpts) -> Result<()> { let auth_unit = unit_name("auth", &o.instance); let vmm_unit = unit_name("vmm", &o.instance); - // 5. auth webhook systemd unit (idempotent). + // 6. auth webhook systemd unit (idempotent). if unit_active(&auth_unit) { println!(" [ok] {auth_unit}.service already active"); } else { @@ -461,7 +475,7 @@ async fn cmd_install(o: InstallOpts) -> Result<()> { } st.auth_unit = auth_unit.clone(); - // 6. VMM systemd unit (idempotent). + // 7. VMM systemd unit (idempotent). if vmm_reachable(&client_url).await { println!(" [ok] VMM already serving at {client_url}"); } else { @@ -484,7 +498,7 @@ async fn cmd_install(o: InstallOpts) -> Result<()> { st.kp_own_project = kp_own_project; write_state(prefix, &st)?; - // 7. deploy + bootstrap the KMS-in-CVM (idempotent). + // 8. deploy + bootstrap the KMS-in-CVM (idempotent). if o.no_kms { println!(" (--no-kms: skipping KMS deploy)"); } else { @@ -633,11 +647,23 @@ async fn cmd_destroy(prefix: &str, purge: bool) -> Result<()> { Some(st) => { // gracefully stop the KMS CVM first so its keys flush to disk // (unless we purge). Stopping the VMM unit below reaps the supervisor - // and the CVM qemu via the unit's cgroup. - if let Some(id) = &st.kms_vm_id { - if let Ok(vmm) = Vmm::connect(&st.client_url) { - if vmm.has_vm(id).await { - let _ = vmm.stop_vm(id).await; + // and the CVM qemu via the unit's cgroup. Look it up by recorded id + // AND by name, so an install that died before persisting kms_vm_id + // (or a torn state file) doesn't leave the CVM orphaned. + if let Ok(vmm) = Vmm::connect(&st.client_url) { + let mut target = st.kms_vm_id.clone(); + if target.is_none() { + if let Ok(s) = vmm.status().await { + target = s + .vms + .iter() + .find(|v| v.name == "dstack-kms") + .map(|v| v.id.clone()); + } + } + if let Some(id) = target { + if vmm.has_vm(&id).await { + let _ = vmm.stop_vm(&id).await; println!(" stopping KMS CVM (vm {id})"); } } @@ -684,8 +710,11 @@ async fn cmd_destroy(prefix: &str, purge: bool) -> Result<()> { Ok(()) } +/// write a file atomically (temp + rename), so a crash mid-write never leaves +/// a torn config or state file. fn write(path: &Path, body: &str) -> Result<()> { - fs::write(path, body).with_context(|| format!("writing {}", path.display())) + dstack_core::fsutil::write_atomic(path, body) + .with_context(|| format!("writing {}", path.display())) } /// one-shot liveness probe of the VMM. From 3ca62cd2ec8a2965434e4ab801299383ed75f08e Mon Sep 17 00:00:00 2001 From: Hang Yin Date: Thu, 18 Jun 2026 09:43:18 +0000 Subject: [PATCH 4/8] feat(cli): pin the app OS image in the allowlist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `dstackup install` now reads the guest image's digest.txt and renders it into the webhook allowlist's `osImages`, so the auth webhook enforces which OS image an app may boot — even though the KMS's own download-verify stays off for the single-node flow. Previously both gates were open: an app could boot under a different, unmeasured image and still receive keys. `bootAuth/kms` ignores `osImages`, so the KMS bootstrap is unaffected. Validated on a TDX host with the official meta-dstack v0.5.11 image: the pin (digest.txt c2aa0186…) matches the KMS-reported image hash — nginx still gets keys and serves HTTP 200 with the pin active — while a wrong image hash is now denied ("os image not allowed"); 0x/case variants normalize correctly. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/dstackup/src/main.rs | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/crates/dstackup/src/main.rs b/crates/dstackup/src/main.rs index b6febeca..c017bd30 100644 --- a/crates/dstackup/src/main.rs +++ b/crates/dstackup/src/main.rs @@ -425,10 +425,22 @@ async fn cmd_install(o: InstallOpts) -> Result<()> { ..Default::default() }); // the KMS-in-CVM reaches the host auth webhook at 10.0.2.2:. - // os-image re-verify is off for the single-node flow (the operator controls - // the image at install time; turning it on needs a published image source). + // The KMS's own image download-verify stays off for the single-node flow + // (it would need a published image source), but we PIN the app OS image in + // the webhook allowlist: digest.txt holds the measured image hash the KMS + // reports for an app, so an app cannot boot under a different, unmeasured + // image and still receive keys. bootAuth/kms ignores osImages, so the KMS + // bootstrap itself is unaffected. + let os_image_hash = resolve_os_image_hash(&images, o.image.as_deref()); + match &os_image_hash { + Some(h) => println!(" [ok] pinning app os image {h}"), + None => println!( + " [!] app os image NOT pinned (no digest.txt) — apps' image will be unchecked" + ), + } let host_cfg = HostConfig { auth_webhook_url: format!("http://10.0.2.2:{}", o.auth_port), + os_image_hash: os_image_hash.unwrap_or_default(), verify_os_image: false, ..Default::default() }; @@ -608,6 +620,17 @@ fn split_addr_port(ep: &str) -> Result<(String, u16)> { )) } +/// read the measured OS-image hash from the guest image's `digest.txt` +/// (`//digest.txt`), used to pin which image apps may boot. +/// Returns None when there's no image selected or no readable digest (e.g. +/// `--no-kms` with no `--image`), in which case apps are left unpinned. +fn resolve_os_image_hash(images: &str, image: Option<&str>) -> Option { + let img = image?; + let path = Path::new(images).join(img).join("digest.txt"); + let hash = fs::read_to_string(path).ok()?.trim().to_string(); + (!hash.is_empty()).then_some(hash) +} + /// poll the KMS `GetMeta` RPC (self-signed TLS) via curl until it bootstraps. async fn wait_kms_ready(port: u16, timeout: Duration) -> bool { let deadline = tokio::time::Instant::now() + timeout; From 8624ac837364424f1b82b0950c29b24a1b05f726 Mon Sep 17 00:00:00 2001 From: Hang Yin Date: Thu, 18 Jun 2026 09:48:55 +0000 Subject: [PATCH 5/8] fix(cli): clarify remaining review minors (messages + docs) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `dstack run`: the "registered" line no longer implies the KMS will honor it regardless of path — it states keys are issued only if the file is the allowlist the auth webhook actually serves. - `dstack logs`: clearer gate for a remote endpoint (remote support lands with the TLS+token transport) instead of a terse "unix only". - `dstackup`: document that the auth webhook's 127.0.0.1 bind is deliberate (it decides key release; CVMs still reach it at 10.0.2.2 via user-mode networking). Message/comment-only; no behavior change. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/dstack-core/src/vmm.rs | 11 ++++++++--- crates/dstack/src/main.rs | 3 ++- crates/dstackup/src/main.rs | 3 +++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/crates/dstack-core/src/vmm.rs b/crates/dstack-core/src/vmm.rs index 27693f0f..8f53418e 100644 --- a/crates/dstack-core/src/vmm.rs +++ b/crates/dstack-core/src/vmm.rs @@ -107,11 +107,16 @@ impl Vmm { /// fetch the last `lines` log lines for a VM (non-following). /// - /// `/logs` is a plain-HTTP endpoint; only the local unix-socket transport is - /// supported today (the shared http helper posts on the remote http path). + /// `/logs` is a plain-HTTP `GET` endpoint. Only the local unix-socket + /// transport is wired today; remote `dstack logs` lands with the TLS+token + /// transport (the shared http helper only `POST`s, and an unauthenticated + /// remote log endpoint shouldn't be reachable before that exists). pub async fn logs(&self, id: &str, lines: u32) -> Result { if !self.is_local() { - bail!("`dstack logs` currently supports a local VMM (unix socket) only"); + bail!( + "`dstack logs` over a remote endpoint isn't wired yet (lands with the \ + TLS+token transport); use the local VMM socket for now" + ); } let path = format!("/logs?id={id}&follow=false&ansi=false&lines={lines}"); let (status, body) = http_request("GET", &self.base, &path, b"").await?; diff --git a/crates/dstack/src/main.rs b/crates/dstack/src/main.rs index 1ce162a4..31dd24ad 100644 --- a/crates/dstack/src/main.rs +++ b/crates/dstack/src/main.rs @@ -195,7 +195,8 @@ async fn cmd_run( .with_context(|| { format!("registering app in {path} (it is usually root-owned — run with sudo, or make it writable)") })?; - println!("registered in allowlist: {path}"); + println!("registered compose hash in {path}"); + println!(" (the KMS issues keys only if this is the allowlist its auth webhook serves)"); } else if !no_kms { println!("note: no --allowlist given; a KMS-mode app needs its compose hash registered to get keys"); } diff --git a/crates/dstackup/src/main.rs b/crates/dstackup/src/main.rs index c017bd30..a4a2b711 100644 --- a/crates/dstackup/src/main.rs +++ b/crates/dstackup/src/main.rs @@ -93,6 +93,9 @@ fn remove_unit(unit: &str) { } fn auth_unit_file(bin: &str, allowlist: &Path, port: u16, prefix: &Path) -> String { + // bind 127.0.0.1 deliberately: the webhook decides key release, so it must + // never be reachable off-host. CVMs still reach it at 10.0.2.2: via + // user-mode networking (NAT), which maps to the host loopback. format!( "[Unit]\nDescription=dstack auth webhook\nAfter=network.target\n\n[Service]\n\ ExecStart={bin} --config {cfg} --address 127.0.0.1 --port {port}\n\ From e602cd0c7574d65e54d14993ae2f76550018a7b4 Mon Sep 17 00:00:00 2001 From: Hang Yin Date: Thu, 18 Jun 2026 13:17:34 +0000 Subject: [PATCH 6/8] =?UTF-8?q?fix(cli):=20address=20review=20Majors=20?= =?UTF-8?q?=E2=80=94=20CID=20coexistence,=20safe=20shelling,=20robust=20re?= =?UTF-8?q?adiness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `dstackup install` auto-picks a CID window that avoids any VMM already running on the host: it reads other `dstack-vmm` processes' configs for their reserved `[cid_start, cid_start+pool)` and any live `guest-cid`, then offsets past them. `--cid-start` is now optional (auto by default) and refuses an explicit value that overlaps a reserved range. - external tools (systemctl/docker/curl) run with a sanitized `PATH`, so a hijacked environment can't substitute a binary while we run as root. - KMS readiness now requires curl to succeed AND a parsed, non-empty `ca_cert` field rather than a substring match (an error body can't read as "ready"), which also confirms our KMS is bound to the expected port. - dstack-auth: a BootInfo wire-contract test pins the camelCase field names the webhook depends on, so a future KMS rename breaks a test, not production. Re-validated end-to-end on a TDX host: with an existing VMM reserving [1000,2000), install with no --cid-start auto-picks 2000; KMS + app CVMs land at 2001/2002 (no collision), nginx serves HTTP 200 with the os-image pin active, clean destroy to baseline. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/dstack-auth/src/main.rs | 35 ++++++++++ crates/dstack-core/src/host.rs | 87 +++++++++++++++++++++++ crates/dstackup/src/main.rs | 124 ++++++++++++++++++++++++++++++--- 3 files changed, 236 insertions(+), 10 deletions(-) diff --git a/crates/dstack-auth/src/main.rs b/crates/dstack-auth/src/main.rs index d976d69a..f6e1c3d7 100644 --- a/crates/dstack-auth/src/main.rs +++ b/crates/dstack-auth/src/main.rs @@ -272,4 +272,39 @@ mod tests { // fail closed: empty allowlist denies (the single-node case never calls this). assert!(!check_kms(&info, &Allowlist::default()).is_allowed); } + + // wire-contract snapshot: BootInfo as the KMS serializes it (camelCase). + // Keep these field names in sync with the kms BootInfo. `#[serde(default)]` + // means extra fields are ignored AND a renamed field deserializes to "" — + // which fails closed, but silently — so this test pins the names we depend + // on: if the KMS renames one, the matching assertion here breaks first. + #[test] + fn deserializes_the_kms_bootinfo_wire_contract() { + let wire = r#"{ + "attestationMode": "dstack", + "mrAggregated": "0xAABB", + "osImageHash": "0xC2AA", + "mrSystem": "0xdead", + "appId": "0xApp1", + "composeHash": "0xHASH", + "instanceId": "0x01", + "deviceId": "0xDEV", + "keyProviderInfo": "kp", + "tcbStatus": "UpToDate", + "advisoryIds": [] + }"#; + let info: BootInfo = serde_json::from_str(wire).expect("kms BootInfo must deserialize"); + assert_eq!(norm(&info.mr_aggregated), "aabb"); + assert_eq!(norm(&info.os_image_hash), "c2aa"); + assert_eq!(norm(&info.app_id), "app1"); + assert_eq!(norm(&info.compose_hash), "hash"); + assert_eq!(norm(&info.device_id), "dev"); + // a check using this payload should pass against a matching allowlist. + let info2: BootInfo = serde_json::from_str(wire).unwrap(); + let al: Allowlist = serde_json::from_str( + r#"{"osImages":["0xC2AA"],"apps":{"0xApp1":{"composeHashes":["0xHASH"],"allowAnyDevice":true}}}"#, + ) + .unwrap(); + assert!(check_app(&info2, &al).is_allowed); + } } diff --git a/crates/dstack-core/src/host.rs b/crates/dstack-core/src/host.rs index cbc49b59..a7ef7ff5 100644 --- a/crates/dstack-core/src/host.rs +++ b/crates/dstack-core/src/host.rs @@ -64,3 +64,90 @@ pub fn detect_host_ip() -> Result { pub fn is_link_local(ip: &IpAddr) -> bool { matches!(ip, IpAddr::V4(v4) if v4.is_link_local()) } + +/// CID windows already spoken for on this host. vsock CIDs are a global +/// resource, so a second VMM must avoid these. Two sources, unioned: +/// +/// * the `[cid_start, cid_start+cid_pool_size)` pool of every other running +/// `dstack-vmm` (read from the `-c ` it was launched with) — this +/// catches the reserved pool even when that VMM has no live CVM right now, +/// and +/// * any live `guest-cid=` from a running QEMU, as a 1-wide range (covers a +/// VMM whose config we couldn't read). +/// +/// Best-effort: unreadable cmdlines/configs are skipped. Ranges are half-open +/// `[start, end)`. +pub fn occupied_cid_ranges() -> Vec<(u32, u32)> { + let mut ranges = Vec::new(); + let Ok(entries) = std::fs::read_dir("/proc") else { + return ranges; + }; + for entry in entries.flatten() { + let Ok(data) = std::fs::read(entry.path().join("cmdline")) else { + continue; + }; + // cmdline is NUL-separated argv. + let args: Vec = data + .split(|&b| b == 0) + .filter(|s| !s.is_empty()) + .map(|s| String::from_utf8_lossy(s).into_owned()) + .collect(); + if args.is_empty() { + continue; + } + // (a) another dstack-vmm's reserved pool, from its config. + let is_vmm = Path::new(&args[0]).file_name().and_then(|f| f.to_str()) == Some("dstack-vmm"); + if is_vmm { + if let Some(cfg) = arg_value(&args, "-c").or_else(|| arg_value(&args, "--config")) { + if let Some((start, size)) = read_cid_pool(&cfg) { + ranges.push((start, start.saturating_add(size))); + } + } + } + // (b) any live guest-cid token. + for arg in &args { + for tok in arg.split([',', ' ']) { + if let Some(rest) = tok.strip_prefix("guest-cid=") { + if let Ok(n) = rest.trim().parse::() { + ranges.push((n, n.saturating_add(1))); + } + } + } + } + } + ranges +} + +/// value following `flag` in an argv (`-c foo` → `foo`). +fn arg_value(args: &[String], flag: &str) -> Option { + args.windows(2).find(|w| w[0] == flag).map(|w| w[1].clone()) +} + +/// read `[cvm]` `cid_start` / `cid_pool_size` from a vmm.toml by line scan +/// (avoids a toml dependency; tolerates partial configs — size defaults 1000). +fn read_cid_pool(config_path: &str) -> Option<(u32, u32)> { + let text = std::fs::read_to_string(config_path).ok()?; + let mut start = None; + let mut size = None; + for line in text.lines() { + let l = line.trim(); + if let Some(v) = l.strip_prefix("cid_start") { + start = parse_toml_u32(v); + } else if let Some(v) = l.strip_prefix("cid_pool_size") { + size = parse_toml_u32(v); + } + } + Some((start?, size.unwrap_or(1000))) +} + +/// parse the `= ` that follows a key (tolerating a trailing `# comment`). +fn parse_toml_u32(after_key: &str) -> Option { + after_key + .trim_start() + .strip_prefix('=')? + .split('#') + .next()? + .trim() + .parse() + .ok() +} diff --git a/crates/dstackup/src/main.rs b/crates/dstackup/src/main.rs index a4a2b711..974f59f0 100644 --- a/crates/dstackup/src/main.rs +++ b/crates/dstackup/src/main.rs @@ -62,14 +62,68 @@ fn unit_name(base: &str, instance: &Option) -> String { } } +/// spawn an external tool (systemctl/docker/curl) with a sanitized `PATH`, so a +/// hijacked environment can't substitute a different binary while we run as root. +fn tool(bin: &str) -> PCommand { + let mut c = PCommand::new(bin); + c.env("PATH", "/usr/sbin:/usr/bin:/sbin:/bin"); + c +} + fn systemctl(args: &[&str]) -> bool { - PCommand::new("systemctl") + tool("systemctl") .args(args) .status() .map(|s| s.success()) .unwrap_or(false) } +/// size of a VMM's CID pool (matches `config::VmmRender` default). +const CID_POOL_SIZE: u32 = 1000; +/// default CID pool start when nothing else is using that range. +const DEFAULT_CID_START: u32 = 1000; + +/// whether `[start, start+CID_POOL_SIZE)` intersects any occupied range. +fn cid_window_overlaps(start: u32, occupied: &[(u32, u32)]) -> bool { + let end = start.saturating_add(CID_POOL_SIZE); + occupied.iter().any(|&(s, e)| start < e && s < end) +} + +/// the lowest pool-aligned CID block at or above every occupied range. +fn next_free_cid_block(occupied: &[(u32, u32)]) -> u32 { + let max_end = occupied + .iter() + .map(|&(_, e)| e) + .max() + .unwrap_or(DEFAULT_CID_START); + (max_end.div_ceil(CID_POOL_SIZE) * CID_POOL_SIZE).max(DEFAULT_CID_START) +} + +/// choose a CID window `[start, start+CID_POOL_SIZE)` that won't collide with a +/// VMM already running on this host. With an explicit `--cid-start`, honor it +/// but refuse on overlap; without one, use the default unless it's taken, then +/// move to the next free block. +fn pick_cid_start(explicit: Option, occupied: &[(u32, u32)]) -> Result { + match explicit { + Some(n) => { + if cid_window_overlaps(n, occupied) { + bail!( + "--cid-start {n} overlaps a CID range already reserved on this host; \ + pick a free start, e.g. --cid-start {}", + next_free_cid_block(occupied) + ); + } + Ok(n) + } + None if !cid_window_overlaps(DEFAULT_CID_START, occupied) => Ok(DEFAULT_CID_START), + None => { + let start = next_free_cid_block(occupied); + println!(" [ok] cid-start {start} (avoids CIDs already reserved by another VMM)"); + Ok(start) + } + } +} + fn unit_active(unit: &str) -> bool { systemctl(&["is-active", "--quiet", &format!("{unit}.service")]) } @@ -189,9 +243,10 @@ enum Command { #[arg(long, default_value_t = 10000)] host_api_port: u32, - /// CID pool start (raise to coexist with an existing VMM). - #[arg(long, default_value_t = 1000)] - cid_start: u32, + /// CID pool start (default: auto — the first free block, so it coexists + /// with any VMM already running on this host). + #[arg(long)] + cid_start: Option, /// use an existing key provider at ADDR:PORT instead of running our own. #[arg(long, value_name = "ADDR:PORT")] @@ -334,7 +389,7 @@ struct InstallOpts { dashboard_port: u16, auth_port: u16, host_api_port: u32, - cid_start: u32, + cid_start: Option, use_existing_key_provider: Option, key_provider_port: u16, key_provider_src: Option, @@ -413,6 +468,9 @@ async fn cmd_install(o: InstallOpts) -> Result<()> { let dashboard_addr = format!("tcp:{bind}:{}", o.dashboard_port); let client_url = format!("http://{bind}:{}", o.dashboard_port); + // pick a CID window that doesn't collide with a VMM already running here. + let cid_start = pick_cid_start(o.cid_start, &host::occupied_cid_ranges())?; + let vmm = config::vmm_toml(&VmmRender { dashboard_addr: dashboard_addr.clone(), image_path: images.clone(), @@ -420,7 +478,7 @@ async fn cmd_install(o: InstallOpts) -> Result<()> { run_dir: run_dir.display().to_string(), vm_path: prefix.join("vm").display().to_string(), supervisor_exe: o.supervisor_bin.clone(), - cid_start: o.cid_start, + cid_start, host_api_port: o.host_api_port, key_provider_addr: kp_addr, key_provider_port: kp_port as u32, @@ -590,7 +648,7 @@ fn resolve_key_provider(o: &InstallOpts) -> Result<(String, u16, Option) "no key provider: pass --use-existing-key-provider ADDR:PORT, or --key-provider-src DIR to run our own", )?; let project = format!("dstack-kp-{}", o.key_provider_port); - let status = PCommand::new("docker") + let status = tool("docker") .args([ "compose", "-p", @@ -648,8 +706,14 @@ async fn wait_kms_ready(port: u16, timeout: Duration) -> bool { } } +/// is the KMS bootstrapped and answering on `port`? curl (not the typed +/// client) because the KMS serves self-signed RA-TLS we deliberately don't +/// verify here (`-k`); but require curl to actually succeed AND a parsed, +/// non-empty `ca_cert` field — so an error body or a partial response that +/// merely contains the substring can't read as "ready". A real success here +/// also confirms it's our KMS bound to this exact port (port-verification). fn kms_get_meta_ok(port: u16) -> bool { - let out = PCommand::new("curl") + let out = tool("curl") .args([ "-sk", "--max-time", @@ -661,7 +725,18 @@ fn kms_get_meta_ok(port: u16) -> bool { "{}", ]) .output(); - matches!(out, Ok(o) if String::from_utf8_lossy(&o.stdout).contains("ca_cert")) + let Ok(out) = out else { return false }; + if !out.status.success() { + return false; + } + serde_json::from_slice::(&out.stdout) + .ok() + .and_then(|v| { + v.get("ca_cert") + .and_then(|c| c.as_str()) + .map(|s| !s.is_empty()) + }) + .unwrap_or(false) } /// tear down what `install` started; idempotent. Keeps the prefix (configs + @@ -708,7 +783,7 @@ async fn cmd_destroy(prefix: &str, purge: bool) -> Result<()> { systemctl(&["daemon-reload"]); // stop our own key provider, if we started one. if let Some(project) = &st.kp_own_project { - let _ = PCommand::new("docker") + let _ = tool("docker") .args(["compose", "-p", project, "down"]) .status(); println!(" stopped key provider (project {project})"); @@ -766,3 +841,32 @@ async fn wait_ready(client_url: &str, timeout: Duration) -> bool { tokio::time::sleep(Duration::from_millis(500)).await; } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cid_default_when_range_free() { + assert_eq!(pick_cid_start(None, &[]).unwrap(), 1000); + // a pool entirely above the default window leaves it free. + assert_eq!(pick_cid_start(None, &[(2000, 3000)]).unwrap(), 1000); + } + + #[test] + fn cid_auto_offsets_past_an_existing_vmm() { + // another VMM reserving [1000,2000) -> jump to 2000. + assert_eq!(pick_cid_start(None, &[(1000, 2000)]).unwrap(), 2000); + // its reserved pool plus a stray live CVM at 2500 -> jump past it. + assert_eq!( + pick_cid_start(None, &[(1000, 2000), (2500, 2501)]).unwrap(), + 3000 + ); + } + + #[test] + fn cid_explicit_honored_or_refused() { + assert_eq!(pick_cid_start(Some(2000), &[(1000, 2000)]).unwrap(), 2000); + assert!(pick_cid_start(Some(1000), &[(1000, 2000)]).is_err()); + } +} From 50f6017287ef8b9b69fbf3010ff56bd7d00effc9 Mon Sep 17 00:00:00 2001 From: Hang Yin Date: Thu, 18 Jun 2026 18:07:35 +0000 Subject: [PATCH 7/8] =?UTF-8?q?fix(cli):=20close=202nd-review=20blockers?= =?UTF-8?q?=20=E2=80=94=20fail-closed=20os-image=20pin,=20port=20coexisten?= =?UTF-8?q?ce?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From a second clean-context review of the hardened branch: - B1 (fail-open): a missing/empty digest.txt silently disabled the OS-image pin (osImages=[] => webhook allow-any-image) while install only warned. Now the pin is resolved in preflight and `dstackup install` BAILS in KMS mode unless --allow-unpinned-image is passed. Fail-closed. - B2 (half-install): coexistence was handled only for CIDs; the dashboard/auth TCP ports and the host-api vsock port had fixed defaults with no detection, so a second install half-installed then failed on bind. All collision checks (CIDs + ports) now run in a preflight BEFORE any side effect — TCP ports are bind-tested, the host-api port is checked against other dstack-vmm configs — and refuse with guidance. - M1: `dstack run --allowlist ` no longer misreports ENOENT as a permissions problem; distinct "run dstackup install first" message. - M2: TCB-status enforcement intentionally NOT added (single-node operator trusts their own host; real TDX hosts often report non-UpToDate) — documented as a deliberate deviation from auth-simple. - minors: device_ok matches auth-simple (empty list = any device); write_atomic fsyncs the parent dir (rename durability); lowercase two error messages; comments on the CID-block math, the compose-hash clone, and the /logs transport. Validated on a TDX host: missing-pin and port-collision installs both bail in preflight with zero side effects; --allow-unpinned-image opt-out works; happy path (real image) -> KMS bootstrap -> nginx HTTP 200 -> clean destroy. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/dstack-auth/src/main.rs | 12 +++- crates/dstack-core/src/config.rs | 18 +++++- crates/dstack-core/src/fsutil.rs | 13 +++- crates/dstack-core/src/host.rs | 54 ++++++++++++++++ crates/dstack-core/src/ports.rs | 7 +++ crates/dstack-core/src/vmm.rs | 8 ++- crates/dstack/src/main.rs | 4 +- crates/dstackup/src/main.rs | 103 +++++++++++++++++++++++-------- 8 files changed, 182 insertions(+), 37 deletions(-) diff --git a/crates/dstack-auth/src/main.rs b/crates/dstack-auth/src/main.rs index f6e1c3d7..0bf62c2b 100644 --- a/crates/dstack-auth/src/main.rs +++ b/crates/dstack-auth/src/main.rs @@ -11,6 +11,14 @@ //! allowlist). The allowlist JSON is re-read on every request, so `dstack run` //! can add an app without a restart. Fails closed: a missing/invalid allowlist //! denies everything. +//! +//! Deliberate Tier-1 deviation from `auth-simple`: it does NOT enforce +//! `tcbStatus == UpToDate`. Real TDX hosts routinely report a non-`UpToDate` +//! TCB (microcode / TDX-module behind), and in the single-node model the +//! operator already controls and trusts their own host, so a hard TCB gate +//! would be friction without a corresponding trust gain here. Re-add the check +//! (capture `tcbStatus`, deny unless `UpToDate`) if this grows into a +//! multi-tenant / hosted deployment. use anyhow::Result; use clap::Parser; @@ -103,8 +111,10 @@ fn contains(list: &[String], value: &str) -> bool { list.iter().any(|x| norm(x) == v) } +/// matches auth-simple: an empty `devices` list means "any device" even when +/// `allowAnyDevice` is false (it only enforces a non-empty list). fn device_ok(allow_any: bool, devices: &[String], device_id: &str) -> bool { - allow_any || contains(devices, device_id) + allow_any || devices.is_empty() || contains(devices, device_id) } fn deny(al: &Allowlist, reason: &str) -> BootResponse { diff --git a/crates/dstack-core/src/config.rs b/crates/dstack-core/src/config.rs index 0c2e31f8..a5b61d48 100644 --- a/crates/dstack-core/src/config.rs +++ b/crates/dstack-core/src/config.rs @@ -40,8 +40,22 @@ pub fn norm_hex(s: &str) -> String { /// accumulate visually-distinct-but-equal entries. pub fn register_app_in_allowlist(path: &Path, app_id: &str, compose_hash: &str) -> Result<()> { let _lock = crate::fsutil::lock_exclusive(path)?; - let body = std::fs::read_to_string(path) - .with_context(|| format!("reading allowlist {}", path.display()))?; + let body = match std::fs::read_to_string(path) { + Ok(b) => b, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => anyhow::bail!( + "allowlist {} does not exist — run `dstackup install` first, or check the --allowlist path", + path.display() + ), + Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => { + return Err(e).with_context(|| { + format!( + "reading allowlist {} (it is usually root-owned — run with sudo)", + path.display() + ) + }) + } + Err(e) => return Err(e).with_context(|| format!("reading allowlist {}", path.display())), + }; let mut v: serde_json::Value = serde_json::from_str(&body).context("parsing allowlist json")?; let apps = v .get_mut("apps") diff --git a/crates/dstack-core/src/fsutil.rs b/crates/dstack-core/src/fsutil.rs index 0d955a5e..0fa20ac2 100644 --- a/crates/dstack-core/src/fsutil.rs +++ b/crates/dstack-core/src/fsutil.rs @@ -26,9 +26,10 @@ fn sibling(path: &Path, suffix: &str) -> PathBuf { } /// atomically replace `path`'s contents: write a sibling temp file, fsync it, -/// then rename it over the target. A crash or partial write never leaves the -/// target torn — a reader sees either the old file or the new one, never a -/// fragment. `tmp` and `path` are in the same directory so the rename is atomic. +/// rename it over the target, then fsync the directory. A reader (or a crash) +/// sees either the old file or the new one, never a fragment, and the rename is +/// durable across a power loss. `tmp` and `path` are in the same directory so +/// the rename is atomic. pub fn write_atomic(path: &Path, contents: &str) -> Result<()> { let tmp = sibling(path, ".tmp"); let mut f = @@ -40,6 +41,12 @@ pub fn write_atomic(path: &Path, contents: &str) -> Result<()> { drop(f); std::fs::rename(&tmp, path) .with_context(|| format!("renaming {} -> {}", tmp.display(), path.display()))?; + // fsync the containing directory so the rename itself survives a crash. + if let Some(dir) = path.parent().filter(|d| !d.as_os_str().is_empty()) { + if let Ok(d) = File::open(dir) { + let _ = d.sync_all(); + } + } Ok(()) } diff --git a/crates/dstack-core/src/host.rs b/crates/dstack-core/src/host.rs index a7ef7ff5..6ac39bf9 100644 --- a/crates/dstack-core/src/host.rs +++ b/crates/dstack-core/src/host.rs @@ -151,3 +151,57 @@ fn parse_toml_u32(after_key: &str) -> Option { .parse() .ok() } + +/// host-api vsock ports reserved by other running `dstack-vmm` processes (read +/// from each one's `-c `), so a fresh install can avoid colliding on the +/// host's vsock port space. Best-effort; sorted, deduped. +pub fn other_vmm_host_api_ports() -> Vec { + let mut ports = Vec::new(); + let Ok(entries) = std::fs::read_dir("/proc") else { + return ports; + }; + for entry in entries.flatten() { + let Ok(data) = std::fs::read(entry.path().join("cmdline")) else { + continue; + }; + let args: Vec = data + .split(|&b| b == 0) + .filter(|s| !s.is_empty()) + .map(|s| String::from_utf8_lossy(s).into_owned()) + .collect(); + if args.is_empty() { + continue; + } + if Path::new(&args[0]).file_name().and_then(|f| f.to_str()) != Some("dstack-vmm") { + continue; + } + if let Some(cfg) = arg_value(&args, "-c").or_else(|| arg_value(&args, "--config")) { + if let Some(p) = read_host_api_port(&cfg) { + ports.push(p); + } + } + } + ports.sort_unstable(); + ports.dedup(); + ports +} + +/// read the `[host_api]` `port` from a vmm.toml (section-aware: `port` appears +/// under several tables, so we only read the one inside `[host_api]`). +fn read_host_api_port(config_path: &str) -> Option { + let text = std::fs::read_to_string(config_path).ok()?; + let mut in_host_api = false; + for line in text.lines() { + let l = line.trim(); + if l.starts_with('[') { + in_host_api = l == "[host_api]"; + } else if in_host_api { + if let Some(v) = l.strip_prefix("port") { + if let Some(p) = parse_toml_u32(v) { + return Some(p); + } + } + } + } + None +} diff --git a/crates/dstack-core/src/ports.rs b/crates/dstack-core/src/ports.rs index a3919877..32a65ee0 100644 --- a/crates/dstack-core/src/ports.rs +++ b/crates/dstack-core/src/ports.rs @@ -19,6 +19,13 @@ pub fn free_local_port() -> Result { Ok(port) } +/// whether `addr:port` can be bound right now. Best-effort and racy, but it +/// catches the common "another service already owns this port" case so an +/// install can refuse before it starts changing the host. +pub fn tcp_port_free(addr: &str, port: u16) -> bool { + TcpListener::bind((addr, port)).is_ok() +} + /// parse a `--port` spec into a [`PortMapping`], auto-allocating the host port /// when it is omitted, `0`, or `auto`. Accepted forms: /// diff --git a/crates/dstack-core/src/vmm.rs b/crates/dstack-core/src/vmm.rs index 8f53418e..556a59b1 100644 --- a/crates/dstack-core/src/vmm.rs +++ b/crates/dstack-core/src/vmm.rs @@ -63,7 +63,8 @@ impl Vmm { } /// compute the compose hash for a VM configuration (no side effects). - /// the app id is the first 40 hex chars of this hash. + /// the app id is the first 40 hex chars of this hash. takes `&cfg` and + /// clones once because the generated prpc client consumes its argument. pub async fn get_compose_hash(&self, cfg: &VmConfiguration) -> Result { self.rpc .get_compose_hash(cfg.clone()) @@ -109,8 +110,9 @@ impl Vmm { /// /// `/logs` is a plain-HTTP `GET` endpoint. Only the local unix-socket /// transport is wired today; remote `dstack logs` lands with the TLS+token - /// transport (the shared http helper only `POST`s, and an unauthenticated - /// remote log endpoint shouldn't be reachable before that exists). + /// transport (the shared `http_request` honors the method on the unix/vsock + /// paths but hardcodes `POST` on the remote http path, and an unauthenticated + /// remote log endpoint shouldn't be reachable before that exists anyway). pub async fn logs(&self, id: &str, lines: u32) -> Result { if !self.is_local() { bail!( diff --git a/crates/dstack/src/main.rs b/crates/dstack/src/main.rs index 31dd24ad..b6725bbf 100644 --- a/crates/dstack/src/main.rs +++ b/crates/dstack/src/main.rs @@ -192,9 +192,7 @@ async fn cmd_run( // register the compose hash so the KMS will issue keys (KMS mode, local). if let Some(path) = allowlist { dstack_core::config::register_app_in_allowlist(std::path::Path::new(path), &app_id, &hash) - .with_context(|| { - format!("registering app in {path} (it is usually root-owned — run with sudo, or make it writable)") - })?; + .with_context(|| format!("registering app in {path}"))?; println!("registered compose hash in {path}"); println!(" (the KMS issues keys only if this is the allowlist its auth webhook serves)"); } else if !no_kms { diff --git a/crates/dstackup/src/main.rs b/crates/dstackup/src/main.rs index 974f59f0..26a1593a 100644 --- a/crates/dstackup/src/main.rs +++ b/crates/dstackup/src/main.rs @@ -89,7 +89,9 @@ fn cid_window_overlaps(start: u32, occupied: &[(u32, u32)]) -> bool { occupied.iter().any(|&(s, e)| start < e && s < end) } -/// the lowest pool-aligned CID block at or above every occupied range. +/// the lowest pool-aligned CID block at or above every occupied range. We jump +/// above the highest reservation rather than packing into a free gap below it — +/// simpler, and the result is always collision-free. fn next_free_cid_block(occupied: &[(u32, u32)]) -> u32 { let max_end = occupied .iter() @@ -272,6 +274,11 @@ enum Command { #[arg(long)] no_kms: bool, + /// proceed even if the app OS image can't be pinned (no digest.txt) — + /// apps will boot any unmeasured image and still get keys. NOT recommended. + #[arg(long)] + allow_unpinned_image: bool, + /// render + write configs only; do not start any process. #[arg(long)] no_start: bool, @@ -315,6 +322,7 @@ async fn main() -> Result<()> { kms_image, kms_port, no_kms, + allow_unpinned_image, no_start, } => { let opts = InstallOpts { @@ -337,6 +345,7 @@ async fn main() -> Result<()> { kms_image, kms_port, no_kms, + allow_unpinned_image, no_start, }; let _ = host; // install uses its own prefix-derived endpoint @@ -396,6 +405,7 @@ struct InstallOpts { kms_image: String, kms_port: Option, no_kms: bool, + allow_unpinned_image: bool, no_start: bool, } @@ -429,18 +439,26 @@ async fn cmd_install(o: InstallOpts) -> Result<()> { Err(e) => println!(" [!] could not detect host ip: {e}"), } - // 3. lay out the prefix. + // 3. resolve paths (no side effects yet). let prefix = Path::new(&o.prefix); let run_dir = prefix.join("run"); let images = o .image_path .clone() .unwrap_or_else(|| prefix.join("images").display().to_string()); + + // 4. preflight — fail BEFORE any side effect (key provider, dirs, units), so + // a CID/port clash or a missing os-image pin can't half-install the host. + let cid_start = pick_cid_start(o.cid_start, &host::occupied_cid_ranges())?; + preflight_ports(&o)?; + let os_image_hash = resolve_image_pin(&o, &images)?; + + // 5. lay out the prefix. for dir in [prefix.to_path_buf(), prefix.join("certs"), run_dir.clone()] { fs::create_dir_all(&dir).with_context(|| format!("creating {}", dir.display()))?; } - // 4. resolve the key provider — run our own unless told to use an existing one. + // 6. resolve the key provider — run our own unless told to use an existing one. let (kp_addr, kp_port, kp_own_project) = resolve_key_provider(&o)?; // read prior state up front so re-runs are idempotent. @@ -463,14 +481,11 @@ async fn cmd_install(o: InstallOpts) -> Result<()> { vec![format!("https://10.0.2.2:{kms_port}")] }; - // 5. render configs. + // 7. render configs. let bind = o.expose.clone().unwrap_or_else(|| "127.0.0.1".to_string()); let dashboard_addr = format!("tcp:{bind}:{}", o.dashboard_port); let client_url = format!("http://{bind}:{}", o.dashboard_port); - // pick a CID window that doesn't collide with a VMM already running here. - let cid_start = pick_cid_start(o.cid_start, &host::occupied_cid_ranges())?; - let vmm = config::vmm_toml(&VmmRender { dashboard_addr: dashboard_addr.clone(), image_path: images.clone(), @@ -488,17 +503,10 @@ async fn cmd_install(o: InstallOpts) -> Result<()> { // the KMS-in-CVM reaches the host auth webhook at 10.0.2.2:. // The KMS's own image download-verify stays off for the single-node flow // (it would need a published image source), but we PIN the app OS image in - // the webhook allowlist: digest.txt holds the measured image hash the KMS - // reports for an app, so an app cannot boot under a different, unmeasured - // image and still receive keys. bootAuth/kms ignores osImages, so the KMS - // bootstrap itself is unaffected. - let os_image_hash = resolve_os_image_hash(&images, o.image.as_deref()); - match &os_image_hash { - Some(h) => println!(" [ok] pinning app os image {h}"), - None => println!( - " [!] app os image NOT pinned (no digest.txt) — apps' image will be unchecked" - ), - } + // the webhook allowlist (resolved in preflight, fail-closed): digest.txt + // holds the measured image hash the KMS reports for an app, so an app cannot + // boot under a different, unmeasured image and still receive keys. + // bootAuth/kms ignores osImages, so the KMS bootstrap itself is unaffected. let host_cfg = HostConfig { auth_webhook_url: format!("http://10.0.2.2:{}", o.auth_port), os_image_hash: os_image_hash.unwrap_or_default(), @@ -532,7 +540,7 @@ async fn cmd_install(o: InstallOpts) -> Result<()> { let auth_unit = unit_name("auth", &o.instance); let vmm_unit = unit_name("vmm", &o.instance); - // 6. auth webhook systemd unit (idempotent). + // 8. auth webhook systemd unit (idempotent). if unit_active(&auth_unit) { println!(" [ok] {auth_unit}.service already active"); } else { @@ -548,7 +556,7 @@ async fn cmd_install(o: InstallOpts) -> Result<()> { } st.auth_unit = auth_unit.clone(); - // 7. VMM systemd unit (idempotent). + // 9. VMM systemd unit (idempotent). if vmm_reachable(&client_url).await { println!(" [ok] VMM already serving at {client_url}"); } else { @@ -571,7 +579,7 @@ async fn cmd_install(o: InstallOpts) -> Result<()> { st.kp_own_project = kp_own_project; write_state(prefix, &st)?; - // 8. deploy + bootstrap the KMS-in-CVM (idempotent). + // 10. deploy + bootstrap the KMS-in-CVM (idempotent). if o.no_kms { println!(" (--no-kms: skipping KMS deploy)"); } else { @@ -586,7 +594,7 @@ async fn cmd_install(o: InstallOpts) -> Result<()> { let img = o .image .clone() - .context("KMS deploy needs --image (or pass --no-kms)")?; + .context("kms deploy needs --image (or pass --no-kms)")?; let compose = config::kms_app_compose(&kms, &o.kms_image); let cfg = rpc::VmConfiguration { name: "dstack-kms".into(), @@ -607,7 +615,7 @@ async fn cmd_install(o: InstallOpts) -> Result<()> { let vm_id = vmm .create_vm(cfg) .await - .context("CreateVm for KMS failed")?; + .context("createVm for the kms cvm failed")?; print!(" [..] waiting for KMS bootstrap on :{kms_port} "); if wait_kms_ready(kms_port, Duration::from_secs(240)).await { println!("=> bootstrapped"); @@ -683,8 +691,7 @@ fn split_addr_port(ep: &str) -> Result<(String, u16)> { /// read the measured OS-image hash from the guest image's `digest.txt` /// (`//digest.txt`), used to pin which image apps may boot. -/// Returns None when there's no image selected or no readable digest (e.g. -/// `--no-kms` with no `--image`), in which case apps are left unpinned. +/// Returns None when there's no image selected or no readable digest. fn resolve_os_image_hash(images: &str, image: Option<&str>) -> Option { let img = image?; let path = Path::new(images).join(img).join("digest.txt"); @@ -692,6 +699,52 @@ fn resolve_os_image_hash(images: &str, image: Option<&str>) -> Option { (!hash.is_empty()).then_some(hash) } +/// resolve the OS-image pin, failing CLOSED: in KMS mode a missing/empty +/// `digest.txt` is a hard error (an unpinned app could boot any unmeasured +/// image and still get keys), unless the operator opts out with +/// `--allow-unpinned-image`. Returns Some(hash) to pin, or None when pinning is +/// deliberately off (`--no-kms`, or the explicit opt-out). +fn resolve_image_pin(o: &InstallOpts, images: &str) -> Result> { + let hash = resolve_os_image_hash(images, o.image.as_deref()); + match &hash { + Some(h) => println!(" [ok] pinning app os image {h}"), + None if o.no_kms => {} + None if o.allow_unpinned_image => { + println!(" [!] app os image NOT pinned (--allow-unpinned-image) — apps' image is unchecked") + } + None => bail!( + "no os-image pin: could not read a digest.txt for image {:?} under {images} — an app \ + could boot any unmeasured image and still get keys. fix --image/--image-path, or pass \ + --allow-unpinned-image to proceed unpinned (not recommended)", + o.image.as_deref().unwrap_or("") + ), + } + Ok(hash) +} + +/// fail BEFORE any side effect if a port we need is already taken, so a clash +/// refuses cleanly instead of half-installing. CIDs auto-offset (see +/// `pick_cid_start`); ports are user-facing, so we refuse with guidance rather +/// than silently moving the address the operator will connect to. +fn preflight_ports(o: &InstallOpts) -> Result<()> { + let bind = o.expose.clone().unwrap_or_else(|| "127.0.0.1".to_string()); + for (what, flag, port) in [ + ("dashboard", "--dashboard-port", o.dashboard_port), + ("auth webhook", "--auth-port", o.auth_port), + ] { + if !ports::tcp_port_free(&bind, port) { + bail!("{what} port {bind}:{port} is already in use; pass {flag} "); + } + } + if !o.no_kms && host::other_vmm_host_api_ports().contains(&o.host_api_port) { + bail!( + "host-api vsock port {} is already reserved by another dstack-vmm; pass --host-api-port ", + o.host_api_port + ); + } + Ok(()) +} + /// poll the KMS `GetMeta` RPC (self-signed TLS) via curl until it bootstraps. async fn wait_kms_ready(port: u16, timeout: Duration) -> bool { let deadline = tokio::time::Instant::now() + timeout; From e3ad55a4fdecbc6bce85c47222e173bdf78dcff6 Mon Sep 17 00:00:00 2001 From: Hang Yin Date: Thu, 18 Jun 2026 20:26:33 +0000 Subject: [PATCH 8/8] fix(dstack-core): satisfy CI clippy (no expect/unwrap in lib) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI runs `cargo clippy -- -D warnings -D clippy::expect_used -D clippy::unwrap_used` (stricter than the `-D warnings` documented in CLAUDE.md). The three infallible `to_string_pretty(&value).expect(...)` calls now pretty-print via the Value's Display (`{:#}`), which is byte-identical output — so the compose hash and the rendered configs are unchanged — and trips neither lint. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/dstack-core/src/compose.rs | 4 +++- crates/dstack-core/src/config.rs | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/dstack-core/src/compose.rs b/crates/dstack-core/src/compose.rs index caf14b2c..079244ca 100644 --- a/crates/dstack-core/src/compose.rs +++ b/crates/dstack-core/src/compose.rs @@ -24,5 +24,7 @@ pub fn build_app_compose(name: &str, docker_compose_yaml: &str, kms_enabled: boo "public_sysinfo": true, "no_instance_id": false, }); - serde_json::to_string_pretty(&manifest).expect("app-compose manifest is always serializable") + // pretty-print via Value's Display (`{:#}`) — infallible, and byte-identical + // to serde_json::to_string_pretty (avoids an expect on an unfailable Result). + format!("{manifest:#}") } diff --git a/crates/dstack-core/src/config.rs b/crates/dstack-core/src/config.rs index a5b61d48..7be2af99 100644 --- a/crates/dstack-core/src/config.rs +++ b/crates/dstack-core/src/config.rs @@ -186,7 +186,8 @@ pub fn auth_allowlist_json(cfg: &HostConfig) -> String { }, "apps": {} }); - serde_json::to_string_pretty(&allowlist).expect("allowlist is always serializable") + // infallible pretty-print via Value's Display; see compose::build_app_compose. + format!("{allowlist:#}") } /// default pinned, reproducibly-built KMS image (Docker Hub). @@ -231,7 +232,8 @@ volumes: "secure_time": false, "allowed_envs": [] }); - serde_json::to_string_pretty(&manifest).expect("kms app-compose is always serializable") + // infallible pretty-print via Value's Display; see compose::build_app_compose. + format!("{manifest:#}") } /// inputs for rendering `vmm.toml`. Defaults target a localhost dashboard and