From d02f15ffdc3cc66a71302673c5c7f547e473864b Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 11:10:41 +0700 Subject: [PATCH 01/24] fix: keep the SQLSTATE of PostGIS provisioning failures Signed-off-by: Jeremi Joslin --- crates/registry-breg/src/postgres/roles.rs | 6 +++-- .../tests/postgres_spatial_migration.rs | 26 ++++++++++++++++++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/crates/registry-breg/src/postgres/roles.rs b/crates/registry-breg/src/postgres/roles.rs index 7a674cb341..527e50d513 100644 --- a/crates/registry-breg/src/postgres/roles.rs +++ b/crates/registry-breg/src/postgres/roles.rs @@ -144,7 +144,8 @@ pub async fn provision_postgis_prerequisites( migration_role.quoted(), runtime_role.quoted(), )) - .await?; + .await + .map_err(|error| PostgresKernelError::from_statement_error(&error))?; let bbox_role = provision_spatial_bbox_role(admin, migration_role, runtime_role).await?; Ok(bbox_role) } @@ -178,7 +179,8 @@ pub async fn provision_spatial_bbox_role( migration_role.quoted(), bbox_role.quoted(), )) - .await?; + .await + .map_err(|error| PostgresKernelError::from_statement_error(&error))?; Ok(bbox_role) } diff --git a/crates/registry-breg/tests/postgres_spatial_migration.rs b/crates/registry-breg/tests/postgres_spatial_migration.rs index 0bac9c1689..58fbde782e 100644 --- a/crates/registry-breg/tests/postgres_spatial_migration.rs +++ b/crates/registry-breg/tests/postgres_spatial_migration.rs @@ -31,7 +31,7 @@ use registry_breg::postgres::{ begin_record_transaction, install_compiled_schema, managed_schema_fingerprint, provision_postgis_prerequisites, verify_catalog_identity_for_catalog, verify_postgis, ClaimContext, ExpectedManagedCatalog, ExpectedRegistryIdentity, PostgresFailure, - RegistryLockKey, SqlIdentifier, + PostgresKernelError, RegistryLockKey, SqlIdentifier, }; use registry_breg::startup::{prepare_startup, StartupError}; use registry_breg::CompiledRegistry; @@ -2680,6 +2680,30 @@ async fn cleanup_bbox_role(database: &TestDatabase, bbox_role: &SqlIdentifier) { .expect("test cleanup removes temporary spatial bbox role"); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn postgis_provisioning_keeps_the_sqlstate_of_a_refused_statement() { + let database = TestDatabase::create(1).await; + // The migration role may not create schemas in the database, which is + // what an administrator without `CREATE EXTENSION postgis` rights meets. + let (unprivileged, task) = database.connect_migration().await; + let error = provision_postgis_prerequisites( + &unprivileged, + &database.migration_role, + &database.runtime_role, + ) + .await + .expect_err("a role without schema creation rights cannot provision PostGIS"); + task.abort(); + match &error { + PostgresKernelError::Statement(failure) => { + assert_eq!(failure.sqlstate.as_deref(), Some("42501")); + } + other => panic!("expected a statement failure, got {other:?}"), + } + assert!(error.to_string().contains("42501"), "{error}"); + database.cleanup().await; +} + async fn cleanup_postgis_prerequisites(database: &TestDatabase, bbox_role: &SqlIdentifier) { cleanup_bbox_role(database, bbox_role).await; database From b740b0de7027fdded990aeb4d3e6641f4cfdfa64 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 11:48:37 +0700 Subject: [PATCH 02/24] feat: add rstack dev stop --recover and one JSON error envelope Stop --recover releases the processes, databases and roles of a run whose supervisor is lost, signalling a recorded process group only while its leader's start time matches the one recorded at start. Every rstack dev refusal or failure now prints registry.rstack.error.v1 with a class (domain, usage, operational) and, for runs needing attention, the run ID and recovery facts. Operational failures exit 3. Signed-off-by: Jeremi Joslin --- Cargo.lock | 1 + .../src/project_dev/database.rs | 11 + .../registry-bregctl/src/project_dev/mod.rs | 267 ++++++++++++++-- .../src/project_dev/products.rs | 40 ++- .../src/project_dev/recovery.rs | 299 ++++++++++++++++++ crates/registry-bregctl/tests/project_dev.rs | 230 +++++++++++++- crates/registry-rstack/Cargo.toml | 1 + crates/registry-rstack/src/cli.rs | 20 +- crates/registry-rstack/tests/dev.rs | 72 +++++ products/platform/PROJECT-DEV.md | 52 ++- .../schemas/rstack-dev-error.v1.schema.json | 66 ++++ products/platform/scripts/test-rstack-dev.py | 92 +++++- 12 files changed, 1092 insertions(+), 59 deletions(-) create mode 100644 crates/registry-bregctl/src/project_dev/recovery.rs create mode 100644 products/platform/schemas/rstack-dev-error.v1.schema.json diff --git a/Cargo.lock b/Cargo.lock index fc659c6a13..1020ceb27d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8171,6 +8171,7 @@ dependencies = [ "clap", "fs2", "hex", + "jsonschema", "registry-platform-buildinfo", "registry-platform-canonical-json", "registry-platform-config", diff --git a/crates/registry-bregctl/src/project_dev/database.rs b/crates/registry-bregctl/src/project_dev/database.rs index 4fb99a8924..771db9d5df 100644 --- a/crates/registry-bregctl/src/project_dev/database.rs +++ b/crates/registry-bregctl/src/project_dev/database.rs @@ -601,6 +601,17 @@ impl Database { self.endpoint.product_url(database, user, password) } + /// Take ownership of everything `provision` can create for one label, so + /// that `cleanup` releases the objects of a run whose supervisor is lost. + pub(super) fn adopt(&mut self, label: &str, spatial: bool) -> Result<()> { + let mut names = run_object_names(&self.run_id, label, spatial)?; + let database = names.pop().context("run has no database name")?; + self.acquisitions + .extend(names.into_iter().map(Acquisition::Role)); + self.acquisitions.push(Acquisition::Database(database)); + Ok(()) + } + /// Names of the databases and roles this run still owns, oldest first. pub(super) fn pending_names(&self) -> Vec { self.acquisitions diff --git a/crates/registry-bregctl/src/project_dev/mod.rs b/crates/registry-bregctl/src/project_dev/mod.rs index 31b4ac4a24..23149d11fa 100644 --- a/crates/registry-bregctl/src/project_dev/mod.rs +++ b/crates/registry-bregctl/src/project_dev/mod.rs @@ -8,6 +8,7 @@ mod issuer; mod journey; pub(crate) mod plan; mod products; +mod recovery; use anyhow::{bail, ensure, Context, Result}; use registry_platform_yaml::{Diagnostic, Report}; @@ -43,6 +44,7 @@ use self::{ }; const SCHEMA: &str = "registry.rstack.dev.v1"; +const ERROR_SCHEMA: &str = "registry.rstack.error.v1"; const REQUEST: &str = "RSTACK_PROJECT_DEV_REQUEST"; const DATABASE_URL: &str = "RSTACK_DEV_DATABASE_URL"; const CA_FILE: &str = "RSTACK_DEV_DATABASE_CA"; @@ -90,6 +92,8 @@ enum Request { Stop { run_id: String, json: bool, + #[serde(default)] + recover: bool, }, } @@ -139,7 +143,7 @@ enum WorkerReply { } #[derive(Debug, thiserror::Error)] -#[error("run {run_id} still needs cleanup: {detail}")] +#[error("run {run_id} still needs cleanup: {detail}; run rstack dev stop {run_id} --recover to release its resources")] struct CleanupPending { run_id: String, detail: String, @@ -155,13 +159,17 @@ struct RecoveryNeeded { } fn recovery_message(run_id: &str, facts: &Value) -> String { - let mut text = match facts["kind"].as_str() { - Some("cleanup-pending") => { - format!("run {run_id} gave up cleanup; release these resources by hand") + let mut text = if let Some(reason) = facts["reason"].as_str() { + format!("run {run_id} could not be recovered: {reason}; release these resources by hand") + } else { + match facts["kind"].as_str() { + Some("cleanup-pending") => format!( + "run {run_id} gave up cleanup; run rstack dev stop {run_id} --recover or release these resources by hand" + ), + _ => format!( + "run {run_id} lost its supervisor before cleanup; run rstack dev stop {run_id} --recover or release these resources by hand" + ), } - _ => format!( - "run {run_id} lost its supervisor before cleanup; release these resources by hand" - ), }; let mut line = |entry: String| { text.push_str("\n "); @@ -226,7 +234,9 @@ fn recorded_processes(root: &Path) -> Value { .unwrap_or_else(|| json!([])) } -fn provisioned_objects(root: &Path, run_id: &str) -> Result> { +/// The database label and spatial flag of each database the run's plan +/// provisions. +fn provisioned_labels(root: &Path) -> Result> { let input: RunInput = read_json(&root.join("input.json"), MAXIMUM_CAPTURED_INPUT_BYTES) .context("cannot read the captured development plan")?; let spatial = input @@ -243,7 +253,11 @@ fn provisioned_objects(root: &Path, run_id: &str) -> Result> { Product::Casework => labels.push(("casework", false)), } } - labels + Ok(labels) +} + +fn provisioned_objects(root: &Path, run_id: &str) -> Result> { + provisioned_labels(root)? .into_iter() .map(|(label, spatial)| { let mut names = database::run_object_names(run_id, label, spatial)?; @@ -330,28 +344,29 @@ pub(crate) fn run(args: Args, stdout: &mut dyn Write, stderr: &mut dyn Write) -> Request::Start { project, ttl, .. } => start(&project, ttl.as_deref()), Request::Status { run_id, .. } => status(&run_id), Request::Token { run_id, person, .. } => token(&run_id, &person), - Request::Stop { run_id, .. } => stop(&run_id), + Request::Stop { + run_id, recover, .. + } => { + if recover { + recover_run(&run_id) + } else { + stop(&run_id) + } + } }; let (result, code) = match outcome { Ok(value) => { let code = u8::from(starting && value["journey"]["result"] == "failed"); (emit(stdout, &value, machine), code) } - Err(error) if machine => { - let mut report = json!({ - "schema_version": SCHEMA, "status": "error", "error": format!("{error:#}") - }); - if let Some(pending) = error.downcast_ref::() { - report["run_id"] = json!(pending.run_id); - report["cleanup_pending"] = json!(true); - } - if let Some(recovery) = error.downcast_ref::() { - report["run_id"] = json!(recovery.run_id); - report["recovery"] = recovery.facts.clone(); + Err(error) => { + let (report, code) = error_report(&error); + if machine { + (writeln!(stdout, "{report}"), code) + } else { + (writeln!(stderr, "rstack dev: {error:#}"), code) } - (writeln!(stdout, "{report}"), 1) } - Err(error) => (writeln!(stderr, "rstack dev: {error:#}"), 1), }; if result.is_ok() { code @@ -360,6 +375,38 @@ pub(crate) fn run(args: Args, stdout: &mut dyn Write, stderr: &mut dyn Write) -> } } +/// The failure of an environment or of a run's resources, as opposed to a +/// request the operator can correct. +#[derive(Debug, thiserror::Error)] +#[error("{0}")] +struct Operational(String); + +/// The error envelope and exit code for a failed command. It is the launcher's +/// `registry.rstack.error.v1` object, plus `run_id` and `recovery` when a run +/// still has resources to release. +fn error_report(error: &anyhow::Error) -> (Value, u8) { + let operational = + error.is::() || error.is::() || error.is::(); + let (class, code) = if operational { + ("operational", crate::OPERATIONAL_FAILURE_EXIT) + } else { + ("domain", crate::DOMAIN_REFUSAL_EXIT) + }; + let mut report = json!({ + "schema_version": ERROR_SCHEMA, + "status": "refused", + "error": {"class": class, "message": format!("{error:#}")}, + }); + if let Some(pending) = error.downcast_ref::() { + report["run_id"] = json!(pending.run_id); + } + if let Some(recovery) = error.downcast_ref::() { + report["run_id"] = json!(recovery.run_id); + report["recovery"] = recovery.facts.clone(); + } + (report, code) +} + fn emit(output: &mut dyn Write, report: &Value, machine: bool) -> std::io::Result<()> { if machine { return writeln!(output, "{report}"); @@ -373,6 +420,36 @@ fn emit(output: &mut dyn Write, report: &Value, machine: bool) -> std::io::Resul writeln!(output, "{line}")?; } } + if let Some(recovered) = report.get("recovered") { + for group in recovered["stopped_process_groups"] + .as_array() + .into_iter() + .flatten() + { + writeln!(output, " stopped process group {group}")?; + } + for group in recovered["skipped_process_groups"] + .as_array() + .into_iter() + .flatten() + { + writeln!( + output, + " left process group {group} running: its ID now belongs to another process" + )?; + } + for name in recovered["databases_and_roles"] + .as_array() + .into_iter() + .flatten() + { + writeln!( + output, + " released database or role {}", + name.as_str().unwrap_or("?") + )?; + } + } if let Some(products) = report["products"].as_object() { for (product, url) in products { writeln!(output, " {product} {}", url.as_str().expect("product URL"))?; @@ -892,10 +969,12 @@ fn stop(run_id: &str) -> Result { } let deadline = Instant::now() + Duration::from_secs(60); while root.exists() { - ensure!( - Instant::now() < deadline, - "run {run_id} is still cleaning up or its cleanup has stalled; run rstack dev status {run_id} to see its state" - ); + if Instant::now() >= deadline { + return Err(Operational(format!( + "run {run_id} is still cleaning up or its cleanup has stalled; run rstack dev status {run_id} to see its state" + )) + .into()); + } thread::sleep(POLL); if !root.exists() { break; @@ -919,6 +998,71 @@ fn stop(run_id: &str) -> Result { Ok(simple_report(run_id, "stopped")) } +/// Release the resources of a run whose supervisor is gone: stop its recorded +/// processes, drop its databases and roles, and remove its directory. Every +/// precondition is checked before anything is signalled or dropped, and a +/// failure reports the facts needed to finish by hand. +fn recover_run(run_id: &str) -> Result { + let root = run_dir(run_id)?; + if !root.exists() { + return Ok(simple_report(run_id, "stopped")); + } + ensure_private(&root, false)?; + ensure!( + root.join("worker.lock").exists(), + "run {run_id} has no supervisor lock, so it is still starting or never started; run rstack dev stop {run_id}, or delete {} if the run never started", + root.display() + ); + ensure!( + !supervisor_running(&root)?, + "run {run_id} still has a running supervisor; run rstack dev stop {run_id} without --recover" + ); + let unrecovered = |reason: String| -> anyhow::Error { + let mut facts = lost_supervisor_facts(&root, run_id); + facts["reason"] = json!(reason); + RecoveryNeeded { + run_id: run_id.to_owned(), + facts, + } + .into() + }; + let labels = provisioned_labels(&root).map_err(|error| unrecovered(format!("{error:#}")))?; + let plan = recovery::process_table() + .and_then(|table| recovery::plan(&recorded_processes(&root), &table)) + .map_err(|error| unrecovered(format!("{error:#}")))?; + let admin = Zeroizing::new(env::var(DATABASE_URL).map_err(|_| { + unrecovered( + "no PostgreSQL administrator URL; pass --database-url or set RSTACK_DEV_DATABASE_URL" + .to_owned(), + ) + })?); + let ca = env::var_os(CA_FILE) + .map(|path| plan::read(Path::new(&path), 1024 * 1024)) + .transpose() + .map_err(|error| unrecovered(format!("cannot read the PostgreSQL CA file: {error:#}")))?; + // Connecting takes the run's ownership lock, which a live supervisor holds. + let mut database = Database::connect(&admin, ca.as_deref(), run_id) + .map_err(|error| unrecovered(format!("{error:#}")))?; + recovery::stop_groups(&plan.ours).map_err(|error| unrecovered(format!("{error:#}")))?; + for (label, spatial) in &labels { + database + .adopt(label, *spatial) + .map_err(|error| unrecovered(format!("{error:#}")))?; + } + let dropped = database.pending_names(); + database + .cleanup() + .map_err(|error| unrecovered(format!("{error:#}")))?; + remove_run_dir(&root).map_err(|error| unrecovered(format!("{error:#}")))?; + let mut report = simple_report(run_id, "stopped"); + report["recovered"] = json!({ + "stopped_process_groups": plan.ours, + "skipped_process_groups": plan.reused, + "databases_and_roles": dropped, + }); + Ok(report) +} + fn simple_report(run_id: &str, status: &str) -> Value { json!({"schema_version": SCHEMA, "run_id": run_id, "status": status}) } @@ -1281,6 +1425,75 @@ mod tests { ); } + #[test] + fn errors_map_to_the_envelope_class_and_exit_code() { + let (report, code) = error_report(&anyhow::anyhow!("invalid test person name")); + assert_eq!( + (report["error"]["class"].as_str(), code), + (Some("domain"), 1) + ); + assert_eq!(report["schema_version"], "registry.rstack.error.v1"); + assert_eq!(report["status"], "refused"); + assert!(report.get("run_id").is_none()); + + let (report, code) = error_report(&anyhow::Error::new(Operational("stalled".into()))); + assert_eq!( + (report["error"]["class"].as_str(), code), + (Some("operational"), 3) + ); + + let pending = anyhow::Error::new(CleanupPending { + run_id: RUN_ID.to_owned(), + detail: "database unreachable".to_owned(), + }) + .context("start failed"); + let (report, code) = error_report(&pending); + assert_eq!( + (report["error"]["class"].as_str(), code), + (Some("operational"), 3) + ); + assert_eq!(report["run_id"], RUN_ID); + let message = report["error"]["message"].as_str().unwrap(); + assert!( + message.contains(&format!("stop {RUN_ID} --recover")), + "{message}" + ); + + let facts = json!({"kind": "lost-supervisor", "run_directory": "/run", "reason": "no URL"}); + let recovery = anyhow::Error::new(RecoveryNeeded { + run_id: RUN_ID.to_owned(), + facts: facts.clone(), + }); + let (report, code) = error_report(&recovery); + assert_eq!( + (report["error"]["class"].as_str(), code), + (Some("operational"), 3) + ); + assert_eq!(report["recovery"], facts); + let message = report["error"]["message"].as_str().unwrap(); + assert!( + message.contains("could not be recovered: no URL"), + "{message}" + ); + } + + #[test] + fn provisioned_labels_follow_the_plan() { + let scratch = tempfile::tempdir().unwrap(); + fake_run( + scratch.path(), + json!([ + {"product": "breg", "project": ".", "audience": "registry"}, + {"product": "casework", "project": ".", "audience": "casework"} + ]), + true, + ); + assert_eq!( + provisioned_labels(scratch.path()).unwrap(), + [("breg_test", true), ("breg", true), ("casework", false)] + ); + } + #[test] fn lost_supervisor_facts_survive_missing_files() { let scratch = tempfile::tempdir().unwrap(); diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs index 79ce78d7c1..b919f926bc 100644 --- a/crates/registry-bregctl/src/project_dev/products.rs +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -627,12 +627,21 @@ fn run_logged_tool( ); } -/// Record each child's process group. Children start as group leaders, so the -/// group ID is the PID. +/// Record each child's process group and the start time of its leader. +/// Children start as group leaders, so the group ID is the PID. Recovery after +/// a lost supervisor signals a group only while its leader still has this +/// start time, because process IDs are reused. fn write_processes(root: &Path, children: &[(Product, u32)]) -> Result<()> { + let table = super::recovery::process_table()?; let entries: Vec<_> = children .iter() - .map(|(product, pid)| json!({"product": product.name(), "pid": pid, "pgid": pid})) + .map(|(product, pid)| { + let started = table + .iter() + .find(|row| row.pid == *pid) + .map(|row| &row.started); + json!({"product": product.name(), "pid": pid, "pgid": pid, "started": started}) + }) .collect(); write_private(&root.join("processes.json"), &serde_json::to_vec(&entries)?) } @@ -1828,8 +1837,8 @@ mod tests { assert_eq!( serde_json::from_slice::(&fs::read(&path).unwrap()).unwrap(), json!([ - {"product": "breg", "pid": 4242, "pgid": 4242}, - {"product": "casework", "pid": 4343, "pgid": 4343} + {"product": "breg", "pid": 4242, "pgid": 4242, "started": null}, + {"product": "casework", "pid": 4343, "pgid": 4343, "started": null} ]) ); assert_eq!( @@ -1838,6 +1847,27 @@ mod tests { ); } + #[test] + fn a_live_process_is_recorded_with_its_start_time() { + let scratch = tempfile::tempdir().unwrap(); + let mut child = std::process::Command::new("sleep") + .arg("60") + .spawn() + .unwrap(); + write_processes(scratch.path(), &[(Product::Breg, child.id())]).unwrap(); + let recorded: Value = + serde_json::from_slice(&fs::read(scratch.path().join("processes.json")).unwrap()) + .unwrap(); + let started = recorded[0]["started"].as_str().map(str::to_owned); + assert!(started.is_some(), "{recorded}"); + assert_eq!( + started, + crate::project_dev::recovery::started_at(child.id()) + ); + child.kill().unwrap(); + child.wait().unwrap(); + } + fn minimal_registry(edit: impl FnOnce(&str) -> String) -> registry_breg::CompiledRegistry { let source = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../products/breg/examples/minimal"); diff --git a/crates/registry-bregctl/src/project_dev/recovery.rs b/crates/registry-bregctl/src/project_dev/recovery.rs new file mode 100644 index 0000000000..5c4e8744f8 --- /dev/null +++ b/crates/registry-bregctl/src/project_dev/recovery.rs @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Releasing the processes of a run whose supervisor is lost. +//! +//! A recorded process group is signalled only while it still belongs to the +//! run: the leader's start time must equal the one recorded when the run +//! started. Process IDs are reused, so a recorded ID alone proves nothing. + +use anyhow::{bail, ensure, Context, Result}; +use rustix::process::{kill_process_group, Pid, Signal}; +use serde_json::Value; +use std::{ + process::{Command, Stdio}, + thread, + time::{Duration, Instant}, +}; + +const TERM_GRACE: Duration = Duration::from_secs(5); +const KILL_GRACE: Duration = Duration::from_secs(5); +const POLL: Duration = Duration::from_millis(100); + +/// One row of the process table. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct Row { + pub pid: u32, + pub pgid: u32, + pub started: String, +} + +/// What a recorded process group is now. +#[derive(Debug, Eq, PartialEq)] +pub(super) enum Group { + /// No process of the group remains. + Gone, + /// The group is the run's own. + Ours, + /// The recorded ID belongs to a different process now. + Reused, +} + +/// Every process with its group and start time. `ps` is the one portable +/// source of start times on Linux and macOS. +pub(super) fn process_table() -> Result> { + let output = Command::new("ps") + .args(["-A", "-o", "pid=,pgid=,lstart="]) + .env("LC_ALL", "C") + .stdin(Stdio::null()) + .stderr(Stdio::null()) + .output() + .context("cannot run ps to read process start times")?; + ensure!( + output.status.success(), + "ps could not list processes with their start times" + ); + Ok(parse_table(&String::from_utf8_lossy(&output.stdout))) +} + +fn parse_table(text: &str) -> Vec { + text.lines() + .filter_map(|line| { + let mut fields = line.split_whitespace(); + let pid = fields.next()?.parse().ok()?; + let pgid = fields.next()?.parse().ok()?; + // The start time is several words; spacing differs between + // platforms, so it is kept with single spaces. + let started = fields.collect::>().join(" "); + (!started.is_empty()).then_some(Row { pid, pgid, started }) + }) + .collect() +} + +/// The start time of `pid`, or `None` when it cannot be read. +#[cfg(test)] +pub(super) fn started_at(pid: u32) -> Option { + process_table() + .ok()? + .into_iter() + .find(|row| row.pid == pid) + .map(|row| row.started) +} + +/// Decide whether the recorded group `pgid` is still the run's. A group ID +/// stays reserved while any member lives, so a group whose leader has gone +/// but whose members remain is the original group. +pub(super) fn classify(table: &[Row], pgid: u32, recorded_start: Option<&str>) -> Result { + match table.iter().find(|row| row.pid == pgid) { + Some(leader) => { + let recorded = recorded_start.with_context(|| { + format!("process group {pgid} cannot be verified: its start time was not recorded") + })?; + Ok(if leader.started == recorded { + Group::Ours + } else { + Group::Reused + }) + } + None if table.iter().any(|row| row.pgid == pgid) => Ok(Group::Ours), + None => Ok(Group::Gone), + } +} + +fn signal(pgid: u32, signal: Signal) -> Result<()> { + let pid = i32::try_from(pgid) + .ok() + .and_then(Pid::from_raw) + .context("invalid process group ID")?; + match kill_process_group(pid, signal) { + Ok(()) | Err(rustix::io::Errno::SRCH) => Ok(()), + // Darwin reports EPERM for a group whose only members are zombies. + Err(rustix::io::Errno::PERM) if cfg!(target_os = "macos") => Ok(()), + Err(error) => Err(error).with_context(|| format!("cannot signal process group {pgid}")), + } +} + +fn wait_until_gone(groups: &[u32], grace: Duration) -> Result> { + let deadline = Instant::now() + grace; + loop { + let table = process_table()?; + let alive: Vec = groups + .iter() + .copied() + .filter(|group| table.iter().any(|row| row.pgid == *group)) + .collect(); + if alive.is_empty() || Instant::now() >= deadline { + return Ok(alive); + } + thread::sleep(POLL); + } +} + +/// Stop each group: TERM, then KILL for any that survive the grace period. +pub(super) fn stop_groups(groups: &[u32]) -> Result<()> { + if groups.is_empty() { + return Ok(()); + } + for group in groups { + signal(*group, Signal::TERM)?; + } + let survivors = wait_until_gone(groups, TERM_GRACE)?; + for group in &survivors { + signal(*group, Signal::KILL)?; + } + let remaining = wait_until_gone(&survivors, KILL_GRACE)?; + if !remaining.is_empty() { + bail!( + "process group {} survived SIGKILL", + remaining + .iter() + .map(u32::to_string) + .collect::>() + .join(", ") + ); + } + Ok(()) +} + +/// The recorded processes of a run, sorted into those to stop and those whose +/// recorded ID now belongs to something else. +pub(super) struct Plan { + pub ours: Vec, + pub reused: Vec, +} + +/// Classify every recorded process group before anything is signalled, so one +/// unverifiable entry stops the recovery with nothing touched. +pub(super) fn plan(recorded: &Value, table: &[Row]) -> Result { + let mut plan = Plan { + ours: Vec::new(), + reused: Vec::new(), + }; + for entry in recorded.as_array().into_iter().flatten() { + let pgid = entry["pgid"] + .as_u64() + .and_then(|value| u32::try_from(value).ok()) + .context("a recorded process has no valid group ID")?; + match classify(table, pgid, entry["started"].as_str())? { + Group::Ours => plan.ours.push(pgid), + Group::Reused => plan.reused.push(pgid), + Group::Gone => {} + } + } + Ok(plan) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use std::os::unix::process::CommandExt; + + fn row(pid: u32, pgid: u32, started: &str) -> Row { + Row { + pid, + pgid, + started: started.to_owned(), + } + } + + #[test] + fn the_process_table_parses_spaced_start_times() { + let table = parse_table( + " 1 1 Tue Nov 3 10:00:00 2026\n 4242 4242 Tue Nov 11 12:00:01 2026\nbad line\n", + ); + assert_eq!( + table, + [ + row(1, 1, "Tue Nov 3 10:00:00 2026"), + row(4242, 4242, "Tue Nov 11 12:00:01 2026"), + ] + ); + } + + #[test] + fn a_group_is_ours_only_when_its_leader_started_when_recorded() { + let table = [row(7, 7, "Tue Nov 11 12:00:00 2026")]; + assert_eq!( + classify(&table, 7, Some("Tue Nov 11 12:00:00 2026")).unwrap(), + Group::Ours + ); + assert_eq!( + classify(&table, 7, Some("Mon Nov 10 08:00:00 2026")).unwrap(), + Group::Reused, + "a reused process ID has a different start time" + ); + let unverified = classify(&table, 7, None).unwrap_err().to_string(); + assert!( + unverified.contains("start time was not recorded"), + "{unverified}" + ); + } + + #[test] + fn a_group_without_a_leader_is_ours_while_members_remain() { + let members = [row(8, 7, "Tue Nov 11 12:00:00 2026")]; + assert_eq!(classify(&members, 7, None).unwrap(), Group::Ours); + assert_eq!(classify(&[], 7, None).unwrap(), Group::Gone); + } + + #[test] + fn nothing_is_signalled_when_one_recorded_process_cannot_be_verified() { + let table = [row(7, 7, "A"), row(9, 9, "B")]; + let recorded = json!([ + {"product": "breg", "pid": 7, "pgid": 7, "started": "A"}, + {"product": "casework", "pid": 9, "pgid": 9, "started": null}, + ]); + assert!(plan(&recorded, &table).is_err()); + let recorded = json!([ + {"product": "breg", "pid": 7, "pgid": 7, "started": "A"}, + {"product": "casework", "pid": 9, "pgid": 9, "started": "other"}, + {"product": "gone", "pid": 11, "pgid": 11, "started": "C"}, + ]); + let plan = plan(&recorded, &table).unwrap(); + assert_eq!((plan.ours, plan.reused), (vec![7], vec![9])); + } + + fn is_alive(pgid: u32) -> bool { + process_table().unwrap().iter().any(|row| row.pgid == pgid) + } + + /// A sleeping process in its own group, and a thread that reaps it once it + /// ends, since an unreaped child stays in the process table. + fn sleeper() -> (u32, thread::JoinHandle<()>) { + let mut child = Command::new("sleep") + .arg("60") + .process_group(0) + .spawn() + .unwrap(); + let pid = child.id(); + ( + pid, + thread::spawn(move || { + child.wait().unwrap(); + }), + ) + } + + #[test] + fn a_live_group_with_its_recorded_start_time_is_stopped() { + let (pid, reaper) = sleeper(); + let started = started_at(pid).expect("a live process has a start time"); + let recorded = json!([{"product": "breg", "pid": pid, "pgid": pid, "started": started}]); + let plan = plan(&recorded, &process_table().unwrap()).unwrap(); + assert_eq!(plan.ours, [pid]); + stop_groups(&plan.ours).unwrap(); + reaper.join().unwrap(); + assert!(!is_alive(pid)); + } + + #[test] + fn a_live_group_with_another_start_time_is_left_running() { + let (pid, reaper) = sleeper(); + let recorded = json!([{"product": "breg", "pid": pid, "pgid": pid, "started": "Mon Jan 1 00:00:00 2001"}]); + let plan = plan(&recorded, &process_table().unwrap()).unwrap(); + assert!(plan.ours.is_empty()); + assert_eq!(plan.reused, [pid]); + assert!(is_alive(pid), "the unrelated process must keep running"); + signal(pid, Signal::KILL).unwrap(); + reaper.join().unwrap(); + } +} diff --git a/crates/registry-bregctl/tests/project_dev.rs b/crates/registry-bregctl/tests/project_dev.rs index f895516d1a..dc5a2f051a 100644 --- a/crates/registry-bregctl/tests/project_dev.rs +++ b/crates/registry-bregctl/tests/project_dev.rs @@ -27,6 +27,209 @@ journey:\n id: smoke\n steps:\n - {{id: read, product: breg, as: operator, .unwrap(); } +const RUN_ID: &str = "0123456789abcdef0123456789abcdef"; + +/// Validates a helper failure against the documented error schema and checks +/// its class. +fn assert_error_envelope(report: &serde_json::Value, class: &str) { + let schema: serde_json::Value = serde_json::from_str(include_str!( + "../../../products/platform/schemas/rstack-dev-error.v1.schema.json" + )) + .unwrap(); + let schema = jsonschema::JSONSchema::options() + .with_draft(jsonschema::Draft::Draft202012) + .compile(&schema) + .unwrap(); + if let Err(errors) = schema.validate(report) { + let errors: Vec<_> = errors.map(|error| error.to_string()).collect(); + panic!("{report} violates the error schema: {errors:?}"); + } + assert_eq!(report["error"]["class"], class, "{report}"); +} + +fn helper(request: serde_json::Value, state: &std::path::Path) -> std::process::Output { + Command::new(env!("CARGO_BIN_EXE_bregctl")) + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request.to_string()) + .env("RSTACK_DEV_STATE_DIR", state) + .env_remove("RSTACK_DEV_DATABASE_URL") + .output() + .unwrap() +} + +/// A run directory whose supervisor is gone: its lock file exists, unlocked. +fn lost_run(state: &std::path::Path) -> std::path::PathBuf { + let root = state.join(RUN_ID); + fs::create_dir_all(&root).unwrap(); + fs::set_permissions(state, fs::Permissions::from_mode(0o700)).unwrap(); + fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap(); + fs::write(root.join("worker.lock"), "").unwrap(); + root +} + +#[test] +fn every_helper_failure_prints_the_error_envelope() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + lost_run(&state); + let missing_project = scratch.path().join("missing"); + let other_run = "fedcba9876543210fedcba9876543210"; + let cases = [ + ( + "start of a missing project", + serde_json::json!({"action": "start", "project": missing_project, "json": true}), + "domain", + 1, + ), + ( + "start with a bad ttl", + serde_json::json!({"action": "start", "project": scratch.path(), "ttl": "9", "json": true}), + "domain", + 1, + ), + ( + "status with a malformed run ID", + serde_json::json!({"action": "status", "run_id": "../x", "json": true}), + "domain", + 1, + ), + ( + "token with a malformed person", + serde_json::json!({"action": "token", "run_id": other_run, "person": "../x", "json": true}), + "domain", + 1, + ), + ( + "token for a run that is not ready", + serde_json::json!({"action": "token", "run_id": other_run, "person": "operator", "json": true}), + "domain", + 1, + ), + ( + "stop with a malformed run ID", + serde_json::json!({"action": "stop", "run_id": "../x", "json": true}), + "domain", + 1, + ), + ( + "status of a run whose supervisor is lost", + serde_json::json!({"action": "status", "run_id": RUN_ID, "json": true}), + "operational", + 3, + ), + ( + "stop of a run whose supervisor is lost", + serde_json::json!({"action": "stop", "run_id": RUN_ID, "json": true}), + "operational", + 3, + ), + ]; + for (name, request, class, exit) in cases { + let output = helper(request, &state); + assert_eq!(output.status.code(), Some(exit), "{name}: {output:?}"); + assert!(output.stderr.is_empty(), "{name}: {output:?}"); + let report: serde_json::Value = serde_json::from_slice(&output.stdout) + .unwrap_or_else(|error| panic!("{name}: {error}: {output:?}")); + assert_error_envelope(&report, class); + if class == "operational" { + assert_eq!(report["run_id"], RUN_ID, "{name}"); + assert_eq!(report["recovery"]["kind"], "lost-supervisor", "{name}"); + } else { + assert!(report.get("run_id").is_none(), "{name}: {report}"); + } + } +} + +#[test] +fn a_helper_failure_without_json_prints_one_line_and_the_same_exit_class() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + lost_run(&state); + let output = helper( + serde_json::json!({"action": "status", "run_id": RUN_ID, "json": false}), + &state, + ); + assert_eq!(output.status.code(), Some(3), "{output:?}"); + assert!(output.stdout.is_empty(), "{output:?}"); + let text = String::from_utf8(output.stderr).unwrap(); + assert!(text.starts_with("rstack dev: run "), "{text}"); + assert!(text.contains("lost its supervisor"), "{text}"); +} + +fn recover(state: &std::path::Path, run_id: &str) -> std::process::Output { + helper( + serde_json::json!({"action": "stop", "run_id": run_id, "json": true, "recover": true}), + state, + ) +} + +#[test] +fn recover_of_an_absent_run_reports_it_stopped() { + let scratch = tempfile::tempdir().unwrap(); + let output = recover(&scratch.path().join("state"), RUN_ID); + assert_eq!(output.status.code(), Some(0), "{output:?}"); + let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(report["status"], "stopped"); +} + +#[test] +fn recover_refuses_a_run_with_a_running_supervisor() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let root = lost_run(&state); + let lock = fs::File::open(root.join("worker.lock")).unwrap(); + rustix::fs::flock(&lock, rustix::fs::FlockOperation::NonBlockingLockExclusive).unwrap(); + + let output = recover(&state, RUN_ID); + + assert_eq!(output.status.code(), Some(1), "{output:?}"); + let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_error_envelope(&report, "domain"); + let message = report["error"]["message"].as_str().unwrap(); + assert!( + message.contains("still has a running supervisor"), + "{message}" + ); + assert!(root.exists()); +} + +#[test] +fn recover_refuses_a_run_without_a_supervisor_lock() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let root = lost_run(&state); + fs::remove_file(root.join("worker.lock")).unwrap(); + + let output = recover(&state, RUN_ID); + + assert_eq!(output.status.code(), Some(1), "{output:?}"); + let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_error_envelope(&report, "domain"); + assert!(root.exists()); +} + +#[test] +fn recover_that_cannot_proceed_reports_the_facts_and_changes_nothing() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let root = lost_run(&state); + + let output = recover(&state, RUN_ID); + + assert_eq!(output.status.code(), Some(3), "{output:?}"); + let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_error_envelope(&report, "operational"); + assert_eq!(report["run_id"], RUN_ID); + assert_eq!(report["recovery"]["kind"], "lost-supervisor"); + assert!(report["recovery"]["run_directory"] + .as_str() + .unwrap() + .ends_with(RUN_ID)); + let message = report["error"]["message"].as_str().unwrap(); + assert!(message.contains("could not be recovered"), "{message}"); + assert!(root.exists()); +} + fn semantic_project(root: &std::path::Path, grant: &str, step_extra: &str) { fs::write(root.join("registry.yaml"), "fixture\n").unwrap(); fs::write( @@ -232,7 +435,8 @@ fn assert_dangling_schema_assignment( assert_eq!(started.status.code(), Some(1), "{started:?}"); let report: serde_json::Value = serde_json::from_slice(&started.stdout).unwrap(); - let error = report["error"].as_str().unwrap(); + assert_error_envelope(&report, "domain"); + let error = report["error"]["message"].as_str().unwrap(); assert!(error.contains(path), "{error}"); assert!(error.contains(message), "{error}"); assert!(error.contains(suggested_action), "{error}"); @@ -363,7 +567,8 @@ fn check_and_start_report_an_explicit_assignment_with_no_fixture_file() { assert_eq!(started.status.code(), Some(1), "{started:?}"); let report: serde_json::Value = serde_json::from_slice(&started.stdout).unwrap(); - let error = report["error"].as_str().unwrap(); + assert_error_envelope(&report, "domain"); + let error = report["error"]["message"].as_str().unwrap(); assert!(error.contains("/schemaTestPeople"), "{error}"); assert!( error.contains("fixture journeys file is missing"), @@ -416,7 +621,8 @@ fn check_and_start_refuse_a_fixture_symlink_outside_the_breg_project() { assert_eq!(started.status.code(), Some(1), "{started:?}"); let report: serde_json::Value = serde_json::from_slice(&started.stdout).unwrap(); - let error = report["error"].as_str().unwrap(); + assert_error_envelope(&report, "domain"); + let error = report["error"]["message"].as_str().unwrap(); assert!(error.contains("/products/0/project"), "{error}"); assert!(error.contains("fixture journeys file must be"), "{error}"); assert!(error.contains("without linking it outside"), "{error}"); @@ -678,7 +884,8 @@ fn start_reports_an_actionable_value_free_unstarted_grant() { assert_eq!(result.status.code(), Some(1)); let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); - let error = report["error"].as_str().unwrap(); + assert_error_envelope(&report, "domain"); + let error = report["error"]["message"].as_str().unwrap(); assert!( error.contains("/people/operator/grants/casework"), "{error}" @@ -753,7 +960,8 @@ journey:\n id: smoke\n steps:\n - {id: read, product: breg, as: editor, met assert_eq!(result.status.code(), Some(1), "{result:?}"); let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); - let error = report["error"].as_str().unwrap(); + assert_error_envelope(&report, "domain"); + let error = report["error"]["message"].as_str().unwrap(); assert!( error.contains("/people/unused/grants/breg/profile"), "{error}" @@ -820,7 +1028,8 @@ fn check_and_start_resolve_casework_profiles_and_bindings_with_the_owning_report assert_eq!(started.status.code(), Some(1), "{started:?}"); let report: serde_json::Value = serde_json::from_slice(&started.stdout).unwrap(); - let error = report["error"].as_str().unwrap(); + assert_error_envelope(&report, "domain"); + let error = report["error"]["message"].as_str().unwrap(); assert!(error.contains(path), "{error}"); assert!(error.contains(message), "{error}"); assert!(error.contains(action), "{error}"); @@ -947,10 +1156,10 @@ fn a_malformed_plan_names_its_field_before_allocating_run_state() { .unwrap(); assert!(!result.status.success()); let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); - assert_eq!(report["schema_version"], "registry.rstack.dev.v1"); + assert_error_envelope(&report, "domain"); assert!(report.get("diagnostics").is_none()); assert!( - report["error"] + report["error"]["message"] .as_str() .unwrap() .contains("/products/0/product"), @@ -973,7 +1182,10 @@ fn a_run_id_cannot_name_a_path() { .unwrap(); assert!(!result.status.success()); let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); - assert!(report["error"].as_str().unwrap().contains("run ID")); + assert!(report["error"]["message"] + .as_str() + .unwrap() + .contains("run ID")); assert_eq!(fs::read_dir(scratch.path()).unwrap().count(), 0); } diff --git a/crates/registry-rstack/Cargo.toml b/crates/registry-rstack/Cargo.toml index 6ee9fdebfa..af0ca1ec4d 100644 --- a/crates/registry-rstack/Cargo.toml +++ b/crates/registry-rstack/Cargo.toml @@ -32,3 +32,4 @@ tempfile.workspace = true ureq.workspace = true [dev-dependencies] +jsonschema.workspace = true diff --git a/crates/registry-rstack/src/cli.rs b/crates/registry-rstack/src/cli.rs index cda7a57406..8a6b47c5b9 100644 --- a/crates/registry-rstack/src/cli.rs +++ b/crates/registry-rstack/src/cli.rs @@ -218,6 +218,14 @@ enum DevCommand { /// Run ID returned by rstack dev start. #[arg(value_name = "RUN_ID")] run_id: String, + /// Release the processes, databases and roles of a run whose supervisor + /// was lost. + #[arg(long)] + recover: bool, + /// TLS PostgreSQL administrator URL for --recover. Defaults to + /// RSTACK_DEV_DATABASE_URL. + #[arg(long, value_name = "URL", requires = "recover")] + database_url: Option, /// Emit the runner's JSON report. #[arg(long)] json: bool, @@ -249,6 +257,8 @@ enum DevRequest<'a> { }, Stop { run_id: &'a str, + #[serde(skip_serializing_if = "std::ops::Not::not")] + recover: bool, json: bool, }, } @@ -1463,12 +1473,18 @@ fn dispatch_dev(cli: &Cli, action: &DevCommand, format: ReportFormat) -> Result< None, Needs::default(), ), - DevCommand::Stop { run_id, json } => ( + DevCommand::Stop { + run_id, + recover, + database_url, + json, + } => ( DevRequest::Stop { run_id, + recover: *recover, json: *json || launcher_json, }, - None, + database_url.as_deref(), Needs::default(), ), }; diff --git a/crates/registry-rstack/tests/dev.rs b/crates/registry-rstack/tests/dev.rs index 360a1b3a8f..70025c2300 100644 --- a/crates/registry-rstack/tests/dev.rs +++ b/crates/registry-rstack/tests/dev.rs @@ -427,6 +427,10 @@ fn dev_actions_encode_the_public_contract_and_launcher_json_format() { vec!["dev", "stop", "run-4", "--json"], serde_json::json!({"action":"stop","run_id":"run-4","json":true}), ), + ( + vec!["dev", "stop", "run-5", "--recover", "--json"], + serde_json::json!({"action":"stop","run_id":"run-5","recover":true,"json":true}), + ), ] { let output = tools .empty_command() @@ -439,6 +443,52 @@ fn dev_actions_encode_the_public_contract_and_launcher_json_format() { } } +#[test] +fn recover_passes_the_administrator_url_through_the_environment_only() { + let tools = SourceTools::new(); + let database_url = "postgresql://example.invalid/dev-secret"; + let output = tools + .empty_command() + .args([ + "dev", + "stop", + "run-6", + "--recover", + "--database-url", + database_url, + ]) + .env("RSTACK_BREGCTL_BIN", &tools.bregctl) + .env("EXPECTED_DATABASE_URL", database_url) + .output() + .unwrap(); + + assert!(output.status.success(), "{output:?}"); + let stdout = String::from_utf8(output.stdout.clone()).unwrap(); + assert!(stdout.contains("database:matched\n"), "{stdout}"); + assert!(!stdout.contains(database_url)); + assert!(!request(&output).to_string().contains(database_url)); +} + +#[test] +fn database_url_is_refused_for_stop_without_recover() { + let tools = SourceTools::new(); + let output = tools + .empty_command() + .args([ + "dev", + "stop", + "run-7", + "--database-url", + "postgresql://example.invalid/x", + "--json", + ]) + .env("RSTACK_BREGCTL_BIN", &tools.bregctl) + .output() + .unwrap(); + + assert_eq!(output.status.code(), Some(2), "{output:?}"); +} + #[test] fn start_requires_the_complete_source_binary_set_without_an_installation() { let tools = SourceTools::new(); @@ -474,6 +524,10 @@ fn root_and_dev_help_need_no_installation_or_binary_overrides() { &["dev", "start", "--help"][..], &["--database-url", "--ttl", "--json"][..], ), + ( + &["dev", "stop", "--help"][..], + &["--recover", "--database-url", "--json"][..], + ), ] { let output = tools.empty_command().args(arguments).output().unwrap(); assert!(output.status.success(), "{arguments:?}: {output:?}"); @@ -502,6 +556,22 @@ fn write_plan(project: &Path, products: &[&str]) { .unwrap(); } +/// Validates a launcher failure against the documented error schema. +fn assert_matches_error_schema(report: &Value) { + let schema: Value = serde_json::from_str(include_str!( + "../../../products/platform/schemas/rstack-dev-error.v1.schema.json" + )) + .unwrap(); + let schema = jsonschema::JSONSchema::options() + .with_draft(jsonschema::Draft::Draft202012) + .compile(&schema) + .unwrap(); + if let Err(errors) = schema.validate(report) { + let errors: Vec<_> = errors.map(|error| error.to_string()).collect(); + panic!("{report} violates the error schema: {errors:?}"); + }; +} + #[test] fn dev_subcommand_json_makes_launcher_failures_json() { let tools = SourceTools::new(); @@ -517,6 +587,7 @@ fn dev_subcommand_json_makes_launcher_failures_json() { assert_eq!(report["schema_version"], "registry.rstack.error.v1"); assert_eq!(report["status"], "refused"); assert_eq!(report["error"]["class"], "domain"); + assert_matches_error_schema(&report); let usage = tools .empty_command() @@ -527,6 +598,7 @@ fn dev_subcommand_json_makes_launcher_failures_json() { let report: Value = serde_json::from_slice(&usage.stdout).unwrap(); assert_eq!(report["schema_version"], "registry.rstack.error.v1"); assert_eq!(report["error"]["class"], "usage"); + assert_matches_error_schema(&report); let human = tools .empty_command() diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index 8859aed6ab..ca4fdaa681 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -228,7 +228,29 @@ directory, and `status` reports `cleanup-pending` with the remaining databases, roles and processes. If the supervisor is lost, `stop` and `status` report the run directory, the process groups, and the exact database and role names. -Release the listed resources by hand, in this order: +Release a run whose supervisor is gone with: + +```sh +rstack dev stop RUN_ID --recover [--database-url URL] +``` + +`--database-url` defaults to `RSTACK_DEV_DATABASE_URL`, as for `start`, and +the URL needs the same administrator rights. `--recover` first checks +everything it needs, then acts: it refuses a run whose supervisor is still +running (use plain `stop`), stops each recorded process group, drops the +databases and roles, and removes the run directory. It signals a process group +only while the group leader's start time equals the one recorded when the run +started. If the recorded ID now belongs to another process, `--recover` leaves +that process running and lists the group under `skipped_process_groups` in the +report. If a start time was not recorded and the leader is still alive, +`--recover` refuses and signals nothing. When it cannot finish, the error +names the reason and lists the run directory, process groups, databases and +roles to release by hand. Running it again after a partial failure is safe. + +A successful recovery reports `status` `stopped` and a `recovered` object with +`stopped_process_groups`, `skipped_process_groups` and `databases_and_roles`. + +To release the resources by hand instead, in this order: 1. Stop each listed process group, for example `kill -TERM -PGID`. 2. As the administrator, run `DROP DATABASE IF EXISTS "NAME" WITH (FORCE)` for @@ -239,13 +261,27 @@ Release the listed resources by hand, in this order: ### Errors and limitations -With `--json`, a failed `rstack dev` command prints one JSON error object with -`schema_version` (`registry.rstack.dev.v1`), `status` set to `error`, and -`error`. It adds `run_id` when the run needs cleanup and `recovery` with the -resources listed above. Failures that occur before the helper starts, such as a -missing component, print a `registry.rstack.error.v1` object with `status` set -to `refused` and an `error` object holding `class` and `message`. The reports of -`rstack dev check` use `BRegCtlReport` instead. +With `--json`, every refusal or failure of `rstack dev` start, status, token +and stop prints one JSON object on standard output, validated by +`schemas/rstack-dev-error.v1.schema.json`. The launcher prints the same object +for failures before the helper starts, such as a missing component. + +- `schema_version` is `registry.rstack.error.v1` and `status` is `refused`. +- `error.class` is `domain` (exit 1: the request or the run's state was + refused), `usage` (exit 2: the command line is wrong) or `operational` + (exit 3: the environment failed, a cleanup stalled, or a run needs its + resources released). `error.message` is the human-readable reason. +- `run_id` is present when the failure concerns a run that still has + resources to release. Use it with `stop --recover`. +- `recovery` accompanies `run_id` when the resources are known. It has `kind` + (`lost-supervisor` or `cleanup-pending`), `run_directory`, `processes`, + `objects` or `databases_and_roles`, and `reason` when `--recover` could not + finish. + +Without `--json`, the message goes to standard error as `rstack dev: MESSAGE` +with the same exit code. The reports of `rstack dev check` use `BRegCtlReport` +instead. A helper started without the launcher's request prints a plain +message and exits 2. `rstack dev` does not configure field encryption yet. `rstack dev check` and `start` refuse a BReg project with encrypted fields at the product's diff --git a/products/platform/schemas/rstack-dev-error.v1.schema.json b/products/platform/schemas/rstack-dev-error.v1.schema.json new file mode 100644 index 0000000000..263c7ed8c7 --- /dev/null +++ b/products/platform/schemas/rstack-dev-error.v1.schema.json @@ -0,0 +1,66 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "rstack error report", + "description": "The one JSON object that rstack and rstack dev print on stdout when --json is set and a command is refused or fails.", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "status", "error"], + "properties": { + "schema_version": {"const": "registry.rstack.error.v1"}, + "status": {"const": "refused"}, + "error": { + "type": "object", + "additionalProperties": false, + "required": ["class", "message"], + "properties": { + "class": { + "description": "domain (exit 1): the request or the run's state was refused. usage (exit 2): the command line is wrong. operational (exit 3): the environment failed or resources need cleanup.", + "enum": ["domain", "usage", "operational"] + }, + "message": {"type": "string", "minLength": 1} + } + }, + "run_id": { + "description": "Present when the failure concerns a run that still has resources to release.", + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + "recovery": { + "description": "Present with run_id when the resources to release are known.", + "type": "object", + "additionalProperties": false, + "required": ["kind", "run_directory"], + "properties": { + "kind": {"enum": ["lost-supervisor", "cleanup-pending"]}, + "run_directory": {"type": "string"}, + "processes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "product": {"type": "string"}, + "pid": {"type": "integer"}, + "pgid": {"type": "integer"}, + "started": {"type": ["string", "null"]} + } + } + }, + "objects": { + "type": "array", + "items": { + "type": "object", + "required": ["label", "database", "roles"], + "properties": { + "label": {"type": "string"}, + "database": {"type": "string"}, + "roles": {"type": "array", "items": {"type": "string"}} + } + } + }, + "databases_and_roles": {"type": "array", "items": {"type": "string"}}, + "unavailable": {"type": "string"}, + "reason": {"type": "string"} + } + } + } +} diff --git a/products/platform/scripts/test-rstack-dev.py b/products/platform/scripts/test-rstack-dev.py index 97dba72c73..48d4012b90 100644 --- a/products/platform/scripts/test-rstack-dev.py +++ b/products/platform/scripts/test-rstack-dev.py @@ -206,6 +206,10 @@ def dev(bin_dir, env, cwd, *args, expected=0): raise RuntimeError(f"rstack dev {args[0]} exited {result.returncode}: {report}") if args[0] == "check": assert report["kind"] == "BRegCtlReport" and report["command"] == "dev-check" + elif report["schema_version"] == "registry.rstack.error.v1": + assert report["status"] == "refused", report + assert report["error"]["class"] in ("domain", "usage", "operational"), report + assert report["error"]["message"], report else: assert report["schema_version"] == "registry.rstack.dev.v1" return report @@ -510,9 +514,9 @@ def exercise_runs(root, bin_dir, pg, checkout, state, env, pg_env): failing.chmod(0o700) failure_env = {**env, "RSTACK_CASEWORK_BIN": str(failing)} report = dev(bin_dir, failure_env, checkout, "start", ".", expected=1) - assert "casework exited before readiness" in report["error"], report - assert "boom" in report["error"], report - logs = kept_logs(state, report["error"]) + assert "casework exited before readiness" in report["error"]["message"], report + assert "boom" in report["error"]["message"], report + logs = kept_logs(state, report["error"]["message"]) assert any("boom" in log.read_text() for log in logs.glob("casework-*.log")), report assert inventory(pg, pg_env) == baseline, report assert live_runs(state) == [], report @@ -562,7 +566,7 @@ def verify_provisioning_deadline(bin_dir, pg, checkout, state, env, pg_env): time.sleep(0.1) report = dev(bin_dir, env, checkout, "start", ".", expected=1) - error = report["error"] + error = report["error"]["message"] assert "cannot create an isolated PostgreSQL database" in error, report assert "statement deadline" in error, report assert psql(pg, pg_env, lock_query).strip() == "t" @@ -631,9 +635,10 @@ def verify_lost_supervisor(bin_dir, pg, checkout, state, env, pg_env, owner_env) ) [supervisor] = worker.stdout.split() os.kill(int(supervisor), signal.SIGKILL) - status = dev(bin_dir, env, checkout, "status", run, expected=1) - stopped = dev(bin_dir, env, checkout, "stop", run, expected=1) + status = dev(bin_dir, env, checkout, "status", run, expected=3) + stopped = dev(bin_dir, env, checkout, "stop", run, expected=3) for lost in (status, stopped): + assert lost["error"]["class"] == "operational", lost assert lost["recovery"]["kind"] == "lost-supervisor", lost assert lost["run_id"] == run, lost recovery = stopped["recovery"] @@ -664,6 +669,77 @@ def verify_lost_supervisor(bin_dir, pg, checkout, state, env, pg_env, owner_env) assert live_runs(state) == [] assert dev(bin_dir, env, checkout, "status", run)["status"] == "stopped" print("A lost supervisor was recovered by hand from the facts stop reported") + verify_recover(bin_dir, pg, checkout, state, env, pg_env) + + +def lose_supervisor(bin_dir, checkout, state, env): + report = dev(bin_dir, env, checkout, "start", ".") + run = report["run_id"] + worker = command( + "pgrep", "-f", f"project-dev --worker {re.escape(str(state / run))}" + ) + [supervisor] = worker.stdout.split() + os.kill(int(supervisor), signal.SIGKILL) + return report, run + + +def verify_recover(bin_dir, pg, checkout, state, env, pg_env): + baseline = inventory(pg, pg_env) + baseline_locks = administrator_locks(pg, pg_env) + + report, run = lose_supervisor(bin_dir, checkout, state, env) + recovered = dev(bin_dir, env, checkout, "stop", run, "--recover") + assert recovered["status"] == "stopped", recovered + assert len(recovered["recovered"]["stopped_process_groups"]) == 2, recovered + assert recovered["recovered"]["skipped_process_groups"] == [], recovered + assert_nothing_listening(bin_dir, report["products"].values()) + assert inventory(pg, pg_env) == baseline + eventually( + lambda: administrator_locks(pg, pg_env) == baseline_locks, + "the recovery's administrator lock was not released", + ) + assert live_runs(state) == [] + assert dev(bin_dir, env, checkout, "status", run)["status"] == "stopped" + assert dev(bin_dir, env, checkout, "stop", run, "--recover")["status"] == "stopped" + print("stop --recover released a lost supervisor's processes, databases and roles") + + # A recorded group whose leader started at another time is not the run's: + # recovery leaves it running and still releases the rest. + report, run = lose_supervisor(bin_dir, checkout, state, env) + processes_file = state / run / "processes.json" + processes = json.loads(processes_file.read_text()) + [casework] = [p for p in processes if p["product"] == "casework"] + casework["started"] = "Mon Jan 1 00:00:00 2001" + processes_file.write_text(json.dumps(processes)) + try: + recovered = dev(bin_dir, env, checkout, "stop", run, "--recover") + assert recovered["recovered"]["skipped_process_groups"] == [ + casework["pgid"] + ], recovered + os.killpg(casework["pgid"], 0) + assert inventory(pg, pg_env) == baseline + assert live_runs(state) == [] + finally: + try: + os.killpg(casework["pgid"], signal.SIGKILL) + except ProcessLookupError: + pass + assert_nothing_listening(bin_dir, report["products"].values()) + + # Without an administrator URL nothing is signalled or dropped, and the + # error carries the facts for a hand release. + report, run = lose_supervisor(bin_dir, checkout, state, env) + no_url = {k: v for k, v in env.items() if k != "RSTACK_DEV_DATABASE_URL"} + refused = dev(bin_dir, no_url, checkout, "stop", run, "--recover", expected=3) + assert "no PostgreSQL administrator URL" in refused["error"]["message"], refused + assert refused["run_id"] == run, refused + assert len(refused["recovery"]["processes"]) == 2, refused + assert len(product_pids(bin_dir)) == 2, refused + dev(bin_dir, env, checkout, "stop", run, "--recover") + assert_nothing_listening(bin_dir, report["products"].values()) + assert inventory(pg, pg_env) == baseline + assert live_runs(state) == [] + print("stop --recover skips a reused process ID and fails with facts when it cannot proceed") def verify_native_handoff(root, bin_dir, env): @@ -767,9 +843,9 @@ def verify_spatial_refusal(root, bin_dir, pg, state, env, pg_env): supported = bool(offered) and tuple(map(int, offered.split(".")[:2])) >= (3, 5) report = dev(bin_dir, env, checkout, "start", ".", expected=1) if supported: - assert "must be allowed to CREATE EXTENSION postgis" in report["error"], report + assert "must be allowed to CREATE EXTENSION postgis" in report["error"]["message"], report else: - assert "PostGIS 3.5 or later installed" in report["error"], report + assert "PostGIS 3.5 or later installed" in report["error"]["message"], report assert inventory(pg, pg_env) == baseline, report assert live_runs(state) == [], report print("A spatial start without PostGIS rights was refused and left nothing behind") From 981c07c5ab43bca1799241581775894643342f35 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 12:11:45 +0700 Subject: [PATCH 03/24] feat: check rstack dev bindings with the product readers `rstack dev check` now renders each binding fragment with the code `start` uses and validates it with BReg's review-authority reader and Casework's context-binding checks, so a configuration a product would refuse fails before `start` provisions anything. Refusals name the plan field that produced the value. The runner no longer restates the resource and scope rules the readers own, and `check` compares the Casework status profile with the context provider's service-status profile instead of leaving it to Casework start. Signed-off-by: Jeremi Joslin --- Cargo.lock | 1 + crates/registry-breg/src/runtime_config.rs | 49 ++ crates/registry-bregctl/Cargo.toml | 1 + .../src/project_dev/bindings.rs | 463 ++++++++++++------ .../src/project_dev/products.rs | 97 +++- crates/registry-bregctl/tests/project_dev.rs | 95 +++- crates/registry-casework-breg/src/config.rs | 107 +++- products/platform/PROJECT-DEV.md | 9 +- 8 files changed, 647 insertions(+), 175 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1020ceb27d..d7f0d25aa8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6331,6 +6331,7 @@ dependencies = [ "rcgen", "registry-breg", "registry-breg-client", + "registry-casework-breg", "registry-cli-api", "registry-cli-reference", "registry-cli-report", diff --git a/crates/registry-breg/src/runtime_config.rs b/crates/registry-breg/src/runtime_config.rs index 2e79826a38..9972aed606 100644 --- a/crates/registry-breg/src/runtime_config.rs +++ b/crates/registry-breg/src/runtime_config.rs @@ -2759,6 +2759,15 @@ pub(crate) fn parse_secret_reference( SecretReference::parse(value).map_err(|_| error) } +/// Checks one `reviewAuthorities` entry with the reader the runtime uses, so +/// a caller that renders the entry can refuse it before the runtime starts. +/// Secret references are checked for shape only; nothing is resolved. +pub fn check_review_authority(id: &str, authority: &serde_json::Value) -> Result<()> { + let raw = RawReviewAuthorityConfig::deserialize(authority) + .map_err(|_| RuntimeConfigError::InvalidBinding)?; + ReviewAuthorityConfig::from_raw(id, raw).map(|_| ()) +} + #[derive(Clone)] struct ReviewAuthorityConfig { endpoint: reqwest::Url, @@ -4203,6 +4212,46 @@ mod tests { use super::RuntimeConfigError; use crate::attachment_verification_worker::ATTEMPT_BUDGET; + fn review_authority_fragment() -> serde_json::Value { + serde_json::json!({ + "endpoint": "https://casework.example/reviews/", + "profile": "producer", + "producerId": "registry-producer", + "recoveryDays": 7, + "privateKeyJwt": { + "tokenEndpoint": "https://issuer.example/oauth2/token", + "clientIdRef": "secret:file/review-client-id", + "clientAssertionKeyRef": "secret:file/review-client-key", + "assertionAudience": "https://issuer.example", + "resource": "urn:casework:primary", + "scopes": ["casework:reviews:request"] + } + }) + } + + #[test] + fn check_review_authority_applies_the_runtime_reader() { + super::check_review_authority("casework-a", &review_authority_fragment()) + .expect("a well-formed review authority is accepted"); + + let mut bad_endpoint = review_authority_fragment(); + bad_endpoint["endpoint"] = "https://casework.example/reviews/?q=1".into(); + let mut bad_profile = review_authority_fragment(); + bad_profile["profile"] = "not a profile".into(); + let mut unknown_member = review_authority_fragment(); + unknown_member["surprise"] = true.into(); + for fragment in [bad_endpoint, bad_profile, unknown_member] { + assert!( + matches!( + super::check_review_authority("casework-a", &fragment), + Err(RuntimeConfigError::InvalidBinding) + ), + "{fragment}" + ); + } + assert!(super::check_review_authority("", &review_authority_fragment()).is_err()); + } + #[test] fn the_attempt_budget_action_states_the_worker_budget() { let budget = ATTEMPT_BUDGET.as_millis(); diff --git a/crates/registry-bregctl/Cargo.toml b/crates/registry-bregctl/Cargo.toml index 4637a037fd..f65add3ea5 100644 --- a/crates/registry-bregctl/Cargo.toml +++ b/crates/registry-bregctl/Cargo.toml @@ -65,6 +65,7 @@ registry-cli-api.workspace = true registry-cli-reference.workspace = true registry-breg = { workspace = true, features = ["runtime", "tooling"] } registry-breg-client.workspace = true +registry-casework-breg.workspace = true registry-evidence-verifier.workspace = true registry-linkml.workspace = true registry-manifest-core.workspace = true diff --git a/crates/registry-bregctl/src/project_dev/bindings.rs b/crates/registry-bregctl/src/project_dev/bindings.rs index 4a85b97016..04bb8a5524 100644 --- a/crates/registry-bregctl/src/project_dev/bindings.rs +++ b/crates/registry-bregctl/src/project_dev/bindings.rs @@ -114,14 +114,6 @@ impl Bindings { "Products must place BReg before Casework when bindings are configured.", "Move the BReg product before the Casework product.", )?; - // Each binding requests a token for the other product's audience as - // its OAuth resource, which the products accept only as an absolute URI. - if self.breg_review_authority.is_some() { - require_resource_audience(plan, casework, "bregReviewAuthority")?; - } - if self.casework_context_provider.is_some() { - require_resource_audience(plan, breg, "caseworkContextProvider")?; - } if let Some(binding) = &self.breg_review_authority { plan::require( @@ -173,22 +165,6 @@ fn product_position(plan: &Plan, product: Product) -> Option { .position(|entry| entry.product == product) } -fn require_resource_audience( - plan: &Plan, - position: Option, - binding: &str, -) -> plan::SemanticResult<()> { - let Some(index) = position else { - return Ok(()); - }; - plan::require( - registry_platform_httputil::valid_resource_uri(&plan.products[index].audience), - &format!("/products/{index}/audience"), - format!("The audience must be an absolute URI because /bindings/{binding} requests tokens for it as an OAuth resource."), - "Set audience to an absolute URI without a fragment or credentials, such as `urn:example:review`.", - ) -} - fn require_machine_person<'a>( plan: &'a Plan, alias: &str, @@ -220,42 +196,9 @@ fn require_machine_person<'a>( product.name() ), )?; - let grant = &person.grants[&product]; - plan::require( - !grant.scopes.is_empty(), - field, - format!( - "The binding identity must name a person whose {} grant has a scope.", - product.name() - ), - format!( - "Add a scope to the named person's {} grant or choose another identity.", - product.name() - ), - )?; - // The calling product requests these scopes through the shared OAuth - // client, which refuses the list at startup unless it passes this check. - if let Err(error) = registry_platform_httputil::validate_requested_scopes(&grant.scopes) { - let reason = match error { - registry_platform_httputil::TokenError::Configuration { reason } => reason.to_owned(), - other => other.to_string(), - }; - return plan::refuse( - &format!("/people/{alias}/grants/{}/scopes", product.name()), - &format!("A binding requests this grant's scopes over OAuth, and {reason}."), - "Use only OAuth scope characters (no spaces, quotes, or backslashes), and keep the scopes, joined by spaces, within 4096 bytes.", - ); - } Ok(person) } -struct PreparedBinding { - client_id_ref: String, - client_assertion_key_ref: String, - person: String, - product: Product, -} - struct TokenServer { endpoint: String, shutdown: Option>, @@ -273,11 +216,12 @@ impl Drop for TokenServer { } } +const BREG_REVIEW_CLIENT: &str = "breg-review-client"; +const CASEWORK_CONTEXT_CLIENT: &str = "casework-context-client"; + #[derive(Default)] pub(super) struct RuntimeBindings { server: Option, - breg_review: Option, - casework_context: Option, input: Option>, origins: BTreeMap, // Kept outside the server state so Issuer's Tokio runtime is dropped on @@ -304,39 +248,29 @@ impl RuntimeBindings { ); let mut clients = Vec::new(); - let breg_review = bindings - .breg_review_authority - .as_ref() - .map(|binding| { - prepare_binding( - root, - &input, - &binding.identity, - Product::Casework, - "breg-review-client", - &mut clients, - ) - }) - .transpose()?; - let casework_context = bindings - .casework_context_provider - .as_ref() - .map(|binding| { - prepare_binding( - root, - &input, - &binding.identity, - Product::Breg, - "casework-context-client", - &mut clients, - ) - }) - .transpose()?; + if let Some(binding) = &bindings.breg_review_authority { + prepare_binding( + root, + &input, + &binding.identity, + Product::Casework, + BREG_REVIEW_CLIENT, + &mut clients, + )?; + } + if let Some(binding) = &bindings.casework_context_provider { + prepare_binding( + root, + &input, + &binding.identity, + Product::Breg, + CASEWORK_CONTEXT_CLIENT, + &mut clients, + )?; + } let server = start_token_server(input.clone(), issuer.clone(), clients)?; Ok(Self { server: Some(server), - breg_review, - casework_context, input: Some(input), origins: origins.clone(), _issuer: Some(issuer), @@ -352,55 +286,235 @@ impl RuntimeBindings { .as_ref() .expect("prepared token server") .endpoint; - match product { - Product::Breg => { - let (Some(binding), Some(prepared)) = ( - input.plan.bindings.breg_review_authority.as_ref(), - self.breg_review.as_ref(), - ) else { - return; - }; - let person = &input.plan.people[&prepared.person]; - let grant = &person.grants[&prepared.product]; - config["reviewAuthorities"][binding.id.as_str()] = json!({ - "endpoint": self.origins[&Product::Casework], - "profile": grant.profile, - "privateKeyJwt": { - "tokenEndpoint": endpoint, - "clientIdRef": prepared.client_id_ref, - "clientAssertionKeyRef": prepared.client_assertion_key_ref, - "assertionAudience": input.plan.issuer, - "resource": input.plan.product(Product::Casework).audience, - "scopes": grant.scopes, - }, - "producerId": binding.producer, - "recoveryDays": 7, - }); - } - Product::Casework => { - let (Some(binding), Some(prepared)) = ( - input.plan.bindings.casework_context_provider.as_ref(), - self.casework_context.as_ref(), - ) else { - return; - }; - let person = &input.plan.people[&prepared.person]; - let grant = &person.grants[&prepared.product]; - config["contextProviders"][binding.id.as_str()] = json!({ - "baseUrl": self.origins[&Product::Breg], - "statusProfile": grant.profile, - "tokenEndpoint": endpoint, - "clientAssertionAudience": input.plan.issuer, - "resource": input.plan.product(Product::Breg).audience, + render(&input.plan, product, &self.origins, endpoint, config); + } +} + +/// Writes the binding the plan configures for `product` into its runtime +/// configuration. Only the token endpoint and the product origins belong to +/// the run; every other value comes from the plan. +fn render( + plan: &Plan, + product: Product, + origins: &BTreeMap, + token_endpoint: &str, + config: &mut Value, +) { + match product { + Product::Breg => { + let Some(binding) = plan.bindings.breg_review_authority.as_ref() else { + return; + }; + let grant = &plan.people[&binding.identity].grants[&Product::Casework]; + let (client_id_ref, client_assertion_key_ref) = client_refs(BREG_REVIEW_CLIENT); + config["reviewAuthorities"][binding.id.as_str()] = json!({ + "endpoint": origins[&Product::Casework], + "profile": grant.profile, + "privateKeyJwt": { + "tokenEndpoint": token_endpoint, + "clientIdRef": client_id_ref, + "clientAssertionKeyRef": client_assertion_key_ref, + "assertionAudience": plan.issuer, + "resource": plan.product(Product::Casework).audience, "scopes": grant.scopes, - "clientIdRef": prepared.client_id_ref, - "clientAssertionKeyRef": prepared.client_assertion_key_ref, - }); - } + }, + "producerId": binding.producer, + "recoveryDays": 7, + }); + } + Product::Casework => { + let Some(binding) = plan.bindings.casework_context_provider.as_ref() else { + return; + }; + let grant = &plan.people[&binding.identity].grants[&Product::Breg]; + let (client_id_ref, client_assertion_key_ref) = client_refs(CASEWORK_CONTEXT_CLIENT); + config["contextProviders"][binding.id.as_str()] = json!({ + "baseUrl": origins[&Product::Breg], + "statusProfile": grant.profile, + "tokenEndpoint": token_endpoint, + "clientAssertionAudience": plan.issuer, + "resource": plan.product(Product::Breg).audience, + "scopes": grant.scopes, + "clientIdRef": client_id_ref, + "clientAssertionKeyRef": client_assertion_key_ref, + }); + } + } +} + +/// The secret references of the machine client `filename` names. +fn client_refs(filename: &str) -> (String, String) { + ( + format!("secret:file/{filename}-id"), + format!("secret:file/{filename}-key"), + ) +} + +/// Origin and token endpoint `check` renders with. Nothing listens there: the +/// product readers refuse a binding's shape, not the address it names. +const CHECK_ORIGIN: &str = "http://127.0.0.1:1"; +const CHECK_TOKEN_ENDPOINT: &str = "http://127.0.0.1:1/oauth2/token"; + +/// Renders each binding the plan configures exactly as `start` writes it and +/// lets the owning product's reader refuse what it would refuse at startup, +/// before anything is provisioned. A refusal names the plan field that +/// produced the value, not the generated fragment. +pub(super) fn check_with_product_readers(plan: &Plan) -> plan::SemanticResult<()> { + let origins = BTreeMap::from([ + (Product::Breg, CHECK_ORIGIN.to_owned()), + (Product::Casework, CHECK_ORIGIN.to_owned()), + ]); + if let Some(binding) = &plan.bindings.breg_review_authority { + let mut config = json!({}); + render( + plan, + Product::Breg, + &origins, + CHECK_TOKEN_ENDPOINT, + &mut config, + ); + let grant = grant_field(&binding.identity, Product::Casework); + let audience = audience_field(plan, Product::Casework); + require_reader_accepts( + "BReg", + "/bindings/bregReviewAuthority", + &config["reviewAuthorities"][binding.id.as_str()], + &[ + Source::new( + "/profile", + &format!("{grant}/profile"), + json!("placeholder"), + ), + Source::new( + "/privateKeyJwt/assertionAudience", + "/issuer", + json!(STAND_IN_URI), + ), + Source::new("/privateKeyJwt/resource", &audience, json!(STAND_IN_URI)), + Source::new( + "/privateKeyJwt/scopes", + &format!("{grant}/scopes"), + json!(["placeholder"]), + ), + Source::new( + "/producerId", + "/bindings/bregReviewAuthority/producer", + json!("placeholder"), + ), + ], + |fragment| { + registry_breg::runtime_config::check_review_authority(&binding.id, fragment) + .map_err(|error| error.to_string()) + }, + )?; + } + if let Some(binding) = &plan.bindings.casework_context_provider { + let mut config = json!({}); + render( + plan, + Product::Casework, + &origins, + CHECK_TOKEN_ENDPOINT, + &mut config, + ); + let grant = grant_field(&binding.identity, Product::Breg); + let audience = audience_field(plan, Product::Breg); + require_reader_accepts( + "Casework", + "/bindings/caseworkContextProvider", + &config["contextProviders"][binding.id.as_str()], + &[ + Source::new( + "/statusProfile", + &format!("{grant}/profile"), + json!("placeholder"), + ), + Source::new("/clientAssertionAudience", "/issuer", json!(STAND_IN_URI)), + Source::new("/resource", &audience, json!(STAND_IN_URI)), + Source::new( + "/scopes", + &format!("{grant}/scopes"), + json!(["placeholder"]), + ), + ], + |fragment| { + serde_json::from_value::(fragment.clone()) + .map_err(|_| { + "the configuration does not have the shape the reader expects".to_owned() + })? + .check_offline() + .map_err(|error| error.to_string()) + }, + )?; + } + Ok(()) +} + +const STAND_IN_URI: &str = "urn:example:stand-in"; + +/// One member of a rendered fragment and the plan field that produced it, +/// with a value every reader accepts for that member. +struct Source { + member: &'static str, + field: String, + stand_in: Value, +} + +impl Source { + fn new(member: &'static str, field: &str, stand_in: Value) -> Self { + Self { + member, + field: field.to_owned(), + stand_in, } } } +fn grant_field(alias: &str, product: Product) -> String { + format!( + "/people/{}/grants/{}", + registry_platform_yaml::escape_pointer_segment(alias), + product.name() + ) +} + +fn audience_field(plan: &Plan, product: Product) -> String { + let index = product_position(plan, product).expect("validated product"); + format!("/products/{index}/audience") +} + +/// Refuses when `reader` refuses `fragment`. The readers state a rule without +/// saying which member broke it, so the refusal lands on the plan field of the +/// one member whose replacement by a stand-in makes the reader accept. When no +/// single member explains it, the refusal lands on the binding. +fn require_reader_accepts( + product: &str, + binding_field: &str, + fragment: &Value, + sources: &[Source], + reader: impl Fn(&Value) -> Result<(), String>, +) -> plan::SemanticResult<()> { + let Err(refusal) = reader(fragment) else { + return Ok(()); + }; + let field = sources + .iter() + .find(|source| { + let mut probe = fragment.clone(); + probe + .pointer_mut(source.member) + .map(|member| *member = source.stand_in.clone()) + .is_some() + && reader(&probe).is_ok() + }) + .map_or(binding_field, |source| source.field.as_str()); + plan::refuse( + field, + &format!("{product} refuses the binding configuration the run writes from this value: {refusal}."), + "Correct the value so the product accepts the configuration the run writes for this binding.", + ) +} + fn prepare_binding( root: &Path, input: &RunInput, @@ -408,16 +522,17 @@ fn prepare_binding( product: Product, filename: &str, clients: &mut Vec, -) -> Result { +) -> Result<()> { let person = require_machine_person(&input.plan, person_alias, product, "/bindings")?; let grant = &person.grants[&product]; let key = generate_private_jwk(GeneratedKeyAlgorithm::Es256)?; let public_key = key.public(); - let key_name = format!("{filename}-key"); - let id_name = format!("{filename}-id"); let encoded = encode_private_jwk(&key)?; - write_private(&root.join(&key_name), &encoded)?; - write_private(&root.join(&id_name), person.client.as_bytes())?; + write_private(&root.join(format!("{filename}-key")), &encoded)?; + write_private( + &root.join(format!("{filename}-id")), + person.client.as_bytes(), + )?; clients.push(MachineClient { client_id: person.client.clone(), person: person_alias.to_owned(), @@ -426,12 +541,7 @@ fn prepare_binding( resource: input.plan.product(product).audience.clone(), scope: grant.scopes.join(" "), }); - Ok(PreparedBinding { - client_id_ref: format!("secret:file/{id_name}"), - client_assertion_key_ref: format!("secret:file/{key_name}"), - person: person_alias.to_owned(), - product, - }) + Ok(()) } #[derive(Serialize)] @@ -859,26 +969,32 @@ mod tests { } #[test] - fn validation_refuses_a_bound_audience_that_is_not_a_resource_uri() { + fn check_accepts_a_plan_both_product_readers_accept() { + check_with_product_readers(&plan()).unwrap(); + } + + #[test] + fn check_refuses_a_bound_audience_that_is_not_a_resource_uri() { let mut value = plan(); value.products[1].audience = "review".to_owned(); - let error = value.bindings.validate(&value).unwrap_err().to_string(); + let error = check_with_product_readers(&value).unwrap_err().to_string(); assert!(error.contains("/products/1/audience"), "{error}"); - assert!(error.contains("absolute URI"), "{error}"); + assert!(error.contains("BReg refuses"), "{error}"); let mut value = plan(); value.products[0].audience = "registry".to_owned(); - let error = value.bindings.validate(&value).unwrap_err().to_string(); + let error = check_with_product_readers(&value).unwrap_err().to_string(); assert!(error.contains("/products/0/audience"), "{error}"); + assert!(error.contains("Casework refuses"), "{error}"); let mut value = plan(); value.bindings.casework_context_provider = None; value.products[0].audience = "registry".to_owned(); - value.bindings.validate(&value).unwrap(); + check_with_product_readers(&value).unwrap(); } #[test] - fn validation_refuses_bound_scopes_the_products_would_not_request() { + fn check_refuses_bound_scopes_the_products_would_not_request() { // Both products request a binding's scopes through the shared OAuth // client, which refuses this list before the run could start them. let too_long = (0..17) @@ -894,12 +1010,11 @@ mod tests { .unwrap() .scopes .clone_from(&too_long); - let error = value.bindings.validate(&value).unwrap_err().to_string(); + let error = check_with_product_readers(&value).unwrap_err().to_string(); assert!( error.contains("/people/producer/grants/casework/scopes"), "{error}" ); - assert!(error.contains("4096"), "{error}"); let mut value = plan(); value @@ -910,7 +1025,7 @@ mod tests { .get_mut(&Product::Breg) .unwrap() .scopes = vec!["registry:\"quoted\"".to_owned()]; - let error = value.bindings.validate(&value).unwrap_err().to_string(); + let error = check_with_product_readers(&value).unwrap_err().to_string(); assert!( error.contains("/people/context-reader/grants/breg/scopes"), "{error}" @@ -928,6 +1043,48 @@ mod tests { .unwrap() .scopes = too_long; value.bindings.validate(&value).unwrap(); + check_with_product_readers(&value).unwrap(); + } + + #[test] + fn check_refuses_other_binding_values_at_the_plan_field_that_produced_them() { + let mut value = plan(); + value.issuer = "not a uri".to_owned(); + let error = check_with_product_readers(&value).unwrap_err().to_string(); + assert!(error.contains("/issuer"), "{error}"); + + // The profile grammar is a rule only the product readers state: the + // runner restates nothing about it, so each product's reader catches + // it and the refusal lands on the grant that produced the value. + let mut value = plan(); + value + .people + .get_mut("context-reader") + .unwrap() + .grants + .get_mut(&Product::Breg) + .unwrap() + .profile = "not a profile".to_owned(); + let error = check_with_product_readers(&value).unwrap_err().to_string(); + assert!( + error.contains("/people/context-reader/grants/breg/profile"), + "{error}" + ); + + let mut value = plan(); + value + .people + .get_mut("producer") + .unwrap() + .grants + .get_mut(&Product::Casework) + .unwrap() + .profile = "not a profile".to_owned(); + let error = check_with_product_readers(&value).unwrap_err().to_string(); + assert!( + error.contains("/people/producer/grants/casework/profile"), + "{error}" + ); } #[test] diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs index b919f926bc..51cf453c09 100644 --- a/crates/registry-bregctl/src/project_dev/products.rs +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -745,6 +745,7 @@ pub(super) fn inspect( plan: &Plan, terminate: &AtomicBool, ) -> Result { + super::bindings::check_with_product_readers(plan)?; let breg_authority = breg_authority(root, plan)?; let casework_principals = casework_principals(root, plan, breg_authority.as_ref(), terminate)?; Ok(ProductInspection { @@ -986,7 +987,14 @@ fn casework_principals( ).into()); } }; - validate_casework_binding_ids(&plan.bindings, &facts)?; + let status_profile = plan + .bindings + .casework_context_provider + .as_ref() + .and_then(|binding| plan.people.get(&binding.identity)) + .and_then(|person| person.grants.get(&Product::Breg)) + .map(|grant| grant.profile.as_str()); + validate_casework_binding_ids(&plan.bindings, &facts, status_profile)?; Ok(casework_principals_from_facts(plan, breg, &facts)?) } @@ -1112,7 +1120,9 @@ struct InvalidCaseworkProject; struct CaseworkFacts { profiles: BTreeMap, trusted_producers: BTreeSet, - context_providers: BTreeSet, + /// Each context provider's ID with the profile its policy names for the + /// `service-status` capability, when it names one. + context_providers: BTreeMap>, } fn casework_report_facts(report: &Value) -> Result { @@ -1145,10 +1155,30 @@ fn casework_report_facts(report: &Value) -> Result { Ok(CaseworkFacts { profiles: principals, trusted_producers: casework_report_ids(report, "trustedProducers")?, - context_providers: casework_report_ids(report, "contextProviders")?, + context_providers: casework_context_providers(report)?, }) } +fn casework_context_providers(report: &Value) -> Result>> { + let entries = report["contextProviders"] + .as_array() + .context("caseworkctl explain did not report contextProviders")?; + let mut providers = BTreeMap::new(); + for entry in entries { + let id = entry["id"] + .as_str() + .context("a caseworkctl explain contextProviders entry has no ID")?; + let status_profile = entry["serviceProfiles"]["service-status"] + .as_str() + .map(str::to_owned); + ensure!( + providers.insert(id.to_owned(), status_profile).is_none(), + "caseworkctl explain returned duplicate contextProviders IDs" + ); + } + Ok(providers) +} + fn casework_report_ids(report: &Value, member: &str) -> Result> { let entries = report[member] .as_array() @@ -1169,6 +1199,7 @@ fn casework_report_ids(report: &Value, member: &str) -> Result> fn validate_casework_binding_ids( bindings: &Bindings, facts: &CaseworkFacts, + status_profile: Option<&str>, ) -> plan::SemanticResult<()> { if let Some(binding) = &bindings.breg_review_authority { plan::require( @@ -1180,11 +1211,24 @@ fn validate_casework_binding_ids( } if let Some(binding) = &bindings.casework_context_provider { plan::require( - facts.context_providers.contains(&binding.id), + facts.context_providers.contains_key(&binding.id), "/bindings/caseworkContextProvider/id", "The Casework context provider ID does not name a context provider in the Casework project.", "Set id to a context provider defined by the Casework project.", )?; + // The run writes the identity's BReg grant profile as the provider's + // status profile, and Casework refuses a provider whose binding and + // policy disagree. Refusing here keeps that from surfacing at start. + let policy_profile = facts.context_providers[&binding.id].as_deref(); + plan::require( + policy_profile == status_profile, + &format!( + "/people/{}/grants/breg/profile", + registry_platform_yaml::escape_pointer_segment(&binding.identity) + ), + "The BReg grant profile is not the service-status profile the Casework context provider names.", + "Set the grant profile to the service-status profile in the Casework context provider's serviceProfiles.", + )?; } Ok(()) } @@ -1608,7 +1652,7 @@ mod tests { let facts = |principal: &str| CaseworkFacts { profiles: BTreeMap::from([("reviewer".to_owned(), principal.to_owned())]), trusted_producers: BTreeSet::new(), - context_providers: BTreeSet::new(), + context_providers: BTreeMap::new(), }; let plan = authority_plan(json!({}), json!({}), false); @@ -1713,10 +1757,13 @@ mod tests { "reviewPolicies": [], "accessProfiles": [{"id": "reviewer", "principalClaim": "sub"}], "trustedProducers": [{"id": "review-producer"}], - "contextProviders": [{"id": "proposal-context"}] + "contextProviders": [{ + "id": "proposal-context", + "serviceProfiles": {"service-status": "status-reader"} + }] }); let facts = casework_report_facts(&report).unwrap(); - validate_casework_binding_ids(&bindings, &facts).unwrap(); + validate_casework_binding_ids(&bindings, &facts, Some("status-reader")).unwrap(); for (member, field, reason, action) in [ ( @@ -1734,7 +1781,7 @@ mod tests { ] { report[member] = json!([]); let facts = casework_report_facts(&report).unwrap(); - let error = validate_casework_binding_ids(&bindings, &facts) + let error = validate_casework_binding_ids(&bindings, &facts, Some("status-reader")) .unwrap_err() .to_string(); assert!(error.contains(field), "{error}"); @@ -1744,12 +1791,44 @@ mod tests { assert!(!error.contains("proposal-context"), "{error}"); report[member] = match member { "trustedProducers" => json!([{"id": "review-producer"}]), - "contextProviders" => json!([{"id": "proposal-context"}]), + "contextProviders" => json!([{ + "id": "proposal-context", + "serviceProfiles": {"service-status": "status-reader"} + }]), _ => unreachable!(), }; } } + #[test] + fn the_status_profile_the_run_writes_must_match_the_context_provider_policy() { + let bindings = product_bindings(); + let report = |service_profiles: Value| { + json!({ + "apiVersion": "id.registrystack.org/formats/casework/report/v1alpha2", + "kind": "CaseworkReport", "command": "explain", "status": "complete", "ok": true, + "accessProfiles": [], "trustedProducers": [{"id": "review-producer"}], + "contextProviders": [{"id": "proposal-context", "serviceProfiles": service_profiles}] + }) + }; + for (profiles, written) in [ + (json!({"service-status": "other-profile"}), "status-reader"), + (json!({}), "status-reader"), + (json!({"service-status": "status-reader"}), "other-profile"), + ] { + let facts = casework_report_facts(&report(profiles)).unwrap(); + let error = validate_casework_binding_ids(&bindings, &facts, Some(written)) + .unwrap_err() + .to_string(); + assert!( + error.contains("/people/context-reader/grants/breg/profile"), + "{error}" + ); + assert!(error.contains("service-status profile"), "{error}"); + assert!(!error.contains("other-profile"), "{error}"); + } + } + #[test] fn a_report_without_the_access_profile_list_is_refused() { let report = json!({ diff --git a/crates/registry-bregctl/tests/project_dev.rs b/crates/registry-bregctl/tests/project_dev.rs index dc5a2f051a..7b72779430 100644 --- a/crates/registry-bregctl/tests/project_dev.rs +++ b/crates/registry-bregctl/tests/project_dev.rs @@ -333,6 +333,15 @@ fn start_with_casework_tool( } fn fake_casework_tool(root: &std::path::Path) -> std::path::PathBuf { + fake_casework_tool_with_status_profile(root, "operator") +} + +/// A `caseworkctl` whose `proposal-context` provider names `status_profile` +/// for the `service-status` capability. +fn fake_casework_tool_with_status_profile( + root: &std::path::Path, + status_profile: &str, +) -> std::path::PathBuf { let tool = root.join("caseworkctl"); fs::write( &tool, @@ -348,9 +357,12 @@ fn fake_casework_tool(root: &std::path::Path) -> std::path::PathBuf { "\"accessProfiles\":[{\"id\":\"reviewer\",\"principalClaim\":\"sub\"}],", "\"reviewPolicies\":[{\"dependencies\":{\"accessProfiles\":[{\"id\":\"reviewer\",\"principalClaim\":\"sub\"}]}}],", "\"trustedProducers\":[{\"id\":\"review-producer\"}],", - "\"contextProviders\":[{\"id\":\"proposal-context\"}]}", - "'\n", - ), + "\"contextProviders\":[{\"id\":\"proposal-context\",", + "\"serviceProfiles\":{\"service-status\":\"", + ) + .to_owned() + + status_profile + + "\"}}]}'\n", ) .unwrap(); let mut permissions = fs::metadata(&tool).unwrap().permissions(); @@ -1038,6 +1050,83 @@ fn check_and_start_resolve_casework_profiles_and_bindings_with_the_owning_report } } +#[test] +fn check_refuses_binding_configuration_the_products_would_refuse_at_the_plan_field() { + for (plan_edit, from, to, path, product) in [ + ( + "scope", + "casework: {profile: reviewer, scopes: [casework:review]}", + "casework: {profile: reviewer, scopes: ['casework:\"quoted\"']}", + "/people/reviewer/grants/casework/scopes", + "BReg refuses", + ), + ( + "audience", + "audience: \"urn:example:registry\"", + "audience: registry", + "/products/0/audience", + "Casework refuses", + ), + ] { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let tool = casework_reference_project( + scratch.path(), + "reviewer", + concat!( + "bindings:\n", + " bregReviewAuthority: {id: local-review, producer: review-producer, identity: reviewer}\n", + " caseworkContextProvider: {id: proposal-context, identity: operator, action: review-action}\n", + ), + ); + let dev = scratch.path().join("dev.yaml"); + let text = fs::read_to_string(&dev).unwrap(); + assert!(text.contains(from), "{plan_edit}: {text}"); + fs::write(&dev, text.replace(from, to)).unwrap(); + + let checked = check_with_casework_tool(scratch.path(), &state, &tool); + + assert_eq!(checked.status.code(), Some(1), "{plan_edit}: {checked:?}"); + let report: serde_json::Value = serde_json::from_slice(&checked.stdout).unwrap(); + assert_eq!(report["status"], "domain-refusal", "{plan_edit}"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], path, "{plan_edit}: {diagnostic}"); + assert!( + diagnostic["message"].as_str().unwrap().contains(product), + "{plan_edit}: {diagnostic}" + ); + assert!(!state.exists(), "{plan_edit}"); + } +} + +#[test] +fn check_refuses_a_status_profile_the_context_provider_policy_does_not_name() { + let bindings = "bindings:\n caseworkContextProvider: {id: proposal-context, identity: operator, action: review-action}\n"; + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let tool = casework_reference_project(scratch.path(), "reviewer", bindings); + fake_casework_tool_with_status_profile(scratch.path(), "another-status-profile"); + + let checked = check_with_casework_tool(scratch.path(), &state, &tool); + + assert_eq!(checked.status.code(), Some(1), "{checked:?}"); + let report: serde_json::Value = serde_json::from_slice(&checked.stdout).unwrap(); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/people/operator/grants/breg/profile"); + assert!(!diagnostic.to_string().contains("another-status-profile")); + assert!(!state.exists()); + + let started = start_with_casework_tool(scratch.path(), &state, &tool); + assert_eq!(started.status.code(), Some(1), "{started:?}"); + assert!(!state.exists()); + + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let tool = casework_reference_project(scratch.path(), "reviewer", bindings); + let checked = check_with_casework_tool(scratch.path(), &state, &tool); + assert_eq!(checked.status.code(), Some(0), "{checked:?}"); +} + #[test] fn check_distinguishes_an_unavailable_casework_helper_from_an_invalid_project() { let scratch = tempfile::tempdir().unwrap(); diff --git a/crates/registry-casework-breg/src/config.rs b/crates/registry-casework-breg/src/config.rs index 778e7b2a29..50e93748d5 100644 --- a/crates/registry-casework-breg/src/config.rs +++ b/crates/registry-casework-breg/src/config.rs @@ -99,6 +99,52 @@ pub struct BregBinding { #[serde(default, skip_serializing_if = "Option::is_none")] pub delegated_display_exchange: Option, } +impl BregBinding { + /// Checks the binding as provider construction reads it, without + /// resolving a secret or reading a context export. The client identifier + /// is a secret, so the identity check runs with a stand-in; the key and + /// the identifier themselves are checked when the provider is built. + pub fn check_offline(&self) -> Result<(), ReviewContextError> { + use registry_platform_httputil::{ + valid_resource_uri, validate_requested_scopes, PrivateKeyJwtConfig, + }; + self.check_profiles()?; + url::Url::parse(&self.base_url).map_err(|_| ReviewContextError::Invalid)?; + let token_endpoint = + url::Url::parse(&self.token_endpoint).map_err(|_| ReviewContextError::Invalid)?; + let invalid = |_| ReviewContextError::Invalid; + PrivateKeyJwtConfig::check_identity( + &token_endpoint, + "offline-check", + self.client_assertion_audience.as_deref(), + ) + .map_err(invalid)?; + if self + .resource + .as_deref() + .is_some_and(|resource| !valid_resource_uri(resource)) + { + return Err(ReviewContextError::Invalid); + } + if let Some(scopes) = &self.scopes { + validate_requested_scopes(scopes).map_err(invalid)?; + } + Ok(()) + } + + fn check_profiles(&self) -> Result<(), ReviewContextError> { + if !registry_casework_core::valid_profile_identifier(&self.status_profile) + || self.task_grant_profile.as_ref().is_some_and(|profile| { + !registry_casework_core::valid_profile_identifier(profile) + || profile == &self.status_profile + }) + { + return Err(ReviewContextError::Invalid); + } + Ok(()) + } +} + impl std::fmt::Debug for BregBinding { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("BregBinding()") @@ -155,16 +201,10 @@ pub fn build_provider( }; use registry_platform_crypto::PrivateJwk; use std::{sync::Arc, time::Duration}; - if description.is_empty() - || description.len() > 1024 * 1024 - || !registry_casework_core::valid_profile_identifier(&binding.status_profile) - || binding.task_grant_profile.as_ref().is_some_and(|profile| { - !registry_casework_core::valid_profile_identifier(profile) - || profile == &binding.status_profile - }) - { + if description.is_empty() || description.len() > 1024 * 1024 { return Err(ReviewContextError::Invalid); } + binding.check_profiles()?; let export: BregContextExport = serde_json::from_value( registry_breg_client::decode_exact_json(description) .map_err(|_| ReviewContextError::Invalid)?, @@ -286,6 +326,57 @@ mod tests { use super::*; use serde_json::json; + fn offline_binding() -> serde_json::Value { + json!({ + "baseUrl": "http://127.0.0.1:31001", + "statusProfile": "casework-review-status", + "tokenEndpoint": "http://127.0.0.1:31003/oauth2/token", + "clientIdRef": "secret:file/status-client-id", + "clientAssertionKeyRef": "secret:file/status-client-key", + "clientAssertionAudience": "https://idp.example.test", + "resource": "urn:example:registry", + "scopes": ["registry:read"] + }) + } + + fn offline_check(value: serde_json::Value) -> Result<(), ReviewContextError> { + serde_json::from_value::(value) + .expect("the binding parses") + .check_offline() + } + + #[test] + fn offline_check_accepts_a_binding_without_resolving_secrets() { + offline_check(offline_binding()).expect("the binding is acceptable"); + } + + #[test] + fn offline_check_refuses_what_provider_construction_refuses_before_secrets() { + for (member, value) in [ + ("statusProfile", json!("not a profile")), + ("taskGrantProfile", json!("casework-review-status")), + ("taskGrantProfile", json!("not a profile")), + ("resource", json!("registry")), + ("resource", json!("urn:example:registry#fragment")), + ("scopes", json!([])), + ("scopes", json!(["registry:read", "registry:read"])), + ("scopes", json!(["has space"])), + ("clientAssertionAudience", json!(" ")), + ("tokenEndpoint", json!("http://idp.example.test/token")), + ("baseUrl", json!("not a url")), + ] { + let mut binding = offline_binding(); + binding[member] = value.clone(); + let result = serde_json::from_value::(binding) + .map_err(|_| ReviewContextError::Invalid) + .and_then(|binding| binding.check_offline()); + assert!( + matches!(result, Err(ReviewContextError::Invalid)), + "{member} {value} was accepted" + ); + } + } + #[test] fn delegated_display_exchange_is_explicit_and_redacted() { let binding: BregBinding = serde_json::from_value(json!({ diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index ca4fdaa681..7b7a3c616b 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -131,12 +131,17 @@ Casework producer, and `bindings.caseworkContextProvider` identifies the BReg context provider and action. `rstack dev check` refuses a binding that names a review authority or action the BReg project does not declare, or a producer or context provider the Casework project does not declare; `start` refuses it -before it provisions anything. A binding requests tokens for the other +before it provisions anything. `check` also writes each binding's product +configuration exactly as `start` does and lets BReg's review-authority reader +and Casework's context-binding reader refuse what they would refuse at startup, +reporting the refusal at the plan field that produced the value. It refuses a +BReg grant profile that is not the `service-status` profile the Casework context +provider's policy names. A binding requests tokens for the other product's audience as an OAuth resource, so that audience must be an absolute URI, such as `urn:example:review`. It requests the scopes of its identity's grant for that product, so those scopes must be OAuth scope-tokens (no quotes or backslashes) that fit in 4096 bytes when joined by spaces; `check` reports a -violation at the grant's `scopes`. The run pairs the provider on a private copy +violation at the audience or the grant's `scopes`. The run pairs the provider on a private copy before packaging. A reviewer whose Casework context call forwards the bearer to BReg declares both grants, one shared client, and `caseworkContext: true`. Both grants declare the same scopes and additional claims. Casework reads the From 87c07c2e58cb02f1713532eb1b2cf87e2658dc04 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 13:14:43 +0700 Subject: [PATCH 04/24] fix(breg): decode encrypted field envelopes across PostgreSQL line breaks PostgreSQL's base64 encode breaks its output every 76 characters. The record read and history migration projections pass that text through unchanged, so reading an encrypted value whose envelope exceeds 57 bytes failed with source.unavailable. Row decode now drops the line breaks, which covers every projection that feeds it. Signed-off-by: Jeremi Joslin --- crates/registry-breg/src/mutation.rs | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/crates/registry-breg/src/mutation.rs b/crates/registry-breg/src/mutation.rs index 6f2f0c49aa..f10bc01f62 100644 --- a/crates/registry-breg/src/mutation.rs +++ b/crates/registry-breg/src/mutation.rs @@ -3659,8 +3659,10 @@ pub(crate) fn envelope_member_from_projection(value: Value) -> Result>(); + let encoded = base64::engine::general_purpose::STANDARD.encode(&envelope); + let wrapped = encoded + .as_bytes() + .chunks(76) + .map(|line| std::str::from_utf8(line).expect("base64 is ASCII")) + .collect::>() + .join("\n"); + assert_eq!( + envelope_member_from_projection(Value::String(wrapped)), + Ok(envelope_member_json(&envelope)) + ); + assert_eq!( + envelope_member_from_projection(Value::String(format!(" {encoded}"))), + Err(MutationError::Unavailable) + ); + } } From 3b9ba41df2f2d05b4166d55223e36ca9babbfe2f Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 13:14:52 +0700 Subject: [PATCH 05/24] feat: run rstack dev projects with encrypted fields on a per-run key `rstack dev` no longer refuses a BReg project with encrypted fields. `start` draws a data key for the run with BReg's key generator, owner-only in the owner-only run directory, and configures BReg's local-file key custody to read it by reference. The key never reaches output, the plan, the error envelope or the checkout, and goes with the run directory on stop, TTL expiry, a failed start and `stop --recover`. A run that needs recovery names the key file in `recovery.field_encryption_key`. Environment failures now report the operational class (exit 3): an unreachable PostgreSQL server or refused administrator session, failed provisioning, an unreadable CA file, a product process that cannot be started, and no free loopback port. The worker carries the class to the supervisor. Project and plan refusals stay domain (exit 1). The admin connection deadline is tested against a server that accepts and never answers, for the first connection and the cleanup reconnection, and the large-plan test starts through the supervisor instead of writing the worker input by hand. Signed-off-by: Jeremi Joslin --- .../src/project_dev/bindings.rs | 4 +- .../src/project_dev/database.rs | 139 +++++++++- .../src/project_dev/journey.rs | 1 + .../registry-bregctl/src/project_dev/mod.rs | 160 +++++++++-- .../src/project_dev/products.rs | 254 +++++++++++++---- crates/registry-bregctl/tests/project_dev.rs | 189 +++++++++---- products/platform/PROJECT-DEV.md | 47 +++- .../schemas/rstack-dev-error.v1.schema.json | 6 +- products/platform/scripts/test-rstack-dev.py | 262 +++++++++++++++++- 9 files changed, 918 insertions(+), 144 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/bindings.rs b/crates/registry-bregctl/src/project_dev/bindings.rs index 04bb8a5524..290fbf7aff 100644 --- a/crates/registry-bregctl/src/project_dev/bindings.rs +++ b/crates/registry-bregctl/src/project_dev/bindings.rs @@ -618,7 +618,8 @@ fn start_token_server( ) -> Result { ensure!(!clients.is_empty(), "a binding token client is required"); let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) - .context("cannot bind the run-local token endpoint")?; + .context("cannot bind the run-local token endpoint") + .map_err(super::operational)?; listener.set_nonblocking(true)?; let origin = format!("http://{}", listener.local_addr()?); let endpoint = format!("{origin}/oauth2/token"); @@ -945,6 +946,7 @@ mod tests { principal: "sub".to_owned(), purpose: false, requires_postgis: false, + encrypts_fields: false, named_clients: BTreeSet::new(), }), casework_principals: BTreeMap::from([("producer".to_owned(), "sub".to_owned())]), diff --git a/crates/registry-bregctl/src/project_dev/database.rs b/crates/registry-bregctl/src/project_dev/database.rs index 771db9d5df..5cf5ffab74 100644 --- a/crates/registry-bregctl/src/project_dev/database.rs +++ b/crates/registry-bregctl/src/project_dev/database.rs @@ -275,7 +275,10 @@ impl Database { &ownership_lock, session_deadline, ) - .context("cannot establish the PostgreSQL administrator session over verified TLS")?; + .context("cannot establish the PostgreSQL administrator session over verified TLS") + // The server is unreachable or refused the session: the environment + // failed, not the request. + .map_err(super::operational)?; Ok(Self { client, _connection_task: connection_task, @@ -928,6 +931,140 @@ mod tests { ); } + /// A loopback server that accepts one connection and never answers it, as a + /// PostgreSQL server does whose TLS negotiation stalls. The thread ends when + /// the client closes the connection. + fn stalled_server() -> (u16, std::thread::JoinHandle<()>) { + let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + std::io::copy(&mut stream, &mut std::io::sink()).unwrap(); + }); + (port, server) + } + + /// The administrator configuration `Database::connect` builds for `port`. + fn stalled_admin_config(port: u16) -> Config { + let (mut config, _) = admin_config(&format!( + "postgresql://admin:secret@127.0.0.1:{port}/postgres" + )) + .unwrap(); + config.ssl_mode(SslMode::Require); + apply_session_options(&mut config); + config + } + + /// A TLS connector that trusts nothing; a stalled server never reaches TLS. + fn untrusting_tls() -> MakeRustlsConnect { + let _ = rustls::crypto::ring::default_provider().install_default(); + MakeRustlsConnect::new( + rustls::ClientConfig::builder() + .with_root_certificates(rustls::RootCertStore::empty()) + .with_no_client_auth(), + ) + } + + #[test] + fn a_stalled_server_cannot_hold_the_first_administrator_connection() { + let (port, server) = stalled_server(); + let runtime = Builder::new_current_thread().enable_all().build().unwrap(); + let started = Instant::now(); + + let Err(error) = open_admin_session( + &runtime, + &stalled_admin_config(port), + &untrusting_tls(), + "SELECT 1", + Instant::now() + Duration::from_millis(300), + ) else { + panic!("a server that never answers opened a session"); + }; + + assert_eq!( + format!("{error:#}"), + "the PostgreSQL connection handshake timed out" + ); + assert!(started.elapsed() < Duration::from_secs(5)); + // The abandoned attempt closed its connection. + server.join().unwrap(); + } + + /// A plaintext server that completes one PostgreSQL startup without + /// authentication, so a test can hold a real administrator `Client`. + fn trusting_server() -> (u16, std::thread::JoinHandle<()>) { + use std::io::{Read, Write}; + let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut length = [0_u8; 4]; + stream.read_exact(&mut length).unwrap(); + let mut startup = vec![0_u8; u32::from_be_bytes(length) as usize - 4]; + stream.read_exact(&mut startup).unwrap(); + // AuthenticationOk, then ReadyForQuery while idle. + stream + .write_all(b"R\0\0\0\x08\0\0\0\0Z\0\0\0\x05I") + .unwrap(); + std::io::copy(&mut stream, &mut std::io::sink()).unwrap(); + }); + (port, server) + } + + #[test] + fn a_stalled_server_cannot_hold_the_cleanup_reconnection() { + let (trusting_port, trusting) = trusting_server(); + let (stalled_port, stalled) = stalled_server(); + let runtime = Builder::new_current_thread().enable_all().build().unwrap(); + let mut first = Config::new(); + first + .host("127.0.0.1") + .port(trusting_port) + .user("admin") + .dbname("postgres") + .ssl_mode(SslMode::Disable); + let (client, connection) = runtime + .block_on(first.connect(tokio_postgres::NoTls)) + .unwrap(); + let connection_task = runtime.spawn(connection); + // The first session failed mid-operation, so cleanup must reconnect + // first, and the replacement server never answers. + let mut database = Database { + client, + _connection_task: connection_task, + admin_session_failed: true, + tls: untrusting_tls(), + admin_config: stalled_admin_config(stalled_port), + ownership_lock: "SELECT 1".to_owned(), + endpoint: Endpoint { + host: UrlHost::Ipv4(std::net::Ipv4Addr::LOCALHOST), + port: Some(stalled_port), + }, + run_id: RUN_ID.to_owned(), + acquisitions: Vec::new(), + runtime, + }; + let started = Instant::now(); + + let error = database + .execute_cleanup( + "DROP ROLE IF EXISTS rstack_stalled_role", + Instant::now() + Duration::from_millis(300), + ) + .unwrap_err(); + + assert_eq!( + format!("{error:#}"), + "cannot reconnect to the PostgreSQL administrator database: \ + the PostgreSQL connection handshake timed out" + ); + assert!(started.elapsed() < Duration::from_secs(5)); + assert!(database.admin_session_failed); + stalled.join().unwrap(); + drop(database); + trusting.join().unwrap(); + } + #[test] fn post_handshake_operation_deadline_marks_the_effect_unconfirmed() { let runtime = Builder::new_current_thread().enable_time().build().unwrap(); diff --git a/crates/registry-bregctl/src/project_dev/journey.rs b/crates/registry-bregctl/src/project_dev/journey.rs index 0cd7596abf..316a531b4b 100644 --- a/crates/registry-bregctl/src/project_dev/journey.rs +++ b/crates/registry-bregctl/src/project_dev/journey.rs @@ -732,6 +732,7 @@ mod tests { principal: "sub".to_owned(), purpose: false, requires_postgis: false, + encrypts_fields: false, named_clients: Default::default(), }), casework_principals: BTreeMap::new(), diff --git a/crates/registry-bregctl/src/project_dev/mod.rs b/crates/registry-bregctl/src/project_dev/mod.rs index 23149d11fa..4145fa9131 100644 --- a/crates/registry-bregctl/src/project_dev/mod.rs +++ b/crates/registry-bregctl/src/project_dev/mod.rs @@ -126,6 +126,9 @@ struct BregAuthority { principal: String, purpose: bool, requires_postgis: bool, + /// The compiled project declares encrypted fields, so the run draws its + /// own development data key. + encrypts_fields: bool, named_clients: BTreeSet, } @@ -138,8 +141,36 @@ struct Binaries { #[derive(Deserialize, Serialize)] #[serde(tag = "outcome", rename_all = "snake_case")] enum WorkerReply { - Ready { report: Value }, - Failed { error: String }, + Ready { + report: Value, + }, + Failed { + error: String, + /// The environment failed rather than the project or plan, so the + /// supervisor reports the operational class. + #[serde(default)] + operational: bool, + }, +} + +impl WorkerReply { + fn failed(error: &anyhow::Error) -> Self { + Self::failure(format!("{error:#}"), error.is::()) + } + + fn failure(error: String, operational: bool) -> Self { + Self::Failed { error, operational } + } +} + +/// The supervisor's error for a failure its worker reported, in the class the +/// worker gave it. +fn restore_failure(error: String, operational: bool) -> anyhow::Error { + if operational { + Operational(error).into() + } else { + anyhow::anyhow!(error) + } } #[derive(Debug, thiserror::Error)] @@ -205,6 +236,9 @@ fn recovery_message(run_id: &str, facts: &Value) -> String { { line(format!("database or role {}", name.as_str().unwrap_or("?"))); } + if let Some(key) = facts["field_encryption_key"].as_str() { + line(format!("development field-encryption key {key}")); + } if let Some(reason) = facts["unavailable"].as_str() { line(format!("some facts are unavailable: {reason}")); } @@ -224,9 +258,17 @@ fn lost_supervisor_facts(root: &Path, run_id: &str) -> Value { Ok(objects) => facts["objects"] = json!(objects), Err(error) => facts["unavailable"] = json!(format!("{error:#}")), } + name_field_key(root, &mut facts); facts } +/// Name, never read, the development data key a run left behind. +fn name_field_key(root: &Path, facts: &mut Value) { + if let Some(key) = products::field_key_file(root) { + facts["field_encryption_key"] = json!(key); + } +} + fn recorded_processes(root: &Path) -> Value { read_json::(&root.join("processes.json"), MAXIMUM_RUN_REPORT_BYTES) .ok() @@ -313,12 +355,7 @@ pub(crate) fn run(args: Args, stdout: &mut dyn Write, stderr: &mut dyn Write) -> return match worker(&root, stdout) { Ok(()) => 0, Err(error) => { - let _ = reply( - stdout, - &WorkerReply::Failed { - error: format!("{error:#}"), - }, - ); + let _ = reply(stdout, &WorkerReply::failed(&error)); 3 } }; @@ -381,6 +418,13 @@ pub(crate) fn run(args: Args, stdout: &mut dyn Write, stderr: &mut dyn Write) -> #[error("{0}")] struct Operational(String); +/// Classify `error` as an environment failure, such as an unreachable +/// PostgreSQL server, a product binary that cannot be started, or no free +/// loopback port. The message is unchanged. +fn operational(error: anyhow::Error) -> anyhow::Error { + Operational(format!("{error:#}")).into() +} + /// The error envelope and exit code for a failed command. It is the launcher's /// `registry.rstack.error.v1` object, plus `run_id` and `recovery` when a run /// still has resources to release. @@ -661,8 +705,8 @@ fn start(project: &Path, ttl: Option<&str>) -> Result { loop { match receiver.recv_timeout(POLL) { Ok(Ok(WorkerReply::Ready { report })) => return Ok(report), - Ok(Ok(WorkerReply::Failed { error })) => { - let error = anyhow::anyhow!(error); + Ok(Ok(WorkerReply::Failed { error, operational })) => { + let error = restore_failure(error, operational); let kept_logs = failed_logs_dir(&base, &input.run_id); if root.join("worker.lock").exists() { if let Err(cleanup) = stop(&input.run_id) { @@ -743,7 +787,8 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { let ca = env::var_os(CA_FILE) .map(|path| { let bytes = plan::read(Path::new(&path), 1024 * 1024) - .context("cannot read the PostgreSQL CA file")?; + .context("cannot read the PostgreSQL CA file") + .map_err(operational)?; write_private(&root.join("postgres-ca"), &bytes)?; Ok::<_, anyhow::Error>(bytes) }) @@ -776,7 +821,10 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { match prepared { Ok(report) => { if let Err(error) = reply(output, &WorkerReply::Ready { report }) { - failure = Some(format!("cannot report development run readiness: {error}")); + failure = Some(WorkerReply::failure( + format!("cannot report development run readiness: {error}"), + true, + )); } else { let deadline = Instant::now() + Duration::from_secs(input.ttl_seconds); while Instant::now() < deadline @@ -790,7 +838,7 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { } } } - Err(error) => failure = Some(format!("{error:#}")), + Err(error) => failure = Some(WorkerReply::failed(&error)), } let stopped = simple_report(&input.run_id, "stopping"); let mark_stopping = write_private(&root.join(REPORT), &serde_json::to_vec(&stopped)?); @@ -805,13 +853,8 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { if cleanup.is_ok() { break; } - if let Some(error) = failure.as_ref().filter(|_| !told_parent) { - let _ = reply( - output, - &WorkerReply::Failed { - error: error.clone(), - }, - ); + if let Some(failed) = failure.as_ref().filter(|_| !told_parent) { + let _ = reply(output, failed); told_parent = true; } let Some(delay) = delays.next() else { @@ -836,12 +879,12 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { Ok(()) } .and_then(|()| remove_run_dir(root)); - if let Some(error) = failure.filter(|_| !told_parent) { + if let Some(WorkerReply::Failed { error, operational }) = failure.filter(|_| !told_parent) { let error = match removed.as_ref() { Ok(()) => error, Err(cleanup) => format!("{error}; run {} {cleanup}", input.run_id), }; - reply(output, &WorkerReply::Failed { error })?; + reply(output, &WorkerReply::failure(error, operational))?; } removed?; // If the root is gone, no stale ready report could have survived. @@ -896,6 +939,7 @@ fn pending_status(root: &Path, run_id: &str, pending: Value) -> Value { "databases_and_roles": pending["databases_and_roles"], "processes": pending["processes"], }); + name_field_key(root, &mut report["recovery"]); report } @@ -1300,7 +1344,10 @@ fn remove_run_dir(root: &Path) -> Result<()> { fn note_kept_logs(error: anyhow::Error, kept: Option) -> anyhow::Error { match kept { - Some(path) => anyhow::anyhow!("{error:#}; logs kept at {}", path.display()), + Some(path) => restore_failure( + format!("{error:#}; logs kept at {}", path.display()), + error.is::(), + ), None => error, } } @@ -1346,7 +1393,8 @@ mod tests { "schema_people": [], "breg_authority": { "principal": "p", "purpose": false, - "requires_postgis": postgis, "named_clients": [] + "requires_postgis": postgis, "encrypts_fields": false, + "named_clients": [] }, "casework_principals": {}, "binaries": {} @@ -1425,6 +1473,45 @@ mod tests { ); } + #[test] + fn recovery_names_a_field_key_left_in_the_run_directory() { + let scratch = tempfile::tempdir().unwrap(); + let root = scratch.path(); + fake_run( + root, + json!([{"product": "breg", "project": ".", "audience": "registry"}]), + false, + ); + let facts = lost_supervisor_facts(root, RUN_ID); + assert!(facts.get("field_encryption_key").is_none(), "{facts}"); + let pending = pending_status(root, RUN_ID, pending_report(RUN_ID, vec![], json!([]))); + assert!( + pending["recovery"].get("field_encryption_key").is_none(), + "{pending}" + ); + + let canary = "field-key-canary-value"; + fs::write(root.join(products::FIELD_KEY), canary).unwrap(); + let key = root.join(products::FIELD_KEY); + for facts in [ + lost_supervisor_facts(root, RUN_ID), + pending_status(root, RUN_ID, pending_report(RUN_ID, vec![], json!([])))["recovery"] + .clone(), + ] { + assert_eq!(facts["field_encryption_key"], json!(key), "{facts}"); + assert!(!facts.to_string().contains(canary), "{facts}"); + let text = recovery_message(RUN_ID, &facts); + assert!( + text.contains(&format!( + "development field-encryption key {}", + key.display() + )), + "{text}" + ); + assert!(!text.contains(canary), "{text}"); + } + } + #[test] fn errors_map_to_the_envelope_class_and_exit_code() { let (report, code) = error_report(&anyhow::anyhow!("invalid test person name")); @@ -1477,6 +1564,31 @@ mod tests { ); } + #[test] + fn a_worker_failure_keeps_its_class_through_the_supervisor() { + let class = |error: &anyhow::Error| error_report(error).0["error"]["class"].clone(); + for (failure, expected) in [ + ( + anyhow::Error::new(Operational("server unreachable".into())).context("start"), + "operational", + ), + (anyhow::anyhow!("breg check failed"), "domain"), + ] { + let mut line = Vec::new(); + reply(&mut line, &WorkerReply::failed(&failure)).unwrap(); + let WorkerReply::Failed { error, operational } = serde_json::from_slice(&line).unwrap() + else { + panic!("the worker reported readiness"); + }; + let restored = restore_failure(error, operational); + assert_eq!(restored.to_string(), format!("{failure:#}")); + assert_eq!(class(&restored), expected); + let noted = note_kept_logs(restored, Some(PathBuf::from("/kept/logs"))); + assert!(noted.to_string().ends_with("; logs kept at /kept/logs")); + assert_eq!(class(¬ed), expected); + } + } + #[test] fn provisioned_labels_follow_the_plan() { let scratch = tempfile::tempdir().unwrap(); diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs index 51cf453c09..a3d57f2d15 100644 --- a/crates/registry-bregctl/src/project_dev/products.rs +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -27,10 +27,22 @@ use super::{ clock, database::{Database, DatabaseUrls}, issuer::Issuer, + operational, plan::{self, Plan, Product}, - private_directory, stopping, write_private, BregAuthority, RunInput, POLL, + private_directory, stopping, write_private, BregAuthority, Operational, RunInput, POLL, }; +/// The run's development field-encryption data key, inside the owner-only run +/// directory and removed with it. +pub(super) const FIELD_KEY: &str = "field-key"; + +/// The run's field-encryption key file, when the run wrote one, so recovery +/// can name it for removal. +pub(super) fn field_key_file(root: &Path) -> Option { + let path = root.join(FIELD_KEY); + fs::symlink_metadata(&path).is_ok().then_some(path) +} + pub(super) struct ProductRunner<'a> { input: Arc, root: &'a Path, @@ -97,7 +109,8 @@ impl<'a> ProductRunner<'a> { .iter() .map(|_| TcpListener::bind((Ipv4Addr::LOCALHOST, 0))) .collect::>() - .context("reserve loopback listeners")?; + .context("reserve loopback listeners") + .map_err(operational)?; let urls = self .input .plan @@ -148,7 +161,8 @@ impl<'a> ProductRunner<'a> { drop(reservation); let child = command .spawn() - .with_context(|| format!("cannot start {}", product.name()))?; + .with_context(|| format!("cannot start {}", product.name())) + .map_err(operational)?; self.children.push((product, child)); self.record_processes()?; self.ready(product, url)?; @@ -169,7 +183,12 @@ impl<'a> ProductRunner<'a> { .as_ref() .expect("BReg authority") .requires_postgis; - let rehearsal = database.provision("breg_test", true, requires_postgis)?; + self.write_field_key()?; + // A server that cannot provision, for example without PostGIS or the + // privilege to create it, is an environment failure. + let rehearsal = database + .provision("breg_test", true, requires_postgis) + .map_err(operational)?; let empty_package = self.root.join("breg-test-package"); fs::create_dir(&empty_package)?; let config = self.config( @@ -207,7 +226,9 @@ impl<'a> ProductRunner<'a> { ], )?; let package = build.join("package"); - let urls = database.provision("breg", true, requires_postgis)?; + let urls = database + .provision("breg", true, requires_postgis) + .map_err(operational)?; let config = self.config(Product::Breg, "breg", bind, &package, urls)?; self.tool( Product::Breg, @@ -228,6 +249,33 @@ impl<'a> ProductRunner<'a> { Ok((config, package)) } + /// Draw the run's own data key when the project encrypts fields. BReg's + /// key generator creates the file owner-only from the platform random + /// source; the key is read only by the runtime through its `secret:file` + /// reference. + fn write_field_key(&self) -> Result<()> { + if !self.encrypts_fields() { + return Ok(()); + } + crate::field_encryption::keygen(&self.root.join(FIELD_KEY)) + .map(drop) + .map_err(|_| { + Operational( + "cannot write the development run's field-encryption key into its run \ + directory" + .to_owned(), + ) + .into() + }) + } + + fn encrypts_fields(&self) -> bool { + self.input + .breg_authority + .as_ref() + .is_some_and(|authority| authority.encrypts_fields) + } + fn schema_credentials(&self, issuer: &Issuer) -> Result { let mut bindings = Vec::with_capacity(self.input.schema_people.len()); for (index, binding) in self.input.schema_people.iter().enumerate() { @@ -301,7 +349,9 @@ impl<'a> ProductRunner<'a> { package.as_os_str(), ], )?; - let urls = database.provision("casework", false, false)?; + let urls = database + .provision("casework", false, false) + .map_err(operational)?; let config = self.config(Product::Casework, "casework", bind, &package, urls)?; let arguments = ["--runtime-config".as_ref(), config.as_os_str()]; self.tool(Product::Casework, "apply", &arguments)?; @@ -379,10 +429,18 @@ impl<'a> ProductRunner<'a> { if name == "breg" && authority.requires_postgis { config["listener"]["publicOrigin"] = json!(format!("http://{bind}")); } + // A development run initializes a disposable local database, as + // `bregctl dev` does; BReg accepts local-file key custody only + // for that environment. config["identity"] = json!({ - "environment": "development", "instanceId": identity, - "databaseId": identity, "databaseInitializationEnvironment": "development" + "environment": "local", "instanceId": identity, + "databaseId": identity, "databaseInitializationEnvironment": "local" }); + if authority.encrypts_fields { + config["fieldEncryption"] = json!({"provider": { + "type": "local-file", "dekRef": format!("secret:file/{FIELD_KEY}") + }}); + } let mut authority_claims = json!({"principal": authority.principal}); if authority.purpose { authority_claims["purpose"] = json!("registry_purpose"); @@ -615,7 +673,8 @@ fn run_logged_tool( .stdout(output.try_clone()?) .stderr(open_log(root, &name)?) .spawn() - .with_context(|| format!("cannot start {product} {stage}"))?; + .with_context(|| format!("cannot start {product} {stage}")) + .map_err(operational)?; if wait_for_tool(&mut child, active)?.success() { return Ok(()); } @@ -816,7 +875,6 @@ fn breg_authority(root: &Path, plan: &Plan) -> Result> { "Run `rstack breg check` for authored diagnostics, then correct the BReg project.", ) })?; - refuse_encrypted_fields(®istry, &project.field)?; let review_authorities = registry .review_authorities() .iter() @@ -837,6 +895,7 @@ fn breg_authority(root: &Path, plan: &Plan) -> Result> { purpose: inventory.purpose_required, named_clients: crate::identity_facts::named_requester_clients(®istry), requires_postgis: breg_requires_postgis(®istry), + encrypts_fields: breg_encrypts_fields(®istry), })) } @@ -896,28 +955,15 @@ fn validate_breg_binding_ids( Ok(()) } -/// The generated runtime configuration has no field-encryption provider, so a -/// project with encrypted fields would only fail late and obscurely. -fn refuse_encrypted_fields( - registry: ®istry_breg::CompiledRegistry, - product_field: &str, -) -> plan::SemanticResult<()> { - for entity in registry.entities().values() { - for field in entity.fields.values() { - plan::require( - field.encryption.is_none(), - product_field, - format!( - "The BReg project encrypts {}.{}, and rstack dev does not configure field \ - encryption yet.", - entity.id, field.id - ), - "Run this BReg project with `rstack breg` and a field-encryption provider, or \ - select a project without encrypted fields.", - )?; - } - } - Ok(()) +/// The runtime needs a field-encryption provider exactly when the compiled +/// project declares an encrypted field. +fn breg_encrypts_fields(registry: ®istry_breg::CompiledRegistry) -> bool { + registry.entities().values().any(|entity| { + entity + .fields + .values() + .any(|field| field.encryption.is_some()) + }) } /// Every BReg grant must name an access profile the project declares, @@ -1684,6 +1730,7 @@ mod tests { principal: "registry_principal".to_owned(), purpose: false, requires_postgis: false, + encrypts_fields: false, named_clients: BTreeSet::new(), }; let error = casework_principals_from_facts(&plan, Some(&breg), &facts("custom_principal")) @@ -2003,30 +2050,133 @@ mod tests { check_breg_grants(®istry, &people).unwrap(); } + fn encrypt_contact_email(source: &str) -> String { + let source = source.replacen( + "mutationMode: mutable\n display: {label: Contact", + "mutationMode: mutable\n classification: restricted\n display: {label: Contact", + 1, + ); + source.replacen( + "{id: email, type: string, required: true, minimumLength: 1, maximumLength: 254, classification: internal}\n - {id: jurisdiction", + "{id: email, type: string, required: true, minimumLength: 1, maximumLength: 254, classification: restricted, encrypted: true}\n - {id: jurisdiction", + 1, + ) + } + #[test] - fn encrypted_fields_are_refused_before_provisioning() { - let registry = minimal_registry(|source| { - let source = source.replacen( - "mutationMode: mutable\n display: {label: Contact", - "mutationMode: mutable\n classification: restricted\n display: {label: Contact", - 1, - ); - source.replacen( - "{id: email, type: string, required: true, minimumLength: 1, maximumLength: 254, classification: internal}\n - {id: jurisdiction", - "{id: email, type: string, required: true, minimumLength: 1, maximumLength: 254, classification: restricted, encrypted: true}\n - {id: jurisdiction", - 1, + fn the_compiled_project_decides_whether_a_run_encrypts_fields() { + assert!(breg_encrypts_fields(&minimal_registry( + encrypt_contact_email + ))); + assert!(!breg_encrypts_fields(&minimal_registry(str::to_owned))); + } + + fn breg_input(encrypts_fields: bool) -> RunInput { + RunInput { + run_id: "0123456789abcdef0123456789abcdef".to_owned(), + project: PathBuf::from("/tmp/project"), + plan: serde_json::from_value(json!({ + "issuer": "https://issuer.example.invalid", + "products": [{"product": "breg", "project": ".", "audience": "registry"}], + "people": {"reader": { + "subject": "reader", "client": "reader-client", + "grants": {"breg": {"profile": "reader", "scopes": []}} + }}, + "journey": {"id": "read", "steps": [{ + "id": "read", "product": "breg", "as": "reader", "method": "GET", + "path": "/v1/records", "expect": {"status": 200} + }]} + })) + .unwrap(), + ttl_seconds: 300, + schema_people: Vec::new(), + breg_authority: Some(BregAuthority { + principal: "sub".to_owned(), + purpose: false, + requires_postgis: false, + encrypts_fields, + named_clients: BTreeSet::new(), + }), + casework_principals: BTreeMap::new(), + binaries: BTreeMap::new(), + } + } + + fn breg_config(runner: &ProductRunner<'_>, root: &Path, name: &str) -> (Value, String) { + let urls = DatabaseUrls { + runtime: "postgresql://runtime:password@127.0.0.1:5432/run".to_owned(), + migration: "postgresql://migration:password@127.0.0.1:5432/run".to_owned(), + }; + let path = runner + .config( + Product::Breg, + name, + "127.0.0.1:0", + &root.join("package"), + urls, ) - }); + .unwrap(); + let text = fs::read_to_string(path).unwrap(); + (serde_json::from_str(&text).unwrap(), text) + } + + #[test] + fn an_encrypting_run_configures_local_file_custody_with_an_owner_only_run_key() { + use base64::{engine::general_purpose::STANDARD, Engine}; + let scratch = tempfile::tempdir().unwrap(); + let root = scratch.path().join("run"); + super::super::private_directory(&root).unwrap(); + let terminate = AtomicBool::new(false); + let runner = ProductRunner::new(Arc::new(breg_input(true)), &root, &terminate).unwrap(); + + runner.write_field_key().unwrap(); + let key = root.join(FIELD_KEY); + assert_eq!(field_key_file(&root), Some(key.clone())); assert_eq!( - refuse_encrypted_fields(®istry, "/products/0/project") - .unwrap_err() - .to_string(), - "dev.yaml /products/0/project: The BReg project encrypts contact.email, and rstack \ - dev does not configure field encryption yet. Next: Run this BReg project with \ - `rstack breg` and a field-encryption provider, or select a project without \ - encrypted fields." + fs::metadata(&key).unwrap().permissions().mode() & 0o777, + 0o600 ); - refuse_encrypted_fields(&minimal_registry(str::to_owned), "/products/0/project").unwrap(); + let encoded = fs::read_to_string(&key).unwrap(); + assert_eq!(STANDARD.decode(encoded.trim()).unwrap().len(), 32); + + // The rehearsal and the served database read the same run key, by + // reference only; local-file custody requires local initialization. + for name in ["breg-test", "breg"] { + let (config, text) = breg_config(&runner, &root, name); + assert_eq!( + config["fieldEncryption"], + json!({"provider": {"type": "local-file", "dekRef": format!("secret:file/{FIELD_KEY}")}}), + "{name}" + ); + assert_eq!(config["secretProviders"]["file"]["root"], json!(root)); + assert_eq!(config["identity"]["environment"], "local"); + assert_eq!( + config["identity"]["databaseInitializationEnvironment"], + "local" + ); + assert!(!text.contains(encoded.trim()), "{name}"); + } + + // Each run draws its own key. + let other = scratch.path().join("other"); + super::super::private_directory(&other).unwrap(); + let second = ProductRunner::new(Arc::new(breg_input(true)), &other, &terminate).unwrap(); + second.write_field_key().unwrap(); + assert_ne!(fs::read_to_string(other.join(FIELD_KEY)).unwrap(), encoded); + } + + #[test] + fn a_run_without_encrypted_fields_writes_no_key_and_no_provider() { + let scratch = tempfile::tempdir().unwrap(); + let root = scratch.path().join("run"); + super::super::private_directory(&root).unwrap(); + let terminate = AtomicBool::new(false); + let runner = ProductRunner::new(Arc::new(breg_input(false)), &root, &terminate).unwrap(); + runner.write_field_key().unwrap(); + assert_eq!(field_key_file(&root), None); + let (config, _) = breg_config(&runner, &root, "breg"); + assert!(config.get("fieldEncryption").is_none(), "{config}"); + assert_eq!(config["identity"]["environment"], "local"); } #[test] diff --git a/crates/registry-bregctl/tests/project_dev.rs b/crates/registry-bregctl/tests/project_dev.rs index 7b72779430..9354b0f373 100644 --- a/crates/registry-bregctl/tests/project_dev.rs +++ b/crates/registry-bregctl/tests/project_dev.rs @@ -474,6 +474,35 @@ fn check_validates_the_closed_plan_without_allocating_run_state() { assert!(!state.exists()); } +#[test] +fn check_accepts_a_project_with_encrypted_fields_without_allocating_state() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + project(scratch.path(), ""); + let registry = scratch.path().join("registry.yaml"); + let source = fs::read_to_string(®istry).unwrap(); + let encrypted = source + .replacen( + "mutationMode: mutable\n display: {label: Record", + "mutationMode: mutable\n classification: restricted\n display: {label: Record", + 1, + ) + .replacen( + "- {id: name, type: string, required: true, minimumLength: 1, maximumLength: 100, classification: internal}\n - {id: owner", + "- {id: name, type: string, required: true, minimumLength: 1, maximumLength: 100, classification: restricted, encrypted: true}\n - {id: owner", + 1, + ); + assert_eq!(encrypted.matches("encrypted: true").count(), 1); + fs::write(®istry, encrypted).unwrap(); + + let result = check(scratch.path(), &state, true); + + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + assert!(result.status.success(), "{report}"); + assert_eq!(report["diagnostics"], serde_json::json!([])); + assert!(!state.exists()); +} + #[test] fn check_and_start_reject_an_unknown_fixture_journey_without_allocating_state() { assert_dangling_schema_assignment( @@ -1243,7 +1272,7 @@ fn a_malformed_plan_names_its_field_before_allocating_run_state() { .env_remove("RSTACK_DEV_DATABASE_URL") .output() .unwrap(); - assert!(!result.status.success()); + assert_eq!(result.status.code(), Some(1), "{result:?}"); let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); assert_error_envelope(&report, "domain"); assert!(report.get("diagnostics").is_none()); @@ -1278,20 +1307,71 @@ fn a_run_id_cannot_name_a_path() { assert_eq!(fs::read_dir(scratch.path()).unwrap().count(), 0); } +/// A system binary such as `true` or `false`, wherever this platform keeps it. +fn system_binary(name: &str) -> std::path::PathBuf { + ["/usr/bin", "/bin"] + .into_iter() + .map(|directory| std::path::Path::new(directory).join(name)) + .find(|path| path.is_file()) + .unwrap() +} + +/// A loopback port nothing listens on: it is reserved and released at once. +fn closed_loopback_port() -> u16 { + std::net::TcpListener::bind(("127.0.0.1", 0)) + .unwrap() + .local_addr() + .unwrap() + .port() +} + +/// Start a BReg run whose product tool and runtime are the given binaries and +/// whose PostgreSQL administrator URL is `database_url`. +fn start_with_breg_binaries( + root: &std::path::Path, + state: &std::path::Path, + (tool, runtime): (&std::path::Path, &std::path::Path), + database_url: &str, +) -> std::process::Output { + let request = serde_json::json!({ + "action": "start", "project": root, "json": true, + }); + Command::new(env!("CARGO_BIN_EXE_bregctl")) + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request.to_string()) + .env("RSTACK_DEV_STATE_DIR", state) + .env("RSTACK_DEV_DATABASE_URL", database_url) + .env_remove("RSTACK_DEV_DATABASE_CA") + .env("RSTACK_BREGCTL_BIN", tool) + .env("RSTACK_BREG_BIN", runtime) + .output() + .unwrap() +} + +/// The run directories `state` still holds; kept logs of failed runs are not +/// runs. +fn live_runs(state: &std::path::Path) -> Vec { + match fs::read_dir(state) { + Ok(entries) => entries + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|name| name != "failed") + .collect(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(), + Err(error) => panic!("cannot read the run state: {error}"), + } +} + #[test] fn worker_restores_a_valid_plan_whose_captured_input_exceeds_512_kib() { let scratch = tempfile::tempdir().unwrap(); let project_root = scratch.path().join("project"); fs::create_dir(&project_root).unwrap(); - fs::write( - project_root.join("registry.yaml"), - include_str!( - "../../../products/platform/conformance/projects/breg-project-dev/registry/registry.yaml" - ), - ) - .unwrap(); + fixture_project(&project_root, ""); let raw = BASE64.encode(vec![b'x'; 400 * 1024]); - let mut plan = serde_json::json!({ + // The captured input carries the plan's base64 body verbatim, so it is + // larger than the body itself. + assert!(raw.len() > 512 * 1024, "raw body: {}", raw.len()); + let plan = serde_json::json!({ "apiVersion": "id.registrystack.org/formats/breg/project-dev/v1alpha1", "kind": "BRegProjectDev", "issuer": "https://issuer.example.invalid", @@ -1320,48 +1400,57 @@ fn worker_restores_a_valid_plan_whose_captured_input_exceeds_512_kib() { "the substantial development plan was refused" ); - let members = plan.as_object_mut().unwrap(); - members.remove("apiVersion"); - members.remove("kind"); - let run_id = "0123456789abcdef0123456789abcdef"; + // The supervisor captures the plan for its worker; a refusing product tool + // proves the worker restored that capture and reached the product check. let state = scratch.path().join("state"); - let root = state.join(run_id); - fs::create_dir_all(&root).unwrap(); - fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap(); - let root = root.canonicalize().unwrap(); - let false_binary = ["/usr/bin/false", "/bin/false"] - .into_iter() - .map(std::path::Path::new) - .find(|path| path.is_file()) - .unwrap() - .canonicalize() - .unwrap(); - let input = serde_json::to_vec(&serde_json::json!({ - "run_id": run_id, - "project": project_root, - "plan": plan, - "ttl_seconds": 1, - "schema_people": [], - "breg_authority": { - "principal": "sub", "purpose": false, "requires_postgis": false, - "named_clients": [] - }, - "casework_principals": {}, - "binaries": {"breg": {"tool": false_binary, "runtime": false_binary}} - })) - .unwrap(); - assert!(input.len() > 512 * 1024, "captured input: {}", input.len()); - fs::write(root.join("input.json"), input).unwrap(); - - let result = Command::new(env!("CARGO_BIN_EXE_bregctl")) - .args(["project-dev", "--worker"]) - .arg(&root) - .env("RSTACK_DEV_STATE_DIR", &state) - .env_remove("RSTACK_DEV_DATABASE_URL") - .output() - .unwrap(); - let reply: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); - let error = reply["error"].as_str().unwrap(); + let false_binary = system_binary("false"); + let result = start_with_breg_binaries( + &project_root, + &state, + (&false_binary, &false_binary), + &format!("postgresql://127.0.0.1:{}/postgres", closed_loopback_port()), + ); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + assert_error_envelope(&report, "domain"); + assert_eq!(result.status.code(), Some(1), "{report}"); + let error = report["error"]["message"].as_str().unwrap(); assert!(error.contains("breg check failed"), "{error}"); assert!(!error.contains("cannot restore the captured development plan")); + assert_eq!(live_runs(&state), Vec::::new()); +} + +#[test] +fn start_reports_an_unreachable_postgresql_server_as_operational() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + fixture_project(scratch.path(), ""); + let canary = "rstack-dev-admin-password-canary"; + + let result = start_with_breg_binaries( + scratch.path(), + &state, + (&system_binary("true"), &system_binary("false")), + &format!( + "postgresql://admin:{canary}@127.0.0.1:{}/postgres", + closed_loopback_port() + ), + ); + + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + assert_error_envelope(&report, "operational"); + assert_eq!(result.status.code(), Some(3), "{report}"); + assert!( + report["error"]["message"] + .as_str() + .unwrap() + .contains("cannot establish the PostgreSQL administrator session"), + "{report}" + ); + // The run released everything itself, so there is nothing to recover. + assert!(report.get("run_id").is_none(), "{report}"); + assert!(report.get("recovery").is_none(), "{report}"); + assert_eq!(live_runs(&state), Vec::::new()); + for output in [&result.stdout, &result.stderr] { + assert!(!String::from_utf8_lossy(output).contains(canary)); + } } diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index 7b7a3c616b..2446593574 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -209,6 +209,28 @@ Stop and TTL expiry terminate the services and remove the databases, roles, keys, and run files. A spatial BReg run also creates a `NOLOGIN` role whose name starts with `breg_bbox_`; cleanup removes it with the others. +### Encrypted fields + +A BReg project with encrypted fields runs without extra setup. `start` draws a +development data key for the run from the operating system's random source and +writes it as `field-key` in the run directory, readable only by your user. BReg +reads it through its `local-file` key custody, which it accepts only for a +local development database. The key is never printed, never written to the +checkout, and is removed with the run directory on stop, TTL expiry and a +failed start. When a run needs recovery, the report names the key file under +`recovery.field_encryption_key` until `stop --recover` removes it. Every run +gets a new key, so encrypted data does not outlive its run. + +The schema test that `start` runs before activation has no field-encryption +key, so a step in `tests/journeys.yaml` that touches a Record type with an +encrypted field fails it, and `start` reports `breg test failed +(test.step.failed)`. Exercise those Record types in the plan's `journey` +instead. + +This key custody is for development only. A deployment supplies its key as +[BReg field encryption](../../docs/site/src/content/docs/configure/breg-field-encryption.mdx) +describes. + The state directory holds one directory per run. It defaults to `rstack-dev-UID` in the system temporary directory. Set `RSTACK_DEV_STATE_DIR` to use another location. A run directory is renamed to `..removing` @@ -262,7 +284,8 @@ To release the resources by hand instead, in this order: each listed database. 3. As the administrator, run `DROP ROLE IF EXISTS "NAME"` for each listed role, including a `breg_bbox_*` role from a spatial run. -4. Remove the run directory. +4. Remove the run directory. It holds the run's development field-encryption + key when `field_encryption_key` is listed. ### Errors and limitations @@ -272,23 +295,23 @@ and stop prints one JSON object on standard output, validated by for failures before the helper starts, such as a missing component. - `schema_version` is `registry.rstack.error.v1` and `status` is `refused`. -- `error.class` is `domain` (exit 1: the request or the run's state was - refused), `usage` (exit 2: the command line is wrong) or `operational` - (exit 3: the environment failed, a cleanup stalled, or a run needs its - resources released). `error.message` is the human-readable reason. +- `error.class` is `domain` (exit 1: the request, the project, the plan or the + run's state was refused), `usage` (exit 2: the command line is wrong) or + `operational` (exit 3: the environment failed, a cleanup stalled, or a run + needs its resources released). Environment failures include a PostgreSQL + server that is unreachable, refuses the administrator session, or cannot + provision a run's databases and roles, a product binary that cannot be + started, and no free loopback port. `error.message` is the human-readable + reason. - `run_id` is present when the failure concerns a run that still has resources to release. Use it with `stop --recover`. - `recovery` accompanies `run_id` when the resources are known. It has `kind` (`lost-supervisor` or `cleanup-pending`), `run_directory`, `processes`, - `objects` or `databases_and_roles`, and `reason` when `--recover` could not - finish. + `objects` or `databases_and_roles`, `field_encryption_key` when the run + directory still holds the run's key file (the path, never the key), and + `reason` when `--recover` could not finish. Without `--json`, the message goes to standard error as `rstack dev: MESSAGE` with the same exit code. The reports of `rstack dev check` use `BRegCtlReport` instead. A helper started without the launcher's request prints a plain message and exits 2. - -`rstack dev` does not configure field encryption yet. `rstack dev check` and -`start` refuse a BReg project with encrypted fields at the product's -`/products/N/project` pointer, and the error names the encrypted -`ENTITY.FIELD`; `start` refuses it before it provisions anything. diff --git a/products/platform/schemas/rstack-dev-error.v1.schema.json b/products/platform/schemas/rstack-dev-error.v1.schema.json index 263c7ed8c7..2b0c799162 100644 --- a/products/platform/schemas/rstack-dev-error.v1.schema.json +++ b/products/platform/schemas/rstack-dev-error.v1.schema.json @@ -14,7 +14,7 @@ "required": ["class", "message"], "properties": { "class": { - "description": "domain (exit 1): the request or the run's state was refused. usage (exit 2): the command line is wrong. operational (exit 3): the environment failed or resources need cleanup.", + "description": "domain (exit 1): the request or the run's state was refused. usage (exit 2): the command line is wrong. operational (exit 3): the environment failed, for example PostgreSQL was unreachable or refused the connection, a product process could not be started, or no loopback port was available, or resources need cleanup.", "enum": ["domain", "usage", "operational"] }, "message": {"type": "string", "minLength": 1} @@ -58,6 +58,10 @@ } }, "databases_and_roles": {"type": "array", "items": {"type": "string"}}, + "field_encryption_key": { + "description": "Present when the run directory still holds the run's development field-encryption key file. The path only; the key itself is never reported.", + "type": "string" + }, "unavailable": {"type": "string"}, "reason": {"type": "string"} } diff --git a/products/platform/scripts/test-rstack-dev.py b/products/platform/scripts/test-rstack-dev.py index 48d4012b90..75657f6072 100644 --- a/products/platform/scripts/test-rstack-dev.py +++ b/products/platform/scripts/test-rstack-dev.py @@ -25,6 +25,7 @@ STACK = Path(__file__).resolve().parents[3] NATIVE_REVIEW = STACK / "products/breg/acceptance/reviewed-evidence-journey" SPATIAL = STACK / "products/breg/acceptance/spatial-service-sites" +MINIMAL = STACK / "products/breg/examples/minimal" HTTP = build_opener(ProxyHandler({})) @@ -184,6 +185,11 @@ def files(root): def dev(bin_dir, env, cwd, *args, expected=0): + return dev_output(bin_dir, env, cwd, *args, expected=expected)[0] + + +def dev_output(bin_dir, env, cwd, *args, expected=0): + """Run `rstack dev` and return its report with everything it printed.""" install_root = Path(env["RSTACK_DEV_STATE_DIR"]).with_name("rstack-install") result = command( bin_dir / "rstack", @@ -212,7 +218,7 @@ def dev(bin_dir, env, cwd, *args, expected=0): assert report["error"]["message"], report else: assert report["schema_version"] == "registry.rstack.dev.v1" - return report + return report, result.stdout + result.stderr def credential(bin_dir, env, cwd, run, alias): @@ -418,6 +424,7 @@ def exercise(root, bin_dir, pg, project, ca, url, pg_env, spatial): verify_lost_supervisor(bin_dir, pg, checkout, state, env, pg_env, owner_env) verify_provisioning_deadline(bin_dir, pg, checkout, state, env, pg_env) verify_spatial_refusal(root, bin_dir, pg, state, env, pg_env) + verify_field_encryption(root, bin_dir, pg, state, env, pg_env) if spatial: verify_spatial( root, bin_dir, pg, {**env, "RSTACK_DEV_DATABASE_URL": url}, pg_env @@ -565,7 +572,8 @@ def verify_provisioning_deadline(bin_dir, pg, checkout, state, env, pg_env): assert time.monotonic() < deadline, "the PostgreSQL lock was not acquired" time.sleep(0.1) - report = dev(bin_dir, env, checkout, "start", ".", expected=1) + report = dev(bin_dir, env, checkout, "start", ".", expected=3) + assert report["error"]["class"] == "operational", report error = report["error"]["message"] assert "cannot create an isolated PostgreSQL database" in error, report assert "statement deadline" in error, report @@ -841,7 +849,8 @@ def verify_spatial_refusal(root, bin_dir, pg, state, env, pg_env): "SELECT default_version FROM pg_available_extensions WHERE name = 'postgis';", ).strip() supported = bool(offered) and tuple(map(int, offered.split(".")[:2])) >= (3, 5) - report = dev(bin_dir, env, checkout, "start", ".", expected=1) + report = dev(bin_dir, env, checkout, "start", ".", expected=3) + assert report["error"]["class"] == "operational", report if supported: assert "must be allowed to CREATE EXTENSION postgis" in report["error"]["message"], report else: @@ -851,6 +860,253 @@ def verify_spatial_refusal(root, bin_dir, pg, state, env, pg_env): print("A spatial start without PostGIS rights was refused and left nothing behind") +def encrypted_project(root, name): + """Copy the minimal registry with a note Record type whose body is + encrypted, and write a plan that creates a note and reads it back. + + The fixture journeys leave notes alone: `bregctl test` runs them without + field-encryption key state, so a fixture step that wrote an encrypted + field would be refused there before the run reached its own key.""" + checkout = root / name + registry = checkout / "registry" + shutil.copytree(MINIMAL, registry) + source = (registry / "registry.yaml").read_text() + for anchor, addition in [ + ( + "\nactions:\n", + " - id: note\n" + " route: notes\n" + " primaryDataset: directory\n" + " mutationMode: mutable\n" + " classification: restricted\n" + " display: {label: Note, description: A note whose body is encrypted.}\n" + " fields:\n" + " - {id: title, type: string, required: true, minimumLength: 1," + " maximumLength: 160, classification: internal}\n" + " - {id: body, type: string, required: true, minimumLength: 1," + " maximumLength: 2000, classification: restricted, encrypted: true}\n" + " - {id: jurisdiction, type: string, required: true, minimumLength: 1," + " maximumLength: 40, classification: internal}\n", + ), + ( + " actions:\n - action: update-contact-email\n", + " - recordType: note\n" + " operations: [create, get]\n" + " readableFields: [title, body, jurisdiction]\n" + " writableFields: [title, body, jurisdiction]\n" + " rowBoundaries:\n" + " - {field: jurisdiction, claim: jurisdiction, operator: equals}\n", + ), + ]: + # Each addition goes in front of its anchor: the note after the + # contact Record type, and its grant after the contact grant. + assert source.count(anchor) == 1, anchor + source = source.replace(anchor, addition + anchor) + (registry / "registry.yaml").write_text(source) + # Both fixture principals hold the same grant, so each fixture step names + # the person whose jurisdiction it expects. + people = { + "synthetic-directory-editor": ("editor", "zone-a"), + "synthetic-foreign-editor": ("foreign-editor", "zone-b"), + } + schema_test_people = {} + steps = 0 + for line in (registry / "tests/journeys.yaml").read_text().splitlines(): + if match := re.fullmatch(r" - id: (\S+)", line): + journey = schema_test_people.setdefault(match[1], {}) + elif match := re.fullmatch(r" - id: (\S+)", line): + step = match[1] + steps += 1 + elif match := re.fullmatch(r" +principal: (\S+)", line): + journey[step] = people[match[1]][0] + assert steps and sum(map(len, schema_test_people.values())) == steps, ( + schema_test_people + ) + body = f"encrypted-note-{secrets.token_hex(8)}" + plan = { + "apiVersion": "id.registrystack.org/formats/breg/project-dev/v1alpha1", + "kind": "BRegProjectDev", + "issuer": "https://directory-smoke.example.invalid", + "products": [ + {"product": "breg", "project": "registry", "audience": "directory"} + ], + "people": { + alias: { + "subject": principal, + "client": "directory-smoke", + "grants": { + "breg": { + "profile": "directory-editor", + "scopes": ["registry:directory:write"], + "claims": { + "jurisdiction": jurisdiction, + "registry_actor_kind": "human", + "registry_purpose": "directory-administration", + }, + } + }, + } + for principal, (alias, jurisdiction) in people.items() + }, + "schemaTestPeople": schema_test_people, + "journey": { + "id": "encrypted-note", + "steps": [ + { + "id": "create-note", + "product": "breg", + "as": "editor", + "method": "POST", + "path": "/v1/records/notes", + "headers": {"Idempotency-Key": "encrypted-note"}, + "body": { + "data": { + "title": "Encrypted note", + "body": body, + "jurisdiction": "zone-a", + } + }, + "expect": {"status": 201}, + "capture": { + "note": { + "type": "pointer", + "pointer": "/data/recordIdentifier", + } + }, + }, + { + "id": "read-note", + "product": "breg", + "as": "editor", + "method": "GET", + "path": "/v1/records/notes/{{note}}", + "expect": { + "status": 200, + "json": {"/data/domainData/body": body}, + }, + }, + ], + }, + } + (checkout / "dev.yaml").write_text(json.dumps(plan)) + return checkout, body + + +def field_key(state, run): + """The run's key, after checking its custody: an owner-only file in the + owner-only run directory holding 32 random bytes in base64.""" + key = state / run / "field-key" + assert key.parent.stat().st_mode & 0o777 == 0o700, key + assert key.stat().st_mode & 0o777 == 0o600, key + encoded = key.read_bytes() + assert len(base64.b64decode(encoded, validate=True)) == 32, key + return encoded + + +def tree_bytes(root, skip=()): + return [ + path.read_bytes() + for path in root.rglob("*") + if path.is_file() and path not in skip + ] + + +def without_key(encoded, outputs): + """No output carries the key, as base64 or as raw bytes. The assertion + names no value so a failure cannot print the key.""" + raw = base64.b64decode(encoded) + for output in outputs: + assert encoded not in output and raw not in output, "the field key leaked" + + +def stored_values(pg, pg_env, run): + """Every row of the run's BReg database, as text with byte columns hex + encoded, so plaintext is found whether it sits in text or bytes.""" + dump = checked( + pg / "pg_dump", + "--data-only", + env={**pg_env, "PGDATABASE": f"rstack_{run}_breg"}, + ) + return lambda value: value.encode() in dump or value.encode().hex().encode() in dump + + +def verify_field_encryption(root, bin_dir, pg, state, env, pg_env): + """A project with encrypted fields runs on a per-run key that stays in the + run directory and is removed on every cleanup path.""" + checkout, body = encrypted_project(root, "encrypted") + before = files(checkout) + baseline = inventory(pg, pg_env) + + report, printed = dev_output(bin_dir, env, checkout, "start", ".") + run = report["run_id"] + try: + assert report["journey"]["result"] == "passed", report + key = field_key(state, run) + credential(bin_dir, env, checkout, run, "editor") + _, token = dev_output(bin_dir, env, checkout, "token", run, "--as", "editor") + _, status = dev_output(bin_dir, env, checkout, "status", run) + without_key( + key, + [printed, token, status] + + tree_bytes(state / run, skip={state / run / "field-key"}) + + tree_bytes(checkout), + ) + stored = stored_values(pg, pg_env, run) + assert stored("Encrypted note") and stored("__bregEncryptedV1") + assert not stored(body), "the encrypted body is stored in plaintext" + finally: + stopped, printed = dev_output(bin_dir, env, checkout, "stop", run) + assert stopped["status"] == "stopped", stopped + assert not (state / run).exists() + without_key(key, [printed]) + assert inventory(pg, pg_env) == baseline + print("An encrypted field round-tripped on an owner-only run key that stop removed") + + short = dev(bin_dir, env, checkout, "start", ".", "--ttl", "5s") + field_key(state, short["run_id"]) + wait_for_stop(bin_dir, env, checkout, short["run_id"]) + assert not (state / short["run_id"]).exists() + assert live_runs(state) == [] + + # The runtime fixture copies the key it was given so the test can look + # for it in everything the failed start left behind. + seen = root / "failed-run-key" + failing = root / "failing-breg" + failing.write_text( + "#!/bin/sh\numask 077\n" + f"cp \"$(dirname \"$2\")/field-key\" '{seen}'\n" + "printf 'boom\\n' >&2\nexit 1\n" + ) + failing.chmod(0o700) + failure_env = {**env, "RSTACK_BREG_BIN": str(failing)} + report, printed = dev_output( + bin_dir, failure_env, checkout, "start", ".", expected=1 + ) + assert "breg exited before readiness" in report["error"]["message"], report + logs = kept_logs(state, report["error"]["message"]) + key = seen.read_bytes() + assert len(base64.b64decode(key, validate=True)) == 32 + assert not list(state.rglob("field-key")), "a failed start kept its key" + without_key(key, [printed] + tree_bytes(logs.parent)) + assert live_runs(state) == [] + assert inventory(pg, pg_env) == baseline + + report, run = lose_supervisor(bin_dir, checkout, state, env) + lost, printed = dev_output(bin_dir, env, checkout, "stop", run, expected=3) + key = field_key(state, run) + assert lost["recovery"]["field_encryption_key"] == str(state / run / "field-key") + without_key(key, [printed]) + recovered, printed = dev_output(bin_dir, env, checkout, "stop", run, "--recover") + assert recovered["status"] == "stopped", recovered + assert not (state / run).exists() + without_key(key, [printed]) + assert_nothing_listening(bin_dir, report["products"].values()) + assert inventory(pg, pg_env) == baseline + assert live_runs(state) == [] + assert files(checkout) == before + print("TTL expiry, a failed start and stop --recover removed the run's key") + + def verify_spatial(root, bin_dir, pg, env, pg_env): checkout, registry, bbox = spatial_project(root, "spatial") before = files(registry) From 0a3e3679c25c775dbf99c66ba28f76fb5c3c231e Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 13:29:55 +0700 Subject: [PATCH 06/24] test(breg): read long encrypted envelopes back through every read projection Signed-off-by: Jeremi Joslin --- .../tests/postgres_field_encryption.rs | 101 ++++++++++++++++++ products/breg/CHANGELOG.md | 6 ++ 2 files changed, 107 insertions(+) diff --git a/crates/registry-breg/tests/postgres_field_encryption.rs b/crates/registry-breg/tests/postgres_field_encryption.rs index 3239826a47..9f8174c791 100644 --- a/crates/registry-breg/tests/postgres_field_encryption.rs +++ b/crates/registry-breg/tests/postgres_field_encryption.rs @@ -1312,6 +1312,107 @@ async fn encrypted_fields_round_trip_and_store_only_envelopes() { server.shutdown().await; } +/// An encrypted value long enough that its stored envelope exceeds 57 bytes, +/// so PostgreSQL's `encode(..., 'base64')` wraps the projection at 76 +/// characters. Every read projection must still hand back the plaintext. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn long_envelopes_read_back_through_every_read_projection() { + let server = boot_live_server().await; + let long_secret = "long-secret-value-".repeat(10); + let (status, created, _etag) = create_holder( + &server, + "long-envelope-create", + json!({ + "jurisdiction": "area-a", + "label": "long-envelope", + "secret": long_secret, + }), + ) + .await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(created["data"]["domainData"]["secret"], long_secret); + let record_id = created["data"]["recordIdentifier"] + .as_str() + .expect("record identifier is present") + .to_owned(); + + let envelope = stored_column( + &server, + &envelope_column(server.registry(), "secret"), + &record_id, + ) + .await; + assert!( + envelope.len() > 57, + "the envelope must wrap PostgreSQL's base64 output: {} bytes", + envelope.len() + ); + + let fetched = send( + &server, + Method::GET, + &format!("/v1/records/holders/{record_id}"), + &[], + None, + ) + .await; + assert_eq!(fetched.status(), StatusCode::OK); + let fetched = body_json(fetched).await; + assert_eq!(fetched["data"]["domainData"]["secret"], long_secret); + + let listed = send( + &server, + Method::GET, + "/v1/records/holders?$select=jurisdiction,label,secret", + &[], + None, + ) + .await; + assert_eq!(listed.status(), StatusCode::OK); + let listed = body_json(listed).await; + let items = listed["items"].as_array().expect("list items"); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["domainData"]["secret"], long_secret); + + let snapshot = send( + &server, + Method::GET, + "/v1/records/holders:snapshot?$select=jurisdiction,label,secret", + &[], + None, + ) + .await; + assert_eq!(snapshot.status(), StatusCode::OK); + let snapshot = body_json(snapshot).await; + assert_eq!(snapshot["items"][0]["domainData"]["secret"], long_secret); + + let revisions = send( + &server, + Method::GET, + &format!("/v1/records/holders/{record_id}/revisions"), + &[], + None, + ) + .await; + assert_eq!(revisions.status(), StatusCode::OK); + let revisions = body_json(revisions).await; + assert_eq!(revisions["items"][0]["domainData"]["secret"], long_secret); + + let detail = send( + &server, + Method::GET, + &format!("/v1/records/holders/{record_id}/revisions/1"), + &[], + None, + ) + .await; + assert_eq!(detail.status(), StatusCode::OK); + let detail = body_json(detail).await; + assert_eq!(detail["data"]["domainData"]["secret"], long_secret); + + server.shutdown().await; +} + /// The single stored idempotency body of one result kind, as UTF-8 text. async fn cached_response_body(server: &LiveServer, result_kind: &str) -> String { let rows = server diff --git a/products/breg/CHANGELOG.md b/products/breg/CHANGELOG.md index e593ae3bb0..c2ed22df75 100644 --- a/products/breg/CHANGELOG.md +++ b/products/breg/CHANGELOG.md @@ -21,6 +21,12 @@ on `geometry.type === "Point"` before reading `coordinates`. In Python the `.pyi` stubs describe the union, so a type checker flags code that assumes a Point. +- Fix reading back an encrypted field on PostgreSQL. A registry that uses + field encryption failed a record, list, snapshot or revision read with + `503 source.unavailable` once a stored value was long enough for its + encrypted envelope to exceed 57 bytes, and the value could not be read + until this fix. Writes and shorter values were unaffected. The fault was in the read path since v0.33.0, through v0.46.0; no + stored data changes and nothing needs migrating. ## 0.46.0 - 2026-11-08 From 58e436640d6665c1c307628e5a54a17221c01263 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 13:39:26 +0700 Subject: [PATCH 07/24] docs: add a how-to page for running a project with rstack dev Add "Run a project with rstack dev" under configure/: install the components, start a local TLS PostgreSQL server, adapt the licensing dev.yaml, check, start, call both products with a development token, stop, and recover a failed run. Register it in the Operations sidebar group, link it from PROJECT-DEV.md, and link it from the rstack dev command help so the generated CLI reference carries the link. The regenerated crates/registry-cli-docs/catalog.json also picks up `stop --recover` and `--database-url`, which the earlier command change had not regenerated. Closes #2641 Signed-off-by: Jeremi Joslin --- crates/registry-cli-docs/catalog.json | 32 +- crates/registry-rstack/src/cli.rs | 3 + docs/site/astro.config.mjs | 1 + .../run-a-project-with-rstack-dev.mdx | 275 ++++++++++++++++++ products/platform/PROJECT-DEV.md | 4 + 5 files changed, 312 insertions(+), 3 deletions(-) create mode 100644 docs/site/src/content/docs/configure/run-a-project-with-rstack-dev.mdx diff --git a/crates/registry-cli-docs/catalog.json b/crates/registry-cli-docs/catalog.json index 8002f85a12..80e5db9aa3 100644 --- a/crates/registry-cli-docs/catalog.json +++ b/crates/registry-cli-docs/catalog.json @@ -25865,7 +25865,7 @@ "name": "dev", "invocation": "rstack dev", "about": "Run and inspect a local Registry Stack development environment", - "long_about": null, + "long_about": "Run and inspect a local Registry Stack development environment. For a walk-through from a project checkout to a running registry and review service, see `https://docs.registrystack.org/configure/run-a-project-with-rstack-dev/`", "usage": "rstack dev [OPTIONS] ", "arguments": [], "options": [ @@ -25912,7 +25912,7 @@ }, { "display": "-h, --help", - "description": "Print help", + "description": "Print help (see a summary with '-h')", "always_required": false, "repeatable": false, "default_values": [], @@ -26300,6 +26300,24 @@ } ], "options": [ + { + "display": "--recover", + "description": "Release the processes, databases and roles of a run whose supervisor was lost", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--database-url ", + "description": "TLS PostgreSQL administrator URL for --recover. Defaults to RSTACK_DEV_DATABASE_URL", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, { "display": "--json", "description": "Emit the runner's JSON report", @@ -26360,7 +26378,15 @@ "environment": null } ], - "constraints": [], + "constraints": [ + { + "kind": "requires-all", + "when": "--database-url ", + "arguments": [ + "--recover" + ] + } + ], "subcommands": [] } ] diff --git a/crates/registry-rstack/src/cli.rs b/crates/registry-rstack/src/cli.rs index 8a6b47c5b9..fa57a6304b 100644 --- a/crates/registry-rstack/src/cli.rs +++ b/crates/registry-rstack/src/cli.rs @@ -155,6 +155,9 @@ enum Command { command: IdentityCommand, }, /// Run and inspect a local Registry Stack development environment. + #[command( + long_about = "Run and inspect a local Registry Stack development environment.\n\nFor a walk-through from a project checkout to a running registry and review service, see `https://docs.registrystack.org/configure/run-a-project-with-rstack-dev/`" + )] Dev { #[command(subcommand)] command: DevCommand, diff --git a/docs/site/astro.config.mjs b/docs/site/astro.config.mjs index 6237c9764f..4ddbefbbf3 100644 --- a/docs/site/astro.config.mjs +++ b/docs/site/astro.config.mjs @@ -910,6 +910,7 @@ export default defineConfig({ { label: 'Overview', slug: 'operate/advanced' }, { label: 'Operator handoff', slug: 'operate' }, { label: 'Call an API with rstack', slug: 'configure/api-connections' }, + { label: 'Run a project with rstack dev', slug: 'configure/run-a-project-with-rstack-dev' }, { label: 'Rotate credentials and trust', slug: 'operate/advanced/rotate-credentials-and-trust' }, { label: 'Connect an OIDC issuer', slug: 'operate/advanced/connect-an-oidc-issuer' }, { label: 'Generate ThunderID configuration', slug: 'operate/advanced/generate-identity-configuration' }, diff --git a/docs/site/src/content/docs/configure/run-a-project-with-rstack-dev.mdx b/docs/site/src/content/docs/configure/run-a-project-with-rstack-dev.mdx new file mode 100644 index 0000000000..fc940049b3 --- /dev/null +++ b/docs/site/src/content/docs/configure/run-a-project-with-rstack-dev.mdx @@ -0,0 +1,275 @@ +--- +title: Run a project with rstack dev +description: Start a Base Registry Engine and Registry Casework pair from a project checkout on a local TLS PostgreSQL server, call both with development tokens, and recover a failed run. +status: current +owner: registry-docs +source_repos: + - registry-stack +last_reviewed: "2026-11-11" +doc_type: how-to +persona: + - operator +locale: en +standards_referenced: [] +--- + +If you are writing a registry project and want to see Base Registry Engine (BReg) and Registry +Casework serve it together, `rstack dev` starts both from your project directory, runs a short +journey against them, and removes everything when you stop. It does not start PostgreSQL and does +not use Docker: you supply a development PostgreSQL server over TLS. This page takes you from a +checkout to a running pair, using the licensing example that ships with Registry Stack. + +## Before you start + +You need: + +- PostgreSQL 17 or newer command-line tools (`initdb`, `pg_ctl`, `psql`) and `openssl` on your + `PATH`, to run the local server below. A server you already have works too if it serves TLS and + has an administrator role with `LOGIN CREATEROLE CREATEDB`. +- `curl` and `jq`. +- A checkout of Registry Stack at the release you installed, for the example project. + +## Install the components + +Install `rstack` from one verified release, then add the two components the example uses. The +steps are in [Decide your first Base Registry Engine registry](../../tutorials/first-breg/#install-registry-stack); +install `casework` the same way. + +```sh test-skip="uses a downloaded, independently verified release installer" +rstack install breg casework +``` + +Check that the commands resolve before you continue: + +```sh test-skip="needs the installed release" +rstack dev --help +``` + +Success is the list of `check`, `start`, `status`, `token` and `stop`. If `rstack` is missing, run +`rstack doctor`; if a component is missing, `start` names the `rstack install` command to run. + +## Start a local TLS PostgreSQL server + +`rstack dev` always verifies the server certificate and host name, so the server needs a +certificate your machine trusts. This recipe makes a private certificate authority, starts the +server on `127.0.0.1`, and creates a least-privilege administrator. Keep the working directory +outside any repository. If port 5432 is taken, change `PORT`. + +```sh test-skip="needs a PostgreSQL installation and starts a server" +export DEV="$HOME/rstack-dev-postgres" +PORT=5432 +mkdir -p -m 700 "$DEV" +umask 077 +openssl rand -hex 16 > "$DEV/postgres.pass" +openssl rand -hex 16 > "$DEV/devadmin.pass" +initdb -D "$DEV/data" -U postgres --auth=scram-sha-256 --pwfile="$DEV/postgres.pass" -E UTF8 + +openssl req -new -x509 -newkey rsa:2048 -nodes -days 30 -subj /CN=rstack-dev-ca \ + -addext basicConstraints=critical,CA:TRUE -keyout "$DEV/ca.key" -out "$DEV/ca.pem" +openssl req -new -newkey rsa:2048 -nodes -subj /CN=localhost \ + -keyout "$DEV/data/server.key" -out "$DEV/server.csr" +printf 'subjectAltName=DNS:localhost,IP:127.0.0.1\nextendedKeyUsage=serverAuth\n' \ + > "$DEV/server.ext" +openssl x509 -req -in "$DEV/server.csr" -CA "$DEV/ca.pem" -CAkey "$DEV/ca.key" \ + -CAcreateserial -days 30 -extfile "$DEV/server.ext" -out "$DEV/data/server.crt" + +printf "listen_addresses = '127.0.0.1'\nport = %s\nssl = on\n" "$PORT" \ + >> "$DEV/data/postgresql.conf" +pg_ctl -D "$DEV/data" -l "$DEV/postgres.log" -w start +``` + +The passwords are random and stay in owner-only files, so nothing secret reaches your shell +history or a process list. Create the administrator with the password from its file: + +```sh test-skip="needs the running development server" +CONNECTION="host=127.0.0.1 port=$PORT user=postgres dbname=postgres sslmode=verify-full sslrootcert=$DEV/ca.pem" +PGPASSWORD="$(cat "$DEV/postgres.pass")" psql "$CONNECTION" <<'SQL' +\set pw `cat "$DEV/devadmin.pass"` +CREATE ROLE devadmin LOGIN CREATEROLE CREATEDB PASSWORD :'pw'; +SQL +``` + +Success is `CREATE ROLE`. The server must be running before you start a project. Point `rstack dev` +at it: + +```sh test-skip="needs the running development server" +export RSTACK_DEV_DATABASE_CA="$DEV/ca.pem" +export RSTACK_DEV_DATABASE_URL="postgresql://devadmin:$(cat "$DEV/devadmin.pass")@127.0.0.1:$PORT/postgres?sslmode=verify-full" +``` + +Set the URL through an environment variable, not `--database-url`, because a flag is visible to +other local users in the process list. For a spatial project, install PostGIS for this server and +create the administrator as a superuser instead. + +{/* Evidence: products/platform/PROJECT-DEV.md; crates/registry-rstack/src/cli.rs, dispatch_dev. */} + +## Adapt the development plan + +Copy the licensing example from your checkout, then work in the copy: + +```sh test-skip="needs a Registry Stack checkout" +cp -R path/to/registry-stack/products/platform/examples/licensing ./licensing +``` + +The directory holds a BReg project (`registry/`), a Casework project (`casework/`) and the plan +`dev.yaml`. Open `dev.yaml`: `rstack dev` reads only this file to decide what to start. Product +files stay byte-for-byte unchanged, and generated settings, keys and packages live in the run's +owner-only scratch directory. + +To adapt the plan to your own project, change these parts and leave the structure alone: + +- `issuer` names the identity issuer the local test tokens claim. Use a name that is yours. +- `products` lists each product's project directory and token audience, in startup order. +- `people` declares synthetic test people, each with a subject and one grant per product. A grant + selects an access profile of that product's project, its scopes, and extra claims. A BReg grant + must name a profile the BReg project declares. +- `journey` is the sequence of requests the run makes before it reports ready. Each step names a + person, a product, a method, a path starting with `/v1/`, and what it expects back. + +The [`rstack dev` reference](https://github.com/registrystack/registry-stack/blob/main/products/platform/PROJECT-DEV.md) +describes every key, capture type and limit. + +## Check, then start + +`check` validates the plan and resolves its fixtures, grants and references with the installed +authoring tools. It works with the database offline. + +```sh test-skip="needs the installed release and a running development server" +rstack dev check ./licensing +``` + +Success is `Development plan is valid` and exit status 0. Add `--deny-warnings` to refuse any +warning, and `--json` for a machine-readable report. Fix every error before you start: the report +names the plan field that caused it. + +Start the run. The time to live (TTL) begins when the run is ready and defaults to 30 minutes: + +```sh test-skip="needs the installed release and a running development server" +rstack dev start ./licensing --ttl 30m +``` + +The command creates separate databases and logins for the run, starts both products on loopback +ports, runs the journey, and returns while the services keep running: + +```text test-excerpt +Run 2f17770cb47d4502ad0b809a529f4309 ready + breg http://127.0.0.1:64930 + casework http://127.0.0.1:64931 + Journey licensing-approval passed + Logs STATE_DIRECTORY/2f17770cb47d4502ad0b809a529f4309/logs + rstack dev token 2f17770cb47d4502ad0b809a529f4309 --as administrator + rstack dev token 2f17770cb47d4502ad0b809a529f4309 --as applicant + rstack dev token 2f17770cb47d4502ad0b809a529f4309 --as intake + rstack dev token 2f17770cb47d4502ad0b809a529f4309 --as reviewer +``` + +Your run ID and ports differ, and `STATE_DIRECTORY` stands for the state directory your run reports. Copy the run ID: every later command takes it. The +journey has already created an application in BReg and had a reviewer approve it in Casework. +`rstack dev status RUN_ID` repeats this report at any time. + +## Call the products with a token + +`token` writes a short-lived bearer header file for a test person and reports where it is. Tokens +expire five minutes after issue, or with the run if that is sooner, so ask again when one lapses. +Pass the header file to curl with `--header @FILE`: + +```sh test-skip="needs a running development run" +RUN=2f17770cb47d4502ad0b809a529f4309 +HEADERS="$(rstack dev token "$RUN" --as applicant --json | jq -r '.credentials[0].header_file')" +curl -sS --header @"$HEADERS" \ + "http://127.0.0.1:64930/v1/records/applications?accessProfile=applicant" +``` + +Use the BReg URL from your own report. BReg needs the selected profile as the `accessProfile` +query parameter, and the response lists the application the journey created: + +```json test-excerpt +{"items":[{"domainData":{"profession":"Surveyor","reference":"application-001"},"recordIdentifier":"23257c7f-1acd-4fbf-8a3c-4b253238043c","revisionIdentifier":"1"}],"meta":{"datasetIdentifier":"licensing","recordTypeIdentifier":"application","registryIdentifier":"licensing"},"pageInfo":{"nextCursor":null}} +``` + +Casework needs no query parameter, because its header file carries the profile. This asks for the +reviewer's open tasks; the queue is empty because the journey already decided the request: + +```sh test-skip="needs a running development run" +HEADERS="$(rstack dev token "$RUN" --as reviewer --json | jq -r '.credentials[0].header_file')" +curl -sS --header @"$HEADERS" "http://127.0.0.1:64931/v1/review-tasks?queue=applications" +``` + +```json test-excerpt +{"items":[],"status":"complete"} +``` + +A BReg project with encrypted fields runs without extra setup: each run gets its own development +data key, removed when the run ends, so encrypted data does not outlive its run. A step in the +project's `tests/journeys.yaml` that touches an encrypted Record type fails the pre-activation +test, because that test has no key. Exercise those Record types in the plan's `journey` instead. + +{/* Evidence: products/platform/PROJECT-DEV.md; crates/registry-rstack/src/cli.rs, dispatch_dev. */} + +## Stop the run + +```sh test-skip="needs a running development run" +rstack dev stop "$RUN" +``` + +Success is `Run RUN_ID stopped`. Stop, and TTL expiry, terminate both services and remove the run's +databases, roles, keys and files. Stop the PostgreSQL server itself when you finish: + +```sh test-skip="needs the running development server" +pg_ctl -D "$DEV/data" stop +``` + +## When a run fails + +Read the error first. With `--json`, a refusal prints one object on standard output and the +process exit status tells you whose problem it is: + +| Exit | `error.class` | Meaning | +| --- | --- | --- | +| 1 | `domain` | The request, the project, the plan or the run's state was refused. Fix the plan or the command. | +| 2 | `usage` | The command line is wrong. | +| 3 | `operational` | The environment failed: PostgreSQL is unreachable or refuses the administrator, a product binary cannot start, no loopback port is free, or a run needs its resources released. | + +For example, `rstack dev start ./licensing --json` with the PostgreSQL server stopped (the operating +system error number varies by platform): + +```text test-excerpt +{"error":{"class":"operational","message":"cannot establish the PostgreSQL administrator session over verified TLS: the PostgreSQL connection handshake failed: error connecting to server: Connection refused (os error 61)"},"schema_version":"registry.rstack.error.v1","status":"refused"} +``` + +Start the server and run the command again. Without `--json`, the same message goes to standard +error as `rstack dev: MESSAGE` with the same exit status. + +A failed journey exits 1 and leaves the run alive, so you can inspect it and then stop it. The +report includes the run ID. When a start fails before that, the run directory is removed but its +logs are kept at `/failed//logs`, and the error names the path. Otherwise, a live +run's logs are in `/logs`: `supervisor.log` for the supervisor and one file per product stage, +such as `breg-check.log` or `casework-runtime.log`. `rstack dev status RUN_ID` names the logs +directory. Delete kept logs yourself. + +When a run still holds databases, roles or processes, the error carries its `run_id` and a +`recovery` object listing them. This happens if PostgreSQL is unavailable during cleanup for +longer than the supervisor retries (up to 30 minutes), or if the supervisor was lost. Release the +run with: + +```sh test-skip="needs a run that lost its supervisor" +rstack dev stop RUN_ID --recover +``` + +`--recover` checks everything it needs, then stops the recorded process groups, drops the +databases and roles, and removes the run directory, including the run's development key file. It +refuses a run whose supervisor is still running; use plain `stop` for that. Running it again +after a partial failure is safe. Success is `status` `stopped` and a `recovered` object. When it +cannot finish, the error names what to release by hand. The +[reference](https://github.com/registrystack/registry-stack/blob/main/products/platform/PROJECT-DEV.md) +lists the manual steps and the recovery fields. + +{/* Evidence: products/platform/PROJECT-DEV.md; crates/registry-rstack/src/cli.rs, dispatch_dev. */} + +## Next + +- [`rstack dev` reference](https://github.com/registrystack/registry-stack/blob/main/products/platform/PROJECT-DEV.md) + for the full plan format, journey steps and recovery behaviour. +- [Call an API with rstack](../api-connections/) to make short requests against the running pair. +- [Test with journeys](../breg-journeys/) to write the BReg project's own journey tests. +- [Encrypt a field](../breg-field-encryption/) for the key custody a deployment uses. diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index 2446593574..7fe6602e3b 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -6,6 +6,10 @@ PostgreSQL server over TLS. `rstack dev` does not start PostgreSQL and does not use Docker: you supply the server. The products' own development modes are unchanged by `rstack dev`. +For a walk-through from a checkout to a running pair, see +[Run a project with rstack dev](https://docs.registrystack.org/configure/run-a-project-with-rstack-dev/). +This file is the reference. + ```sh rstack install breg casework # Set RSTACK_DEV_DATABASE_URL through your shell's secret provider. From fd93194f672cc79c160616f35b77b24afd3d8606 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 14:24:19 +0700 Subject: [PATCH 08/24] fix: check Casework context bindings through caseworkctl, not a crate link `rstack dev check` linked registry-casework-breg into bregctl to run Casework's context-binding reader in process, which breaks the rule that BReg tooling never depends on a Casework crate. The dependency-direction gate already refuses that closure. Casework now refuses an invalid `contextProviders` binding in its own offline check, at the member and naming the rule, through the same validation provider construction runs (including the HTTPS base URL and the delegated display exchange). `rstack dev check` renders the Casework configuration the run writes, runs `caseworkctl check --runtime-config` on it, and reports a refusal at the plan field that produced the member. A refusal it cannot attribute to the binding is reported as an operational failure rather than a pass. Signed-off-by: Jeremi Joslin --- Cargo.lock | 1 - crates/registry-bregctl/Cargo.toml | 1 - .../src/project_dev/bindings.rs | 253 ++++----- .../src/project_dev/products.rs | 491 +++++++++++++++--- crates/registry-bregctl/tests/project_dev.rs | 96 +++- crates/registry-casework-breg/src/config.rs | 315 +++++++++-- crates/registry-casework-breg/src/lib.rs | 22 +- crates/registry-casework/src/config.rs | 102 ++++ products/casework/CHANGELOG.md | 6 + products/platform/PROJECT-DEV.md | 7 +- products/platform/scripts/test-rstack-dev.py | 18 + 11 files changed, 1027 insertions(+), 285 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d7f0d25aa8..1020ceb27d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6331,7 +6331,6 @@ dependencies = [ "rcgen", "registry-breg", "registry-breg-client", - "registry-casework-breg", "registry-cli-api", "registry-cli-reference", "registry-cli-report", diff --git a/crates/registry-bregctl/Cargo.toml b/crates/registry-bregctl/Cargo.toml index f65add3ea5..4637a037fd 100644 --- a/crates/registry-bregctl/Cargo.toml +++ b/crates/registry-bregctl/Cargo.toml @@ -65,7 +65,6 @@ registry-cli-api.workspace = true registry-cli-reference.workspace = true registry-breg = { workspace = true, features = ["runtime", "tooling"] } registry-breg-client.workspace = true -registry-casework-breg.workspace = true registry-evidence-verifier.workspace = true registry-linkml.workspace = true registry-manifest-core.workspace = true diff --git a/crates/registry-bregctl/src/project_dev/bindings.rs b/crates/registry-bregctl/src/project_dev/bindings.rs index 290fbf7aff..01790f831e 100644 --- a/crates/registry-bregctl/src/project_dev/bindings.rs +++ b/crates/registry-bregctl/src/project_dev/bindings.rs @@ -351,109 +351,89 @@ fn client_refs(filename: &str) -> (String, String) { } /// Origin and token endpoint `check` renders with. Nothing listens there: the -/// product readers refuse a binding's shape, not the address it names. +/// product checks refuse a binding's shape, not the address it names. const CHECK_ORIGIN: &str = "http://127.0.0.1:1"; const CHECK_TOKEN_ENDPOINT: &str = "http://127.0.0.1:1/oauth2/token"; -/// Renders each binding the plan configures exactly as `start` writes it and -/// lets the owning product's reader refuse what it would refuse at startup, -/// before anything is provisioned. A refusal names the plan field that -/// produced the value, not the generated fragment. -pub(super) fn check_with_product_readers(plan: &Plan) -> plan::SemanticResult<()> { +/// Writes the binding the plan configures for `product` into `config` as +/// `start` writes it, at the origin and token endpoint `check` renders with. +pub(super) fn render_for_check(plan: &Plan, product: Product, config: &mut Value) { let origins = BTreeMap::from([ (Product::Breg, CHECK_ORIGIN.to_owned()), (Product::Casework, CHECK_ORIGIN.to_owned()), ]); - if let Some(binding) = &plan.bindings.breg_review_authority { - let mut config = json!({}); - render( - plan, - Product::Breg, - &origins, - CHECK_TOKEN_ENDPOINT, - &mut config, - ); - let grant = grant_field(&binding.identity, Product::Casework); - let audience = audience_field(plan, Product::Casework); - require_reader_accepts( - "BReg", - "/bindings/bregReviewAuthority", - &config["reviewAuthorities"][binding.id.as_str()], - &[ - Source::new( - "/profile", - &format!("{grant}/profile"), - json!("placeholder"), - ), - Source::new( - "/privateKeyJwt/assertionAudience", - "/issuer", - json!(STAND_IN_URI), - ), - Source::new("/privateKeyJwt/resource", &audience, json!(STAND_IN_URI)), - Source::new( - "/privateKeyJwt/scopes", - &format!("{grant}/scopes"), - json!(["placeholder"]), - ), - Source::new( - "/producerId", - "/bindings/bregReviewAuthority/producer", - json!("placeholder"), - ), - ], - |fragment| { - registry_breg::runtime_config::check_review_authority(&binding.id, fragment) - .map_err(|error| error.to_string()) - }, - )?; - } - if let Some(binding) = &plan.bindings.casework_context_provider { - let mut config = json!({}); - render( - plan, - Product::Casework, - &origins, - CHECK_TOKEN_ENDPOINT, - &mut config, - ); - let grant = grant_field(&binding.identity, Product::Breg); - let audience = audience_field(plan, Product::Breg); - require_reader_accepts( - "Casework", - "/bindings/caseworkContextProvider", - &config["contextProviders"][binding.id.as_str()], - &[ - Source::new( - "/statusProfile", - &format!("{grant}/profile"), - json!("placeholder"), - ), - Source::new("/clientAssertionAudience", "/issuer", json!(STAND_IN_URI)), - Source::new("/resource", &audience, json!(STAND_IN_URI)), - Source::new( - "/scopes", - &format!("{grant}/scopes"), - json!(["placeholder"]), - ), - ], - |fragment| { - serde_json::from_value::(fragment.clone()) - .map_err(|_| { - "the configuration does not have the shape the reader expects".to_owned() - })? - .check_offline() - .map_err(|error| error.to_string()) - }, - )?; + render(plan, product, &origins, CHECK_TOKEN_ENDPOINT, config); +} + +/// Renders the review authority binding exactly as `start` writes it and lets +/// BReg's reader refuse what it would refuse at startup, before anything is +/// provisioned. A refusal names the plan field that produced the value, not +/// the generated fragment. Casework checks the context provider binding +/// itself, through `caseworkctl check`. +pub(super) fn check_with_breg_reader(plan: &Plan) -> plan::SemanticResult<()> { + let Some(binding) = &plan.bindings.breg_review_authority else { + return Ok(()); + }; + let mut config = json!({}); + render_for_check(plan, Product::Breg, &mut config); + let grant = grant_field(&binding.identity, Product::Casework); + let audience = audience_field(plan, Product::Casework); + require_reader_accepts( + "BReg", + "/bindings/bregReviewAuthority", + &config["reviewAuthorities"][binding.id.as_str()], + &[ + Source::new( + "/profile", + &format!("{grant}/profile"), + json!("placeholder"), + ), + Source::new( + "/privateKeyJwt/assertionAudience", + "/issuer", + json!(STAND_IN_URI), + ), + Source::new("/privateKeyJwt/resource", &audience, json!(STAND_IN_URI)), + Source::new( + "/privateKeyJwt/scopes", + &format!("{grant}/scopes"), + json!(["placeholder"]), + ), + Source::new( + "/producerId", + "/bindings/bregReviewAuthority/producer", + json!("placeholder"), + ), + ], + |fragment| { + registry_breg::runtime_config::check_review_authority(&binding.id, fragment) + .map_err(|error| error.to_string()) + }, + ) +} + +/// The plan field that produced the member at `member`, a JSON pointer +/// relative to the context provider binding the run writes for Casework. +/// Members only the run supplies land on the binding. +pub(super) fn casework_binding_field( + plan: &Plan, + binding: &CaseworkContextProvider, + member: &str, +) -> String { + let grant = grant_field(&binding.identity, Product::Breg); + match member.split('/').nth(1).unwrap_or_default() { + "statusProfile" => format!("{grant}/profile"), + "clientAssertionAudience" => "/issuer".to_owned(), + "resource" => audience_field(plan, Product::Breg), + "scopes" => format!("{grant}/scopes"), + _ => "/bindings/caseworkContextProvider".to_owned(), } - Ok(()) } const STAND_IN_URI: &str = "urn:example:stand-in"; /// One member of a rendered fragment and the plan field that produced it, -/// with a value every reader accepts for that member. +/// with a value the reader accepts for that member. struct Source { member: &'static str, field: String, @@ -483,7 +463,7 @@ fn audience_field(plan: &Plan, product: Product) -> String { format!("/products/{index}/audience") } -/// Refuses when `reader` refuses `fragment`. The readers state a rule without +/// Refuses when `reader` refuses `fragment`. The reader states a rule without /// saying which member broke it, so the refusal lands on the plan field of the /// one member whose replacement by a stand-in makes the reader accept. When no /// single member explains it, the refusal lands on the binding. @@ -971,28 +951,23 @@ mod tests { } #[test] - fn check_accepts_a_plan_both_product_readers_accept() { - check_with_product_readers(&plan()).unwrap(); + fn check_accepts_a_plan_the_breg_reader_accepts() { + check_with_breg_reader(&plan()).unwrap(); } #[test] fn check_refuses_a_bound_audience_that_is_not_a_resource_uri() { let mut value = plan(); value.products[1].audience = "review".to_owned(); - let error = check_with_product_readers(&value).unwrap_err().to_string(); + let error = check_with_breg_reader(&value).unwrap_err().to_string(); assert!(error.contains("/products/1/audience"), "{error}"); assert!(error.contains("BReg refuses"), "{error}"); + // The context provider resource is Casework's rule, which + // `caseworkctl check` applies to the configuration the run writes. let mut value = plan(); value.products[0].audience = "registry".to_owned(); - let error = check_with_product_readers(&value).unwrap_err().to_string(); - assert!(error.contains("/products/0/audience"), "{error}"); - assert!(error.contains("Casework refuses"), "{error}"); - - let mut value = plan(); - value.bindings.casework_context_provider = None; - value.products[0].audience = "registry".to_owned(); - check_with_product_readers(&value).unwrap(); + check_with_breg_reader(&value).unwrap(); } #[test] @@ -1012,67 +987,37 @@ mod tests { .unwrap() .scopes .clone_from(&too_long); - let error = check_with_product_readers(&value).unwrap_err().to_string(); + let error = check_with_breg_reader(&value).unwrap_err().to_string(); assert!( error.contains("/people/producer/grants/casework/scopes"), "{error}" ); - let mut value = plan(); - value - .people - .get_mut("context-reader") - .unwrap() - .grants - .get_mut(&Product::Breg) - .unwrap() - .scopes = vec!["registry:\"quoted\"".to_owned()]; - let error = check_with_product_readers(&value).unwrap_err().to_string(); - assert!( - error.contains("/people/context-reader/grants/breg/scopes"), - "{error}" - ); - // A person no binding names keeps the plan's own scope rule. let mut value = plan(); - value.bindings.casework_context_provider = None; + value.bindings.breg_review_authority = None; value .people - .get_mut("context-reader") + .get_mut("producer") .unwrap() .grants - .get_mut(&Product::Breg) + .get_mut(&Product::Casework) .unwrap() .scopes = too_long; value.bindings.validate(&value).unwrap(); - check_with_product_readers(&value).unwrap(); + check_with_breg_reader(&value).unwrap(); } #[test] fn check_refuses_other_binding_values_at_the_plan_field_that_produced_them() { let mut value = plan(); value.issuer = "not a uri".to_owned(); - let error = check_with_product_readers(&value).unwrap_err().to_string(); + let error = check_with_breg_reader(&value).unwrap_err().to_string(); assert!(error.contains("/issuer"), "{error}"); // The profile grammar is a rule only the product readers state: the - // runner restates nothing about it, so each product's reader catches - // it and the refusal lands on the grant that produced the value. - let mut value = plan(); - value - .people - .get_mut("context-reader") - .unwrap() - .grants - .get_mut(&Product::Breg) - .unwrap() - .profile = "not a profile".to_owned(); - let error = check_with_product_readers(&value).unwrap_err().to_string(); - assert!( - error.contains("/people/context-reader/grants/breg/profile"), - "{error}" - ); - + // runner restates nothing about it, so BReg's reader catches it and + // the refusal lands on the grant that produced the value. let mut value = plan(); value .people @@ -1082,13 +1027,39 @@ mod tests { .get_mut(&Product::Casework) .unwrap() .profile = "not a profile".to_owned(); - let error = check_with_product_readers(&value).unwrap_err().to_string(); + let error = check_with_breg_reader(&value).unwrap_err().to_string(); assert!( error.contains("/people/producer/grants/casework/profile"), "{error}" ); } + #[test] + fn a_casework_refusal_lands_on_the_plan_field_that_produced_the_member() { + let value = plan(); + let binding = value.bindings.casework_context_provider.as_ref().unwrap(); + for (member, field) in [ + ( + "/statusProfile", + "/people/context-reader/grants/breg/profile", + ), + ("/clientAssertionAudience", "/issuer"), + ("/resource", "/products/0/audience"), + ("/scopes", "/people/context-reader/grants/breg/scopes"), + ("/scopes/0", "/people/context-reader/grants/breg/scopes"), + ("/baseUrl", "/bindings/caseworkContextProvider"), + ("/tokenEndpoint", "/bindings/caseworkContextProvider"), + ("/clientIdRef", "/bindings/caseworkContextProvider"), + ("", "/bindings/caseworkContextProvider"), + ] { + assert_eq!( + casework_binding_field(&value, binding, member), + field, + "{member}" + ); + } + } + #[test] fn stopping_the_token_server_does_not_wait_for_an_unfinished_request() { use std::io::Write; diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs index a3d57f2d15..e3de50e860 100644 --- a/crates/registry-bregctl/src/project_dev/products.rs +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -367,45 +367,22 @@ impl<'a> ProductRunner<'a> { package: &Path, urls: DatabaseUrls, ) -> Result { - let mut clients: BTreeSet<_> = self - .input - .plan - .people - .values() - .filter(|person| person.grants.contains_key(&product)) - .map(|person| person.client.as_str()) - .collect(); - if product == Product::Breg { - let authority = self.input.breg_authority.as_ref().expect("BReg authority"); - clients.extend(authority.named_clients.iter().map(String::as_str)); - } - let mut oidc = json!({ - "issuer": self.input.plan.issuer, - "audience": self.input.plan.product(product).audience, - "allowedClients": clients, - "scopeClaim": "scope", - "jwksSource": {"type": "static", "documentRef": "secret:file/jwks"} - }); - let mut database = json!({ - "runtimeUrlRef": self.secret(&format!("{name}-runtime-url"), urls.runtime.as_bytes())?, - "migrationUrlRef": self.secret(&format!("{name}-migration-url"), urls.migration.as_bytes())? - }); - if self.root.join("postgres-ca").is_file() { - database["trustedRootCertificateRef"] = json!("secret:file/postgres-ca"); - } - let audit = json!({ - "hashKeyRef": self.secret(&format!("{name}-audit-key"), random_secret()?.as_bytes())?, - "path": self.root.join(format!("{name}-audit")).join("audit.jsonl"), - }); + let plan = &self.input.plan; + self.secret(&format!("{name}-runtime-url"), urls.runtime.as_bytes())?; + self.secret(&format!("{name}-migration-url"), urls.migration.as_bytes())?; + self.secret(&format!("{name}-audit-key"), random_secret()?.as_bytes())?; let identity = format!("dev-{}-{name}", self.input.run_id); - let mut config = json!({ - "package": {"root": package}, - "secretProviders": {"file": {"root": self.root}}, - "database": database, - "audit": audit, - }); - match product { + let mut config = match product { Product::Breg => { + let authority = self.input.breg_authority.as_ref().expect("BReg authority"); + let (mut config, mut oidc) = shared_config( + plan, + product, + name, + self.root, + package, + &authority.named_clients, + ); oidc["allowedAlgorithm"] = json!("ES256"); oidc["accessTokenType"] = json!("at+jwt"); oidc["scopeSeparator"] = json!(" "); @@ -425,7 +402,6 @@ impl<'a> ProductRunner<'a> { }); config["audit"]["destination"] = json!("file"); config["listener"] = json!({"bind": bind}); - let authority = self.input.breg_authority.as_ref().expect("BReg authority"); if name == "breg" && authority.requires_postgis { config["listener"]["publicOrigin"] = json!(format!("http://{bind}")); } @@ -458,17 +434,10 @@ impl<'a> ProductRunner<'a> { "recordLockMilliseconds": 5000, "migrationLockMilliseconds": 30000, "migrationStatementMilliseconds": 60000 }); + config } - Product::Casework => { - config["apiVersion"] = - json!("id.registrystack.org/formats/casework/runtime/v1alpha1"); - config["kind"] = json!("CaseworkRuntimeConfig"); - config["listener"] = - json!({"bind": bind, "tlsTermination": "development-loopback"}); - config["identity"] = json!({"databaseId": identity}); - config["authentication"] = json!({"oidc": oidc}); - } - } + Product::Casework => casework_config(plan, name, self.root, package, &identity, bind), + }; if let Some(bindings) = &self.bindings { bindings.configure(product, &mut config); } @@ -479,7 +448,7 @@ impl<'a> ProductRunner<'a> { fn secret(&self, name: &str, bytes: &[u8]) -> Result { write_private(&self.root.join(name), bytes)?; - Ok(format!("secret:file/{name}")) + Ok(secret_ref(name)) } fn tool(&self, product: Product, stage: &str, args: &[&std::ffi::OsStr]) -> Result<()> { @@ -705,6 +674,83 @@ fn write_processes(root: &Path, children: &[(Product, u32)]) -> Result<()> { write_private(&root.join("processes.json"), &serde_json::to_vec(&entries)?) } +/// The reference to the secret the run writes as `name` in its directory. +fn secret_ref(name: &str) -> String { + format!("secret:file/{name}") +} + +/// The members both products' runtime configurations share, and the OIDC +/// settings each completes. They name the run's secrets by reference only: +/// `config` writes the secrets beside them. +fn shared_config( + plan: &Plan, + product: Product, + name: &str, + root: &Path, + package: &Path, + extra_clients: &BTreeSet, +) -> (Value, Value) { + let clients: BTreeSet<_> = plan + .people + .values() + .filter(|person| person.grants.contains_key(&product)) + .map(|person| person.client.as_str()) + .chain(extra_clients.iter().map(String::as_str)) + .collect(); + let oidc = json!({ + "issuer": plan.issuer, + "audience": plan.product(product).audience, + "allowedClients": clients, + "scopeClaim": "scope", + "jwksSource": {"type": "static", "documentRef": secret_ref("jwks")} + }); + let mut database = json!({ + "runtimeUrlRef": secret_ref(&format!("{name}-runtime-url")), + "migrationUrlRef": secret_ref(&format!("{name}-migration-url")) + }); + if root.join("postgres-ca").is_file() { + database["trustedRootCertificateRef"] = json!(secret_ref("postgres-ca")); + } + let audit = json!({ + "hashKeyRef": secret_ref(&format!("{name}-audit-key")), + "path": root.join(format!("{name}-audit")).join("audit.jsonl"), + }); + let config = json!({ + "package": {"root": package}, + "secretProviders": {"file": {"root": root}}, + "database": database, + "audit": audit, + }); + (config, oidc) +} + +/// The Casework runtime configuration the run writes, before its bindings. +/// `check` renders it with stand-in paths, so the configuration Casework +/// checks is the one `start` writes. +fn casework_config( + plan: &Plan, + name: &str, + root: &Path, + package: &Path, + identity: &str, + bind: &str, +) -> Value { + let (mut config, oidc) = shared_config( + plan, + Product::Casework, + name, + root, + package, + &BTreeSet::new(), + ); + config["apiVersion"] = json!("id.registrystack.org/formats/casework/runtime/v1alpha1"); + config["kind"] = json!("CaseworkRuntimeConfig"); + config["listener"] = json!({"bind": bind, "tlsTermination": "development-loopback"}); + config["identity"] = json!({"databaseId": identity}); + config["authentication"] = json!({"oidc": oidc}); + config +} + fn random_secret() -> Result> { use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; let mut bytes = zeroize::Zeroizing::new([0u8; 32]); @@ -804,7 +850,7 @@ pub(super) fn inspect( plan: &Plan, terminate: &AtomicBool, ) -> Result { - super::bindings::check_with_product_readers(plan)?; + super::bindings::check_with_breg_reader(plan)?; let breg_authority = breg_authority(root, plan)?; let casework_principals = casework_principals(root, plan, breg_authority.as_ref(), terminate)?; Ok(ProductInspection { @@ -1041,6 +1087,9 @@ fn casework_principals( .and_then(|person| person.grants.get(&Product::Breg)) .map(|grant| grant.profile.as_str()); validate_casework_binding_ids(&plan.bindings, &facts, status_profile)?; + if let Some(binding) = &plan.bindings.casework_context_provider { + check_casework_binding(&tool, &project, plan, binding, terminate)?; + } Ok(casework_principals_from_facts(plan, breg, &facts)?) } @@ -1094,15 +1143,33 @@ fn explain_casework_facts( project: &Path, terminate: &AtomicBool, ) -> Result { + let (status, report) = casework_report(tool, "explain", &[project.as_os_str()], terminate)?; + if status.success() { + return casework_report_facts(&report); + } + if status.code() == Some(1) && casework_domain_refusal(&report, "explain") { + return Err(InvalidCaseworkProject.into()); + } + anyhow::bail!("caseworkctl explain did not complete successfully") +} + +/// Runs `caseworkctl --format json ` and returns its +/// exit status with the JSON report it printed. +fn casework_report( + tool: &Path, + command: &str, + arguments: &[&std::ffi::OsStr], + terminate: &AtomicBool, +) -> Result<(ExitStatus, Value)> { const MAX_REPORT: u64 = 4 * 1024 * 1024; let mut child = child_command(tool) - .args(["--format", "json", "explain"]) - .arg(project) + .args(["--format", "json", command]) + .args(arguments) .stdout(Stdio::piped()) .spawn() - .context("cannot start caseworkctl explain")?; - let stdout = child.stdout.take().expect("the explain output was piped"); + .with_context(|| format!("cannot start caseworkctl {command}"))?; + let stdout = child.stdout.take().expect("the report output was piped"); let reader = thread::spawn(move || { let mut bytes = Vec::new(); stdout.take(MAX_REPORT + 1).read_to_end(&mut bytes)?; @@ -1114,31 +1181,25 @@ fn explain_casework_facts( !terminate.load(Ordering::Relaxed), "development product inspection was interrupted" ); - ensure!(Instant::now() < deadline, "caseworkctl explain timed out"); + ensure!(Instant::now() < deadline, "caseworkctl {command} timed out"); Ok(()) })?; let bytes = reader .join() - .map_err(|_| anyhow::anyhow!("cannot collect the Casework authoring report"))??; + .map_err(|_| anyhow::anyhow!("cannot collect the caseworkctl {command} report"))??; ensure!( bytes.len() as u64 <= MAX_REPORT, - "the Casework authoring report exceeds the size limit" + "the caseworkctl {command} report exceeds the size limit" ); - let report: Value = serde_json::from_slice(&bytes) - .map_err(|_| anyhow::anyhow!("caseworkctl explain did not emit a JSON report"))?; - if status.success() { - return casework_report_facts(&report); - } - if status.code() == Some(1) && casework_domain_refusal(&report) { - return Err(InvalidCaseworkProject.into()); - } - anyhow::bail!("caseworkctl explain did not complete successfully") + let report = serde_json::from_slice(&bytes) + .map_err(|_| anyhow::anyhow!("caseworkctl {command} did not emit a JSON report"))?; + Ok((status, report)) } -fn casework_domain_refusal(report: &Value) -> bool { +fn casework_domain_refusal(report: &Value, command: &str) -> bool { report["apiVersion"] == "id.registrystack.org/formats/casework/report/v1alpha2" && report["kind"] == "CaseworkReport" - && report["command"] == "explain" + && report["command"] == command && report["status"] == "domain-refusal" && report["ok"] == false && report["diagnostics"].as_array().is_some_and(|diagnostics| { @@ -1158,6 +1219,137 @@ fn casework_domain_refusal(report: &Value) -> bool { }) } +/// The listener `check` renders the Casework configuration with. Nothing +/// binds it. +const CHECK_BIND: &str = "127.0.0.1:1"; + +/// Renders the Casework runtime configuration the run writes and lets the +/// selected `caseworkctl check` decide whether Casework would accept the +/// context provider binding in it. A refusal lands on the plan field that +/// produced the member Casework names. +fn check_casework_binding( + tool: &Path, + project: &SelectedProduct, + plan: &Plan, + binding: &super::bindings::CaseworkContextProvider, + terminate: &AtomicBool, +) -> Result<()> { + let unavailable = || { + InspectionUnavailable::new( + &project.field, + "The selected `caseworkctl` helper could not check the Casework configuration the run writes.", + "Install the matching Casework tools or provide RSTACK_CASEWORKCTL_BIN, then run the check again.", + ) + }; + let checked = (|| { + let directory = tempfile::tempdir()?; + // The runtime loader refuses a file reached through a symbolic link, + // as the default temporary directory is on some systems. + let root = directory.path().canonicalize()?; + let mut config = casework_config( + plan, + "casework", + &root, + &root.join("casework-package"), + "dev-check-casework", + CHECK_BIND, + ); + super::bindings::render_for_check(plan, Product::Casework, &mut config); + let path = root.join("casework-runtime.yaml"); + write_private(&path, &serde_json::to_vec(&config)?)?; + casework_report( + tool, + "check", + &[ + project.root.as_os_str(), + "--runtime-config".as_ref(), + path.as_os_str(), + ], + terminate, + ) + })(); + let Ok((status, report)) = checked else { + return Err(unavailable().into()); + }; + match casework_binding_check(status.code(), &report, &binding.id) { + BindingCheck::Accepted => Ok(()), + BindingCheck::Refused { member, message } => Err(plan::SemanticRefusal::new( + super::bindings::casework_binding_field(plan, binding, &member), + format!( + "Casework refuses the binding configuration the run writes from this value: {}.", + message.trim_end_matches('.') + ), + "Correct the value so the product accepts the configuration the run writes for this binding.", + ) + .into()), + BindingCheck::Unchecked => Err(unavailable().into()), + } +} + +#[derive(Debug, Eq, PartialEq)] +enum BindingCheck { + Accepted, + /// Casework refused `member`, a JSON pointer relative to the binding. + Refused { + member: String, + message: String, + }, + /// Casework did not read the binding. + Unchecked, +} + +/// What a `caseworkctl check` that exited with `code` and printed `report` +/// decided about the context provider binding `id` in the configuration the +/// run writes. Only refusals at the binding count: the rest of the file names +/// stand-in paths and the project is not yet paired, and `start` checks both +/// as they are. A refusal by the shared loader elsewhere in the file means +/// Casework never read the binding. +fn casework_binding_check(code: Option, report: &Value, id: &str) -> BindingCheck { + if code == Some(0) + && report["apiVersion"] == "id.registrystack.org/formats/casework/report/v1alpha2" + && report["kind"] == "CaseworkReport" + && report["command"] == "check" + && report["status"] == "complete" + && report["ok"] == true + { + return BindingCheck::Accepted; + } + if code != Some(1) || !casework_domain_refusal(report, "check") { + return BindingCheck::Unchecked; + } + let binding = format!( + "/contextProviders/{}", + registry_platform_yaml::escape_pointer_segment(id) + ); + let mut unread = false; + for diagnostic in report["diagnostics"].as_array().into_iter().flatten() { + if diagnostic["severity"] != "error" || diagnostic["artifact"] != "CaseworkRuntimeConfig" { + continue; + } + let (Some(path), Some(code), Some(message)) = ( + diagnostic["path"].as_str(), + diagnostic["code"].as_str(), + diagnostic["message"].as_str(), + ) else { + return BindingCheck::Unchecked; + }; + if let Some(member) = path.strip_prefix(&binding) { + if member.is_empty() || member.starts_with('/') { + return BindingCheck::Refused { + member: member.to_owned(), + message: message.to_owned(), + }; + } + } + unread |= !code.starts_with("casework.runtime."); + } + if unread { + BindingCheck::Unchecked + } else { + BindingCheck::Accepted + } +} + #[derive(Debug, thiserror::Error)] #[error("the Casework project did not pass its owning validation")] struct InvalidCaseworkProject; @@ -1876,6 +2068,169 @@ mod tests { } } + #[test] + fn only_a_casework_refusal_at_the_binding_refuses_the_binding() { + let report = |status: &str, diagnostics: Value| { + json!({ + "apiVersion": "id.registrystack.org/formats/casework/report/v1alpha2", + "kind": "CaseworkReport", "command": "check", "status": status, + "ok": status == "complete", "diagnostics": diagnostics + }) + }; + let diagnostic = |artifact: &str, code: &str, path: &str| { + json!({ + "severity": "error", "code": code, "artifact": artifact, "path": path, + "message": "the rule", "suggestedAction": "Correct it." + }) + }; + let runtime = "CaseworkRuntimeConfig"; + let refused = |member: &str| BindingCheck::Refused { + member: member.to_owned(), + message: "the rule".to_owned(), + }; + for (code, diagnostics, expected) in [ + (Some(0), json!([]), BindingCheck::Accepted), + ( + Some(1), + json!([diagnostic( + runtime, + "casework.runtime.invalid-context-provider", + "/contextProviders/registry/resource" + )]), + refused("/resource"), + ), + ( + Some(1), + json!([diagnostic( + runtime, + "config.invalid-value", + "/contextProviders/registry/baseUrl" + )]), + refused("/baseUrl"), + ), + ( + Some(1), + json!([diagnostic( + runtime, + "casework.runtime.unknown", + "/contextProviders/registry" + )]), + refused(""), + ), + // Another provider, Casework rules about the stand-in files or + // the unpaired project, and project diagnostics are not this + // binding's. + ( + Some(1), + json!([ + diagnostic( + runtime, + "casework.runtime.invalid-context-provider", + "/contextProviders/registry-other/resource" + ), + diagnostic( + runtime, + "casework.runtime.context-providers-mismatch", + "/contextProviders" + ), + diagnostic( + "CaseworkProject", + "casework.project.invalid", + "/reviewPolicies" + ), + ]), + BindingCheck::Accepted, + ), + // The loader refused the file elsewhere, so Casework never read + // the binding. + ( + Some(1), + json!([diagnostic(runtime, "config.unknown-key", "/listener/port")]), + BindingCheck::Unchecked, + ), + ( + Some(3), + json!([diagnostic( + runtime, + "casework.runtime.unknown", + "/contextProviders/registry" + )]), + BindingCheck::Unchecked, + ), + (None, json!([]), BindingCheck::Unchecked), + ] { + let status = if code == Some(0) { + "complete" + } else { + "domain-refusal" + }; + assert_eq!( + casework_binding_check(code, &report(status, diagnostics.clone()), "registry"), + expected, + "{code:?} {diagnostics}" + ); + } + let explain = json!({ + "apiVersion": "id.registrystack.org/formats/casework/report/v1alpha2", + "kind": "CaseworkReport", "command": "explain", "status": "complete", "ok": true + }); + assert_eq!( + casework_binding_check(Some(0), &explain, "registry"), + BindingCheck::Unchecked + ); + } + + #[test] + fn check_renders_the_casework_configuration_start_writes() { + let scratch = tempfile::tempdir().unwrap(); + let root = scratch.path().join("run"); + super::super::private_directory(&root).unwrap(); + let terminate = AtomicBool::new(false); + let mut input = breg_input(false); + input.plan = serde_json::from_value(json!({ + "issuer": "https://issuer.example.invalid", + "products": [ + {"product": "breg", "project": ".", "audience": "urn:example:registry"}, + {"product": "casework", "project": "casework", "audience": "urn:example:review"} + ], + "people": {"reviewer": { + "subject": "reviewer", "client": "reviewer-client", + "grants": {"casework": {"profile": "reviewer", "scopes": ["casework:review"]}} + }}, + "journey": {"id": "read", "steps": [{ + "id": "read", "product": "casework", "as": "reviewer", "method": "GET", + "path": "/v1/review-tasks", "expect": {"status": 200} + }]} + })) + .unwrap(); + let identity = format!("dev-{}-casework", input.run_id); + let runner = ProductRunner::new(Arc::new(input), &root, &terminate).unwrap(); + let urls = DatabaseUrls { + runtime: "postgresql://runtime:password@127.0.0.1:5432/run".to_owned(), + migration: "postgresql://migration:password@127.0.0.1:5432/run".to_owned(), + }; + let package = root.join("casework-package"); + let path = runner + .config(Product::Casework, "casework", "127.0.0.1:0", &package, urls) + .unwrap(); + let written: Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + + let rendered = casework_config( + &runner.input.plan, + "casework", + &root, + &package, + &identity, + "127.0.0.1:0", + ); + assert_eq!(written, rendered); + assert_eq!( + written["authentication"]["oidc"]["allowedClients"], + json!(["reviewer-client"]) + ); + assert!(!written.to_string().contains("password"), "{written}"); + } + #[test] fn a_report_without_the_access_profile_list_is_refused() { let report = json!({ diff --git a/crates/registry-bregctl/tests/project_dev.rs b/crates/registry-bregctl/tests/project_dev.rs index 9354b0f373..3233e15b88 100644 --- a/crates/registry-bregctl/tests/project_dev.rs +++ b/crates/registry-bregctl/tests/project_dev.rs @@ -337,7 +337,8 @@ fn fake_casework_tool(root: &std::path::Path) -> std::path::PathBuf { } /// A `caseworkctl` whose `proposal-context` provider names `status_profile` -/// for the `service-status` capability. +/// for the `service-status` capability. Its `check` refuses a runtime file +/// whose context provider resource is `registry`, as Casework does. fn fake_casework_tool_with_status_profile( root: &std::path::Path, status_profile: &str, @@ -348,9 +349,26 @@ fn fake_casework_tool_with_status_profile( concat!( "#!/usr/bin/env bash\n", "set -euo pipefail\n", - "[[ $# -eq 4 ]]\n", - "[[ $1 == --format && $2 == json && $3 == explain ]]\n", - "[[ -f $4/casework.yaml ]]\n", + "[[ $1 == --format && $2 == json && -f $4/casework.yaml ]]\n", + "if [[ $3 == check ]]; then\n", + " [[ $# -eq 6 && $5 == --runtime-config && -f $6 ]]\n", + " if grep -q '\"resource\":\"registry\"' \"$6\"; then\n", + " printf '%s\\n' '", + "{\"apiVersion\":\"id.registrystack.org/formats/casework/report/v1alpha2\",", + "\"kind\":\"CaseworkReport\",\"command\":\"check\",\"status\":\"domain-refusal\",\"ok\":false,", + "\"diagnostics\":[{\"severity\":\"error\",\"code\":\"casework.runtime.invalid-context-provider\",", + "\"artifact\":\"CaseworkRuntimeConfig\",\"path\":\"/contextProviders/proposal-context/resource\",", + "\"message\":\"contextProviders.proposal-context.resource: the resource must be one absolute URI without a fragment or userinfo\",", + "\"suggestedAction\":\"Correct the context provider member so it meets the rule the message names.\"}]}'\n", + " exit 1\n", + " fi\n", + " printf '%s\\n' '", + "{\"apiVersion\":\"id.registrystack.org/formats/casework/report/v1alpha2\",", + "\"kind\":\"CaseworkReport\",\"command\":\"check\",\"status\":\"complete\",\"ok\":true,", + "\"diagnostics\":[]}'\n", + " exit 0\n", + "fi\n", + "[[ $# -eq 4 && $3 == explain ]]\n", "printf '%s\\n' '", "{\"apiVersion\":\"id.registrystack.org/formats/casework/report/v1alpha2\",", "\"kind\":\"CaseworkReport\",\"command\":\"explain\",\"status\":\"complete\",\"ok\":true,", @@ -1128,6 +1146,76 @@ fn check_refuses_binding_configuration_the_products_would_refuse_at_the_plan_fie } } +#[test] +fn check_refuses_a_context_provider_binding_with_the_rule_caseworkctl_check_names() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let tool = casework_reference_project( + scratch.path(), + "reviewer", + "bindings:\n caseworkContextProvider: {id: proposal-context, identity: operator, action: review-action}\n", + ); + let dev = scratch.path().join("dev.yaml"); + let text = fs::read_to_string(&dev).unwrap(); + fs::write( + &dev, + text.replace("audience: \"urn:example:registry\"", "audience: registry"), + ) + .unwrap(); + + let checked = check_with_casework_tool(scratch.path(), &state, &tool); + + assert_eq!(checked.status.code(), Some(1), "{checked:?}"); + let report: serde_json::Value = serde_json::from_slice(&checked.stdout).unwrap(); + assert_eq!(report["status"], "domain-refusal"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/products/0/audience"); + assert_eq!( + diagnostic["message"], + "Casework refuses the binding configuration the run writes from this value: \ +contextProviders.proposal-context.resource: the resource must be one absolute URI without a \ +fragment or userinfo." + ); + assert!(!state.exists()); + + // A helper that cannot check the configuration is unavailable, not a + // refusal of the plan. + fs::rename(&tool, scratch.path().join("caseworkctl-reference")).unwrap(); + fs::write( + &tool, + concat!( + "#!/usr/bin/env bash\n", + "set -euo pipefail\n", + "if [[ $3 == check ]]; then\n", + " printf '%s\\n' '", + "{\"apiVersion\":\"id.registrystack.org/formats/casework/report/v1alpha2\",", + "\"kind\":\"CaseworkReport\",\"command\":\"check\",\"status\":\"domain-refusal\",\"ok\":false,", + "\"diagnostics\":[{\"severity\":\"error\",\"code\":\"config.unknown-key\",", + "\"artifact\":\"CaseworkRuntimeConfig\",\"path\":\"/listener\",", + "\"message\":\"The key is not part of this format.\",", + "\"suggestedAction\":\"Remove the key.\"}]}'\n", + " exit 1\n", + "fi\n", + "exec \"${0%/*}/caseworkctl-reference\" \"$@\"\n", + ), + ) + .unwrap(); + fs::set_permissions(&tool, fs::Permissions::from_mode(0o700)).unwrap(); + + let unavailable = check_with_casework_tool(scratch.path(), &state, &tool); + + assert_eq!(unavailable.status.code(), Some(3), "{unavailable:?}"); + let report: serde_json::Value = serde_json::from_slice(&unavailable.stdout).unwrap(); + assert_eq!(report["status"], "operational-failure"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/products/1/project"); + assert_eq!( + diagnostic["message"], + "The selected `caseworkctl` helper could not check the Casework configuration the run writes." + ); + assert!(!state.exists()); +} + #[test] fn check_refuses_a_status_profile_the_context_provider_policy_does_not_name() { let bindings = "bindings:\n caseworkContextProvider: {id: proposal-context, identity: operator, action: review-action}\n"; diff --git a/crates/registry-casework-breg/src/config.rs b/crates/registry-casework-breg/src/config.rs index 50e93748d5..edf286ad01 100644 --- a/crates/registry-casework-breg/src/config.rs +++ b/crates/registry-casework-breg/src/config.rs @@ -99,52 +99,152 @@ pub struct BregBinding { #[serde(default, skip_serializing_if = "Option::is_none")] pub delegated_display_exchange: Option, } +/// The member of a context-provider binding that provider construction would +/// refuse, and the rule it breaks. The reason names the rule and never the +/// configured value. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BindingRefusal { + /// JSON pointer to the refused member, relative to the binding. + pub member: String, + pub reason: String, +} + +impl BindingRefusal { + fn new(member: &str, reason: impl Into) -> Self { + Self { + member: member.to_owned(), + reason: reason.into(), + } + } + + fn within(mut self, parent: &str) -> Self { + self.member.insert_str(0, parent); + self + } +} + +const PROFILE_IDENTIFIER_RULE: &str = + "the profile identifier must be 1..=128 bytes of ASCII letters, digits, `-`, `_`, `.`, or `:`"; +const RESOURCE_URI_RULE: &str = + "the resource must be one absolute URI without a fragment or userinfo"; + impl BregBinding { /// Checks the binding as provider construction reads it, without /// resolving a secret or reading a context export. The client identifier /// is a secret, so the identity check runs with a stand-in; the key and /// the identifier themselves are checked when the provider is built. - pub fn check_offline(&self) -> Result<(), ReviewContextError> { - use registry_platform_httputil::{ - valid_resource_uri, validate_requested_scopes, PrivateKeyJwtConfig, - }; - self.check_profiles()?; - url::Url::parse(&self.base_url).map_err(|_| ReviewContextError::Invalid)?; - let token_endpoint = - url::Url::parse(&self.token_endpoint).map_err(|_| ReviewContextError::Invalid)?; - let invalid = |_| ReviewContextError::Invalid; - PrivateKeyJwtConfig::check_identity( - &token_endpoint, - "offline-check", + pub fn check_offline(&self) -> Result<(), BindingRefusal> { + use registry_platform_httputil::{valid_resource_uri, validate_requested_scopes}; + if !registry_casework_core::valid_profile_identifier(&self.status_profile) { + return Err(BindingRefusal::new( + "/statusProfile", + PROFILE_IDENTIFIER_RULE, + )); + } + if let Some(profile) = &self.task_grant_profile { + if !registry_casework_core::valid_profile_identifier(profile) { + return Err(BindingRefusal::new( + "/taskGrantProfile", + PROFILE_IDENTIFIER_RULE, + )); + } + if profile == &self.status_profile { + return Err(BindingRefusal::new( + "/taskGrantProfile", + "the task grant profile must differ from the status profile", + )); + } + } + check_client( + &self.token_endpoint, self.client_assertion_audience.as_deref(), - ) - .map_err(invalid)?; + )?; if self .resource .as_deref() .is_some_and(|resource| !valid_resource_uri(resource)) { - return Err(ReviewContextError::Invalid); + return Err(BindingRefusal::new("/resource", RESOURCE_URI_RULE)); } if let Some(scopes) = &self.scopes { - validate_requested_scopes(scopes).map_err(invalid)?; + validate_requested_scopes(scopes) + .map_err(|error| BindingRefusal::new("/scopes", token_rule(error)))?; + } + let base_url = url::Url::parse(&self.base_url) + .map_err(|_| BindingRefusal::new("/baseUrl", "the service base URL must be a URL"))?; + registry_platform_httputil::ServiceBaseUrl::new(base_url) + .map_err(|error| BindingRefusal::new("/baseUrl", error.to_string()))?; + if let Some(delegated) = &self.delegated_display_exchange { + delegated + .check_offline() + .map_err(|refusal| refusal.within("/delegatedDisplayExchange"))?; } Ok(()) } +} - fn check_profiles(&self) -> Result<(), ReviewContextError> { - if !registry_casework_core::valid_profile_identifier(&self.status_profile) - || self.task_grant_profile.as_ref().is_some_and(|profile| { - !registry_casework_core::valid_profile_identifier(profile) - || profile == &self.status_profile - }) - { - return Err(ReviewContextError::Invalid); - } - Ok(()) +/// The identity and transport checks a private-key JWT client runs before it +/// holds a key. The client identifier is a secret, so a stand-in takes its place. +fn check_client(token_endpoint: &str, audience: Option<&str>) -> Result<(), BindingRefusal> { + use registry_platform_httputil::PrivateKeyJwtConfig; + let token_endpoint = url::Url::parse(token_endpoint) + .map_err(|_| BindingRefusal::new("/tokenEndpoint", "the token endpoint must be a URL"))?; + PrivateKeyJwtConfig::check_identity(&token_endpoint, "offline-check", None) + .map_err(|error| BindingRefusal::new("/tokenEndpoint", token_rule(error)))?; + PrivateKeyJwtConfig::check_identity(&token_endpoint, "offline-check", audience) + .map_err(|error| BindingRefusal::new("/clientAssertionAudience", token_rule(error))) +} + +fn token_rule(error: registry_platform_httputil::TokenError) -> String { + match error { + registry_platform_httputil::TokenError::Configuration { reason } => reason.to_owned(), + other => other.to_string(), } } +/// The value rules a delegated display exchange enforces on its resource, +/// scopes and generation before it builds a token client. +pub(crate) fn check_exchange_values( + resource: &str, + scopes: &[String], + generation: &str, +) -> Result<(), BindingRefusal> { + let valid_token = |value: &str, maximum| { + !value.is_empty() + && value.len() <= maximum + && !value + .bytes() + .any(|byte| byte.is_ascii_control() || byte.is_ascii_whitespace()) + }; + if !valid_token(resource, 2048) { + return Err(BindingRefusal::new( + "/resource", + "the resource must be 1..=2048 bytes without control or whitespace characters", + )); + } + if scopes.is_empty() + || scopes.len() > 128 + || scopes.iter().any(|scope| !valid_token(scope, 512)) + || scopes + .iter() + .collect::>() + .len() + != scopes.len() + { + return Err(BindingRefusal::new( + "/scopes", + "the scopes must be 1..=128 distinct values of 1..=512 bytes without control or whitespace characters", + )); + } + if !valid_token(generation, 512) { + return Err(BindingRefusal::new( + "/generation", + "the generation must be 1..=512 bytes without control or whitespace characters", + )); + } + Ok(()) +} + impl std::fmt::Debug for BregBinding { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("BregBinding()") @@ -190,6 +290,24 @@ impl std::fmt::Debug for DelegatedDisplayExchangeBinding { } } +impl DelegatedDisplayExchangeBinding { + /// Checks the exchange as provider construction reads it, without + /// resolving its client identifier or key. + pub fn check_offline(&self) -> Result<(), BindingRefusal> { + use registry_platform_httputil::{valid_resource_uri, validate_requested_scopes}; + check_exchange_values(&self.resource, &self.scopes, &self.generation)?; + check_client( + &self.token_endpoint, + self.client_assertion_audience.as_deref(), + )?; + if !valid_resource_uri(&self.resource) { + return Err(BindingRefusal::new("/resource", RESOURCE_URI_RULE)); + } + validate_requested_scopes(&self.scopes) + .map_err(|error| BindingRefusal::new("/scopes", token_rule(error))) + } +} + pub fn build_provider( binding: &BregBinding, policy: ®istry_casework_core::ReviewContextProviderPolicy, @@ -204,7 +322,9 @@ pub fn build_provider( if description.is_empty() || description.len() > 1024 * 1024 { return Err(ReviewContextError::Invalid); } - binding.check_profiles()?; + binding + .check_offline() + .map_err(|_| ReviewContextError::Invalid)?; let export: BregContextExport = serde_json::from_value( registry_breg_client::decode_exact_json(description) .map_err(|_| ReviewContextError::Invalid)?, @@ -339,41 +459,142 @@ mod tests { }) } - fn offline_check(value: serde_json::Value) -> Result<(), ReviewContextError> { + fn offline_check(value: serde_json::Value) -> Result<(), BindingRefusal> { serde_json::from_value::(value) .expect("the binding parses") .check_offline() } + fn delegated_exchange() -> serde_json::Value { + json!({ + "tokenEndpoint": "http://127.0.0.1:31003/oauth2/token", + "clientIdRef": "secret:file/source-client-id", + "clientAssertionKeyRef": "secret:file/source-client-key", + "resource": "urn:example:registry", + "scopes": ["registry:proposal:review"], + "generation": "casework-source-v1" + }) + } + #[test] fn offline_check_accepts_a_binding_without_resolving_secrets() { offline_check(offline_binding()).expect("the binding is acceptable"); + let mut binding = offline_binding(); + binding["delegatedDisplayExchange"] = delegated_exchange(); + offline_check(binding).expect("the delegated exchange is acceptable"); } #[test] - fn offline_check_refuses_what_provider_construction_refuses_before_secrets() { - for (member, value) in [ - ("statusProfile", json!("not a profile")), - ("taskGrantProfile", json!("casework-review-status")), - ("taskGrantProfile", json!("not a profile")), - ("resource", json!("registry")), - ("resource", json!("urn:example:registry#fragment")), - ("scopes", json!([])), - ("scopes", json!(["registry:read", "registry:read"])), - ("scopes", json!(["has space"])), - ("clientAssertionAudience", json!(" ")), - ("tokenEndpoint", json!("http://idp.example.test/token")), - ("baseUrl", json!("not a url")), + fn offline_check_names_the_member_and_rule_provider_construction_refuses() { + for (member, value, rule) in [ + ( + "/statusProfile", + json!("not a profile"), + "profile identifier", + ), + ( + "/taskGrantProfile", + json!("casework-review-status"), + "differ from the status profile", + ), + ( + "/taskGrantProfile", + json!("not a profile"), + "profile identifier", + ), + ("/resource", json!("registry"), "absolute URI"), + ( + "/resource", + json!("urn:example:registry#fragment"), + "absolute URI", + ), + ("/scopes", json!([]), "at least one scope"), + ( + "/scopes", + json!(["registry:read", "registry:read"]), + "repeat", + ), + ("/scopes", json!(["has space"]), "scope-tokens"), + ( + "/clientAssertionAudience", + json!(" "), + "audience must not be empty", + ), + ( + "/tokenEndpoint", + json!("http://idp.example.test/token"), + "HTTPS", + ), + ( + "/tokenEndpoint", + json!("https://idp.example.test/token#top"), + "fragment", + ), + ("/baseUrl", json!("http://breg.example.test"), "HTTPS"), + ( + "/baseUrl", + json!("https://breg.example.test/?view=all"), + "query", + ), + ( + "/baseUrl", + json!("https://breg.example.test/#top"), + "fragment", + ), + ( + "/baseUrl", + json!("https://breg.example.test//api"), + "empty segment", + ), + ( + "/delegatedDisplayExchange/tokenEndpoint", + json!("http://idp.example.test/token"), + "HTTPS", + ), + ( + "/delegatedDisplayExchange/clientAssertionAudience", + json!(""), + "audience must not be empty", + ), + ( + "/delegatedDisplayExchange/resource", + json!("registry"), + "absolute URI", + ), + ( + "/delegatedDisplayExchange/resource", + json!("urn:example:registry two"), + "whitespace", + ), + ("/delegatedDisplayExchange/scopes", json!([]), "1..=128"), + ( + "/delegatedDisplayExchange/scopes", + json!(["registry:\"quoted\""]), + "scope-tokens", + ), + ( + "/delegatedDisplayExchange/generation", + json!(""), + "generation", + ), ] { let mut binding = offline_binding(); - binding[member] = value.clone(); - let result = serde_json::from_value::(binding) - .map_err(|_| ReviewContextError::Invalid) - .and_then(|binding| binding.check_offline()); + binding["delegatedDisplayExchange"] = delegated_exchange(); + let mut target = &mut binding; + for segment in member.split('/').skip(1) { + target = &mut target[segment]; + } + *target = value.clone(); + let refusal = + offline_check(binding).expect_err(&format!("{member} {value} was accepted")); + assert_eq!(refusal.member, member, "{value}: {refusal:?}"); assert!( - matches!(result, Err(ReviewContextError::Invalid)), - "{member} {value} was accepted" + refusal.reason.contains(rule), + "{member} {value}: {refusal:?}" ); + if let Some(text) = value.as_str().filter(|text| text.trim().len() > 1) { + assert!(!refusal.reason.contains(text), "{member}: {refusal:?}"); + } } } diff --git a/crates/registry-casework-breg/src/lib.rs b/crates/registry-casework-breg/src/lib.rs index f592eb9d3f..7c3951788a 100644 --- a/crates/registry-casework-breg/src/lib.rs +++ b/crates/registry-casework-breg/src/lib.rs @@ -41,26 +41,8 @@ impl DelegatedDisplayExchange { scopes: Vec, generation: String, ) -> Result { - let valid_token = |value: &str, maximum| { - !value.is_empty() - && value.len() <= maximum - && !value - .bytes() - .any(|byte| byte.is_ascii_control() || byte.is_ascii_whitespace()) - }; - if !valid_token(&resource, 2048) - || scopes.is_empty() - || scopes.len() > 128 - || scopes.iter().any(|scope| !valid_token(scope, 512)) - || scopes - .iter() - .collect::>() - .len() - != scopes.len() - || !valid_token(&generation, 512) - { - return Err(ReviewContextError::Invalid); - } + config::check_exchange_values(&resource, &scopes, &generation) + .map_err(|_| ReviewContextError::Invalid)?; let actor = Arc::new( PrivateKeyJwt::new(Self::client_config( &token_endpoint, diff --git a/crates/registry-casework/src/config.rs b/crates/registry-casework/src/config.rs index bc38974eb7..24797a22ab 100644 --- a/crates/registry-casework/src/config.rs +++ b/crates/registry-casework/src/config.rs @@ -1423,6 +1423,23 @@ impl RuntimeConfig { } } + /// Refuses a context-provider binding that provider construction would + /// refuse, at the member that breaks the rule, without resolving a secret. + fn check_context_providers(&self, findings: &mut Vec) { + for (deployment, binding) in &self.context_providers { + let Err(refusal) = binding.check_offline() else { + continue; + }; + let mut segments = vec!["contextProviders", deployment.as_str()]; + segments.extend(refusal.member.split('/').skip(1)); + findings.push(RuntimeConfigError::InvalidContextProvider { + path: segments.join("."), + pointer: pointer_to(&segments), + reason: refusal.reason, + }); + } + } + fn check_principal_claims( &self, project: &CaseworkProject, @@ -1803,6 +1820,7 @@ const FILE_RULES: &[FileRule] = &[ RuntimeConfig::check_oidc, RuntimeConfig::check_database, RuntimeConfig::check_review_completion_destinations, + RuntimeConfig::check_context_providers, ]; /// The rules a runtime file satisfies against the project it binds. @@ -2698,6 +2716,78 @@ mod tests { .all(|diagnostic| !diagnostic.code.ends_with("-id-collision"))); } + #[test] + fn context_provider_bindings_are_refused_at_the_member_and_rule_provider_construction_refuses() + { + let directory = tempfile::tempdir().unwrap(); + let binding = json!({ + "baseUrl": "https://registry.example.test", + "statusProfile": "review-status", + "tokenEndpoint": "https://identity.example.test/token", + "clientIdRef": "secret:file/client-id", + "clientAssertionKeyRef": "secret:file/client-key", + "resource": "urn:example:registry", + "scopes": ["registry:proposal:review-status"] + }); + let mut value = operator_value(directory.path(), directory.path()); + value["contextProviders"] = json!({ "registry-main": binding.clone() }); + let path = write_operator(&value, directory.path()); + assert!(check_runtime(&path, None, false) + .diagnostics + .iter() + .all(|diagnostic| diagnostic.code != "casework.runtime.invalid-context-provider")); + + for (member, refused, pointer, rule) in [ + ( + "resource", + json!("registry"), + "/contextProviders/registry-main/resource", + "absolute URI", + ), + ( + "baseUrl", + json!("http://registry.example.test"), + "/contextProviders/registry-main/baseUrl", + "HTTPS", + ), + ( + "delegatedDisplayExchange", + json!({ + "tokenEndpoint": "https://identity.example.test/token", + "clientIdRef": "secret:file/source-id", + "clientAssertionKeyRef": "secret:file/source-key", + "resource": "urn:example:registry", + "scopes": ["registry:proposal:review"], + "generation": "" + }), + "/contextProviders/registry-main/delegatedDisplayExchange/generation", + "generation", + ), + ] { + let mut binding = binding.clone(); + binding[member] = refused; + value["contextProviders"] = json!({ "registry-main": binding }); + let config = read_operator(&value, directory.path()); + let error = first(config.findings(None)).expect_err("the binding is refused"); + assert_eq!(error.code(), "casework.runtime.invalid-context-provider"); + assert_eq!(error.pointer(), pointer); + assert!(error.in_file(), "{error}"); + let message = error.to_string(); + assert!(message.contains(rule), "{message}"); + assert!(!message.contains("response"), "{message}"); + let path = write_operator(&value, directory.path()); + let check = check_runtime(&path, None, false); + let diagnostic = check + .diagnostics + .iter() + .find(|diagnostic| diagnostic.code == "casework.runtime.invalid-context-provider") + .expect("the offline check reports the refusal"); + assert_eq!(diagnostic.path, pointer); + assert!(diagnostic.message.contains(rule), "{diagnostic:?}"); + assert!(diagnostic.source.is_some(), "{diagnostic:?}"); + } + } + #[test] fn environment_expressions_substitute_values_but_never_secret_references() { let temp = @@ -2938,6 +3028,12 @@ pub enum RuntimeConfigError { InvalidAuditDestination(#[source] AuditDestinationError), #[error("contextProviders contains a deploymentRef casework.yaml does not declare")] InvalidContextProviders, + #[error("{path}: {reason}")] + InvalidContextProvider { + path: String, + pointer: String, + reason: String, + }, #[error( "{path} must name exactly one completion secret, in a header the runtime does not reserve" )] @@ -2997,6 +3093,7 @@ impl RuntimeConfigError { | Self::BindingSecret { .. } | Self::InvalidReviewCompletionAuth { .. } | Self::InvalidReviewCompletionDestination { .. } + | Self::InvalidContextProvider { .. } | Self::InvalidTaskAuthority { .. } | Self::InvalidDelegatedStaffReader { .. } | Self::MissingIdentity @@ -3020,6 +3117,7 @@ impl RuntimeConfigError { Self::BindingSecret { pointer, .. } | Self::InvalidReviewCompletionAuth { pointer, .. } | Self::InvalidReviewCompletionDestination { pointer, .. } + | Self::InvalidContextProvider { pointer, .. } | Self::InvalidTaskAuthority { pointer, .. } | Self::InvalidDelegatedStaffReader { pointer, .. } => pointer.clone(), Self::MissingIdentity => "/identity".to_owned(), @@ -3066,6 +3164,7 @@ impl RuntimeConfigError { Self::InvalidDatabaseReference => "casework.runtime.invalid-database-reference", Self::InvalidAuditDestination(_) => "casework.runtime.invalid-audit", Self::InvalidContextProviders => "casework.runtime.context-providers-mismatch", + Self::InvalidContextProvider { .. } => "casework.runtime.invalid-context-provider", Self::InvalidReviewCompletionAuth { .. } => { "casework.runtime.invalid-review-completion-auth" } @@ -3141,6 +3240,9 @@ impl RuntimeConfigError { Self::InvalidContextProviders => { "Remove the unrecognized contextProviders key, or declare its deploymentRef in casework.yaml. Omitted providers remain unavailable to their current-source reviews." } + Self::InvalidContextProvider { .. } => { + "Correct the context provider member so it meets the rule the message names." + } Self::InvalidReviewCompletionAuth { .. } => { "Configure exactly one of bearerTokenRef and auth.secretRef, and name in auth.header only a header the runtime does not reserve." } diff --git a/products/casework/CHANGELOG.md b/products/casework/CHANGELOG.md index 925bbb49a4..5625577e3b 100644 --- a/products/casework/CHANGELOG.md +++ b/products/casework/CHANGELOG.md @@ -11,6 +11,12 @@ with each access profile's `id` and `principalClaim`, so a caller no longer reads them from a review policy's `dependencies`. The member is additive. +- `caseworkctl check --runtime-config` refuses a `contextProviders` binding + that startup would refuse, such as a `resource` that is not an absolute URI + or a `baseUrl` that is not HTTPS, without resolving a secret. The diagnostic + `casework.runtime.invalid-context-provider` sits at the member and names the + rule it breaks. + ## 0.46.0 - 2026-11-08 - Each access profile may carry an optional `label` and `description`. Neither is matched against a caller, a profile that writes neither reads as before, and rewording them leaves a policy's behavioral binding unchanged. Casework keeps both in the review policy description, which any caller who may read that description, requesters included, can see. diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index 7fe6602e3b..57aeb68324 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -136,9 +136,10 @@ context provider and action. `rstack dev check` refuses a binding that names a review authority or action the BReg project does not declare, or a producer or context provider the Casework project does not declare; `start` refuses it before it provisions anything. `check` also writes each binding's product -configuration exactly as `start` does and lets BReg's review-authority reader -and Casework's context-binding reader refuse what they would refuse at startup, -reporting the refusal at the plan field that produced the value. It refuses a +configuration exactly as `start` does. BReg's review-authority reader and +`caseworkctl check --runtime-config` on the Casework configuration refuse what +each product would refuse at startup, and `check` reports the refusal, with the +rule it broke, at the plan field that produced the value. It refuses a BReg grant profile that is not the `service-status` profile the Casework context provider's policy names. A binding requests tokens for the other product's audience as an OAuth resource, so that audience must be an absolute diff --git a/products/platform/scripts/test-rstack-dev.py b/products/platform/scripts/test-rstack-dev.py index 75657f6072..1b28722af6 100644 --- a/products/platform/scripts/test-rstack-dev.py +++ b/products/platform/scripts/test-rstack-dev.py @@ -1200,6 +1200,24 @@ def verify_offline_plan(root, bin_dir, project): assert diagnostic["suggestedAction"] assert unknown not in diagnostic["message"] + diagnostic["suggestedAction"] assert not state.exists() + + # Casework's own check refuses the context provider binding the run would + # write, and the refusal lands on the plan field that produced it. + binding = root / "offline-context-binding" + binding.mkdir() + for product in ("registry", "casework"): + shutil.copytree(NATIVE_REVIEW / product, binding / product) + authored = (NATIVE_REVIEW / "dev.yaml").read_text() + audience = 'audience: "urn:journey:breg"' + assert authored.count(audience) == 1 + (binding / "dev.yaml").write_text(authored.replace(audience, "audience: registry")) + refused = dev(bin_dir, check_env, binding, "check", ".", expected=1) + [diagnostic] = refused["diagnostics"] + assert refused["status"] == "domain-refusal" + assert diagnostic["path"] == "/products/0/audience" + assert diagnostic["message"].startswith("Casework refuses"), diagnostic + assert "contextProviders.registry.resource: " in diagnostic["message"], diagnostic + assert not state.exists() print("Offline checks resolve product profiles with only the authoring tools") From f2e175ae7945455312cd4ae49f08ae5fe716ba0c Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 14:32:51 +0700 Subject: [PATCH 09/24] fix: name the rule a BReg review authority binding breaks `rstack dev check` reported a refused review authority binding as "runtime configuration contains an invalid deployment binding", so a developer saw the plan field but not the rule. `check_review_authority` now returns the rule each check states (the endpoint suffixes, the profile and producer ID grammar, the absolute resource and assertion audience URIs, and the shared OAuth scope limits), never the value. The runtime keeps its existing refusal. Signed-off-by: Jeremi Joslin --- crates/registry-breg/src/runtime_config.rs | 187 +++++++++++++----- .../src/project_dev/bindings.rs | 3 + 2 files changed, 139 insertions(+), 51 deletions(-) diff --git a/crates/registry-breg/src/runtime_config.rs b/crates/registry-breg/src/runtime_config.rs index 9972aed606..36528f46f8 100644 --- a/crates/registry-breg/src/runtime_config.rs +++ b/crates/registry-breg/src/runtime_config.rs @@ -1013,7 +1013,9 @@ impl RuntimeConfig { .review_authorities .into_iter() .map(|(id, authority)| { - ReviewAuthorityConfig::from_raw(&id, authority).map(|authority| (id, authority)) + ReviewAuthorityConfig::from_raw(&id, authority) + .map(|authority| (id, authority)) + .map_err(|_| RuntimeConfigError::InvalidBinding) }) .collect::>>()?; let operational_timeouts = OperationalTimeouts::from_raw(raw.operational_timeouts)?; @@ -2759,15 +2761,33 @@ pub(crate) fn parse_secret_reference( SecretReference::parse(value).map_err(|_| error) } +/// Why a `reviewAuthorities` entry is refused: the rule it breaks, stated +/// without repeating a value. +#[derive(Debug, Error, Clone, Copy, Eq, PartialEq)] +#[error("{0}")] +pub struct ReviewAuthorityRefusal(&'static str); + /// Checks one `reviewAuthorities` entry with the reader the runtime uses, so /// a caller that renders the entry can refuse it before the runtime starts. /// Secret references are checked for shape only; nothing is resolved. -pub fn check_review_authority(id: &str, authority: &serde_json::Value) -> Result<()> { - let raw = RawReviewAuthorityConfig::deserialize(authority) - .map_err(|_| RuntimeConfigError::InvalidBinding)?; +pub fn check_review_authority( + id: &str, + authority: &serde_json::Value, +) -> std::result::Result<(), ReviewAuthorityRefusal> { + let raw = RawReviewAuthorityConfig::deserialize(authority).map_err(|_| { + ReviewAuthorityRefusal( + "the review authority must hold only its documented members, each with a value of the documented type", + ) + })?; ReviewAuthorityConfig::from_raw(id, raw).map(|_| ()) } +/// The rule for a review authority ID and its producer ID: non-blank text of +/// at most 128 bytes without control characters. +fn bounded_identity_text(value: &str) -> bool { + !value.trim().is_empty() && value.len() <= 128 && !value.chars().any(char::is_control) +} + #[derive(Clone)] struct ReviewAuthorityConfig { endpoint: reqwest::Url, @@ -2780,12 +2800,17 @@ struct ReviewAuthorityConfig { } impl ReviewAuthorityConfig { - fn from_raw(id: &str, raw: RawReviewAuthorityConfig) -> Result { - if id.trim().is_empty() || id.len() > 128 || id.chars().any(char::is_control) { - return Err(RuntimeConfigError::InvalidBinding); + fn from_raw( + id: &str, + raw: RawReviewAuthorityConfig, + ) -> std::result::Result { + if !bounded_identity_text(id) { + return Err(ReviewAuthorityRefusal( + "the review authority ID must be non-blank text of at most 128 bytes without control characters", + )); } - let endpoint = - reqwest::Url::parse(&raw.endpoint).map_err(|_| RuntimeConfigError::InvalidBinding)?; + let endpoint = reqwest::Url::parse(&raw.endpoint) + .map_err(|_| ReviewAuthorityRefusal("endpoint must be an absolute URL"))?; // The review client performs the complete credential-destination // validation during activation. Refuse URL suffixes here as well so a // typo never survives configuration parsing. @@ -2794,7 +2819,9 @@ impl ReviewAuthorityConfig { || endpoint.query().is_some() || endpoint.fragment().is_some() { - return Err(RuntimeConfigError::InvalidBinding); + return Err(ReviewAuthorityRefusal( + "endpoint must not carry user information, a query or a fragment", + )); } let completion = match (raw.completion_token_ref, raw.completion_recipient) { (None, None) => None, @@ -2803,24 +2830,36 @@ impl ReviewAuthorityConfig { { Some((token_ref, recipient)) } - _ => return Err(RuntimeConfigError::InvalidBinding), + _ => { + return Err(ReviewAuthorityRefusal( + "completionTokenRef and completionRecipient must be written together, with a recipient of 1 to 256 visible ASCII characters", + )) + } }; let token_ref = raw.token_ref; let private_key_jwt = raw .private_key_jwt .map(ReviewPrivateKeyJwtConfig::from_raw) .transpose()?; - if token_ref.is_some() == private_key_jwt.is_some() - || raw.profile.trim().is_empty() + if token_ref.is_some() == private_key_jwt.is_some() { + return Err(ReviewAuthorityRefusal( + "exactly one of tokenRef and privateKeyJwt must be written", + )); + } + if raw.profile.trim().is_empty() || raw.profile.len() > 128 || !raw.profile.bytes().all(|byte| { byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b':') }) - || raw.producer_id.trim().is_empty() - || raw.producer_id.len() > 128 - || raw.producer_id.chars().any(char::is_control) { - return Err(RuntimeConfigError::InvalidBinding); + return Err(ReviewAuthorityRefusal( + "the profile must be 1 to 128 ASCII letters, digits, `-`, `_`, `.` or `:`", + )); + } + if !bounded_identity_text(&raw.producer_id) { + return Err(ReviewAuthorityRefusal( + "producerId must be non-blank text of at most 128 bytes without control characters", + )); } Ok(Self { endpoint, @@ -2883,25 +2922,28 @@ struct ReviewPrivateKeyJwtConfig { } impl ReviewPrivateKeyJwtConfig { - fn from_raw(raw: RawReviewPrivateKeyJwtConfig) -> Result { - let token_endpoint = raw - .token_endpoint - .parse() - .map_err(|_| RuntimeConfigError::InvalidBinding)?; - if !registry_platform_httputil::valid_resource_uri(&raw.assertion_audience) - || !registry_platform_httputil::valid_resource_uri(&raw.resource) - || raw.scopes.is_empty() - || raw.scopes.len() > registry_platform_httputil::MAXIMUM_REQUESTED_SCOPES - || raw.scopes.iter().collect::>().len() != raw.scopes.len() - || raw.scopes.join(" ").len() - > registry_platform_httputil::MAXIMUM_SCOPE_PARAMETER_BYTES - || raw.scopes.iter().any(|scope| { - scope.len() > registry_platform_httputil::MAXIMUM_REQUESTED_SCOPE_BYTES - || !registry_platform_httputil::valid_scope_token(scope) - }) - { - return Err(RuntimeConfigError::InvalidBinding); + fn from_raw( + raw: RawReviewPrivateKeyJwtConfig, + ) -> std::result::Result { + let token_endpoint = raw.token_endpoint.parse().map_err(|_| { + ReviewAuthorityRefusal("the privateKeyJwt token endpoint must be an absolute URL") + })?; + if !registry_platform_httputil::valid_resource_uri(&raw.assertion_audience) { + return Err(ReviewAuthorityRefusal( + "the privateKeyJwt assertion audience must be one absolute URI without a fragment or userinfo", + )); } + if !registry_platform_httputil::valid_resource_uri(&raw.resource) { + return Err(ReviewAuthorityRefusal( + "the privateKeyJwt resource must be one absolute URI without a fragment or userinfo", + )); + } + registry_platform_httputil::validate_requested_scopes(&raw.scopes).map_err(|error| { + ReviewAuthorityRefusal(match error { + registry_platform_httputil::TokenError::Configuration { reason } => reason, + _ => "the privateKeyJwt scopes are not a scope list the token client requests", + }) + })?; Ok(Self { token_endpoint, client_id_ref: raw.client_id_ref, @@ -4230,26 +4272,69 @@ mod tests { } #[test] - fn check_review_authority_applies_the_runtime_reader() { + fn check_review_authority_applies_the_runtime_reader_and_names_the_rule() { super::check_review_authority("casework-a", &review_authority_fragment()) .expect("a well-formed review authority is accepted"); - let mut bad_endpoint = review_authority_fragment(); - bad_endpoint["endpoint"] = "https://casework.example/reviews/?q=1".into(); - let mut bad_profile = review_authority_fragment(); - bad_profile["profile"] = "not a profile".into(); - let mut unknown_member = review_authority_fragment(); - unknown_member["surprise"] = true.into(); - for fragment in [bad_endpoint, bad_profile, unknown_member] { - assert!( - matches!( - super::check_review_authority("casework-a", &fragment), - Err(RuntimeConfigError::InvalidBinding) - ), - "{fragment}" - ); + let too_long = (0..17) + .map(|index| format!("{index:02}{}", "s".repeat(254))) + .collect::>(); + for (pointer, value, rule) in [ + ( + "/endpoint", + serde_json::json!("https://casework.example/reviews/?q=1"), + "query", + ), + ("/endpoint", serde_json::json!("not-a-url"), "absolute URL"), + ("/profile", serde_json::json!("not a profile"), "profile"), + ("/producerId", serde_json::json!(" "), "producerId"), + ("/surprise", serde_json::json!(true), "documented members"), + ( + "/tokenRef", + serde_json::json!("secret:file/review-token"), + "exactly one of tokenRef and privateKeyJwt", + ), + ( + "/completionRecipient", + serde_json::json!("registry"), + "completionTokenRef and completionRecipient", + ), + ( + "/privateKeyJwt/resource", + serde_json::json!("casework"), + "absolute URI", + ), + ( + "/privateKeyJwt/assertionAudience", + serde_json::json!("issuer"), + "assertion audience", + ), + ( + "/privateKeyJwt/scopes", + serde_json::json!(too_long), + "within 4096 bytes", + ), + ( + "/privateKeyJwt/scopes", + serde_json::json!(["has space"]), + "scope-tokens", + ), + ] { + let mut fragment = review_authority_fragment(); + let (parent, member) = pointer.rsplit_once('/').unwrap(); + fragment.pointer_mut(parent).unwrap()[member] = value.clone(); + let refusal = super::check_review_authority("casework-a", &fragment) + .expect_err(pointer) + .to_string(); + assert!(refusal.contains(rule), "{pointer}: {refusal}"); + if let Some(text) = value.as_str().filter(|text| text.trim().len() > 1) { + assert!(!refusal.contains(text), "{pointer}: {refusal}"); + } } - assert!(super::check_review_authority("", &review_authority_fragment()).is_err()); + let refusal = super::check_review_authority("", &review_authority_fragment()) + .unwrap_err() + .to_string(); + assert!(refusal.contains("review authority ID"), "{refusal}"); } #[test] diff --git a/crates/registry-bregctl/src/project_dev/bindings.rs b/crates/registry-bregctl/src/project_dev/bindings.rs index 01790f831e..29e00d81d6 100644 --- a/crates/registry-bregctl/src/project_dev/bindings.rs +++ b/crates/registry-bregctl/src/project_dev/bindings.rs @@ -992,6 +992,7 @@ mod tests { error.contains("/people/producer/grants/casework/scopes"), "{error}" ); + assert!(error.contains("4096"), "{error}"); // A person no binding names keeps the plan's own scope rule. let mut value = plan(); @@ -1014,6 +1015,7 @@ mod tests { value.issuer = "not a uri".to_owned(); let error = check_with_breg_reader(&value).unwrap_err().to_string(); assert!(error.contains("/issuer"), "{error}"); + assert!(error.contains("assertion audience"), "{error}"); // The profile grammar is a rule only the product readers state: the // runner restates nothing about it, so BReg's reader catches it and @@ -1032,6 +1034,7 @@ mod tests { error.contains("/people/producer/grants/casework/profile"), "{error}" ); + assert!(error.contains("the profile must be"), "{error}"); } #[test] From 8a0b4b38c7baf102147abd4167f52bd8c78e1a89 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 14:46:08 +0700 Subject: [PATCH 10/24] fix: class rstack dev timeouts as operational and malformed arguments as usage A run that misses a readiness or start deadline, or whose supervisor panics, failed in the environment, so the envelope now says operational (exit 3) instead of domain. An interrupted start stays domain: the operator asked for it. A malformed run ID, --ttl value or test person name is refused from the command line alone, before any project or run state is read, so it is usage (exit 2). start now checks --ttl before loading the project, so a bad value is reported as such rather than as a missing dev.yaml. A well-formed run ID that names no run is unchanged. Signed-off-by: Jeremi Joslin --- .../registry-bregctl/src/project_dev/mod.rs | 90 +++++++++++++++++-- .../src/project_dev/products.rs | 76 +++++++++++++--- crates/registry-bregctl/tests/project_dev.rs | 22 +++-- .../run-a-project-with-rstack-dev.mdx | 4 +- products/platform/PROJECT-DEV.md | 16 ++-- 5 files changed, 169 insertions(+), 39 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/mod.rs b/crates/registry-bregctl/src/project_dev/mod.rs index 4145fa9131..8a91c9d2d4 100644 --- a/crates/registry-bregctl/src/project_dev/mod.rs +++ b/crates/registry-bregctl/src/project_dev/mod.rs @@ -425,6 +425,11 @@ fn operational(error: anyhow::Error) -> anyhow::Error { Operational(format!("{error:#}")).into() } +/// A malformed argument, refused before any project or run state is read. +#[derive(Debug, thiserror::Error)] +#[error("{0}")] +struct Usage(&'static str); + /// The error envelope and exit code for a failed command. It is the launcher's /// `registry.rstack.error.v1` object, plus `run_id` and `recovery` when a run /// still has resources to release. @@ -433,6 +438,8 @@ fn error_report(error: &anyhow::Error) -> (Value, u8) { error.is::() || error.is::() || error.is::(); let (class, code) = if operational { ("operational", crate::OPERATIONAL_FAILURE_EXIT) + } else if error.is::() { + ("usage", crate::USAGE_EXIT) } else { ("domain", crate::DOMAIN_REFUSAL_EXIT) }; @@ -632,6 +639,7 @@ fn write_check_report( } fn start(project: &Path, ttl: Option<&str>) -> Result { + let ttl_seconds = parse_ttl(ttl.unwrap_or("30m"))?; let project = project .canonicalize() .context("cannot open the development project directory")?; @@ -639,7 +647,6 @@ fn start(project: &Path, ttl: Option<&str>) -> Result { let schema_people = products::schema_people(&project, &plan, true)?; let signals = signals()?; let inspection = products::inspect(&project, &plan, &signals)?; - let ttl_seconds = parse_ttl(ttl.unwrap_or("30m"))?; ensure!( env::var_os(DATABASE_URL).is_some_and(|value| !value.is_empty()), "supply --database-url or {DATABASE_URL} for a TLS PostgreSQL development server" @@ -746,13 +753,38 @@ fn start(project: &Path, ttl: Option<&str>) -> Result { } Err(mpsc::RecvTimeoutError::Timeout) => {} } - if signals.load(Ordering::Relaxed) || Instant::now() >= deadline { + if let Some(error) = start_abandoned(&signals, deadline) { stop(&input.run_id).map_err(|error| startup_error(&root, &input.run_id, error))?; - bail!("development run start was interrupted or timed out"); + return Err(error); } } } +/// Why `start` stops waiting for the supervisor's startup report, if it does. +/// An interrupt is the operator's request; a missed deadline means the host or +/// the supervisor failed. +fn start_abandoned(signals: &AtomicBool, deadline: Instant) -> Option { + if signals.load(Ordering::Relaxed) { + Some(anyhow::anyhow!("development run start was interrupted")) + } else if Instant::now() >= deadline { + Some(operational(anyhow::anyhow!( + "development run did not start within six minutes" + ))) + } else { + None + } +} + +/// Runs the supervisor's startup, turning a panic into an operational failure +/// that carries no panic payload. +fn catch_startup(startup: impl FnOnce() -> Result) -> Result { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(startup)).unwrap_or_else(|_| { + Err(operational(anyhow::anyhow!( + "development run startup failed internally" + ))) + }) +} + fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { let name = root .file_name() @@ -782,7 +814,7 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { flock(&lock, FlockOperation::LockExclusive)?; let mut database = None; let mut expiry = None; - let prepared = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let prepared = catch_startup(|| { products.check()?; let ca = env::var_os(CA_FILE) .map(|path| { @@ -814,8 +846,7 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { }); write_private(&root.join(REPORT), &capture_report(&report)?)?; Ok::<_, anyhow::Error>(report) - })) - .unwrap_or_else(|_| Err(anyhow::anyhow!("development run startup failed internally"))); + }); let mut failure = None; match prepared { @@ -944,7 +975,7 @@ fn pending_status(root: &Path, run_id: &str, pending: Value) -> Value { } fn token(run_id: &str, person: &str) -> Result { - ensure!(plan::identifier(person), "invalid test person name"); + ensure!(plan::identifier(person), Usage("invalid test person name")); let report = status(run_id)?; ensure!( report["status"] == "ready", @@ -1187,7 +1218,11 @@ fn binaries(product: Product) -> Result { } fn parse_ttl(value: &str) -> Result { - let error = || anyhow::anyhow!("--ttl must be between 1s and 24h, with an s, m, or h suffix"); + let error = || { + anyhow::Error::new(Usage( + "--ttl must be between 1s and 24h, with an s, m, or h suffix", + )) + }; let (number, unit) = value .split_at_checked(value.len().saturating_sub(1)) .ok_or_else(error)?; @@ -1218,7 +1253,7 @@ fn run_dir(run_id: &str) -> Result { && run_id .bytes() .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)), - "invalid run ID; use the ID printed by rstack dev start" + Usage("invalid run ID; use the ID printed by rstack dev start") ); Ok(state_dir().join(run_id)) } @@ -1523,6 +1558,15 @@ mod tests { assert_eq!(report["status"], "refused"); assert!(report.get("run_id").is_none()); + for malformed in [run_dir("../x").unwrap_err(), parse_ttl("9").unwrap_err()] { + let (report, code) = error_report(&malformed); + assert_eq!( + (report["error"]["class"].as_str(), code), + (Some("usage"), 2), + "{malformed:#}" + ); + } + let (report, code) = error_report(&anyhow::Error::new(Operational("stalled".into()))); assert_eq!( (report["error"]["class"].as_str(), code), @@ -1564,6 +1608,34 @@ mod tests { ); } + #[test] + fn an_interrupted_start_is_refused_and_a_late_or_panicked_start_is_operational() { + let class = |error: &anyhow::Error| error_report(error).0["error"]["class"].clone(); + let later = Instant::now() + Duration::from_secs(60); + assert!(start_abandoned(&AtomicBool::new(false), later).is_none()); + + let interrupted = start_abandoned(&AtomicBool::new(true), later).unwrap(); + assert_eq!(class(&interrupted), "domain"); + assert_eq!( + interrupted.to_string(), + "development run start was interrupted" + ); + + let late = start_abandoned(&AtomicBool::new(false), Instant::now()).unwrap(); + assert_eq!(class(&late), "operational"); + assert_eq!( + late.to_string(), + "development run did not start within six minutes" + ); + + let panicked = catch_startup(|| -> Result<()> { panic!("panic payload") }).unwrap_err(); + assert_eq!(class(&panicked), "operational"); + assert_eq!( + panicked.to_string(), + "development run startup failed internally" + ); + } + #[test] fn a_worker_failure_keeps_its_class_through_the_supervisor() { let class = |error: &anyhow::Error| error_report(error).0["error"]["class"].clone(); diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs index e3de50e860..e2d9062bf7 100644 --- a/crates/registry-bregctl/src/project_dev/products.rs +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -36,6 +36,9 @@ use super::{ /// directory and removed with it. pub(super) const FIELD_KEY: &str = "field-key"; +/// How long a started product has to answer `/ready`. +const READINESS_WAIT: Duration = Duration::from_secs(30); + /// The run's field-encryption key file, when the run wrote one, so recovery /// can name it for removal. pub(super) fn field_key_file(root: &Path) -> Option { @@ -165,7 +168,7 @@ impl<'a> ProductRunner<'a> { .map_err(operational)?; self.children.push((product, child)); self.record_processes()?; - self.ready(product, url)?; + self.ready(product, url, READINESS_WAIT)?; } Ok(urls) } @@ -476,8 +479,8 @@ impl<'a> ProductRunner<'a> { write_processes(self.root, &entries) } - fn ready(&mut self, product: Product, base: &str) -> Result<()> { - let end = Instant::now() + Duration::from_secs(30); + fn ready(&mut self, product: Product, base: &str, wait: Duration) -> Result<()> { + let end = Instant::now() + wait; let url = format!("{base}/ready"); loop { self.ensure_active()?; @@ -490,12 +493,13 @@ impl<'a> ProductRunner<'a> { self.ensure_alive(product)?; return Ok(()); } - ensure!( - Instant::now() < end, - "{} did not become ready{}", - product.name(), - self.runtime_log_tail(product) - ); + if Instant::now() >= end { + return Err(operational(anyhow::anyhow!( + "{} did not become ready{}", + product.name(), + self.runtime_log_tail(product) + ))); + } thread::sleep(POLL); } } @@ -536,10 +540,11 @@ impl<'a> ProductRunner<'a> { !stopping(self.root, self.terminate), "development run was stopped" ); - ensure!( - Instant::now() < self.deadline, - "development run did not start within five minutes" - ); + if Instant::now() >= self.deadline { + return Err(operational(anyhow::anyhow!( + "development run did not start within five minutes" + ))); + } Ok(()) } @@ -2520,6 +2525,51 @@ mod tests { assert_ne!(fs::read_to_string(other.join(FIELD_KEY)).unwrap(), encoded); } + #[test] + fn missed_readiness_and_start_deadlines_are_operational() { + let class = + |error: &anyhow::Error| super::super::error_report(error).0["error"]["class"].clone(); + let scratch = tempfile::tempdir().unwrap(); + let root = scratch.path().join("run"); + super::super::private_directory(&root).unwrap(); + let terminate = AtomicBool::new(false); + let mut runner = + ProductRunner::new(Arc::new(breg_input(false)), &root, &terminate).unwrap(); + let unused = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .unwrap() + .local_addr() + .unwrap() + .port(); + + let not_ready = runner + .ready( + Product::Breg, + &format!("http://127.0.0.1:{unused}"), + Duration::ZERO, + ) + .unwrap_err(); + assert_eq!(class(¬_ready), "operational", "{not_ready:#}"); + assert!( + not_ready + .to_string() + .starts_with("breg did not become ready"), + "{not_ready:#}" + ); + + runner.deadline = Instant::now(); + let late = runner.ensure_active().unwrap_err(); + assert_eq!(class(&late), "operational", "{late:#}"); + assert_eq!( + late.to_string(), + "development run did not start within five minutes" + ); + + // A stop requested during startup stays a refusal. + terminate.store(true, Ordering::Relaxed); + let stopped = runner.ensure_active().unwrap_err(); + assert_eq!(class(&stopped), "domain", "{stopped:#}"); + } + #[test] fn a_run_without_encrypted_fields_writes_no_key_and_no_provider() { let scratch = tempfile::tempdir().unwrap(); diff --git a/crates/registry-bregctl/tests/project_dev.rs b/crates/registry-bregctl/tests/project_dev.rs index 3233e15b88..b443f64db7 100644 --- a/crates/registry-bregctl/tests/project_dev.rs +++ b/crates/registry-bregctl/tests/project_dev.rs @@ -84,20 +84,20 @@ fn every_helper_failure_prints_the_error_envelope() { ( "start with a bad ttl", serde_json::json!({"action": "start", "project": scratch.path(), "ttl": "9", "json": true}), - "domain", - 1, + "usage", + 2, ), ( "status with a malformed run ID", serde_json::json!({"action": "status", "run_id": "../x", "json": true}), - "domain", - 1, + "usage", + 2, ), ( "token with a malformed person", serde_json::json!({"action": "token", "run_id": other_run, "person": "../x", "json": true}), - "domain", - 1, + "usage", + 2, ), ( "token for a run that is not ready", @@ -108,8 +108,14 @@ fn every_helper_failure_prints_the_error_envelope() { ( "stop with a malformed run ID", serde_json::json!({"action": "stop", "run_id": "../x", "json": true}), - "domain", - 1, + "usage", + 2, + ), + ( + "recover with a malformed run ID", + serde_json::json!({"action": "stop", "run_id": "../x", "json": true, "recover": true}), + "usage", + 2, ), ( "status of a run whose supervisor is lost", diff --git a/docs/site/src/content/docs/configure/run-a-project-with-rstack-dev.mdx b/docs/site/src/content/docs/configure/run-a-project-with-rstack-dev.mdx index fc940049b3..d753f4274d 100644 --- a/docs/site/src/content/docs/configure/run-a-project-with-rstack-dev.mdx +++ b/docs/site/src/content/docs/configure/run-a-project-with-rstack-dev.mdx @@ -227,8 +227,8 @@ process exit status tells you whose problem it is: | Exit | `error.class` | Meaning | | --- | --- | --- | | 1 | `domain` | The request, the project, the plan or the run's state was refused. Fix the plan or the command. | -| 2 | `usage` | The command line is wrong. | -| 3 | `operational` | The environment failed: PostgreSQL is unreachable or refuses the administrator, a product binary cannot start, no loopback port is free, or a run needs its resources released. | +| 2 | `usage` | The command line is wrong, such as a malformed run ID, `--ttl` or test person name. | +| 3 | `operational` | The environment failed: PostgreSQL is unreachable or refuses the administrator, a product binary cannot start, no loopback port is free, a product or the run does not start in time, or a run needs its resources released. | For example, `rstack dev start ./licensing --json` with the PostgreSQL server stopped (the operating system error number varies by platform): diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index 57aeb68324..057b2d6059 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -301,13 +301,15 @@ for failures before the helper starts, such as a missing component. - `schema_version` is `registry.rstack.error.v1` and `status` is `refused`. - `error.class` is `domain` (exit 1: the request, the project, the plan or the - run's state was refused), `usage` (exit 2: the command line is wrong) or - `operational` (exit 3: the environment failed, a cleanup stalled, or a run - needs its resources released). Environment failures include a PostgreSQL - server that is unreachable, refuses the administrator session, or cannot - provision a run's databases and roles, a product binary that cannot be - started, and no free loopback port. `error.message` is the human-readable - reason. + run's state was refused), `usage` (exit 2: the command line is wrong, such + as a malformed run ID, `--ttl` or test person name) or `operational` (exit 3: + the environment failed, a cleanup stalled, or a run needs its resources + released). Environment failures include a PostgreSQL server that is + unreachable, refuses the administrator session, or cannot provision a run's + databases and roles, a product binary that cannot be started, no free + loopback port, a product or run that does not start in time, and an internal + failure of the run's supervisor. An interrupted `start` is `domain`. + `error.message` is the human-readable reason. - `run_id` is present when the failure concerns a run that still has resources to release. Use it with `stop --recover`. - `recovery` accompanies `run_id` when the resources are known. It has `kind` From db33379dc22b2953b616c424afcde79092fdd6c6 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 14:48:29 +0700 Subject: [PATCH 11/24] fix: keep rstack dev key material and start times safe across interrupts and hosts remove_run_dir deletes the run's field-encryption key before it renames the run directory away, so a deletion that is interrupted leaves a .removing directory without key material for the operator to delete. ps reports start times in the host's time zone, and recovery compares them with the times a run recorded. Reading them with TZ=UTC beside LC_ALL=C keeps that comparison independent of the caller's environment. The run's key-writing refusal now names the key generator's reason, which carries no key material, instead of a fixed message. Signed-off-by: Jeremi Joslin --- .../registry-bregctl/src/field_encryption.rs | 6 +++- .../registry-bregctl/src/project_dev/mod.rs | 29 ++++++++++++++++- .../src/project_dev/products.rs | 17 +++++++--- .../src/project_dev/recovery.rs | 31 ++++++++++++++++--- products/platform/PROJECT-DEV.md | 3 +- 5 files changed, 73 insertions(+), 13 deletions(-) diff --git a/crates/registry-bregctl/src/field_encryption.rs b/crates/registry-bregctl/src/field_encryption.rs index 9e3e479eee..a62d03396d 100644 --- a/crates/registry-bregctl/src/field_encryption.rs +++ b/crates/registry-bregctl/src/field_encryption.rs @@ -29,15 +29,19 @@ pub(crate) struct KeygenOutcome { } /// Value-free generation refusals. -#[derive(Debug, Eq, PartialEq)] +#[derive(Debug, Eq, PartialEq, thiserror::Error)] pub(crate) enum KeygenError { /// A relative path cannot name a deliberate deployment secret location. + #[error("the key path is not absolute")] RelativeOutput, /// The output already exists; key material is never regenerated over it. + #[error("a key file already exists at that path")] OutputExists, /// The platform random source refused. + #[error("the platform random source refused")] RandomSource, /// The filesystem refused a parent directory or the key file itself. + #[error("the filesystem refused a parent directory or the key file")] Write, } diff --git a/crates/registry-bregctl/src/project_dev/mod.rs b/crates/registry-bregctl/src/project_dev/mod.rs index 8a91c9d2d4..f2e3796240 100644 --- a/crates/registry-bregctl/src/project_dev/mod.rs +++ b/crates/registry-bregctl/src/project_dev/mod.rs @@ -1369,9 +1369,17 @@ fn removing_dir(root: &Path) -> Result { } /// Renames the run directory away before deleting it, so `status` sees either -/// the whole run or no run, never a run whose report is already deleted. +/// the whole run or no run, never a run whose report is already deleted. The +/// run's field-encryption key goes first, so an interrupted deletion never +/// leaves key material behind. fn remove_run_dir(root: &Path) -> Result<()> { let doomed = removing_dir(root)?; + match fs::remove_file(root.join(products::FIELD_KEY)) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => { + return Err(error).context("cannot remove the run's field-encryption key"); + } + _ => {} + } fs::rename(root, &doomed).context("cannot remove run scratch files")?; fs::remove_dir_all(&doomed) .with_context(|| format!("cannot remove run scratch files at {}", doomed.display())) @@ -1726,6 +1734,25 @@ mod tests { assert_eq!(fs::read_dir(&base).unwrap().count(), 0); } + #[test] + fn a_run_directory_loses_its_field_key_before_it_is_renamed() { + let scratch = tempfile::tempdir().unwrap(); + let base = scratch.path().canonicalize().unwrap(); + let root = base.join(RUN_ID); + private_directory(&root).unwrap(); + fs::write(root.join(products::FIELD_KEY), "key").unwrap(); + fs::write(root.join(REPORT), "{}").unwrap(); + // A non-empty directory at the removal name makes the rename fail, as + // an interrupted deletion would leave the run behind. + let doomed = removing_dir(&root).unwrap(); + private_directory(&doomed).unwrap(); + fs::write(doomed.join("blocker"), "").unwrap(); + + assert!(remove_run_dir(&root).is_err()); + assert!(!root.join(products::FIELD_KEY).exists()); + assert!(root.join(REPORT).exists()); + } + #[test] fn failed_run_logs_move_out_of_the_run_directory() { let scratch = tempfile::tempdir().unwrap(); diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs index e2d9062bf7..506b7ae01b 100644 --- a/crates/registry-bregctl/src/project_dev/products.rs +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -262,12 +262,11 @@ impl<'a> ProductRunner<'a> { } crate::field_encryption::keygen(&self.root.join(FIELD_KEY)) .map(drop) - .map_err(|_| { - Operational( + .map_err(|error| { + Operational(format!( "cannot write the development run's field-encryption key into its run \ - directory" - .to_owned(), - ) + directory: {error}" + )) .into() }) } @@ -2492,6 +2491,14 @@ mod tests { runner.write_field_key().unwrap(); let key = root.join(FIELD_KEY); assert_eq!(field_key_file(&root), Some(key.clone())); + let refused = runner.write_field_key().unwrap_err(); + assert!(refused.is::(), "{refused:#}"); + assert!( + refused + .to_string() + .ends_with("run directory: a key file already exists at that path"), + "{refused:#}" + ); assert_eq!( fs::metadata(&key).unwrap().permissions().mode() & 0o777, 0o600 diff --git a/crates/registry-bregctl/src/project_dev/recovery.rs b/crates/registry-bregctl/src/project_dev/recovery.rs index 5c4e8744f8..2263b8b8f5 100644 --- a/crates/registry-bregctl/src/project_dev/recovery.rs +++ b/crates/registry-bregctl/src/project_dev/recovery.rs @@ -40,11 +40,7 @@ pub(super) enum Group { /// Every process with its group and start time. `ps` is the one portable /// source of start times on Linux and macOS. pub(super) fn process_table() -> Result> { - let output = Command::new("ps") - .args(["-A", "-o", "pid=,pgid=,lstart="]) - .env("LC_ALL", "C") - .stdin(Stdio::null()) - .stderr(Stdio::null()) + let output = ps_command() .output() .context("cannot run ps to read process start times")?; ensure!( @@ -54,6 +50,18 @@ pub(super) fn process_table() -> Result> { Ok(parse_table(&String::from_utf8_lossy(&output.stdout))) } +/// The `ps` call behind [`process_table`]. +fn ps_command() -> Command { + let mut command = Command::new("ps"); + command + .args(["-A", "-o", "pid=,pgid=,lstart="]) + .env("LC_ALL", "C") + .env("TZ", "UTC") + .stdin(Stdio::null()) + .stderr(Stdio::null()); + command +} + fn parse_table(text: &str) -> Vec { text.lines() .filter_map(|line| { @@ -187,6 +195,19 @@ mod tests { use serde_json::json; use std::os::unix::process::CommandExt; + #[test] + fn start_times_are_read_in_one_locale_and_time_zone() { + use std::ffi::OsStr; + let command = ps_command(); + let envs: Vec<_> = command.get_envs().collect(); + for (name, value) in [("LC_ALL", "C"), ("TZ", "UTC")] { + assert!( + envs.contains(&(OsStr::new(name), Some(OsStr::new(value)))), + "{name}: {envs:?}" + ); + } + } + fn row(pid: u32, pgid: u32, started: &str) -> Row { Row { pid, diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index 057b2d6059..b5add6e03b 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -240,7 +240,8 @@ The state directory holds one directory per run. It defaults to `rstack-dev-UID` in the system temporary directory. Set `RSTACK_DEV_STATE_DIR` to use another location. A run directory is renamed to `..removing` before it is deleted, and `status` and `stop` ignore that name. If the deletion -is interrupted, delete that directory yourself. +is interrupted, delete that directory yourself; the run's field-encryption key +is deleted before the rename, so it holds no key material. ## When a run fails From 43b9c942bf5ab3a39d58b7ac60c97c8b6bf81b52 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 14:53:34 +0700 Subject: [PATCH 12/24] docs: record rstack dev recovery, error envelope and run keys in the changelogs The platform changelog names what rstack dev gained before its first release: stop --recover, the registry.rstack.error.v1 envelope with its three classes and exit codes, and a per-run key for projects with encrypted fields. The BReg entry for the long-envelope fix is rewrapped and names the history journal capture of reviewed migrations, which reads through the same decoder. The field-encryption flip test now seals values whose envelopes PostgreSQL wraps, so that path is covered. The PostGIS provisioning test also checks that the bbox role batch keeps the SQLSTATE of a refused statement. Signed-off-by: Jeremi Joslin --- .../registry-breg/tests/postgres_migration.rs | 5 +++- .../tests/postgres_spatial_migration.rs | 23 +++++++++++++++---- products/breg/CHANGELOG.md | 7 ++++-- products/platform/CHANGELOG.md | 12 ++++++++++ 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/crates/registry-breg/tests/postgres_migration.rs b/crates/registry-breg/tests/postgres_migration.rs index 841b3c2527..40b6bbfe0c 100644 --- a/crates/registry-breg/tests/postgres_migration.rs +++ b/crates/registry-breg/tests/postgres_migration.rs @@ -4481,8 +4481,11 @@ async fn real_postgres_field_encryption_flip_seals_resumes_and_drops_the_plainte let active = apply(&database, &initial, ApplyPrecondition::InitialActivation) .await .expect("flip scenario initial package activates"); + // Each value seals to an envelope longer than 57 bytes, whose base64 + // PostgreSQL breaks across lines, so the journal capture of every sealed + // chunk decodes a wrapped projection. let secrets = (0..5) - .map(|index| format!("flip-secret-{index}")) + .map(|index| format!("flip-secret-{index}-{}", "long-envelope-".repeat(6))) .collect::>(); seed_flip_rows(&database, &prior, &secrets, true).await; diff --git a/crates/registry-breg/tests/postgres_spatial_migration.rs b/crates/registry-breg/tests/postgres_spatial_migration.rs index 58fbde782e..3ebadb21bd 100644 --- a/crates/registry-breg/tests/postgres_spatial_migration.rs +++ b/crates/registry-breg/tests/postgres_spatial_migration.rs @@ -29,9 +29,9 @@ use registry_breg::package::{ }; use registry_breg::postgres::{ begin_record_transaction, install_compiled_schema, managed_schema_fingerprint, - provision_postgis_prerequisites, verify_catalog_identity_for_catalog, verify_postgis, - ClaimContext, ExpectedManagedCatalog, ExpectedRegistryIdentity, PostgresFailure, - PostgresKernelError, RegistryLockKey, SqlIdentifier, + provision_postgis_prerequisites, provision_spatial_bbox_role, + verify_catalog_identity_for_catalog, verify_postgis, ClaimContext, ExpectedManagedCatalog, + ExpectedRegistryIdentity, PostgresFailure, PostgresKernelError, RegistryLockKey, SqlIdentifier, }; use registry_breg::startup::{prepare_startup, StartupError}; use registry_breg::CompiledRegistry; @@ -2693,15 +2693,28 @@ async fn postgis_provisioning_keeps_the_sqlstate_of_a_refused_statement() { ) .await .expect_err("a role without schema creation rights cannot provision PostGIS"); + assert_insufficient_privilege(&error); + // Nor may it create roles, so the bbox role batch is refused too. + let error = provision_spatial_bbox_role( + &unprivileged, + &database.migration_role, + &database.runtime_role, + ) + .await + .expect_err("a role without role creation rights cannot provision the bbox role"); task.abort(); - match &error { + assert_insufficient_privilege(&error); + database.cleanup().await; +} + +fn assert_insufficient_privilege(error: &PostgresKernelError) { + match error { PostgresKernelError::Statement(failure) => { assert_eq!(failure.sqlstate.as_deref(), Some("42501")); } other => panic!("expected a statement failure, got {other:?}"), } assert!(error.to_string().contains("42501"), "{error}"); - database.cleanup().await; } async fn cleanup_postgis_prerequisites(database: &TestDatabase, bbox_role: &SqlIdentifier) { diff --git a/products/breg/CHANGELOG.md b/products/breg/CHANGELOG.md index c2ed22df75..9dea79e345 100644 --- a/products/breg/CHANGELOG.md +++ b/products/breg/CHANGELOG.md @@ -25,8 +25,11 @@ field encryption failed a record, list, snapshot or revision read with `503 source.unavailable` once a stored value was long enough for its encrypted envelope to exceed 57 bytes, and the value could not be read - until this fix. Writes and shorter values were unaffected. The fault was in the read path since v0.33.0, through v0.46.0; no - stored data changes and nothing needs migrating. + until this fix. A reviewed migration whose history journal capture read + such a value, such as a field-encryption flip that sealed one, failed its + apply, because the capture uses the same decoder. Writes and shorter values + were unaffected. The fault was in the read path since v0.33.0, through + v0.46.0; no stored data changes and nothing needs migrating. ## 0.46.0 - 2026-11-08 diff --git a/products/platform/CHANGELOG.md b/products/platform/CHANGELOG.md index ac4a9e0fc6..8704834d94 100644 --- a/products/platform/CHANGELOG.md +++ b/products/platform/CHANGELOG.md @@ -6,6 +6,18 @@ projects together against a PostgreSQL server you supply. See [PROJECT-DEV.md](PROJECT-DEV.md). It is an alpha developer tool and may change. +- `rstack dev stop RUN_ID --recover` releases a run whose supervisor is gone. + It stops each recorded process group whose leader still has the start time + recorded for it, drops the run's databases and roles, and removes the run + directory. Running it again after a partial failure is safe. +- With `--json`, every `rstack dev` refusal or failure prints one + `registry.rstack.error.v1` object, published as + `schemas/rstack-dev-error.v1.schema.json`. Its `error.class` is `domain` + (exit 1), `usage` (exit 2) or `operational` (exit 3); a run that still holds + resources adds its `run_id` and a `recovery` object. +- A BReg project with encrypted fields runs under `rstack dev` without extra + setup. Each run draws its own development data key into its run directory, + readable only by your user, and the key is removed with the run. ## 0.46.0 - 2026-11-08 From aec336374b1984787e73cae2e0fd1df61c67da37 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 15:26:22 +0700 Subject: [PATCH 13/24] fix: class an unavailable Casework inspection as operational and keep its cause Signed-off-by: Jeremi Joslin --- .../registry-bregctl/src/project_dev/mod.rs | 6 +- .../src/project_dev/products.rs | 67 ++++++++++++++----- crates/registry-bregctl/tests/project_dev.rs | 4 +- products/casework/CHANGELOG.md | 5 +- products/platform/CHANGELOG.md | 3 +- 5 files changed, 62 insertions(+), 23 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/mod.rs b/crates/registry-bregctl/src/project_dev/mod.rs index f2e3796240..67461654a8 100644 --- a/crates/registry-bregctl/src/project_dev/mod.rs +++ b/crates/registry-bregctl/src/project_dev/mod.rs @@ -434,8 +434,10 @@ struct Usage(&'static str); /// `registry.rstack.error.v1` object, plus `run_id` and `recovery` when a run /// still has resources to release. fn error_report(error: &anyhow::Error) -> (Value, u8) { - let operational = - error.is::() || error.is::() || error.is::(); + let operational = error.is::() + || error.is::() + || error.is::() + || error.is::(); let (class, code) = if operational { ("operational", crate::OPERATIONAL_FAILURE_EXIT) } else if error.is::() { diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs index 506b7ae01b..35ab958a9c 100644 --- a/crates/registry-bregctl/src/project_dev/products.rs +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -830,7 +830,20 @@ impl InspectionUnavailable { remedy: remedy.to_owned(), } } +} +/// An inspection that could not complete, keeping the cause in the reason. The +/// cause comes from spawning the tool, its timeout, temporary files and the +/// report parse; none of those messages carries a configured value. +fn inspection_failed(field: &str, action: &str, error: &anyhow::Error) -> InspectionUnavailable { + InspectionUnavailable::new( + field, + &format!("The selected `caseworkctl` helper could not {action}: {error:#}."), + "Install the matching Casework tools or provide RSTACK_CASEWORKCTL_BIN, then run the check again.", + ) +} + +impl InspectionUnavailable { pub(super) fn field(&self) -> &str { &self.field } @@ -1075,12 +1088,10 @@ fn casework_principals( "Run `rstack casework check` for authored diagnostics, then correct the Casework project.", ).into()); } - Err(_) => { - return Err(InspectionUnavailable::new( - &project.field, - "The selected `caseworkctl` helper could not inspect the Casework project.", - "Install the matching Casework tools or provide RSTACK_CASEWORKCTL_BIN, then run the check again.", - ).into()); + Err(error) => { + return Err( + inspection_failed(&project.field, "inspect the Casework project", &error).into(), + ); } }; let status_profile = plan @@ -1238,13 +1249,7 @@ fn check_casework_binding( binding: &super::bindings::CaseworkContextProvider, terminate: &AtomicBool, ) -> Result<()> { - let unavailable = || { - InspectionUnavailable::new( - &project.field, - "The selected `caseworkctl` helper could not check the Casework configuration the run writes.", - "Install the matching Casework tools or provide RSTACK_CASEWORKCTL_BIN, then run the check again.", - ) - }; + let action = "check the Casework configuration the run writes"; let checked = (|| { let directory = tempfile::tempdir()?; // The runtime loader refuses a file reached through a symbolic link, @@ -1272,8 +1277,9 @@ fn check_casework_binding( terminate, ) })(); - let Ok((status, report)) = checked else { - return Err(unavailable().into()); + let (status, report) = match checked { + Ok(checked) => checked, + Err(error) => return Err(inspection_failed(&project.field, action, &error).into()), }; match casework_binding_check(status.code(), &report, &binding.id) { BindingCheck::Accepted => Ok(()), @@ -1286,7 +1292,12 @@ fn check_casework_binding( "Correct the value so the product accepts the configuration the run writes for this binding.", ) .into()), - BindingCheck::Unchecked => Err(unavailable().into()), + BindingCheck::Unchecked => Err(inspection_failed( + &project.field, + action, + &anyhow::anyhow!("caseworkctl check did not report a result for the binding"), + ) + .into()), } } @@ -2532,6 +2543,30 @@ mod tests { assert_ne!(fs::read_to_string(other.join(FIELD_KEY)).unwrap(), encoded); } + #[test] + fn an_unavailable_inspection_is_operational_and_names_its_cause() { + let class = + |error: &anyhow::Error| super::super::error_report(error).0["error"]["class"].clone(); + let unavailable = anyhow::Error::new(InspectionUnavailable::new( + "products[0]", + "The Casework project cannot be inspected.", + "Install the tools.", + )); + assert_eq!(class(&unavailable), "operational"); + + let terminate = AtomicBool::new(false); + let missing = Path::new("/nonexistent/caseworkctl"); + let error = explain_casework_facts(missing, Path::new("."), &terminate).unwrap_err(); + let reason = inspection_failed("products[0]", "inspect the Casework project", &error); + assert!( + reason.reason().contains( + "could not inspect the Casework project: cannot start caseworkctl explain" + ), + "{}", + reason.reason() + ); + } + #[test] fn missed_readiness_and_start_deadlines_are_operational() { let class = diff --git a/crates/registry-bregctl/tests/project_dev.rs b/crates/registry-bregctl/tests/project_dev.rs index b443f64db7..5de25de100 100644 --- a/crates/registry-bregctl/tests/project_dev.rs +++ b/crates/registry-bregctl/tests/project_dev.rs @@ -1217,7 +1217,7 @@ fragment or userinfo." assert_eq!(diagnostic["path"], "/products/1/project"); assert_eq!( diagnostic["message"], - "The selected `caseworkctl` helper could not check the Casework configuration the run writes." + "The selected `caseworkctl` helper could not check the Casework configuration the run writes: caseworkctl check did not report a result for the binding." ); assert!(!state.exists()); } @@ -1309,7 +1309,7 @@ fn check_distinguishes_an_unavailable_casework_helper_from_an_invalid_project() assert_eq!(diagnostic["path"], "/products/1/project"); assert_eq!( diagnostic["message"], - "The selected `caseworkctl` helper could not inspect the Casework project." + "The selected `caseworkctl` helper could not inspect the Casework project: caseworkctl explain did not complete successfully." ); assert!(!state.exists()); diff --git a/products/casework/CHANGELOG.md b/products/casework/CHANGELOG.md index 5625577e3b..46216c9b87 100644 --- a/products/casework/CHANGELOG.md +++ b/products/casework/CHANGELOG.md @@ -11,8 +11,9 @@ with each access profile's `id` and `principalClaim`, so a caller no longer reads them from a review policy's `dependencies`. The member is additive. -- `caseworkctl check --runtime-config` refuses a `contextProviders` binding - that startup would refuse, such as a `resource` that is not an absolute URI +- `caseworkctl check --runtime-config`, and every other command that loads the + runtime configuration, refuses a `contextProviders` binding that startup + would refuse, such as a `resource` that is not an absolute URI or a `baseUrl` that is not HTTPS, without resolving a secret. The diagnostic `casework.runtime.invalid-context-provider` sits at the member and names the rule it breaks. diff --git a/products/platform/CHANGELOG.md b/products/platform/CHANGELOG.md index 8704834d94..880125b763 100644 --- a/products/platform/CHANGELOG.md +++ b/products/platform/CHANGELOG.md @@ -14,7 +14,8 @@ `registry.rstack.error.v1` object, published as `schemas/rstack-dev-error.v1.schema.json`. Its `error.class` is `domain` (exit 1), `usage` (exit 2) or `operational` (exit 3); a run that still holds - resources adds its `run_id` and a `recovery` object. + resources adds its `run_id`, with a `recovery` object when the resources to + release are known. - A BReg project with encrypted fields runs under `rstack dev` without extra setup. Each run draws its own development data key into its run directory, readable only by your user, and the key is removed with the run. From 06e364aa893765924612ec002c68c16b3f6d45ae Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 15:29:09 +0700 Subject: [PATCH 14/24] fix: keep an interrupted Casework inspection as the operator's request An interrupt during the Casework inspection in rstack dev start or check became an unavailable inspection, so it reported the operational class. An interrupt is the operator's request and reports as domain, as an interrupted start does. Only other inspection failures become InspectionUnavailable, with their cause in the reason. Signed-off-by: Jeremi Joslin --- .../src/project_dev/products.rs | 89 ++++++++++++++----- 1 file changed, 65 insertions(+), 24 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs index 35ab958a9c..a44b25fce0 100644 --- a/crates/registry-bregctl/src/project_dev/products.rs +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -830,20 +830,7 @@ impl InspectionUnavailable { remedy: remedy.to_owned(), } } -} -/// An inspection that could not complete, keeping the cause in the reason. The -/// cause comes from spawning the tool, its timeout, temporary files and the -/// report parse; none of those messages carries a configured value. -fn inspection_failed(field: &str, action: &str, error: &anyhow::Error) -> InspectionUnavailable { - InspectionUnavailable::new( - field, - &format!("The selected `caseworkctl` helper could not {action}: {error:#}."), - "Install the matching Casework tools or provide RSTACK_CASEWORKCTL_BIN, then run the check again.", - ) -} - -impl InspectionUnavailable { pub(super) fn field(&self) -> &str { &self.field } @@ -857,6 +844,28 @@ impl InspectionUnavailable { } } +/// The error for an inspection that could not complete. An interrupt is the +/// operator's request and stays as it is. Any other failure is an unavailable +/// inspection whose reason keeps the cause; the cause comes from spawning the +/// tool, its timeout, temporary files and the report parse, and none of those +/// messages carries a configured value. +fn inspection_failed( + field: &str, + action: &str, + error: anyhow::Error, + terminate: &AtomicBool, +) -> anyhow::Error { + if terminate.load(Ordering::Relaxed) { + return error; + } + InspectionUnavailable::new( + field, + &format!("The selected `caseworkctl` helper could not {action}: {error:#}."), + "Install the matching Casework tools or provide RSTACK_CASEWORKCTL_BIN, then run the check again.", + ) + .into() +} + struct SelectedProduct { field: String, root: PathBuf, @@ -1089,9 +1098,12 @@ fn casework_principals( ).into()); } Err(error) => { - return Err( - inspection_failed(&project.field, "inspect the Casework project", &error).into(), - ); + return Err(inspection_failed( + &project.field, + "inspect the Casework project", + error, + terminate, + )); } }; let status_profile = plan @@ -1279,7 +1291,7 @@ fn check_casework_binding( })(); let (status, report) = match checked { Ok(checked) => checked, - Err(error) => return Err(inspection_failed(&project.field, action, &error).into()), + Err(error) => return Err(inspection_failed(&project.field, action, error, terminate)), }; match casework_binding_check(status.code(), &report, &binding.id) { BindingCheck::Accepted => Ok(()), @@ -1295,9 +1307,9 @@ fn check_casework_binding( BindingCheck::Unchecked => Err(inspection_failed( &project.field, action, - &anyhow::anyhow!("caseworkctl check did not report a result for the binding"), - ) - .into()), + anyhow::anyhow!("caseworkctl check did not report a result for the binding"), + terminate, + )), } } @@ -2557,13 +2569,42 @@ mod tests { let terminate = AtomicBool::new(false); let missing = Path::new("/nonexistent/caseworkctl"); let error = explain_casework_facts(missing, Path::new("."), &terminate).unwrap_err(); - let reason = inspection_failed("products[0]", "inspect the Casework project", &error); + let failed = inspection_failed( + "products[0]", + "inspect the Casework project", + error, + &terminate, + ); + assert_eq!(class(&failed), "operational"); + let reason = failed + .downcast_ref::() + .expect("an unavailable inspection") + .reason(); assert!( - reason.reason().contains( + reason.contains( "could not inspect the Casework project: cannot start caseworkctl explain" ), - "{}", - reason.reason() + "{reason}" + ); + } + + #[test] + fn an_interrupted_inspection_stays_the_operators_request() { + let class = + |error: &anyhow::Error| super::super::error_report(error).0["error"]["class"].clone(); + let terminate = AtomicBool::new(true); + let interrupted = anyhow::anyhow!("development product inspection was interrupted"); + let failed = inspection_failed( + "products[0]", + "inspect the Casework project", + interrupted, + &terminate, + ); + assert!(!failed.is::()); + assert_eq!(class(&failed), "domain"); + assert_eq!( + failed.to_string(), + "development product inspection was interrupted" ); } From 503136e35b63dacdfdf33b00a2a6f16ca63f2078 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 16:46:20 +0700 Subject: [PATCH 15/24] chore(deps): bump rustix to 1.1.6 for RUSTSEC-2026-0486 cargo deny refuses rustix 1.1.5: its xattr functions mark unwritten bytes as initialized when given a zero-length buffer. Only the rustix entry and its references change. Signed-off-by: Jeremi Joslin --- Cargo.lock | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1020ceb27d..4a21ca40b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1656,7 +1656,7 @@ dependencies = [ "document-features", "mio", "parking_lot", - "rustix 1.1.5", + "rustix 1.1.6", "signal-hook 0.3.18", "signal-hook-mio", "winapi", @@ -4166,7 +4166,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57804b2c9b69967f1536a56f86297e367a33b19e98852ed624b84551cdbc0d90" dependencies = [ - "rustix 1.1.5", + "rustix 1.1.6", ] [[package]] @@ -6011,7 +6011,7 @@ dependencies = [ "registry-platform-sqlite", "registry-platform-yaml", "rhai", - "rustix 1.1.5", + "rustix 1.1.6", "serde", "serde_json", "serde_norway", @@ -6144,7 +6144,7 @@ dependencies = [ "registry-review-protocol", "reqwest 0.12.28", "rhai", - "rustix 1.1.5", + "rustix 1.1.6", "rustls", "schemars", "serde", @@ -6347,7 +6347,7 @@ dependencies = [ "registry-platform-yaml", "registry-thunderid-tooling", "reqwest 0.12.28", - "rustix 1.1.5", + "rustix 1.1.6", "rustls", "schemars", "serde", @@ -6557,7 +6557,7 @@ dependencies = [ "registry-platform-yaml", "registry-thunderid-tooling", "reqwest 0.12.28", - "rustix 1.1.5", + "rustix 1.1.6", "schemars", "serde", "serde_json", @@ -6583,7 +6583,7 @@ dependencies = [ "registry-platform-httputil", "registry-platform-yaml", "reqwest 0.12.28", - "rustix 1.1.5", + "rustix 1.1.6", "schemars", "serde", "serde_json", @@ -6686,7 +6686,7 @@ dependencies = [ "registry-scheduling-core", "reqwest 0.12.28", "rhai", - "rustix 1.1.5", + "rustix 1.1.6", "rustls", "schemars", "serde", @@ -7035,7 +7035,7 @@ dependencies = [ "reqwest 0.12.28", "rhai", "rusqlite", - "rustix 1.1.5", + "rustix 1.1.6", "schemars", "serde", "serde_json", @@ -7320,7 +7320,7 @@ dependencies = [ "registry-platform-yaml", "registry-thunderid-tooling", "rhai", - "rustix 1.1.5", + "rustix 1.1.6", "schemars", "serde", "serde_json", @@ -7344,7 +7344,7 @@ dependencies = [ "futures", "registry-evidence-authoring", "registry-platform-yaml", - "rustix 1.1.5", + "rustix 1.1.6", "serde_json", "serde_norway", "tempfile", @@ -7614,7 +7614,7 @@ dependencies = [ "registry-platform-yaml", "registry-thunderid-tooling", "reqwest 0.12.28", - "rustix 1.1.5", + "rustix 1.1.6", "serde", "serde_json", "serde_norway", @@ -7649,7 +7649,7 @@ version = "0.46.1" dependencies = [ "hmac 0.13.0", "regex", - "rustix 1.1.5", + "rustix 1.1.6", "schemars", "serde", "serde_json", @@ -7710,7 +7710,7 @@ version = "0.46.1" dependencies = [ "registry-platform-canonical-json", "registry-platform-yaml", - "rustix 1.1.5", + "rustix 1.1.6", "schemars", "serde", "serde_json", @@ -7947,7 +7947,7 @@ name = "registry-platform-sqlite" version = "0.46.1" dependencies = [ "rusqlite", - "rustix 1.1.5", + "rustix 1.1.6", "serde", "sha2 0.11.1", "tempfile", @@ -8066,7 +8066,7 @@ dependencies = [ "registry-platform-yaml", "registry-render-client", "reqwest 0.12.28", - "rustix 1.1.5", + "rustix 1.1.6", "schemars", "serde", "serde_json", @@ -8177,7 +8177,7 @@ dependencies = [ "registry-platform-config", "registry-platform-yaml", "registry-thunderid-tooling", - "rustix 1.1.5", + "rustix 1.1.6", "serde", "serde_json", "sha2 0.11.1", @@ -8358,7 +8358,7 @@ dependencies = [ "registry-platform-oidc", "registry-platform-yaml", "reqwest 0.12.28", - "rustix 1.1.5", + "rustix 1.1.6", "schemars", "serde", "serde_json", @@ -8796,9 +8796,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" dependencies = [ "bitflags", "errno", @@ -9778,7 +9778,7 @@ dependencies = [ "fastrand", "getrandom 0.4.3", "once_cell", - "rustix 1.1.5", + "rustix 1.1.6", "windows-sys 0.61.2", ] @@ -11333,7 +11333,7 @@ dependencies = [ "postcard", "pulley-interpreter", "rayon", - "rustix 1.1.5", + "rustix 1.1.6", "serde", "serde_derive", "smallvec", @@ -11432,7 +11432,7 @@ checksum = "96dce7a8ea302c0f1eb4154eb1756525c90c595bbabbc590abad64d4c5ff8bbb" dependencies = [ "cc", "libc", - "rustix 1.1.5", + "rustix 1.1.6", "wasmtime-environ", "wasmtime-internal-versioned-export-macros", "windows-sys 0.61.2", From 447bdbccae7985dc0b17e4027229deb06437e83e Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 16:46:21 +0700 Subject: [PATCH 16/24] fix: register the rstack error report schema as out of configuration scope The configuration conventions check refuses an unregistered schema under products/*/schemas. The rstack error report is a command output, not a configuration format. Signed-off-by: Jeremi Joslin --- products/platform/config-formats.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/products/platform/config-formats.yaml b/products/platform/config-formats.yaml index f64424772b..85daf0a045 100644 --- a/products/platform/config-formats.yaml +++ b/products/platform/config-formats.yaml @@ -4615,6 +4615,8 @@ outOfScope: reason: Coordinator operation input and output contracts, wire payloads governed by each product's API contract. - files: products/breg/contracts/event-record-view/EventRecordView.schema.json reason: Event record view, an inert client projection output governed by the client contract rather than a configuration format. + - files: products/platform/schemas/rstack-dev-error.v1.schema.json + reason: rstack error report printed on stdout with --json, a command output governed by PROJECT-DEV.md rather than a configuration format. - files: products/discovery/profile/schema/*.schema.json reason: Discovery provider description, a published wire resource. - files: products/identifiers/contracts/*.schema.json From 2bb110f41a5337c01fb6beed76416e86d9129474 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 20:17:05 +0700 Subject: [PATCH 17/24] fix: guard rstack dev workloads and bind database recovery to startup Signed-off-by: Jeremi Joslin --- crates/registry-bregctl/src/lib.rs | 3 + .../src/project_dev/database.rs | 149 +++- .../registry-bregctl/src/project_dev/guard.rs | 643 ++++++++++++++++++ .../registry-bregctl/src/project_dev/mod.rs | 120 ++-- .../src/project_dev/products.rs | 149 ++-- .../src/project_dev/recovery.rs | 263 ++----- products/platform/scripts/test-rstack-dev.py | 284 ++++++-- 7 files changed, 1240 insertions(+), 371 deletions(-) create mode 100644 crates/registry-bregctl/src/project_dev/guard.rs diff --git a/crates/registry-bregctl/src/lib.rs b/crates/registry-bregctl/src/lib.rs index 0ab5ac63f3..46af51d1d6 100644 --- a/crates/registry-bregctl/src/lib.rs +++ b/crates/registry-bregctl/src/lib.rs @@ -228,6 +228,8 @@ enum Command { DevSupervisor(dev::SupervisorArgs), #[command(name = "project-dev", hide = true)] ProjectDev(project_dev::Args), + #[command(name = "project-dev-guard", hide = true)] + ProjectDevGuard(project_dev::guard::Args), /// Explain compiled model, access, route, or event inventories. Explain(ExplainArgs), /// Compare an authoring candidate with a rederived closed package. @@ -2908,6 +2910,7 @@ where Command::ProjectDev(args) => { return ExitCode::from(project_dev::run(args, stdout, stderr)); } + Command::ProjectDevGuard(args) => return project_dev::guard::run(args), Command::Init(args) => match (args.from, args.template.as_deref()) { (None, None) => init(&args.destination), (None, Some(id)) => starters::run(&args.destination, id), diff --git a/crates/registry-bregctl/src/project_dev/database.rs b/crates/registry-bregctl/src/project_dev/database.rs index 5cf5ffab74..bfd0223f25 100644 --- a/crates/registry-bregctl/src/project_dev/database.rs +++ b/crates/registry-bregctl/src/project_dev/database.rs @@ -9,6 +9,7 @@ use std::{ use anyhow::{anyhow, bail, Context, Result}; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; use rustls::pki_types::{pem::PemObject as _, CertificateDer}; +use serde::{Deserialize, Serialize}; use tokio::runtime::{Builder, Runtime}; use tokio_postgres::{ config::{Host, SslMode}, @@ -30,6 +31,9 @@ const STATEMENT_TIMEOUT: Duration = Duration::from_secs(10); const OPERATION_TIMEOUT: Duration = Duration::from_secs(15); const CLEANUP_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(30); const KEEPALIVE_IDLE: Duration = Duration::from_secs(30); +const RECOVERY_IDENTITY_QUERY: &str = "SELECT system_identifier, \ + pg_catalog.current_database()::text \ + FROM pg_catalog.pg_control_system()"; /// Lets a role holding CREATEROLE use the roles it creates. PostgreSQL 16 and /// later grant such a creator only ADMIN on them, which is not enough for /// `CREATE SCHEMA ... AUTHORIZATION` or `CREATE DATABASE ... OWNER`. @@ -170,21 +174,84 @@ fn ownership_lock_statement(run_id: &str) -> Result { Ok(format!("SELECT pg_advisory_lock({key})")) } +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(super) struct RecoveryIdentity { + system_identifier: i64, + database: String, +} + +fn require_recovery_identity(expected: &RecoveryIdentity, actual: &RecoveryIdentity) -> Result<()> { + if expected.system_identifier != actual.system_identifier { + bail!( + "this run belongs to a different PostgreSQL cluster; use the administrator URL from startup" + ); + } + if expected.database != actual.database { + bail!( + "this run belongs to a different PostgreSQL maintenance database; use the administrator URL from startup" + ); + } + Ok(()) +} + +fn read_recovery_identity( + runtime: &Runtime, + client: &Client, + deadline: Instant, +) -> Result { + let row = runtime + .block_on(bounded_operation( + "recovery identity check", + client.query_one(RECOVERY_IDENTITY_QUERY, &[]), + remaining_timeout(deadline, OPERATION_TIMEOUT)?, + )) + .map_err(anyhow::Error::new) + .context( + "cannot identify the PostgreSQL cluster; the administrator must be allowed to execute pg_catalog.pg_control_system()", + )?; + Ok(RecoveryIdentity { + system_identifier: row + .try_get(0) + .context("cannot read the PostgreSQL cluster identifier")?, + database: row + .try_get(1) + .context("cannot read the PostgreSQL administrator database name")?, + }) +} + fn open_admin_session( runtime: &Runtime, config: &Config, tls: &MakeRustlsConnect, ownership_lock: &str, + expected_identity: Option<&RecoveryIdentity>, deadline: Instant, ) -> Result<( Client, tokio::task::JoinHandle>, + RecoveryIdentity, )> { let (client, connection) = runtime.block_on(bounded_connection( config.connect(tls.clone()), remaining_timeout(deadline, CONNECTION_TIMEOUT)?, ))?; let connection_task = runtime.spawn(connection); + // PostgreSQL advisory locks are scoped to one cluster and database. Bind + // both before acquiring the lock, or the same run id at another target + // could appear to be exclusively owned. + let identity = match read_recovery_identity(runtime, &client, deadline) { + Ok(identity) => identity, + Err(error) => { + connection_task.abort(); + return Err(error); + } + }; + if let Some(expected) = expected_identity { + if let Err(error) = require_recovery_identity(expected, &identity) { + connection_task.abort(); + return Err(error); + } + } let locked = runtime.block_on(bounded_operation( "run ownership lock acquisition", client.batch_execute(ownership_lock), @@ -194,7 +261,7 @@ fn open_admin_session( connection_task.abort(); return Err(error).context("cannot acquire the PostgreSQL run ownership lock"); } - Ok((client, connection_task)) + Ok((client, connection_task, identity)) } pub(super) struct DatabaseUrls { @@ -215,6 +282,7 @@ pub(super) struct Database { tls: MakeRustlsConnect, admin_config: Config, ownership_lock: String, + recovery_identity: RecoveryIdentity, endpoint: Endpoint, run_id: String, acquisitions: Vec, @@ -257,6 +325,24 @@ impl Endpoint { impl Database { pub(super) fn connect(admin_url: &str, ca_pem: Option<&[u8]>, run_id: &str) -> Result { + Self::connect_checked(admin_url, ca_pem, run_id, None) + } + + pub(super) fn connect_for_recovery( + admin_url: &str, + ca_pem: Option<&[u8]>, + run_id: &str, + expected_identity: &RecoveryIdentity, + ) -> Result { + Self::connect_checked(admin_url, ca_pem, run_id, Some(expected_identity)) + } + + fn connect_checked( + admin_url: &str, + ca_pem: Option<&[u8]>, + run_id: &str, + expected_identity: Option<&RecoveryIdentity>, + ) -> Result { validate_run_id(run_id)?; let (mut admin_config, endpoint) = admin_config(admin_url)?; admin_config.ssl_mode(SslMode::Require); @@ -267,12 +353,13 @@ impl Database { .build() .context("cannot initialize the PostgreSQL executor")?; let ownership_lock = ownership_lock_statement(run_id)?; - let session_deadline = Instant::now() + CONNECTION_TIMEOUT + OPERATION_TIMEOUT; - let (client, connection_task) = open_admin_session( + let session_deadline = Instant::now() + CLEANUP_ATTEMPT_TIMEOUT; + let (client, connection_task, recovery_identity) = open_admin_session( &runtime, &admin_config, &tls, &ownership_lock, + expected_identity, session_deadline, ) .context("cannot establish the PostgreSQL administrator session over verified TLS") @@ -286,6 +373,7 @@ impl Database { tls, admin_config, ownership_lock, + recovery_identity, endpoint, run_id: run_id.to_owned(), acquisitions: Vec::new(), @@ -293,6 +381,10 @@ impl Database { }) } + pub(super) fn recovery_identity(&self) -> &RecoveryIdentity { + &self.recovery_identity + } + pub(super) fn provision( &mut self, label: &str, @@ -379,11 +471,12 @@ impl Database { // session lock orders this replacement session after an unconfirmed // CREATE has committed or rolled back, so cleanup cannot run too early. self.retire_admin_session(); - let (client, connection_task) = open_admin_session( + let (client, connection_task, _) = open_admin_session( &self.runtime, &self.admin_config, &self.tls, &self.ownership_lock, + Some(&self.recovery_identity), deadline, ) .context("cannot reconnect to the PostgreSQL administrator database")?; @@ -976,6 +1069,7 @@ mod tests { &stalled_admin_config(port), &untrusting_tls(), "SELECT 1", + None, Instant::now() + Duration::from_millis(300), ) else { panic!("a server that never answers opened a session"); @@ -1036,6 +1130,10 @@ mod tests { tls: untrusting_tls(), admin_config: stalled_admin_config(stalled_port), ownership_lock: "SELECT 1".to_owned(), + recovery_identity: RecoveryIdentity { + system_identifier: 23, + database: "postgres".to_owned(), + }, endpoint: Endpoint { host: UrlHost::Ipv4(std::net::Ipv4Addr::LOCALHOST), port: Some(stalled_port), @@ -1153,6 +1251,49 @@ mod tests { assert!(ownership_lock_statement("short").is_err()); } + #[test] + fn recovery_refuses_a_different_postgresql_cluster() { + let expected = RecoveryIdentity { + system_identifier: 23, + database: "postgres".to_owned(), + }; + let actual = RecoveryIdentity { + system_identifier: 42, + database: "postgres".to_owned(), + }; + let error = require_recovery_identity(&expected, &actual).unwrap_err(); + assert_eq!( + error.to_string(), + "this run belongs to a different PostgreSQL cluster; use the administrator URL from startup" + ); + } + + #[test] + fn recovery_refuses_a_different_administrator_database_in_the_same_cluster() { + let expected = RecoveryIdentity { + system_identifier: 23, + database: "postgres".to_owned(), + }; + let actual = RecoveryIdentity { + system_identifier: 23, + database: "template1".to_owned(), + }; + let error = require_recovery_identity(&expected, &actual).unwrap_err(); + assert_eq!( + error.to_string(), + "this run belongs to a different PostgreSQL maintenance database; use the administrator URL from startup" + ); + } + + #[test] + fn recovery_accepts_the_original_postgresql_target() { + let identity = RecoveryIdentity { + system_identifier: 23, + database: "postgres".to_owned(), + }; + require_recovery_identity(&identity, &identity).unwrap(); + } + #[test] fn admin_urls_require_one_explicit_password_authenticated_tcp_database() { let password = random_password().unwrap(); diff --git a/crates/registry-bregctl/src/project_dev/guard.rs b/crates/registry-bregctl/src/project_dev/guard.rs new file mode 100644 index 0000000000..a5409cbb3e --- /dev/null +++ b/crates/registry-bregctl/src/project_dev/guard.rs @@ -0,0 +1,643 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Own one development child process group beyond the worker's lifetime. + +use anyhow::{ensure, Context, Result}; +use rustix::{ + fs::{fcntl_getfl, fcntl_setfl, flock, FlockOperation, OFlags}, + process::{kill_process_group, waitid, Pid, Signal, WaitId, WaitIdOptions}, +}; +use std::{ + ffi::{OsStr, OsString}, + fs::{self, File, OpenOptions}, + io::{self, Read}, + os::{ + fd::AsFd, + unix::{ + fs::{MetadataExt, OpenOptionsExt, PermissionsExt}, + process::CommandExt, + }, + }, + path::{Path, PathBuf}, + process::{self, ChildStdin, ExitCode, ExitStatus, Stdio}, + sync::atomic::{AtomicU64, Ordering}, + thread, + time::{Duration, Instant}, +}; + +use super::{private_directory, write_private, POLL}; + +pub(super) const DIRECTORY: &str = "guards"; +const START_WAIT: Duration = Duration::from_secs(5); +const LAUNCHING: &[u8] = b"launching\n"; +const COMPLETE: &[u8] = b"complete\n"; +static NEXT_GUARD: AtomicU64 = AtomicU64::new(0); + +#[derive(clap::Args, Debug)] +pub(crate) struct Args { + #[arg(long)] + lock: PathBuf, + #[arg(long)] + pid_file: PathBuf, + #[arg(long)] + state_file: PathBuf, + #[arg(last = true, required = true, num_args = 1..)] + command: Vec, +} + +/// The hidden bregctl entry point used by a development worker. +pub(crate) fn run(args: Args) -> ExitCode { + match run_inner(args) { + Ok(status) => exit_code(status), + Err(error) => { + eprintln!("development process guard failed: {error:#}"); + ExitCode::FAILURE + } + } +} + +fn run_inner(args: Args) -> Result { + let (program, arguments) = args + .command + .split_first() + .context("the process guard command is empty")?; + let mut command = process::Command::new(program); + command + .args(arguments) + .env_remove("RSTACK_DEV_DATABASE_URL") + .env_remove("RSTACK_PROJECT_DEV_REQUEST") + .process_group(0) + .stdin(Stdio::null()); + let mut controller = io::stdin(); + supervise( + &args.lock, + &args.pid_file, + &args.state_file, + &mut controller, + command, + ) +} + +/// Take the lock and check the lifetime pipe before spawning. Recovery first +/// excludes the worker and then takes these same locks, so a delayed guard +/// cannot start a workload after its worker has died. +fn supervise( + lock_path: &Path, + pid_file: &Path, + state_file: &Path, + controller: &mut (impl Read + AsFd), + mut command: process::Command, +) -> Result { + let lock = open_lock(lock_path)?; + flock(&lock, FlockOperation::LockExclusive).context("cannot lock the process guard")?; + set_nonblocking(&*controller)?; + if !controller_alive(controller)? { + return Ok(success_status()); + } + + // This receipt precedes spawn. If the guard dies unexpectedly after this + // point, recovery sees an incomplete launch after the lock is released and + // refuses to claim cleanup succeeded. + write_private(state_file, LAUNCHING)?; + let child = match command.spawn() { + Ok(child) => child, + Err(error) => { + write_private(state_file, COMPLETE)?; + return Err(error).context("cannot start guarded command"); + } + }; + let mut workload = Workload::new(child); + let operation = (|| -> Result<()> { + write_private(pid_file, workload.child().id().to_string().as_bytes())?; + loop { + if direct_child_exited(workload.child())? || !controller_alive(controller)? { + return Ok(()); + } + thread::sleep(POLL); + } + })(); + let status = workload.stop()?; + write_private(state_file, COMPLETE)?; + operation?; + Ok(status) +} + +fn complete_state(path: &Path) -> Result<()> { + ensure!( + fs::read(path)? == COMPLETE, + "process guard did not record completed cleanup" + ); + Ok(()) +} + +fn incomplete_state(path: &Path) -> Result { + let state = fs::read(path)?; + ensure!( + state == LAUNCHING || state == COMPLETE, + "process guard has an invalid lifecycle receipt" + ); + Ok(state != COMPLETE) +} + +/// Released guard locks with these receipts cannot establish that the +/// workload group was reaped. Recovery must refuse cleanup for the run. +pub(super) fn incomplete_launches(root: &Path) -> Result> { + let directory = root.join(DIRECTORY); + let mut launches = Vec::new(); + let mut pid_stems = Vec::new(); + for entry in fs::read_dir(&directory).with_context(|| { + format!( + "cannot read process guard directory {}", + directory.display() + ) + })? { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.ends_with(".state") && incomplete_state(&entry.path())? { + launches.push(name.trim_end_matches(".state").to_owned()); + } else if name.ends_with(".pid") { + pid_stems.push(name.trim_end_matches(".pid").to_owned()); + } + } + for stem in pid_stems { + if !directory.join(format!("{stem}.state")).is_file() { + launches.push(stem); + } + } + launches.sort(); + launches.dedup(); + Ok(launches) +} + +/// Keep the lock's scope outside this owner, so every return path stops and +/// reaps a spawned workload before the lock is released. +struct Workload { + child: Option, +} + +impl Workload { + fn new(child: process::Child) -> Self { + Self { child: Some(child) } + } + + fn child(&self) -> &process::Child { + self.child.as_ref().expect("workload is live") + } + + fn stop(&mut self) -> Result { + let mut child = self.child.take().expect("workload is live"); + match stop_direct_child(&mut child) { + Ok(status) => Ok(status), + Err(error) => { + self.child = Some(child); + Err(error) + } + } + } +} + +impl Drop for Workload { + fn drop(&mut self) { + let Some(mut child) = self.child.take() else { + return; + }; + // An error after spawn must not release the guard lock while the group + // remains live. If the structured stop failed, retain the lock while + // this final attempt kills the leader and waits for it. + if stop_direct_child(&mut child).is_err() { + let pid = direct_child_pid(&child).ok(); + if let Some(pid) = pid { + let _ = kill_process_group(pid, Signal::KILL); + } + let _ = child.kill(); + // If signaling failed unexpectedly, waiting here retains the + // guard lock instead of letting recovery overclaim cleanup. + let _ = child.wait(); + } + } +} + +fn open_lock(path: &Path) -> Result { + let file = OpenOptions::new() + .read(true) + .write(true) + .custom_flags(OFlags::NOFOLLOW.bits() as i32) + .open(path) + .with_context(|| format!("cannot open process guard lock {}", path.display()))?; + let metadata = file.metadata()?; + ensure!(metadata.is_file(), "the process guard lock is not a file"); + ensure!( + metadata.permissions().mode() & 0o777 == 0o600, + "the process guard lock is not owner-only" + ); + ensure!( + metadata.uid() == rustix::process::getuid().as_raw(), + "the process guard lock has another owner" + ); + Ok(file) +} + +fn set_nonblocking(fd: impl AsFd) -> Result<()> { + let flags = fcntl_getfl(&fd)?; + fcntl_setfl(fd, flags | OFlags::NONBLOCK)?; + Ok(()) +} + +/// A lifetime pipe carries no protocol data. EOF alone means the worker no +/// longer owns the guarded process and the guard must stop it. +fn controller_alive(controller: &mut impl Read) -> Result { + let mut byte = [0_u8; 1]; + loop { + match controller.read(&mut byte) { + Ok(0) => return Ok(false), + Ok(_) => return Ok(true), + Err(error) if error.kind() == io::ErrorKind::WouldBlock => return Ok(true), + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(error) => return Err(error).context("cannot read the process guard lifetime pipe"), + } + } +} + +pub(super) fn direct_child_pid(child: &process::Child) -> Result { + Pid::from_raw(i32::try_from(child.id())?).context("invalid owned process ID") +} + +pub(super) fn direct_child_exited(child: &process::Child) -> Result { + let options = WaitIdOptions::EXITED | WaitIdOptions::NOHANG | WaitIdOptions::NOWAIT; + loop { + match waitid(WaitId::Pid(direct_child_pid(child)?), options) { + Ok(status) => return Ok(status.is_some()), + Err(rustix::io::Errno::INTR) => continue, + Err(error) => return Err(error.into()), + } + } +} + +fn signal_group(child: &process::Child, signal: Signal) -> Result<()> { + match kill_process_group(direct_child_pid(child)?, signal) { + Ok(()) | Err(rustix::io::Errno::SRCH) => Ok(()), + // Darwin reports EPERM when an exited leader has no signalable group + // members. The leader remains our unreaped child, so this is empty. + Err(rustix::io::Errno::PERM) + if cfg!(target_os = "macos") && direct_child_exited(child)? => + { + Ok(()) + } + Err(error) => Err(error.into()), + } +} + +pub(super) fn stop_direct_child(child: &mut process::Child) -> Result { + if !direct_child_exited(child)? { + signal_group(child, Signal::TERM)?; + let deadline = Instant::now() + Duration::from_secs(2); + while Instant::now() < deadline && !direct_child_exited(child)? { + thread::sleep(POLL); + } + } + // Keep the leader PID reserved until the final group signal, then reap it. + signal_group(child, Signal::KILL)?; + Ok(child.wait()?) +} + +fn success_status() -> ExitStatus { + use std::os::unix::process::ExitStatusExt; + ExitStatus::from_raw(0) +} + +fn exit_code(status: ExitStatus) -> ExitCode { + status + .code() + .and_then(|code| u8::try_from(code).ok()) + .map(ExitCode::from) + .unwrap_or(ExitCode::FAILURE) +} + +/// Builder for a command whose process group is owned by a guard. Output and +/// error streams belong to the guard and are inherited unchanged by the +/// workload, preserving the tool's JSON and log behavior. +pub(super) struct Command { + command: process::Command, + pid_file: PathBuf, + state_file: PathBuf, +} + +impl Command { + pub(super) fn new(root: &Path, name: &str, binary: &Path) -> Result { + let directory = root.join(DIRECTORY); + match private_directory(&directory) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error).context("cannot create process guard directory"), + } + ensure!( + fs::metadata(&directory)?.permissions().mode() & 0o777 == 0o700, + "the process guard directory is not owner-only" + ); + + let sequence = NEXT_GUARD.fetch_add(1, Ordering::Relaxed); + let stem = format!("{sequence:016x}-{name}"); + let lock = directory.join(format!("{stem}.lock")); + let lock_file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&lock) + .with_context(|| format!("cannot create process guard lock {}", lock.display()))?; + lock_file.set_permissions(fs::Permissions::from_mode(0o600))?; + let pid_file = directory.join(format!("{stem}.pid")); + let state_file = directory.join(format!("{stem}.state")); + let mut command = process::Command::new(std::env::current_exe()?); + command + .arg("project-dev-guard") + .arg("--lock") + .arg(lock) + .arg("--pid-file") + .arg(&pid_file) + .arg("--state-file") + .arg(&state_file) + .arg("--") + .arg(binary) + .env_remove("RSTACK_DEV_DATABASE_URL") + .env_remove("RSTACK_PROJECT_DEV_REQUEST") + .process_group(0) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + Ok(Self { + command, + pid_file, + state_file, + }) + } + + pub(super) fn arg(&mut self, argument: impl AsRef) -> &mut Self { + self.command.arg(argument); + self + } + + pub(super) fn args(&mut self, arguments: I) -> &mut Self + where + I: IntoIterator, + S: AsRef, + { + self.command.args(arguments); + self + } + + pub(super) fn stdout(&mut self, stream: impl Into) -> &mut Self { + self.command.stdout(stream); + self + } + + pub(super) fn stderr(&mut self, stream: impl Into) -> &mut Self { + self.command.stderr(stream); + self + } + + pub(super) fn spawn(&mut self) -> Result { + let mut guard = self.command.spawn().context("cannot start process guard")?; + let controller = guard + .stdin + .take() + .context("process guard stdin was not piped")?; + let started: Result = (|| -> Result { + let deadline = Instant::now() + START_WAIT; + loop { + match fs::read_to_string(&self.pid_file) { + Ok(value) => { + let raw: i32 = + value.trim().parse().context("invalid guarded process ID")?; + let pid = Pid::from_raw(raw).context("invalid guarded process ID")?; + return Ok(pid); + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error).context("cannot read guarded process ID"), + } + if guard.try_wait()?.is_some() { + anyhow::bail!("process guard exited before starting its workload"); + } + ensure!( + Instant::now() < deadline, + "process guard did not publish its workload ID" + ); + thread::sleep(Duration::from_millis(10)); + } + })(); + match started { + Ok(pid) => Ok(Child { + guard, + controller: Some(controller), + pid, + state_file: self.state_file.clone(), + }), + Err(error) => { + drop(controller); + let _ = guard.wait(); + Err(error) + } + } + } +} + +/// The worker's handle for a guard and its actual workload process group. +pub(super) struct Child { + guard: process::Child, + controller: Option, + pid: Pid, + state_file: PathBuf, +} + +impl Child { + pub(super) fn id(&self) -> u32 { + self.pid + .as_raw_pid() + .try_into() + .expect("positive process ID") + } + + pub(super) fn exited(&self) -> Result { + direct_child_exited(&self.guard) + } + + pub(super) fn stop(&mut self) -> Result { + drop(self.controller.take()); + let status = self.guard.wait()?; + complete_state(&self.state_file)?; + Ok(status) + } +} + +impl Drop for Child { + fn drop(&mut self) { + if self.controller.take().is_some() { + if self.guard.wait().is_ok() { + let _ = complete_state(&self.state_file); + } + } + } +} + +/// The actual runtime groups a run started. Recovery uses this only for its +/// success report after every guard lock is free, never as a signaling source. +pub(super) fn runtime_groups(root: &Path) -> Result> { + let directory = root.join(DIRECTORY); + let mut groups = Vec::new(); + for entry in fs::read_dir(&directory).with_context(|| { + format!( + "cannot read process guard directory {}", + directory.display() + ) + })? { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if !name.contains("-runtime-") || !name.ends_with(".pid") { + continue; + } + complete_state(&entry.path().with_extension("state"))?; + let raw: u32 = fs::read_to_string(entry.path())? + .trim() + .parse() + .context("invalid guarded runtime process ID")?; + groups.push(raw); + } + groups.sort_unstable(); + Ok(groups) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{os::unix::net::UnixStream, sync::mpsc}; + + fn lock_file(root: &Path) -> PathBuf { + let path = root.join("guard.lock"); + OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&path) + .unwrap(); + path + } + + #[test] + fn worker_death_before_pid_publication_spawns_no_workload() { + let scratch = tempfile::tempdir().unwrap(); + let lock = lock_file(scratch.path()); + let pid_file = scratch.path().join("guard.pid"); + let state_file = scratch.path().join("guard.state"); + let marker = scratch.path().join("spawned"); + let (mut controller, worker) = UnixStream::pair().unwrap(); + drop(worker); + let mut command = process::Command::new("/bin/sh"); + command.args(["-c", &format!("touch '{}'", marker.display())]); + + let status = supervise(&lock, &pid_file, &state_file, &mut controller, command).unwrap(); + assert!(status.success()); + assert!(!marker.exists()); + assert!(!pid_file.exists()); + assert!(!state_file.exists()); + } + + #[test] + fn guard_owns_and_reaps_the_workload_process_group() { + let scratch = tempfile::tempdir().unwrap(); + let lock = lock_file(scratch.path()); + let pid_file = scratch.path().join("guard.pid"); + let state_file = scratch.path().join("guard.state"); + let nested_file = scratch.path().join("nested.pid"); + let (mut controller, worker) = UnixStream::pair().unwrap(); + let (finished_tx, finished_rx) = mpsc::sync_channel(1); + let mut command = process::Command::new("/bin/sh"); + command + .args([ + "-c", + &format!( + "sh -c 'trap \"\" TERM; exec sleep 60' & printf '%s' \"$!\" > '{}' ; wait", + nested_file.display() + ), + ]) + .process_group(0) + .stdin(Stdio::null()); + let guard_lock = lock.clone(); + let guard_pid = pid_file.clone(); + let guard_state = state_file.clone(); + let guard_thread = thread::spawn(move || { + let status = supervise( + &guard_lock, + &guard_pid, + &guard_state, + &mut controller, + command, + ) + .unwrap(); + finished_tx.send(status).unwrap(); + }); + + let leader = wait_pid(&pid_file); + let nested = wait_pid(&nested_file); + assert_eq!(rustix::process::getpgid(Some(leader)).unwrap(), leader); + assert_eq!(rustix::process::getpgid(Some(nested)).unwrap(), leader); + let contender = open_lock(&lock).unwrap(); + assert_eq!( + flock(&contender, FlockOperation::NonBlockingLockExclusive).unwrap_err(), + rustix::io::Errno::WOULDBLOCK + ); + + drop(worker); + finished_rx.recv_timeout(Duration::from_secs(4)).unwrap(); + guard_thread.join().unwrap(); + complete_state(&state_file).unwrap(); + flock(&contender, FlockOperation::NonBlockingLockExclusive).unwrap(); + assert_eq!( + rustix::process::getpgid(Some(leader)).unwrap_err(), + rustix::io::Errno::SRCH + ); + assert_eq!( + rustix::process::getpgid(Some(nested)).unwrap_err(), + rustix::io::Errno::SRCH + ); + } + + #[test] + fn runtime_group_report_excludes_provisioning_tools() { + let scratch = tempfile::tempdir().unwrap(); + let directory = scratch.path().join(DIRECTORY); + fs::create_dir(&directory).unwrap(); + fs::write(directory.join("000-runtime-breg.pid"), "41").unwrap(); + fs::write(directory.join("000-runtime-breg.state"), COMPLETE).unwrap(); + fs::write(directory.join("001-tool-breg-check.pid"), "42").unwrap(); + fs::write(directory.join("001-tool-breg-check.state"), COMPLETE).unwrap(); + fs::write(directory.join("002-runtime-casework.pid"), "43").unwrap(); + fs::write(directory.join("002-runtime-casework.state"), COMPLETE).unwrap(); + assert_eq!(runtime_groups(scratch.path()).unwrap(), vec![41, 43]); + } + + #[test] + fn incomplete_launch_receipt_refuses_recovery() { + let scratch = tempfile::tempdir().unwrap(); + let directory = scratch.path().join(DIRECTORY); + fs::create_dir(&directory).unwrap(); + fs::write(directory.join("000-runtime-breg.state"), LAUNCHING).unwrap(); + fs::write(directory.join("001-tool-breg-check.state"), COMPLETE).unwrap(); + fs::write(directory.join("002-runtime-casework.pid"), "43").unwrap(); + assert_eq!( + incomplete_launches(scratch.path()).unwrap(), + vec!["000-runtime-breg", "002-runtime-casework"] + ); + } + + fn wait_pid(path: &Path) -> Pid { + let deadline = Instant::now() + Duration::from_secs(2); + loop { + if let Ok(value) = fs::read_to_string(path) { + let raw = value.parse().unwrap(); + return Pid::from_raw(raw).unwrap(); + } + assert!(Instant::now() < deadline, "PID was not published"); + thread::sleep(Duration::from_millis(10)); + } + } +} diff --git a/crates/registry-bregctl/src/project_dev/mod.rs b/crates/registry-bregctl/src/project_dev/mod.rs index 67461654a8..3d5c0dbea8 100644 --- a/crates/registry-bregctl/src/project_dev/mod.rs +++ b/crates/registry-bregctl/src/project_dev/mod.rs @@ -4,6 +4,7 @@ mod bindings; mod database; +pub(crate) mod guard; mod issuer; mod journey; pub(crate) mod plan; @@ -37,7 +38,7 @@ use uuid::Uuid; use zeroize::Zeroizing; use self::{ - database::Database, + database::{Database, RecoveryIdentity}, issuer::Issuer, plan::{Plan, Product}, products::{ProductRunner, SchemaPerson}, @@ -50,6 +51,7 @@ const DATABASE_URL: &str = "RSTACK_DEV_DATABASE_URL"; const CA_FILE: &str = "RSTACK_DEV_DATABASE_CA"; const REPORT: &str = "report.json"; const PENDING: &str = "cleanup-pending.json"; +const POSTGRES_IDENTITY: &str = "postgres-identity.json"; const CLEANUP_CAP: Duration = Duration::from_secs(30 * 60); const POLL: Duration = Duration::from_millis(100); // A capture can contain the 1 MiB development plan after JSON escaping, @@ -481,16 +483,6 @@ fn emit(output: &mut dyn Write, report: &Value, machine: bool) -> std::io::Resul { writeln!(output, " stopped process group {group}")?; } - for group in recovered["skipped_process_groups"] - .as_array() - .into_iter() - .flatten() - { - writeln!( - output, - " left process group {group} running: its ID now belongs to another process" - )?; - } for name in recovered["databases_and_roles"] .as_array() .into_iter() @@ -675,6 +667,8 @@ fn start(project: &Path, ttl: Option<&str>) -> Result { }; let preparation = (|| { write_private(&root.join("input.json"), &capture_input(&input)?)?; + private_directory(&root.join(guard::DIRECTORY))?; + write_private(&root.join(POSTGRES_IDENTITY), b"null")?; let supervisor_log = open_supervisor_log(&root)?; Command::new(env::current_exe()?) .arg("project-dev") @@ -814,6 +808,7 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { .mode(0o600) .open(root.join("worker.lock"))?; flock(&lock, FlockOperation::LockExclusive)?; + ensure_private(root, false)?; let mut database = None; let mut expiry = None; let prepared = catch_startup(|| { @@ -832,8 +827,15 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { .map_err(|_| anyhow::anyhow!("missing PostgreSQL administrator URL"))?, ); let issuer = Arc::new(Issuer::create(root)?); - let database = database.insert(Database::connect(&admin, ca.as_deref(), &input.run_id)?); - let urls = products.start(database, issuer.clone())?; + let connected = Database::connect(&admin, ca.as_deref(), &input.run_id)?; + // The cluster identity must be recorded before anything can create a database or role. + write_private( + &root.join(POSTGRES_IDENTITY), + &serde_json::to_vec(connected.recovery_identity())?, + ) + .context("cannot record the PostgreSQL cluster for recovery") + .map_err(operational)?; + let urls = products.start(database.insert(connected), issuer.clone())?; let journey = products.journey(&issuer, &urls)?; ensure!( products.services_alive()?, @@ -1075,10 +1077,9 @@ fn stop(run_id: &str) -> Result { Ok(simple_report(run_id, "stopped")) } -/// Release the resources of a run whose supervisor is gone: stop its recorded -/// processes, drop its databases and roles, and remove its directory. Every -/// precondition is checked before anything is signalled or dropped, and a -/// failure reports the facts needed to finish by hand. +/// Wait for the lost supervisor's process guards to finish, then release the +/// databases and roles on the original PostgreSQL cluster and remove the run +/// directory. Hold the local lock across recovery to serialize retries. fn recover_run(run_id: &str) -> Result { let root = run_dir(run_id)?; if !root.exists() { @@ -1090,10 +1091,14 @@ fn recover_run(run_id: &str) -> Result { "run {run_id} has no supervisor lock, so it is still starting or never started; run rstack dev stop {run_id}, or delete {} if the run never started", root.display() ); - ensure!( - !supervisor_running(&root)?, - "run {run_id} still has a running supervisor; run rstack dev stop {run_id} without --recover" - ); + let lock = match try_run_lock(&root) { + Ok(lock) => lock, + Err(_) if !root.exists() => return Ok(simple_report(run_id, "stopped")), + Err(error) => return Err(error), + }; + let _lock = lock.with_context(|| { + format!("run {run_id} still has a running supervisor or recovery; use plain stop for a live supervisor") + })?; let unrecovered = |reason: String| -> anyhow::Error { let mut facts = lost_supervisor_facts(&root, run_id); facts["reason"] = json!(reason); @@ -1103,38 +1108,45 @@ fn recover_run(run_id: &str) -> Result { } .into() }; - let labels = provisioned_labels(&root).map_err(|error| unrecovered(format!("{error:#}")))?; - let plan = recovery::process_table() - .and_then(|table| recovery::plan(&recorded_processes(&root), &table)) - .map_err(|error| unrecovered(format!("{error:#}")))?; - let admin = Zeroizing::new(env::var(DATABASE_URL).map_err(|_| { - unrecovered( - "no PostgreSQL administrator URL; pass --database-url or set RSTACK_DEV_DATABASE_URL" - .to_owned(), + let (stopped_groups, dropped) = (|| -> Result<_> { + // Status can read the last report while recovery holds the worker lock. + write_private( + &root.join(REPORT), + &serde_json::to_vec(&simple_report(run_id, "stopping"))?, ) - })?); - let ca = env::var_os(CA_FILE) - .map(|path| plan::read(Path::new(&path), 1024 * 1024)) - .transpose() - .map_err(|error| unrecovered(format!("cannot read the PostgreSQL CA file: {error:#}")))?; - // Connecting takes the run's ownership lock, which a live supervisor holds. - let mut database = Database::connect(&admin, ca.as_deref(), run_id) - .map_err(|error| unrecovered(format!("{error:#}")))?; - recovery::stop_groups(&plan.ours).map_err(|error| unrecovered(format!("{error:#}")))?; - for (label, spatial) in &labels { - database - .adopt(label, *spatial) - .map_err(|error| unrecovered(format!("{error:#}")))?; - } - let dropped = database.pending_names(); - database - .cleanup() - .map_err(|error| unrecovered(format!("{error:#}")))?; - remove_run_dir(&root).map_err(|error| unrecovered(format!("{error:#}")))?; + .context("cannot mark the run as stopping")?; + // A null identity means the worker never began provisioning. Missing + // or invalid metadata cannot prove that there is nothing to release. + let identity = read_json::>(&root.join(POSTGRES_IDENTITY), 1024) + .context("cannot read the recorded PostgreSQL cluster for this run")?; + let mut database = identity + .map(|expected| { + let admin = Zeroizing::new(env::var(DATABASE_URL).context( + "no PostgreSQL administrator URL; pass --database-url or set RSTACK_DEV_DATABASE_URL", + )?); + let ca = env::var_os(CA_FILE) + .map(|path| plan::read(Path::new(&path), 1024 * 1024)) + .transpose() + .context("cannot read the PostgreSQL CA file")?; + Database::connect_for_recovery(&admin, ca.as_deref(), run_id, &expected) + }) + .transpose()?; + let stopped_groups = recovery::wait_for_guards(&root)?; + let mut dropped = Vec::new(); + if let Some(database) = database.as_mut() { + for (label, spatial) in provisioned_labels(&root)? { + database.adopt(label, spatial)?; + } + dropped = database.pending_names(); + database.cleanup()?; + } + remove_run_dir(&root)?; + Ok((stopped_groups, dropped)) + })() + .map_err(|error| unrecovered(format!("{error:#}")))?; let mut report = simple_report(run_id, "stopped"); report["recovered"] = json!({ - "stopped_process_groups": plan.ours, - "skipped_process_groups": plan.reused, + "stopped_process_groups": stopped_groups, "databases_and_roles": dropped, }); Ok(report) @@ -1160,12 +1172,16 @@ fn supervisor_gone(root: &Path) -> Result { } fn supervisor_running(root: &Path) -> Result { + Ok(try_run_lock(root)?.is_none()) +} + +fn try_run_lock(root: &Path) -> Result> { ensure_private(root, false)?; let lock = File::open(root.join("worker.lock")).context("run supervisor lock is unavailable")?; match flock(&lock, FlockOperation::NonBlockingLockExclusive) { - Ok(()) => Ok(false), - Err(rustix::io::Errno::WOULDBLOCK) => Ok(true), + Ok(()) => Ok(Some(lock)), + Err(rustix::io::Errno::WOULDBLOCK) => Ok(None), Err(error) => Err(error).context("cannot check run supervisor lock"), } } diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs index a44b25fce0..2a9e13ca99 100644 --- a/crates/registry-bregctl/src/project_dev/products.rs +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -3,7 +3,6 @@ use anyhow::{bail, ensure, Context, Result}; use registry_breg::fixtures::journey_step_profiles; -use rustix::process::{kill_process_group, waitid, Pid, Signal, WaitId, WaitIdOptions}; use serde::Serialize; use serde_json::{json, Value}; use std::{ @@ -26,6 +25,7 @@ use super::{ bindings::{Bindings, RuntimeBindings}, clock, database::{Database, DatabaseUrls}, + guard, issuer::Issuer, operational, plan::{self, Plan, Product}, @@ -53,7 +53,7 @@ pub(super) struct ProductRunner<'a> { deadline: Instant, runtime: tokio::runtime::Runtime, http: reqwest::Client, - children: Vec<(Product, Child)>, + children: Vec<(Product, guard::Child)>, bindings: Option, } @@ -150,7 +150,12 @@ impl<'a> ProductRunner<'a> { } }; let binaries = &self.input.binaries[&product]; - let mut command = child_command(&binaries.runtime); + let mut command = guard::Command::new( + self.root, + &format!("runtime-{}", product.name()), + &binaries.runtime, + ) + .map_err(operational)?; command.arg("--runtime-config").arg(config); if product == Product::Casework { command.arg("serve"); @@ -467,8 +472,8 @@ impl<'a> ProductRunner<'a> { ) } - /// Record every live child's process group so a later `stop` can reach - /// processes whose supervisor died. + /// Record every live runtime group for status diagnostics and manual + /// cleanup evidence. Guards, not this record, own process lifetime. fn record_processes(&self) -> Result<()> { let entries: Vec<_> = self .children @@ -506,7 +511,7 @@ impl<'a> ProductRunner<'a> { /// Fail while waiting on `waiting`, naming whichever child exited. fn ensure_alive(&self, waiting: Product) -> Result<()> { for (product, child) in &self.children { - if child_exited(child)? { + if child.exited()? { let detail = self.runtime_log_tail(*product); if *product == waiting { bail!("{} exited before readiness{detail}", product.name()); @@ -527,7 +532,7 @@ impl<'a> ProductRunner<'a> { pub(super) fn services_alive(&self) -> Result { for (_, child) in &self.children { - if child_exited(child)? { + if child.exited()? { return Ok(false); } } @@ -550,17 +555,23 @@ impl<'a> ProductRunner<'a> { pub(super) fn stop(&mut self) -> Result<()> { let mut failures = Vec::new(); self.children.retain_mut(|(product, child)| { - if stop_child(child).is_err() { + if child.stop().is_err() { failures.push(product.name()); true } else { false } }); - // Drop reaped groups from the record so a later recovery cannot signal - // a reused process group ID. + // Drop reaped groups from the diagnostic record. Recovery waits on + // guard locks and lifecycle receipts rather than signaling these IDs. self.record_processes()?; ensure!(failures.is_empty(), "cannot stop {}", failures.join(", ")); + let incomplete = guard::incomplete_launches(self.root)?; + ensure!( + incomplete.is_empty(), + "process guards did not confirm cleanup for {}", + incomplete.join(", ") + ); drop(self.bindings.take()); Ok(()) } @@ -641,28 +652,60 @@ fn run_logged_tool( ) -> Result<()> { let name = format!("{product}-{stage}"); let mut output = tempfile::tempfile_in(root)?; - let mut child = child_command(binary) + let mut command = guard::Command::new(root, &format!("tool-{product}-{stage}"), binary) + .map_err(operational)?; + let mut child = command .args(args) .stdout(output.try_clone()?) .stderr(open_log(root, &name)?) .spawn() .with_context(|| format!("cannot start {product} {stage}")) .map_err(operational)?; - if wait_for_tool(&mut child, active)?.success() { + let status = wait_for_guarded_tool(&mut child, active)?; + logged_tool_result(status, &mut output, root, (product, stage)) +} + +fn logged_tool_result( + status: ExitStatus, + output: &mut fs::File, + root: &Path, + (product, stage): (&str, &str), +) -> Result<()> { + if status.success() { return Ok(()); } output.rewind()?; let codes = report_codes(output)?; bail!( "{product} {stage} failed{codes}{}", - failure_detail(&log_path(root, &name)) + failure_detail(&log_path(root, &format!("{product}-{stage}"))) ); } -/// Record each child's process group and the start time of its leader. -/// Children start as group leaders, so the group ID is the PID. Recovery after -/// a lost supervisor signals a group only while its leader still has this -/// start time, because process IDs are reused. +/// Exercise the log/result boundary in unit tests without routing a shell +/// through the Rust test harness as if it were the bregctl binary. The native +/// lifecycle test covers the real hidden guard dispatch. +#[cfg(test)] +fn run_logged_tool_direct( + binary: &Path, + args: &[&std::ffi::OsStr], + root: &Path, + (product, stage): (&str, &str), +) -> Result<()> { + let name = format!("{product}-{stage}"); + let mut output = tempfile::tempfile_in(root)?; + let mut child = child_command(binary) + .args(args) + .stdout(output.try_clone()?) + .stderr(open_log(root, &name)?) + .spawn()?; + let status = wait_for_tool(&mut child, || Ok(()))?; + logged_tool_result(status, &mut output, root, (product, stage)) +} + +/// Record each runtime process group and the start time of its leader for +/// diagnostics and an operator's manual cleanup decision. Children start as +/// group leaders, so the group ID is the PID. fn write_processes(root: &Path, children: &[(Product, u32)]) -> Result<()> { let table = super::recovery::process_table()?; let entries: Vec<_> = children @@ -1502,7 +1545,7 @@ impl Drop for ProductRunner<'_> { fn drop(&mut self) { // The normal exit reports stop failures explicitly. for (_, child) in &mut self.children { - let _ = stop_child(child); + let _ = child.stop(); } } } @@ -1518,55 +1561,30 @@ pub(super) fn child_command(binary: &Path) -> Command { command } -fn child_pid(child: &Child) -> Result { - Pid::from_raw(i32::try_from(child.id())?).context("invalid owned process ID") -} - -fn child_exited(child: &Child) -> Result { - let pid = child_pid(child)?; - // Keep the leader's PID until the group has received the final signal. - // Reaping the leader first would allow reuse of the group ID. - let options = WaitIdOptions::EXITED | WaitIdOptions::NOHANG | WaitIdOptions::NOWAIT; +fn wait_for_tool(child: &mut Child, active: impl Fn() -> Result<()>) -> Result { loop { - match waitid(WaitId::Pid(pid), options) { - Ok(status) => return Ok(status.is_some()), - Err(rustix::io::Errno::INTR) => continue, - Err(error) => return Err(error.into()), + if let Err(error) = active() { + guard::stop_direct_child(child)?; + return Err(error); } - } -} - -fn signal_group(child: &Child, signal: Signal) -> Result<()> { - match kill_process_group(child_pid(child)?, signal) { - Ok(()) | Err(rustix::io::Errno::SRCH) => Ok(()), - // Darwin excludes zombies from killpg and reports EPERM when there - // are no signalable members. An owned, same-user group with an - // exited leader can already be empty of live processes. - Err(rustix::io::Errno::PERM) if cfg!(target_os = "macos") && child_exited(child)? => Ok(()), - Err(error) => Err(error.into()), - } -} - -fn stop_child(child: &mut Child) -> Result { - if !child_exited(child)? { - signal_group(child, Signal::TERM)?; - let deadline = Instant::now() + Duration::from_secs(2); - while Instant::now() < deadline && !child_exited(child)? { - thread::sleep(POLL); + if guard::direct_child_exited(child)? { + return guard::stop_direct_child(child); } + thread::sleep(POLL); } - signal_group(child, Signal::KILL)?; - Ok(child.wait()?) } -fn wait_for_tool(child: &mut Child, active: impl Fn() -> Result<()>) -> Result { +fn wait_for_guarded_tool( + child: &mut guard::Child, + active: impl Fn() -> Result<()>, +) -> Result { loop { if let Err(error) = active() { - stop_child(child)?; + child.stop()?; return Err(error); } - if child_exited(child)? { - return stop_child(child); + if child.exited()? { + return child.stop(); } thread::sleep(POLL); } @@ -2277,12 +2295,11 @@ mod tests { fn a_failed_tool_leaves_its_stderr_in_an_owner_only_log_and_the_error() { let scratch = tempfile::tempdir().unwrap(); let root = scratch.path(); - let error = run_logged_tool( + let error = run_logged_tool_direct( Path::new("/bin/sh"), &["-c".as_ref(), "echo boom >&2; exit 3".as_ref()], root, ("breg", "test"), - || Ok(()), ) .unwrap_err() .to_string(); @@ -2305,12 +2322,11 @@ mod tests { #[test] fn a_successful_tool_leaves_no_failure_text() { let scratch = tempfile::tempdir().unwrap(); - run_logged_tool( + run_logged_tool_direct( Path::new("/bin/sh"), &["-c".as_ref(), "echo fine >&2".as_ref()], scratch.path(), ("breg", "test"), - || Ok(()), ) .unwrap(); } @@ -2684,12 +2700,15 @@ mod tests { .spawn() .unwrap(); wait_for_leader(&child); - assert_eq!(stop_child(&mut child).unwrap().code(), Some(7)); + assert_eq!( + guard::stop_direct_child(&mut child).unwrap().code(), + Some(7) + ); } fn wait_for_leader(child: &Child) { let deadline = Instant::now() + Duration::from_secs(2); - while !child_exited(child).unwrap() { + while !guard::direct_child_exited(child).unwrap() { assert!(Instant::now() < deadline, "the tool leader did not exit"); thread::sleep(POLL); } @@ -2705,7 +2724,7 @@ mod tests { .stdout(Stdio::piped()) .spawn() .unwrap(); - let pid = child_pid(&child).unwrap(); + let pid = guard::direct_child_pid(&child).unwrap(); assert_eq!(rustix::process::getpgid(Some(pid)).unwrap(), pid); let mut pipe = BufReader::new(child.stdout.take().unwrap()); let mut ready = String::new(); @@ -2714,7 +2733,7 @@ mod tests { if leader_exits { wait_for_leader(&child); } - stop_child(&mut child).unwrap(); + guard::stop_direct_child(&mut child).unwrap(); let (sender, receiver) = mpsc::sync_channel(1); let reader = thread::spawn(move || { diff --git a/crates/registry-bregctl/src/project_dev/recovery.rs b/crates/registry-bregctl/src/project_dev/recovery.rs index 2263b8b8f5..8daef210d0 100644 --- a/crates/registry-bregctl/src/project_dev/recovery.rs +++ b/crates/registry-bregctl/src/project_dev/recovery.rs @@ -1,21 +1,17 @@ // SPDX-License-Identifier: Apache-2.0 -//! Releasing the processes of a run whose supervisor is lost. -//! -//! A recorded process group is signalled only while it still belongs to the -//! run: the leader's start time must equal the one recorded when the run -//! started. Process IDs are reused, so a recorded ID alone proves nothing. +//! Wait for the guards that own a lost worker's child processes. use anyhow::{bail, ensure, Context, Result}; -use rustix::process::{kill_process_group, Pid, Signal}; -use serde_json::Value; +use rustix::fs::{flock, FlockOperation}; use std::{ + fs::{self, File}, + path::Path, process::{Command, Stdio}, thread, time::{Duration, Instant}, }; -const TERM_GRACE: Duration = Duration::from_secs(5); -const KILL_GRACE: Duration = Duration::from_secs(5); +const GUARD_WAIT: Duration = Duration::from_secs(10); const POLL: Duration = Duration::from_millis(100); /// One row of the process table. @@ -26,17 +22,6 @@ pub(super) struct Row { pub started: String, } -/// What a recorded process group is now. -#[derive(Debug, Eq, PartialEq)] -pub(super) enum Group { - /// No process of the group remains. - Gone, - /// The group is the run's own. - Ours, - /// The recorded ID belongs to a different process now. - Reused, -} - /// Every process with its group and start time. `ps` is the one portable /// source of start times on Linux and macOS. pub(super) fn process_table() -> Result> { @@ -86,114 +71,48 @@ pub(super) fn started_at(pid: u32) -> Option { .map(|row| row.started) } -/// Decide whether the recorded group `pgid` is still the run's. A group ID -/// stays reserved while any member lives, so a group whose leader has gone -/// but whose members remain is the original group. -pub(super) fn classify(table: &[Row], pgid: u32, recorded_start: Option<&str>) -> Result { - match table.iter().find(|row| row.pid == pgid) { - Some(leader) => { - let recorded = recorded_start.with_context(|| { - format!("process group {pgid} cannot be verified: its start time was not recorded") - })?; - Ok(if leader.started == recorded { - Group::Ours - } else { - Group::Reused - }) - } - None if table.iter().any(|row| row.pgid == pgid) => Ok(Group::Ours), - None => Ok(Group::Gone), - } +/// The worker creates each lock before launching a guard. Once the worker +/// has exited, a late guard cannot start a workload because its control pipe +/// is closed. A guard already running a workload holds its lock through reap. +pub(super) fn wait_for_guards(root: &Path) -> Result> { + wait_for_guards_until(root, Instant::now() + GUARD_WAIT)?; + super::guard::runtime_groups(root) } -fn signal(pgid: u32, signal: Signal) -> Result<()> { - let pid = i32::try_from(pgid) - .ok() - .and_then(Pid::from_raw) - .context("invalid process group ID")?; - match kill_process_group(pid, signal) { - Ok(()) | Err(rustix::io::Errno::SRCH) => Ok(()), - // Darwin reports EPERM for a group whose only members are zombies. - Err(rustix::io::Errno::PERM) if cfg!(target_os = "macos") => Ok(()), - Err(error) => Err(error).with_context(|| format!("cannot signal process group {pgid}")), - } -} - -fn wait_until_gone(groups: &[u32], grace: Duration) -> Result> { - let deadline = Instant::now() + grace; - loop { - let table = process_table()?; - let alive: Vec = groups - .iter() - .copied() - .filter(|group| table.iter().any(|row| row.pgid == *group)) - .collect(); - if alive.is_empty() || Instant::now() >= deadline { - return Ok(alive); +fn wait_for_guards_until(root: &Path, deadline: Instant) -> Result<()> { + let directory = root.join(super::guard::DIRECTORY); + let entries = fs::read_dir(directory).context("cannot read the run's process guards")?; + for entry in entries { + let path = entry?.path(); + if path.extension().and_then(|value| value.to_str()) != Some("lock") { + continue; } - thread::sleep(POLL); - } -} - -/// Stop each group: TERM, then KILL for any that survive the grace period. -pub(super) fn stop_groups(groups: &[u32]) -> Result<()> { - if groups.is_empty() { - return Ok(()); - } - for group in groups { - signal(*group, Signal::TERM)?; - } - let survivors = wait_until_gone(groups, TERM_GRACE)?; - for group in &survivors { - signal(*group, Signal::KILL)?; - } - let remaining = wait_until_gone(&survivors, KILL_GRACE)?; - if !remaining.is_empty() { - bail!( - "process group {} survived SIGKILL", - remaining - .iter() - .map(u32::to_string) - .collect::>() - .join(", ") - ); - } - Ok(()) -} - -/// The recorded processes of a run, sorted into those to stop and those whose -/// recorded ID now belongs to something else. -pub(super) struct Plan { - pub ours: Vec, - pub reused: Vec, -} - -/// Classify every recorded process group before anything is signalled, so one -/// unverifiable entry stops the recovery with nothing touched. -pub(super) fn plan(recorded: &Value, table: &[Row]) -> Result { - let mut plan = Plan { - ours: Vec::new(), - reused: Vec::new(), - }; - for entry in recorded.as_array().into_iter().flatten() { - let pgid = entry["pgid"] - .as_u64() - .and_then(|value| u32::try_from(value).ok()) - .context("a recorded process has no valid group ID")?; - match classify(table, pgid, entry["started"].as_str())? { - Group::Ours => plan.ours.push(pgid), - Group::Reused => plan.reused.push(pgid), - Group::Gone => {} + let lock = File::open(&path).context("cannot open a run process guard lock")?; + loop { + match flock(&lock, FlockOperation::NonBlockingLockExclusive) { + Ok(()) => break, + Err(rustix::io::Errno::WOULDBLOCK) if Instant::now() < deadline => { + thread::sleep(POLL); + } + Err(rustix::io::Errno::WOULDBLOCK) => { + bail!("a run process guard is still stopping its workload"); + } + Err(error) => return Err(error).context("cannot check a run process guard lock"), + } } } - Ok(plan) + let incomplete = super::guard::incomplete_launches(root)?; + ensure!( + incomplete.is_empty(), + "process guards did not confirm cleanup for {}", + incomplete.join(", ") + ); + Ok(()) } #[cfg(test)] mod tests { use super::*; - use serde_json::json; - use std::os::unix::process::CommandExt; #[test] fn start_times_are_read_in_one_locale_and_time_zone() { @@ -231,90 +150,42 @@ mod tests { } #[test] - fn a_group_is_ours_only_when_its_leader_started_when_recorded() { - let table = [row(7, 7, "Tue Nov 11 12:00:00 2026")]; - assert_eq!( - classify(&table, 7, Some("Tue Nov 11 12:00:00 2026")).unwrap(), - Group::Ours - ); - assert_eq!( - classify(&table, 7, Some("Mon Nov 10 08:00:00 2026")).unwrap(), - Group::Reused, - "a reused process ID has a different start time" - ); - let unverified = classify(&table, 7, None).unwrap_err().to_string(); - assert!( - unverified.contains("start time was not recorded"), - "{unverified}" - ); - } - - #[test] - fn a_group_without_a_leader_is_ours_while_members_remain() { - let members = [row(8, 7, "Tue Nov 11 12:00:00 2026")]; - assert_eq!(classify(&members, 7, None).unwrap(), Group::Ours); - assert_eq!(classify(&[], 7, None).unwrap(), Group::Gone); - } - - #[test] - fn nothing_is_signalled_when_one_recorded_process_cannot_be_verified() { - let table = [row(7, 7, "A"), row(9, 9, "B")]; - let recorded = json!([ - {"product": "breg", "pid": 7, "pgid": 7, "started": "A"}, - {"product": "casework", "pid": 9, "pgid": 9, "started": null}, - ]); - assert!(plan(&recorded, &table).is_err()); - let recorded = json!([ - {"product": "breg", "pid": 7, "pgid": 7, "started": "A"}, - {"product": "casework", "pid": 9, "pgid": 9, "started": "other"}, - {"product": "gone", "pid": 11, "pgid": 11, "started": "C"}, - ]); - let plan = plan(&recorded, &table).unwrap(); - assert_eq!((plan.ours, plan.reused), (vec![7], vec![9])); - } + fn a_held_guard_lock_prevents_recovery() { + let root = tempfile::tempdir().unwrap(); + let guards = root.path().join("guards"); + std::fs::create_dir(&guards).unwrap(); + let path = guards.join("test.lock"); + let lock = std::fs::File::create(&path).unwrap(); + rustix::fs::flock(&lock, rustix::fs::FlockOperation::LockExclusive).unwrap(); - fn is_alive(pgid: u32) -> bool { - process_table().unwrap().iter().any(|row| row.pgid == pgid) - } - - /// A sleeping process in its own group, and a thread that reaps it once it - /// ends, since an unreaped child stays in the process table. - fn sleeper() -> (u32, thread::JoinHandle<()>) { - let mut child = Command::new("sleep") - .arg("60") - .process_group(0) - .spawn() - .unwrap(); - let pid = child.id(); - ( - pid, - thread::spawn(move || { - child.wait().unwrap(); - }), - ) + let error = wait_for_guards_until(root.path(), Instant::now()).unwrap_err(); + assert!(error.to_string().contains("process guard"), "{error:#}"); + drop(lock); + wait_for_guards_until(root.path(), Instant::now()).unwrap(); } #[test] - fn a_live_group_with_its_recorded_start_time_is_stopped() { - let (pid, reaper) = sleeper(); - let started = started_at(pid).expect("a live process has a start time"); - let recorded = json!([{"product": "breg", "pid": pid, "pgid": pid, "started": started}]); - let plan = plan(&recorded, &process_table().unwrap()).unwrap(); - assert_eq!(plan.ours, [pid]); - stop_groups(&plan.ours).unwrap(); - reaper.join().unwrap(); - assert!(!is_alive(pid)); + fn a_released_guard_must_have_confirmed_cleanup() { + let root = tempfile::tempdir().unwrap(); + let guards = root.path().join("guards"); + std::fs::create_dir(&guards).unwrap(); + std::fs::File::create(guards.join("test.lock")).unwrap(); + let receipt = guards.join("test.state"); + std::fs::write(&receipt, "launching\n").unwrap(); + + let error = wait_for_guards_until(root.path(), Instant::now()).unwrap_err(); + assert!( + error.to_string().contains("did not confirm cleanup"), + "{error:#}" + ); + std::fs::write(receipt, "complete\n").unwrap(); + wait_for_guards_until(root.path(), Instant::now()).unwrap(); } #[test] - fn a_live_group_with_another_start_time_is_left_running() { - let (pid, reaper) = sleeper(); - let recorded = json!([{"product": "breg", "pid": pid, "pgid": pid, "started": "Mon Jan 1 00:00:00 2001"}]); - let plan = plan(&recorded, &process_table().unwrap()).unwrap(); - assert!(plan.ours.is_empty()); - assert_eq!(plan.reused, [pid]); - assert!(is_alive(pid), "the unrelated process must keep running"); - signal(pid, Signal::KILL).unwrap(); - reaper.join().unwrap(); + fn a_run_without_guard_metadata_cannot_be_recovered_automatically() { + let root = tempfile::tempdir().unwrap(); + let error = wait_for_guards_until(root.path(), Instant::now()).unwrap_err(); + assert!(error.to_string().contains("process guards"), "{error:#}"); } } diff --git a/products/platform/scripts/test-rstack-dev.py b/products/platform/scripts/test-rstack-dev.py index 1b28722af6..8131cdfbf3 100644 --- a/products/platform/scripts/test-rstack-dev.py +++ b/products/platform/scripts/test-rstack-dev.py @@ -6,6 +6,7 @@ import argparse import base64 import concurrent.futures +import fcntl import hashlib import json import os @@ -422,6 +423,7 @@ def exercise(root, bin_dir, pg, project, ca, url, pg_env, spatial): try: exercise_runs(root, bin_dir, pg, checkout, state, env, pg_env) verify_lost_supervisor(bin_dir, pg, checkout, state, env, pg_env, owner_env) + verify_startup_loss(bin_dir, pg, checkout, state, env, pg_env) verify_provisioning_deadline(bin_dir, pg, checkout, state, env, pg_env) verify_spatial_refusal(root, bin_dir, pg, state, env, pg_env) verify_field_encryption(root, bin_dir, pg, state, env, pg_env) @@ -528,6 +530,7 @@ def exercise_runs(root, bin_dir, pg, checkout, state, env, pg_env): assert inventory(pg, pg_env) == baseline, report assert live_runs(state) == [], report assert_nothing_listening(bin_dir) + verify_failed_journey(root, bin_dir, checkout, state, env) verify_native_handoff(root, bin_dir, env) assert inventory(pg, pg_env) == baseline assert administrator_locks(pg, pg_env) == baseline_locks @@ -537,6 +540,28 @@ def exercise_runs(root, bin_dir, pg, checkout, state, env, pg_env): ) +def verify_failed_journey(root, bin_dir, checkout, state, env): + project = root / "failed-journey" + shutil.copytree(checkout, project) + plan = project / "dev.yaml" + authored = plan.read_text() + expectation = " expect: {status: 201}" + assert expectation in authored + plan.write_text(authored.replace(expectation, " expect: {status: 202}", 1)) + report = dev(bin_dir, env, project, "start", ".", expected=1) + run = report["run_id"] + try: + assert report["schema_version"] == "registry.rstack.dev.v1", report + assert report["status"] == "ready" and report["journey"]["result"] == "failed" + status = dev(bin_dir, env, project, "status", run) + assert status["status"] == "ready" and status["journey"] == report["journey"] + assert all(accepts_connections(url) for url in report["products"].values()) + finally: + assert dev(bin_dir, env, project, "stop", run)["status"] == "stopped" + assert not (state / run).exists() + print("A failed journey exited 1 with an inspectable run report and live products") + + def verify_provisioning_deadline(bin_dir, pg, checkout, state, env, pg_env): baseline = inventory(pg, pg_env) baseline_locks = administrator_locks(pg, pg_env) @@ -602,28 +627,7 @@ def listener_pids(url): def recover_lost_supervisor(pg, owner_env, recovery): - """Release a lost run by hand, from the facts `stop` reports. - - Order: stop each process group (TERM, then KILL), drop every database - (WITH (FORCE) ends leftover sessions), drop every role, then remove the - run directory. - """ - for process in recovery["processes"]: - group = process["pgid"] - os.killpg(group, signal.SIGTERM) - - def gone(group=group): - try: - os.killpg(group, 0) - except ProcessLookupError: - return True - return False - - try: - eventually(gone, "a process group ignored SIGTERM", timeout=10) - except AssertionError: - os.killpg(group, signal.SIGKILL) - eventually(gone, "a process group survived SIGKILL", timeout=10) + """Release the PostgreSQL objects and directory from the recovery report.""" objects = recovery["objects"] statements = [ f'DROP DATABASE IF EXISTS "{item["database"]}" WITH (FORCE);' @@ -638,6 +642,12 @@ def verify_lost_supervisor(bin_dir, pg, checkout, state, env, pg_env, owner_env) baseline_locks = administrator_locks(pg, pg_env) report = dev(bin_dir, env, checkout, "start", ".") run = report["run_id"] + recorded = json.loads((state / run / "processes.json").read_text()) + for process in recorded: + assert os.getpgid(process["pid"]) == process["pgid"], process + assert str(process["pid"]) in listener_pids( + report["products"][process["product"]] + ) worker = command( "pgrep", "-f", f"project-dev --worker {re.escape(str(state / run))}" ) @@ -652,14 +662,9 @@ def verify_lost_supervisor(bin_dir, pg, checkout, state, env, pg_env, owner_env) recovery = stopped["recovery"] assert recovery["run_directory"] == str(state / run), recovery processes = recovery["processes"] + assert processes == recorded assert sorted(p["product"] for p in processes) == ["breg", "casework"], recovery - for process in processes: - os.kill(process["pid"], 0) - assert os.getpgid(process["pid"]) == process["pgid"], recovery - listeners = listener_pids(report["products"][process["product"]]) - assert str(process["pid"]) in listeners, recovery - # The pattern behind assert_nothing_listening sees the live products. - assert len(product_pids(bin_dir)) == 2, recovery + assert_nothing_listening(bin_dir, report["products"].values()) databases, roles = inventory(pg, pg_env) assert {item["database"] for item in recovery["objects"]} == ( databases - baseline[0] @@ -699,7 +704,6 @@ def verify_recover(bin_dir, pg, checkout, state, env, pg_env): recovered = dev(bin_dir, env, checkout, "stop", run, "--recover") assert recovered["status"] == "stopped", recovered assert len(recovered["recovered"]["stopped_process_groups"]) == 2, recovered - assert recovered["recovered"]["skipped_process_groups"] == [], recovered assert_nothing_listening(bin_dir, report["products"].values()) assert inventory(pg, pg_env) == baseline eventually( @@ -711,28 +715,8 @@ def verify_recover(bin_dir, pg, checkout, state, env, pg_env): assert dev(bin_dir, env, checkout, "stop", run, "--recover")["status"] == "stopped" print("stop --recover released a lost supervisor's processes, databases and roles") - # A recorded group whose leader started at another time is not the run's: - # recovery leaves it running and still releases the rest. - report, run = lose_supervisor(bin_dir, checkout, state, env) - processes_file = state / run / "processes.json" - processes = json.loads(processes_file.read_text()) - [casework] = [p for p in processes if p["product"] == "casework"] - casework["started"] = "Mon Jan 1 00:00:00 2001" - processes_file.write_text(json.dumps(processes)) - try: - recovered = dev(bin_dir, env, checkout, "stop", run, "--recover") - assert recovered["recovered"]["skipped_process_groups"] == [ - casework["pgid"] - ], recovered - os.killpg(casework["pgid"], 0) - assert inventory(pg, pg_env) == baseline - assert live_runs(state) == [] - finally: - try: - os.killpg(casework["pgid"], signal.SIGKILL) - except ProcessLookupError: - pass - assert_nothing_listening(bin_dir, report["products"].values()) + verify_unrelated_groups(bin_dir, checkout, state, env) + verify_wrong_cluster(bin_dir, pg, checkout, state, env, pg_env) # Without an administrator URL nothing is signalled or dropped, and the # error carries the facts for a hand release. @@ -742,12 +726,204 @@ def verify_recover(bin_dir, pg, checkout, state, env, pg_env): assert "no PostgreSQL administrator URL" in refused["error"]["message"], refused assert refused["run_id"] == run, refused assert len(refused["recovery"]["processes"]) == 2, refused - assert len(product_pids(bin_dir)) == 2, refused + assert (state / run).is_dir() and inventory(pg, pg_env) != baseline dev(bin_dir, env, checkout, "stop", run, "--recover") assert_nothing_listening(bin_dir, report["products"].values()) assert inventory(pg, pg_env) == baseline assert live_runs(state) == [] - print("stop --recover skips a reused process ID and fails with facts when it cannot proceed") + print("stop --recover preserves the run when no administrator URL is available") + + +def verify_unrelated_groups(bin_dir, checkout, state, env): + # Recovery must never signal based on a persisted numeric PID, even when + # the current process table appears to confirm it. The test owns both + # unrelated groups and leaves one unreaped until recovery has finished. + with ( + subprocess.Popen(["/bin/sleep", "90"], start_new_session=True) as live, + subprocess.Popen(["/usr/bin/true"], start_new_session=True) as zombie, + ): + try: + + def process_field(pid, field): + return ( + checked( + "ps", + "-p", + str(pid), + "-o", + f"{field}=", + env={**env, "LC_ALL": "C", "TZ": "UTC"}, + ) + .decode() + .strip() + ) + + eventually( + lambda: process_field(zombie.pid, "stat").startswith("Z"), + "the unreaped child did not become a zombie", + ) + report, run = lose_supervisor(bin_dir, checkout, state, env) + processes_file = state / run / "processes.json" + processes = json.loads(processes_file.read_text()) + for child in (live, zombie): + processes.append( + { + "product": "breg", + "pid": child.pid, + "pgid": child.pid, + "started": " ".join(process_field(child.pid, "lstart").split()), + } + ) + processes_file.write_text(json.dumps(processes)) + # Hold a guard long enough to observe what status reports while + # recovery holds worker.lock and the old ready report is stale. + guard_path = state / run / "guards" / "test.lock" + with concurrent.futures.ThreadPoolExecutor(1) as pool: + with guard_path.open("wb") as guard: + fcntl.flock(guard, fcntl.LOCK_EX) + recovering = pool.submit( + dev, bin_dir, env, checkout, "stop", run, "--recover" + ) + report_path = state / run / "report.json" + eventually( + lambda: json.loads(report_path.read_text())["status"] == "stopping", + "recovery did not mark the run as stopping", + ) + assert dev(bin_dir, env, checkout, "status", run)["status"] == "stopping" + recovered = recovering.result(timeout=10) + assert recovered["status"] == "stopped", recovered + assert live.poll() is None, "recovery signaled an unrelated process group" + assert process_field(zombie.pid, "stat").startswith("Z") + assert_nothing_listening(bin_dir, report["products"].values()) + finally: + if live.poll() is None: + live.terminate() + print("Recovery completed with unrelated live and unreaped process groups present") + + +def verify_wrong_cluster(bin_dir, pg, checkout, state, env, pg_env): + other = state.parent / "other-postgres" + other.mkdir() + cluster, ca, url, other_env = prepare_postgres(other, pg) + checked(pg / "pg_ctl", "-D", cluster, "-l", other / "postgres.log", "-w", "start") + try: + wrong_url, _ = create_owner(pg, other_env, url) + other_baseline = inventory(pg, other_env) + baseline = inventory(pg, pg_env) + report, run = lose_supervisor(bin_dir, checkout, state, env) + original_objects = inventory(pg, pg_env) + assert original_objects != baseline + try: + refused = dev( + bin_dir, + { + **env, + "RSTACK_DEV_DATABASE_URL": wrong_url, + "RSTACK_DEV_DATABASE_CA": str(ca), + }, + checkout, + "stop", + run, + "--recover", + expected=3, + ) + assert ( + refused["run_id"] == run and refused["error"]["class"] == "operational" + ) + assert "cluster" in refused["error"]["message"], refused + assert (state / run).is_dir() + assert inventory(pg, pg_env) == original_objects + assert inventory(pg, other_env) == other_baseline + + # PostgreSQL advisory locks are scoped to the maintenance database, + # so even the same cluster must use the database held by startup. + alternate = "devowner_alternate_admin" + psql(pg, pg_env, f"CREATE DATABASE {alternate};") + try: + owner_url = env["RSTACK_DEV_DATABASE_URL"] + wrong_database_url = f"{owner_url.rsplit('/', 1)[0]}/{alternate}" + refused = dev( + bin_dir, + {**env, "RSTACK_DEV_DATABASE_URL": wrong_database_url}, + checkout, + "stop", + run, + "--recover", + expected=3, + ) + assert "maintenance database" in refused["error"]["message"], refused + assert (state / run).is_dir() and inventory( + pg, pg_env + ) == original_objects + finally: + psql(pg, pg_env, f"DROP DATABASE {alternate};") + finally: + if (state / run).is_dir(): + assert ( + dev(bin_dir, env, checkout, "stop", run, "--recover")["status"] + == "stopped" + ) + assert inventory(pg, pg_env) == baseline + assert_nothing_listening(bin_dir, report["products"].values()) + finally: + checked(pg / "pg_ctl", "-D", cluster, "-m", "fast", "-w", "stop") + print( + "Recovery refused a different TLS PostgreSQL cluster and retried on the original" + ) + + +def verify_startup_loss(bin_dir, pg, checkout, state, env, pg_env): + baseline = inventory(pg, pg_env) + shim = state.parent / "slow-bregctl" + ready = state.parent / "slow-bregctl-ready" + shim.write_text("""#!/usr/bin/env python3 +import os +import sys +import time +from pathlib import Path + +if sys.argv[1] == "test": + Path(os.environ["RSTACK_TEST_MARKER"]).touch() + time.sleep(90) + raise SystemExit(1) +os.execv(os.environ["RSTACK_TEST_BREGCTL"], sys.argv) +""") + shim.chmod(0o700) + slow_env = { + **env, + "RSTACK_BREGCTL_BIN": str(shim), + "RSTACK_TEST_MARKER": str(ready), + "RSTACK_TEST_BREGCTL": str(bin_dir / "bregctl"), + } + with concurrent.futures.ThreadPoolExecutor(1) as pool: + starting = pool.submit( + dev, bin_dir, slow_env, checkout, "start", ".", expected=3 + ) + eventually(ready.is_file, "the provisioning tool did not start", timeout=30) + [run_root] = live_runs(state) + assert not (run_root / "processes.json").exists() + assert inventory(pg, pg_env) != baseline + worker = command( + "pgrep", "-f", f"project-dev --worker {re.escape(str(run_root))}" + ) + [supervisor] = worker.stdout.split() + os.kill(int(supervisor), signal.SIGKILL) + assert starting.result(timeout=10)["run_id"] == run_root.name + assert ( + dev(bin_dir, env, checkout, "stop", run_root.name, "--recover")["status"] + == "stopped" + ) + eventually( + lambda: ( + command("pgrep", "-f", f"{re.escape(str(shim))} test").returncode == 1 + ), + "the worker left an unrecorded provisioning tool running", + ) + assert live_runs(state) == [] + assert inventory(pg, pg_env) == baseline + print( + "Recovery drained a guarded provisioning tool before any runtime PID was recorded" + ) def verify_native_handoff(root, bin_dir, env): From d185bc08fcbcdd3aae363b189b342b43db415539 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 20:17:06 +0700 Subject: [PATCH 18/24] docs: clarify rstack dev installation, recovery and journey reports Signed-off-by: Jeremi Joslin --- .../run-a-project-with-rstack-dev.mdx | 47 ++++++++------ products/platform/CHANGELOG.md | 15 +++-- products/platform/PROJECT-DEV.md | 64 +++++++++++-------- 3 files changed, 74 insertions(+), 52 deletions(-) diff --git a/docs/site/src/content/docs/configure/run-a-project-with-rstack-dev.mdx b/docs/site/src/content/docs/configure/run-a-project-with-rstack-dev.mdx index d753f4274d..e04a8c82bc 100644 --- a/docs/site/src/content/docs/configure/run-a-project-with-rstack-dev.mdx +++ b/docs/site/src/content/docs/configure/run-a-project-with-rstack-dev.mdx @@ -39,14 +39,17 @@ install `casework` the same way. rstack install breg casework ``` -Check that the commands resolve before you continue: +Check the selected installation before you continue: ```sh test-skip="needs the installed release" -rstack dev --help +rstack components list ``` -Success is the list of `check`, `start`, `status`, `token` and `stop`. If `rstack` is missing, run -`rstack doctor`; if a component is missing, `start` names the `rstack install` command to run. +Success is a `breg` line and a `casework` line that each say `installed`. `available` means that the +selected release provides the component but it is not installed, so run the `rstack install` +command again. `unavailable` means that the selected release does not provide the component for +your platform. If your shell cannot find `rstack`, or the command says `No installation selected`, +repeat the verified `rstack` installation steps linked above. ## Start a local TLS PostgreSQL server @@ -221,8 +224,15 @@ pg_ctl -D "$DEV/data" stop ## When a run fails -Read the error first. With `--json`, a refusal prints one object on standard output and the -process exit status tells you whose problem it is: +A failed journey is different from a startup failure. `start` exits 1 but prints the normal run +report. With `--json`, its `schema_version` is `registry.rstack.dev.v1`, its `status` is `ready`, +and `journey.result` is `failed`; the journey includes the failed step and its reason. The services +keep running, and `rstack dev status RUN_ID` repeats the report, so use the run ID to inspect the +products and logs before you stop the run. + +Other refusals and failures produce an error report instead. With `--json`, the report is one +object on standard output with `schema_version` `registry.rstack.error.v1` and `status` `refused`. +Its process exit status and `error.class` identify the kind of failure: | Exit | `error.class` | Meaning | | --- | --- | --- | @@ -240,12 +250,11 @@ system error number varies by platform): Start the server and run the command again. Without `--json`, the same message goes to standard error as `rstack dev: MESSAGE` with the same exit status. -A failed journey exits 1 and leaves the run alive, so you can inspect it and then stop it. The -report includes the run ID. When a start fails before that, the run directory is removed but its -logs are kept at `/failed//logs`, and the error names the path. Otherwise, a live -run's logs are in `/logs`: `supervisor.log` for the supervisor and one file per product stage, -such as `breg-check.log` or `casework-runtime.log`. `rstack dev status RUN_ID` names the logs -directory. Delete kept logs yourself. +When a start fails before the run becomes available, the run directory is removed but its logs are +kept at `/failed//logs`, and the error names the path. A live run's logs are in +`/logs`: `supervisor.log` for the supervisor and one file per product stage, such as +`breg-check.log` or `casework-runtime.log`. `rstack dev status RUN_ID` names the logs directory. +Delete kept logs yourself. When a run still holds databases, roles or processes, the error carries its `run_id` and a `recovery` object listing them. This happens if PostgreSQL is unavailable during cleanup for @@ -256,15 +265,17 @@ run with: rstack dev stop RUN_ID --recover ``` -`--recover` checks everything it needs, then stops the recorded process groups, drops the -databases and roles, and removes the run directory, including the run's development key file. It -refuses a run whose supervisor is still running; use plain `stop` for that. Running it again -after a partial failure is safe. Success is `status` `stopped` and a `recovered` object. When it -cannot finish, the error names what to release by hand. The +If the supervisor is still running, use plain `stop`. For a lost supervisor, its process guards +stop the products and any product tools still running. Use the original PostgreSQL cluster and +maintenance database for `--recover`. Recovery waits for the guards, drops the databases and +roles, and removes the run directory, including its development key file. If a check or cleanup +step fails, the report explains what to inspect before retrying. The [reference](https://github.com/registrystack/registry-stack/blob/main/products/platform/PROJECT-DEV.md) lists the manual steps and the recovery fields. -{/* Evidence: products/platform/PROJECT-DEV.md; crates/registry-rstack/src/cli.rs, dispatch_dev. */} +{/* Evidence: products/platform/PROJECT-DEV.md; crates/registry-rstack/src/cli.rs, dispatch_dev; + crates/registry-bregctl/src/project_dev/mod.rs; + crates/registry-bregctl/src/project_dev/journey.rs. */} ## Next diff --git a/products/platform/CHANGELOG.md b/products/platform/CHANGELOG.md index 880125b763..9c943d5bcc 100644 --- a/products/platform/CHANGELOG.md +++ b/products/platform/CHANGELOG.md @@ -7,15 +7,16 @@ [PROJECT-DEV.md](PROJECT-DEV.md). It is an alpha developer tool and may change. - `rstack dev stop RUN_ID --recover` releases a run whose supervisor is gone. - It stops each recorded process group whose leader still has the start time - recorded for it, drops the run's databases and roles, and removes the run - directory. Running it again after a partial failure is safe. -- With `--json`, every `rstack dev` refusal or failure prints one + It waits for the process guards to finish, drops the run's databases and roles + on the original PostgreSQL cluster, and removes the run directory. You can + retry after a partial failure. +- With `--json`, refusals and startup failures from `rstack dev` print a `registry.rstack.error.v1` object, published as `schemas/rstack-dev-error.v1.schema.json`. Its `error.class` is `domain` - (exit 1), `usage` (exit 2) or `operational` (exit 3); a run that still holds - resources adds its `run_id`, with a `recovery` object when the resources to - release are known. + (exit 1), `usage` (exit 2) or `operational` (exit 3). If the run still holds + resources, the report includes its `run_id` and any known `recovery` details. + A failed journey exits 1 with the regular run report and leaves the run + available for inspection. - A BReg project with encrypted fields runs under `rstack dev` without extra setup. Each run draws its own development data key into its run directory, readable only by your user, and the key is removed with the run. diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index b5add6e03b..728752c7cd 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -27,9 +27,14 @@ product authoring tools. The check works with the database offline. With Use `--deny-warnings` to refuse any unaccepted warning. Start reports the product URLs and the journey result. -A failed journey exits 1 and includes the live run ID so you can inspect the -result and stop the run. Startup failures clean up or report the run ID if -cleanup is pending. The TTL starts at readiness and defaults to 30 minutes. +A failed journey still produces the normal `registry.rstack.dev.v1` run report: +the report has `status` `ready`, `journey.result` `failed`, the failed step and +its reason, and the live run ID. `start` exits 1, but the supervisor and +products keep running. Use `status` to repeat the report, inspect the products +and logs, and then use `stop` to release the run. This differs from a startup +failure, which produces a `registry.rstack.error.v1` error report and cleans up, +or reports the run ID if cleanup is pending. The TTL starts at readiness and +defaults to 30 minutes. Supply one whole-number value from `1s` to `24h` with an `s`, `m`, or `h` suffix. The [licensing example](examples/licensing/dev.yaml) creates an application in @@ -49,6 +54,8 @@ Superuser is not needed for a non-spatial run. The runner sets `createrole_self_grant` on its own sessions, so you do not configure it. The administrator also installs `btree_gist`, which PostgreSQL treats as a trusted extension. +The runner reads `pg_catalog.pg_control_system()` to identify the cluster for +recovery. PostgreSQL allows this function by default. The administrator URL always carries a password. Set it through `RSTACK_DEV_DATABASE_URL`, because `--database-url` is visible to other local @@ -257,9 +264,9 @@ and the error names that path. Delete kept logs yourself. If PostgreSQL becomes unavailable during cleanup, the supervisor retries with backoff for up to 30 minutes. It then writes `cleanup-pending.json` in the run -directory, and `status` reports `cleanup-pending` with the remaining databases, -roles and processes. If the supervisor is lost, `stop` and `status` report the -run directory, the process groups, and the exact database and role names. +directory. If the supervisor is lost, the run's process guards stop the product +runtimes and any product tool that was still running. `stop` and `status` report +the run directory, the recorded process groups, and the database and role names. Release a run whose supervisor is gone with: @@ -267,38 +274,41 @@ Release a run whose supervisor is gone with: rstack dev stop RUN_ID --recover [--database-url URL] ``` -`--database-url` defaults to `RSTACK_DEV_DATABASE_URL`, as for `start`, and -the URL needs the same administrator rights. `--recover` first checks -everything it needs, then acts: it refuses a run whose supervisor is still -running (use plain `stop`), stops each recorded process group, drops the -databases and roles, and removes the run directory. It signals a process group -only while the group leader's start time equals the one recorded when the run -started. If the recorded ID now belongs to another process, `--recover` leaves -that process running and lists the group under `skipped_process_groups` in the -report. If a start time was not recorded and the leader is still alive, -`--recover` refuses and signals nothing. When it cannot finish, the error -names the reason and lists the run directory, process groups, databases and -roles to release by hand. Running it again after a partial failure is safe. +Use plain `stop` while the supervisor is still running. For recovery, use an +administrator URL for the same PostgreSQL cluster and maintenance database used +at startup, with the same administrator rights. `--database-url` defaults to +`RSTACK_DEV_DATABASE_URL`, as for `start`. Recovery checks the recorded cluster +identifier and maintenance database before dropping any resources. It waits for +the process guards to finish, then drops the databases and roles and removes +the run directory. If any step fails, the report gives the reason and the +resources you can check before retrying. +For runs from an older development build without recovery metadata, use the +manual cleanup steps below. A successful recovery reports `status` `stopped` and a `recovered` object with -`stopped_process_groups`, `skipped_process_groups` and `databases_and_roles`. +`stopped_process_groups` and `databases_and_roles`. -To release the resources by hand instead, in this order: +If you need to finish cleanup by hand, do it in this order: -1. Stop each listed process group, for example `kill -TERM -PGID`. -2. As the administrator, run `DROP DATABASE IF EXISTS "NAME" WITH (FORCE)` for - each listed database. -3. As the administrator, run `DROP ROLE IF EXISTS "NAME"` for each listed role, - including a `breg_bbox_*` role from a spatial run. +1. Verify which product processes still belong to the run and stop those + groups. Use the run's configuration paths and process details to establish + ownership before signaling a group. +2. Connect to the original PostgreSQL cluster as the administrator. Run + `DROP DATABASE IF EXISTS "NAME" WITH (FORCE)` for each listed database. +3. Run `DROP ROLE IF EXISTS "NAME"` for each listed role, including a + `breg_bbox_*` role from a spatial run. 4. Remove the run directory. It holds the run's development field-encryption key when `field_encryption_key` is listed. ### Errors and limitations -With `--json`, every refusal or failure of `rstack dev` start, status, token -and stop prints one JSON object on standard output, validated by +With `--json`, a refusal or startup failure from `rstack dev` start, status, +token or stop prints one JSON object on standard output, validated by `schemas/rstack-dev-error.v1.schema.json`. The launcher prints the same object for failures before the helper starts, such as a missing component. +When a journey assertion fails, `start` exits 1 and prints the regular +`registry.rstack.dev.v1` run report with `status` `ready` and +`journey.result` `failed`. The run remains available for inspection. - `schema_version` is `registry.rstack.error.v1` and `status` is `refused`. - `error.class` is `domain` (exit 1: the request, the project, the plan or the From 39b118e50b7e8a8c23b0f46a44a80556e4c54af1 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 20:18:35 +0700 Subject: [PATCH 19/24] fix: keep guard cleanup errors visible during drop Signed-off-by: Jeremi Joslin --- crates/registry-bregctl/src/project_dev/database.rs | 10 ++++------ crates/registry-bregctl/src/project_dev/guard.rs | 9 +++++---- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/database.rs b/crates/registry-bregctl/src/project_dev/database.rs index bfd0223f25..ebdb4283c8 100644 --- a/crates/registry-bregctl/src/project_dev/database.rs +++ b/crates/registry-bregctl/src/project_dev/database.rs @@ -219,6 +219,8 @@ fn read_recovery_identity( }) } +type AdminConnectionTask = tokio::task::JoinHandle>; + fn open_admin_session( runtime: &Runtime, config: &Config, @@ -226,11 +228,7 @@ fn open_admin_session( ownership_lock: &str, expected_identity: Option<&RecoveryIdentity>, deadline: Instant, -) -> Result<( - Client, - tokio::task::JoinHandle>, - RecoveryIdentity, -)> { +) -> Result<(Client, AdminConnectionTask, RecoveryIdentity)> { let (client, connection) = runtime.block_on(bounded_connection( config.connect(tls.clone()), remaining_timeout(deadline, CONNECTION_TIMEOUT)?, @@ -277,7 +275,7 @@ enum Acquisition { pub(super) struct Database { client: Client, - _connection_task: tokio::task::JoinHandle>, + _connection_task: AdminConnectionTask, admin_session_failed: bool, tls: MakeRustlsConnect, admin_config: Config, diff --git a/crates/registry-bregctl/src/project_dev/guard.rs b/crates/registry-bregctl/src/project_dev/guard.rs index a5409cbb3e..c308b7dee2 100644 --- a/crates/registry-bregctl/src/project_dev/guard.rs +++ b/crates/registry-bregctl/src/project_dev/guard.rs @@ -470,10 +470,11 @@ impl Child { impl Drop for Child { fn drop(&mut self) { - if self.controller.take().is_some() { - if self.guard.wait().is_ok() { - let _ = complete_state(&self.state_file); - } + if self.controller.is_none() { + return; + } + if let Err(error) = self.stop() { + eprintln!("development process guard cleanup failed: {error:#}"); } } } From 5099a7e80c9a1692fd60c355aecc778dc61b278b Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 20:35:42 +0700 Subject: [PATCH 20/24] test: allow reaped workload descendants to remain visible as zombies Signed-off-by: Jeremi Joslin --- .../registry-bregctl/src/project_dev/guard.rs | 40 +++++++++++++++---- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/guard.rs b/crates/registry-bregctl/src/project_dev/guard.rs index c308b7dee2..105b6956de 100644 --- a/crates/registry-bregctl/src/project_dev/guard.rs +++ b/crates/registry-bregctl/src/project_dev/guard.rs @@ -555,11 +555,10 @@ mod tests { command .args([ "-c", - &format!( - "sh -c 'trap \"\" TERM; exec sleep 60' & printf '%s' \"$!\" > '{}' ; wait", - nested_file.display() - ), + r#"sh -c 'trap "" TERM; printf "%s" "$$" > "$1.tmp" && mv "$1.tmp" "$1" && exec sleep 60' sh "$1" & wait"#, + "guard-fixture", ]) + .arg(&nested_file) .process_group(0) .stdin(Stdio::null()); let guard_lock = lock.clone(); @@ -596,10 +595,35 @@ mod tests { rustix::process::getpgid(Some(leader)).unwrap_err(), rustix::io::Errno::SRCH ); - assert_eq!( - rustix::process::getpgid(Some(nested)).unwrap_err(), - rustix::io::Errno::SRCH - ); + // The guard owns the leader. The OS adopts the nested child when + // the leader exits, and may keep it visible as a zombie after SIGKILL. + let deadline = Instant::now() + Duration::from_secs(5); + let pid = nested.as_raw_pid().to_string(); + loop { + let output = process::Command::new("ps") + .args(["-p", &pid, "-o", "stat="]) + .output() + .unwrap(); + if output.status.code() == Some(1) + && output.stdout.is_empty() + && output.stderr.is_empty() + { + break; + } + assert!( + output.status.success(), + "ps could not read the nested process" + ); + let state = String::from_utf8(output.stdout).unwrap(); + if state.trim_start().starts_with('Z') { + break; + } + assert!( + Instant::now() < deadline, + "the nested process is still live: {state}" + ); + thread::sleep(POLL); + } } #[test] From 6763481e673b743b60b2b6d36228e1f62b20a0db Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 21:52:14 +0700 Subject: [PATCH 21/24] ci: fall back to the Ubuntu archive when the Azure mirror cannot resolve Signed-off-by: Jeremi Joslin --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02a92deccb..597d8334b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2702,6 +2702,10 @@ jobs: shell: bash run: | set -euo pipefail + # Use the Ubuntu archive if the runner's Azure mirror cannot resolve. + if [[ -f /etc/apt/apt-mirrors.txt ]] && ! timeout 10 getent ahosts azure.archive.ubuntu.com >/dev/null; then + printf '%s\n' 'https://archive.ubuntu.com/ubuntu/' | sudo tee /etc/apt/apt-mirrors.txt >/dev/null + fi sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y sudo env DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends postgresql-18 postgresql-18-postgis-3 /usr/lib/postgresql/18/bin/postgres --version From f016e87b16eec7a80d7c0952601198679898444e Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 22:03:09 +0700 Subject: [PATCH 22/24] ci: use the Ubuntu archive fallback for docs shell dependencies Signed-off-by: Jeremi Joslin --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 597d8334b0..4a3398e435 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3075,7 +3075,13 @@ jobs: cache-dependency-path: docs/site/package-lock.json - name: Install docs shell dependency + shell: bash run: | + set -euo pipefail + # Use the Ubuntu archive if the runner's Azure mirror cannot resolve. + if [[ -f /etc/apt/apt-mirrors.txt ]] && ! timeout 10 getent ahosts azure.archive.ubuntu.com >/dev/null; then + printf '%s\n' 'https://archive.ubuntu.com/ubuntu/' | sudo tee /etc/apt/apt-mirrors.txt >/dev/null + fi sudo apt-get update sudo apt-get install --yes zsh From 0fee982bc644a532d79f4f568f69340bf28f9163 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 22:15:21 +0700 Subject: [PATCH 23/24] ci: select a reachable Ubuntu archive before installing packages Signed-off-by: Jeremi Joslin --- .github/scripts/select-ubuntu-archive.sh | 18 ++++++++++++++++++ .github/workflows/ci.yml | 6 ++++-- 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 .github/scripts/select-ubuntu-archive.sh diff --git a/.github/scripts/select-ubuntu-archive.sh b/.github/scripts/select-ubuntu-archive.sh new file mode 100644 index 0000000000..7dde9ab967 --- /dev/null +++ b/.github/scripts/select-ubuntu-archive.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +for mirror in \ + https://archive.ubuntu.com/ubuntu/ \ + https://mirrors.edge.kernel.org/ubuntu/ \ + https://mirror.math.princeton.edu/pub/ubuntu/; do + host="${mirror#https://}" + if timeout 5 getent ahosts "${host%%/*}" >/dev/null && + curl --proto '=https' --fail --silent --show-error --head --max-time 10 \ + "${mirror}dists/noble/InRelease" >/dev/null; then + printf '%s\n' "$mirror" + exit 0 + fi +done + +printf '%s\n' 'No Ubuntu Noble archive mirror is reachable.' >&2 +exit 1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a3398e435..cc32fb0c38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2704,7 +2704,8 @@ jobs: set -euo pipefail # Use the Ubuntu archive if the runner's Azure mirror cannot resolve. if [[ -f /etc/apt/apt-mirrors.txt ]] && ! timeout 10 getent ahosts azure.archive.ubuntu.com >/dev/null; then - printf '%s\n' 'https://archive.ubuntu.com/ubuntu/' | sudo tee /etc/apt/apt-mirrors.txt >/dev/null + mirror="$(bash .github/scripts/select-ubuntu-archive.sh)" + printf '%s\n' "$mirror" | sudo tee /etc/apt/apt-mirrors.txt >/dev/null fi sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y sudo env DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends postgresql-18 postgresql-18-postgis-3 @@ -3080,7 +3081,8 @@ jobs: set -euo pipefail # Use the Ubuntu archive if the runner's Azure mirror cannot resolve. if [[ -f /etc/apt/apt-mirrors.txt ]] && ! timeout 10 getent ahosts azure.archive.ubuntu.com >/dev/null; then - printf '%s\n' 'https://archive.ubuntu.com/ubuntu/' | sudo tee /etc/apt/apt-mirrors.txt >/dev/null + mirror="$(bash .github/scripts/select-ubuntu-archive.sh)" + printf '%s\n' "$mirror" | sudo tee /etc/apt/apt-mirrors.txt >/dev/null fi sudo apt-get update sudo apt-get install --yes zsh From 9dc0abd82b86c5ec413a58e2c408a2aad3c01359 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 22:18:07 +0700 Subject: [PATCH 24/24] ci: check every Noble package index when selecting a mirror Signed-off-by: Jeremi Joslin --- .github/scripts/select-ubuntu-archive.sh | 26 +++++++++++++++--------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/scripts/select-ubuntu-archive.sh b/.github/scripts/select-ubuntu-archive.sh index 7dde9ab967..c15e7e8714 100644 --- a/.github/scripts/select-ubuntu-archive.sh +++ b/.github/scripts/select-ubuntu-archive.sh @@ -1,17 +1,23 @@ #!/usr/bin/env bash set -euo pipefail +has_noble_indexes() { + local archive="$1" suite + for suite in noble noble-updates noble-backports noble-security; do + curl --proto '=https' --fail --silent --show-error --head --max-time 10 \ + "${archive}dists/$suite/InRelease" >/dev/null || return 1 + done +} + for mirror in \ - https://archive.ubuntu.com/ubuntu/ \ - https://mirrors.edge.kernel.org/ubuntu/ \ - https://mirror.math.princeton.edu/pub/ubuntu/; do - host="${mirror#https://}" - if timeout 5 getent ahosts "${host%%/*}" >/dev/null && - curl --proto '=https' --fail --silent --show-error --head --max-time 10 \ - "${mirror}dists/noble/InRelease" >/dev/null; then - printf '%s\n' "$mirror" - exit 0 - fi + https://archive.ubuntu.com/ubuntu/ \ + https://mirrors.edge.kernel.org/ubuntu/ \ + https://mirror.math.princeton.edu/pub/ubuntu/; do + host="${mirror#https://}" + if timeout 5 getent ahosts "${host%%/*}" >/dev/null && has_noble_indexes "$mirror"; then + printf '%s\n' "$mirror" + exit 0 + fi done printf '%s\n' 'No Ubuntu Noble archive mirror is reachable.' >&2