diff --git a/.github/scripts/select-ubuntu-archive.sh b/.github/scripts/select-ubuntu-archive.sh new file mode 100644 index 0000000000..c15e7e8714 --- /dev/null +++ b/.github/scripts/select-ubuntu-archive.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +has_noble_indexes() { + local archive="$1" suite + for suite in noble noble-updates noble-backports noble-security; do + curl --proto '=https' --fail --silent --show-error --head --max-time 10 \ + "${archive}dists/$suite/InRelease" >/dev/null || return 1 + done +} + +for mirror in \ + https://archive.ubuntu.com/ubuntu/ \ + https://mirrors.edge.kernel.org/ubuntu/ \ + https://mirror.math.princeton.edu/pub/ubuntu/; do + host="${mirror#https://}" + if timeout 5 getent ahosts "${host%%/*}" >/dev/null && has_noble_indexes "$mirror"; then + printf '%s\n' "$mirror" + exit 0 + fi +done + +printf '%s\n' 'No Ubuntu Noble archive mirror is reachable.' >&2 +exit 1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02a92deccb..cc32fb0c38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2702,6 +2702,11 @@ jobs: shell: bash run: | set -euo pipefail + # Use the Ubuntu archive if the runner's Azure mirror cannot resolve. + if [[ -f /etc/apt/apt-mirrors.txt ]] && ! timeout 10 getent ahosts azure.archive.ubuntu.com >/dev/null; then + mirror="$(bash .github/scripts/select-ubuntu-archive.sh)" + printf '%s\n' "$mirror" | sudo tee /etc/apt/apt-mirrors.txt >/dev/null + fi sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y sudo env DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends postgresql-18 postgresql-18-postgis-3 /usr/lib/postgresql/18/bin/postgres --version @@ -3071,7 +3076,14 @@ jobs: cache-dependency-path: docs/site/package-lock.json - name: Install docs shell dependency + shell: bash run: | + set -euo pipefail + # Use the Ubuntu archive if the runner's Azure mirror cannot resolve. + if [[ -f /etc/apt/apt-mirrors.txt ]] && ! timeout 10 getent ahosts azure.archive.ubuntu.com >/dev/null; then + mirror="$(bash .github/scripts/select-ubuntu-archive.sh)" + printf '%s\n' "$mirror" | sudo tee /etc/apt/apt-mirrors.txt >/dev/null + fi sudo apt-get update sudo apt-get install --yes zsh diff --git a/Cargo.lock b/Cargo.lock index fc659c6a13..4a21ca40b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1656,7 +1656,7 @@ dependencies = [ "document-features", "mio", "parking_lot", - "rustix 1.1.5", + "rustix 1.1.6", "signal-hook 0.3.18", "signal-hook-mio", "winapi", @@ -4166,7 +4166,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57804b2c9b69967f1536a56f86297e367a33b19e98852ed624b84551cdbc0d90" dependencies = [ - "rustix 1.1.5", + "rustix 1.1.6", ] [[package]] @@ -6011,7 +6011,7 @@ dependencies = [ "registry-platform-sqlite", "registry-platform-yaml", "rhai", - "rustix 1.1.5", + "rustix 1.1.6", "serde", "serde_json", "serde_norway", @@ -6144,7 +6144,7 @@ dependencies = [ "registry-review-protocol", "reqwest 0.12.28", "rhai", - "rustix 1.1.5", + "rustix 1.1.6", "rustls", "schemars", "serde", @@ -6347,7 +6347,7 @@ dependencies = [ "registry-platform-yaml", "registry-thunderid-tooling", "reqwest 0.12.28", - "rustix 1.1.5", + "rustix 1.1.6", "rustls", "schemars", "serde", @@ -6557,7 +6557,7 @@ dependencies = [ "registry-platform-yaml", "registry-thunderid-tooling", "reqwest 0.12.28", - "rustix 1.1.5", + "rustix 1.1.6", "schemars", "serde", "serde_json", @@ -6583,7 +6583,7 @@ dependencies = [ "registry-platform-httputil", "registry-platform-yaml", "reqwest 0.12.28", - "rustix 1.1.5", + "rustix 1.1.6", "schemars", "serde", "serde_json", @@ -6686,7 +6686,7 @@ dependencies = [ "registry-scheduling-core", "reqwest 0.12.28", "rhai", - "rustix 1.1.5", + "rustix 1.1.6", "rustls", "schemars", "serde", @@ -7035,7 +7035,7 @@ dependencies = [ "reqwest 0.12.28", "rhai", "rusqlite", - "rustix 1.1.5", + "rustix 1.1.6", "schemars", "serde", "serde_json", @@ -7320,7 +7320,7 @@ dependencies = [ "registry-platform-yaml", "registry-thunderid-tooling", "rhai", - "rustix 1.1.5", + "rustix 1.1.6", "schemars", "serde", "serde_json", @@ -7344,7 +7344,7 @@ dependencies = [ "futures", "registry-evidence-authoring", "registry-platform-yaml", - "rustix 1.1.5", + "rustix 1.1.6", "serde_json", "serde_norway", "tempfile", @@ -7614,7 +7614,7 @@ dependencies = [ "registry-platform-yaml", "registry-thunderid-tooling", "reqwest 0.12.28", - "rustix 1.1.5", + "rustix 1.1.6", "serde", "serde_json", "serde_norway", @@ -7649,7 +7649,7 @@ version = "0.46.1" dependencies = [ "hmac 0.13.0", "regex", - "rustix 1.1.5", + "rustix 1.1.6", "schemars", "serde", "serde_json", @@ -7710,7 +7710,7 @@ version = "0.46.1" dependencies = [ "registry-platform-canonical-json", "registry-platform-yaml", - "rustix 1.1.5", + "rustix 1.1.6", "schemars", "serde", "serde_json", @@ -7947,7 +7947,7 @@ name = "registry-platform-sqlite" version = "0.46.1" dependencies = [ "rusqlite", - "rustix 1.1.5", + "rustix 1.1.6", "serde", "sha2 0.11.1", "tempfile", @@ -8066,7 +8066,7 @@ dependencies = [ "registry-platform-yaml", "registry-render-client", "reqwest 0.12.28", - "rustix 1.1.5", + "rustix 1.1.6", "schemars", "serde", "serde_json", @@ -8171,12 +8171,13 @@ dependencies = [ "clap", "fs2", "hex", + "jsonschema", "registry-platform-buildinfo", "registry-platform-canonical-json", "registry-platform-config", "registry-platform-yaml", "registry-thunderid-tooling", - "rustix 1.1.5", + "rustix 1.1.6", "serde", "serde_json", "sha2 0.11.1", @@ -8357,7 +8358,7 @@ dependencies = [ "registry-platform-oidc", "registry-platform-yaml", "reqwest 0.12.28", - "rustix 1.1.5", + "rustix 1.1.6", "schemars", "serde", "serde_json", @@ -8795,9 +8796,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" dependencies = [ "bitflags", "errno", @@ -9777,7 +9778,7 @@ dependencies = [ "fastrand", "getrandom 0.4.3", "once_cell", - "rustix 1.1.5", + "rustix 1.1.6", "windows-sys 0.61.2", ] @@ -11332,7 +11333,7 @@ dependencies = [ "postcard", "pulley-interpreter", "rayon", - "rustix 1.1.5", + "rustix 1.1.6", "serde", "serde_derive", "smallvec", @@ -11431,7 +11432,7 @@ checksum = "96dce7a8ea302c0f1eb4154eb1756525c90c595bbabbc590abad64d4c5ff8bbb" dependencies = [ "cc", "libc", - "rustix 1.1.5", + "rustix 1.1.6", "wasmtime-environ", "wasmtime-internal-versioned-export-macros", "windows-sys 0.61.2", diff --git a/crates/registry-breg/src/mutation.rs b/crates/registry-breg/src/mutation.rs index 6f2f0c49aa..f10bc01f62 100644 --- a/crates/registry-breg/src/mutation.rs +++ b/crates/registry-breg/src/mutation.rs @@ -3659,8 +3659,10 @@ pub(crate) fn envelope_member_from_projection(value: Value) -> Result>(); + let encoded = base64::engine::general_purpose::STANDARD.encode(&envelope); + let wrapped = encoded + .as_bytes() + .chunks(76) + .map(|line| std::str::from_utf8(line).expect("base64 is ASCII")) + .collect::>() + .join("\n"); + assert_eq!( + envelope_member_from_projection(Value::String(wrapped)), + Ok(envelope_member_json(&envelope)) + ); + assert_eq!( + envelope_member_from_projection(Value::String(format!(" {encoded}"))), + Err(MutationError::Unavailable) + ); + } } diff --git a/crates/registry-breg/src/postgres/roles.rs b/crates/registry-breg/src/postgres/roles.rs index 7a674cb341..527e50d513 100644 --- a/crates/registry-breg/src/postgres/roles.rs +++ b/crates/registry-breg/src/postgres/roles.rs @@ -144,7 +144,8 @@ pub async fn provision_postgis_prerequisites( migration_role.quoted(), runtime_role.quoted(), )) - .await?; + .await + .map_err(|error| PostgresKernelError::from_statement_error(&error))?; let bbox_role = provision_spatial_bbox_role(admin, migration_role, runtime_role).await?; Ok(bbox_role) } @@ -178,7 +179,8 @@ pub async fn provision_spatial_bbox_role( migration_role.quoted(), bbox_role.quoted(), )) - .await?; + .await + .map_err(|error| PostgresKernelError::from_statement_error(&error))?; Ok(bbox_role) } diff --git a/crates/registry-breg/src/runtime_config.rs b/crates/registry-breg/src/runtime_config.rs index 2e79826a38..36528f46f8 100644 --- a/crates/registry-breg/src/runtime_config.rs +++ b/crates/registry-breg/src/runtime_config.rs @@ -1013,7 +1013,9 @@ impl RuntimeConfig { .review_authorities .into_iter() .map(|(id, authority)| { - ReviewAuthorityConfig::from_raw(&id, authority).map(|authority| (id, authority)) + ReviewAuthorityConfig::from_raw(&id, authority) + .map(|authority| (id, authority)) + .map_err(|_| RuntimeConfigError::InvalidBinding) }) .collect::>>()?; let operational_timeouts = OperationalTimeouts::from_raw(raw.operational_timeouts)?; @@ -2759,6 +2761,33 @@ pub(crate) fn parse_secret_reference( SecretReference::parse(value).map_err(|_| error) } +/// Why a `reviewAuthorities` entry is refused: the rule it breaks, stated +/// without repeating a value. +#[derive(Debug, Error, Clone, Copy, Eq, PartialEq)] +#[error("{0}")] +pub struct ReviewAuthorityRefusal(&'static str); + +/// Checks one `reviewAuthorities` entry with the reader the runtime uses, so +/// a caller that renders the entry can refuse it before the runtime starts. +/// Secret references are checked for shape only; nothing is resolved. +pub fn check_review_authority( + id: &str, + authority: &serde_json::Value, +) -> std::result::Result<(), ReviewAuthorityRefusal> { + let raw = RawReviewAuthorityConfig::deserialize(authority).map_err(|_| { + ReviewAuthorityRefusal( + "the review authority must hold only its documented members, each with a value of the documented type", + ) + })?; + ReviewAuthorityConfig::from_raw(id, raw).map(|_| ()) +} + +/// The rule for a review authority ID and its producer ID: non-blank text of +/// at most 128 bytes without control characters. +fn bounded_identity_text(value: &str) -> bool { + !value.trim().is_empty() && value.len() <= 128 && !value.chars().any(char::is_control) +} + #[derive(Clone)] struct ReviewAuthorityConfig { endpoint: reqwest::Url, @@ -2771,12 +2800,17 @@ struct ReviewAuthorityConfig { } impl ReviewAuthorityConfig { - fn from_raw(id: &str, raw: RawReviewAuthorityConfig) -> Result { - if id.trim().is_empty() || id.len() > 128 || id.chars().any(char::is_control) { - return Err(RuntimeConfigError::InvalidBinding); + fn from_raw( + id: &str, + raw: RawReviewAuthorityConfig, + ) -> std::result::Result { + if !bounded_identity_text(id) { + return Err(ReviewAuthorityRefusal( + "the review authority ID must be non-blank text of at most 128 bytes without control characters", + )); } - let endpoint = - reqwest::Url::parse(&raw.endpoint).map_err(|_| RuntimeConfigError::InvalidBinding)?; + let endpoint = reqwest::Url::parse(&raw.endpoint) + .map_err(|_| ReviewAuthorityRefusal("endpoint must be an absolute URL"))?; // The review client performs the complete credential-destination // validation during activation. Refuse URL suffixes here as well so a // typo never survives configuration parsing. @@ -2785,7 +2819,9 @@ impl ReviewAuthorityConfig { || endpoint.query().is_some() || endpoint.fragment().is_some() { - return Err(RuntimeConfigError::InvalidBinding); + return Err(ReviewAuthorityRefusal( + "endpoint must not carry user information, a query or a fragment", + )); } let completion = match (raw.completion_token_ref, raw.completion_recipient) { (None, None) => None, @@ -2794,24 +2830,36 @@ impl ReviewAuthorityConfig { { Some((token_ref, recipient)) } - _ => return Err(RuntimeConfigError::InvalidBinding), + _ => { + return Err(ReviewAuthorityRefusal( + "completionTokenRef and completionRecipient must be written together, with a recipient of 1 to 256 visible ASCII characters", + )) + } }; let token_ref = raw.token_ref; let private_key_jwt = raw .private_key_jwt .map(ReviewPrivateKeyJwtConfig::from_raw) .transpose()?; - if token_ref.is_some() == private_key_jwt.is_some() - || raw.profile.trim().is_empty() + if token_ref.is_some() == private_key_jwt.is_some() { + return Err(ReviewAuthorityRefusal( + "exactly one of tokenRef and privateKeyJwt must be written", + )); + } + if raw.profile.trim().is_empty() || raw.profile.len() > 128 || !raw.profile.bytes().all(|byte| { byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b':') }) - || raw.producer_id.trim().is_empty() - || raw.producer_id.len() > 128 - || raw.producer_id.chars().any(char::is_control) { - return Err(RuntimeConfigError::InvalidBinding); + return Err(ReviewAuthorityRefusal( + "the profile must be 1 to 128 ASCII letters, digits, `-`, `_`, `.` or `:`", + )); + } + if !bounded_identity_text(&raw.producer_id) { + return Err(ReviewAuthorityRefusal( + "producerId must be non-blank text of at most 128 bytes without control characters", + )); } Ok(Self { endpoint, @@ -2874,25 +2922,28 @@ struct ReviewPrivateKeyJwtConfig { } impl ReviewPrivateKeyJwtConfig { - fn from_raw(raw: RawReviewPrivateKeyJwtConfig) -> Result { - let token_endpoint = raw - .token_endpoint - .parse() - .map_err(|_| RuntimeConfigError::InvalidBinding)?; - if !registry_platform_httputil::valid_resource_uri(&raw.assertion_audience) - || !registry_platform_httputil::valid_resource_uri(&raw.resource) - || raw.scopes.is_empty() - || raw.scopes.len() > registry_platform_httputil::MAXIMUM_REQUESTED_SCOPES - || raw.scopes.iter().collect::>().len() != raw.scopes.len() - || raw.scopes.join(" ").len() - > registry_platform_httputil::MAXIMUM_SCOPE_PARAMETER_BYTES - || raw.scopes.iter().any(|scope| { - scope.len() > registry_platform_httputil::MAXIMUM_REQUESTED_SCOPE_BYTES - || !registry_platform_httputil::valid_scope_token(scope) - }) - { - return Err(RuntimeConfigError::InvalidBinding); + fn from_raw( + raw: RawReviewPrivateKeyJwtConfig, + ) -> std::result::Result { + let token_endpoint = raw.token_endpoint.parse().map_err(|_| { + ReviewAuthorityRefusal("the privateKeyJwt token endpoint must be an absolute URL") + })?; + if !registry_platform_httputil::valid_resource_uri(&raw.assertion_audience) { + return Err(ReviewAuthorityRefusal( + "the privateKeyJwt assertion audience must be one absolute URI without a fragment or userinfo", + )); + } + if !registry_platform_httputil::valid_resource_uri(&raw.resource) { + return Err(ReviewAuthorityRefusal( + "the privateKeyJwt resource must be one absolute URI without a fragment or userinfo", + )); } + registry_platform_httputil::validate_requested_scopes(&raw.scopes).map_err(|error| { + ReviewAuthorityRefusal(match error { + registry_platform_httputil::TokenError::Configuration { reason } => reason, + _ => "the privateKeyJwt scopes are not a scope list the token client requests", + }) + })?; Ok(Self { token_endpoint, client_id_ref: raw.client_id_ref, @@ -4203,6 +4254,89 @@ mod tests { use super::RuntimeConfigError; use crate::attachment_verification_worker::ATTEMPT_BUDGET; + fn review_authority_fragment() -> serde_json::Value { + serde_json::json!({ + "endpoint": "https://casework.example/reviews/", + "profile": "producer", + "producerId": "registry-producer", + "recoveryDays": 7, + "privateKeyJwt": { + "tokenEndpoint": "https://issuer.example/oauth2/token", + "clientIdRef": "secret:file/review-client-id", + "clientAssertionKeyRef": "secret:file/review-client-key", + "assertionAudience": "https://issuer.example", + "resource": "urn:casework:primary", + "scopes": ["casework:reviews:request"] + } + }) + } + + #[test] + fn check_review_authority_applies_the_runtime_reader_and_names_the_rule() { + super::check_review_authority("casework-a", &review_authority_fragment()) + .expect("a well-formed review authority is accepted"); + + let too_long = (0..17) + .map(|index| format!("{index:02}{}", "s".repeat(254))) + .collect::>(); + for (pointer, value, rule) in [ + ( + "/endpoint", + serde_json::json!("https://casework.example/reviews/?q=1"), + "query", + ), + ("/endpoint", serde_json::json!("not-a-url"), "absolute URL"), + ("/profile", serde_json::json!("not a profile"), "profile"), + ("/producerId", serde_json::json!(" "), "producerId"), + ("/surprise", serde_json::json!(true), "documented members"), + ( + "/tokenRef", + serde_json::json!("secret:file/review-token"), + "exactly one of tokenRef and privateKeyJwt", + ), + ( + "/completionRecipient", + serde_json::json!("registry"), + "completionTokenRef and completionRecipient", + ), + ( + "/privateKeyJwt/resource", + serde_json::json!("casework"), + "absolute URI", + ), + ( + "/privateKeyJwt/assertionAudience", + serde_json::json!("issuer"), + "assertion audience", + ), + ( + "/privateKeyJwt/scopes", + serde_json::json!(too_long), + "within 4096 bytes", + ), + ( + "/privateKeyJwt/scopes", + serde_json::json!(["has space"]), + "scope-tokens", + ), + ] { + let mut fragment = review_authority_fragment(); + let (parent, member) = pointer.rsplit_once('/').unwrap(); + fragment.pointer_mut(parent).unwrap()[member] = value.clone(); + let refusal = super::check_review_authority("casework-a", &fragment) + .expect_err(pointer) + .to_string(); + assert!(refusal.contains(rule), "{pointer}: {refusal}"); + if let Some(text) = value.as_str().filter(|text| text.trim().len() > 1) { + assert!(!refusal.contains(text), "{pointer}: {refusal}"); + } + } + let refusal = super::check_review_authority("", &review_authority_fragment()) + .unwrap_err() + .to_string(); + assert!(refusal.contains("review authority ID"), "{refusal}"); + } + #[test] fn the_attempt_budget_action_states_the_worker_budget() { let budget = ATTEMPT_BUDGET.as_millis(); diff --git a/crates/registry-breg/tests/postgres_field_encryption.rs b/crates/registry-breg/tests/postgres_field_encryption.rs index 3239826a47..9f8174c791 100644 --- a/crates/registry-breg/tests/postgres_field_encryption.rs +++ b/crates/registry-breg/tests/postgres_field_encryption.rs @@ -1312,6 +1312,107 @@ async fn encrypted_fields_round_trip_and_store_only_envelopes() { server.shutdown().await; } +/// An encrypted value long enough that its stored envelope exceeds 57 bytes, +/// so PostgreSQL's `encode(..., 'base64')` wraps the projection at 76 +/// characters. Every read projection must still hand back the plaintext. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn long_envelopes_read_back_through_every_read_projection() { + let server = boot_live_server().await; + let long_secret = "long-secret-value-".repeat(10); + let (status, created, _etag) = create_holder( + &server, + "long-envelope-create", + json!({ + "jurisdiction": "area-a", + "label": "long-envelope", + "secret": long_secret, + }), + ) + .await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(created["data"]["domainData"]["secret"], long_secret); + let record_id = created["data"]["recordIdentifier"] + .as_str() + .expect("record identifier is present") + .to_owned(); + + let envelope = stored_column( + &server, + &envelope_column(server.registry(), "secret"), + &record_id, + ) + .await; + assert!( + envelope.len() > 57, + "the envelope must wrap PostgreSQL's base64 output: {} bytes", + envelope.len() + ); + + let fetched = send( + &server, + Method::GET, + &format!("/v1/records/holders/{record_id}"), + &[], + None, + ) + .await; + assert_eq!(fetched.status(), StatusCode::OK); + let fetched = body_json(fetched).await; + assert_eq!(fetched["data"]["domainData"]["secret"], long_secret); + + let listed = send( + &server, + Method::GET, + "/v1/records/holders?$select=jurisdiction,label,secret", + &[], + None, + ) + .await; + assert_eq!(listed.status(), StatusCode::OK); + let listed = body_json(listed).await; + let items = listed["items"].as_array().expect("list items"); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["domainData"]["secret"], long_secret); + + let snapshot = send( + &server, + Method::GET, + "/v1/records/holders:snapshot?$select=jurisdiction,label,secret", + &[], + None, + ) + .await; + assert_eq!(snapshot.status(), StatusCode::OK); + let snapshot = body_json(snapshot).await; + assert_eq!(snapshot["items"][0]["domainData"]["secret"], long_secret); + + let revisions = send( + &server, + Method::GET, + &format!("/v1/records/holders/{record_id}/revisions"), + &[], + None, + ) + .await; + assert_eq!(revisions.status(), StatusCode::OK); + let revisions = body_json(revisions).await; + assert_eq!(revisions["items"][0]["domainData"]["secret"], long_secret); + + let detail = send( + &server, + Method::GET, + &format!("/v1/records/holders/{record_id}/revisions/1"), + &[], + None, + ) + .await; + assert_eq!(detail.status(), StatusCode::OK); + let detail = body_json(detail).await; + assert_eq!(detail["data"]["domainData"]["secret"], long_secret); + + server.shutdown().await; +} + /// The single stored idempotency body of one result kind, as UTF-8 text. async fn cached_response_body(server: &LiveServer, result_kind: &str) -> String { let rows = server diff --git a/crates/registry-breg/tests/postgres_migration.rs b/crates/registry-breg/tests/postgres_migration.rs index 841b3c2527..40b6bbfe0c 100644 --- a/crates/registry-breg/tests/postgres_migration.rs +++ b/crates/registry-breg/tests/postgres_migration.rs @@ -4481,8 +4481,11 @@ async fn real_postgres_field_encryption_flip_seals_resumes_and_drops_the_plainte let active = apply(&database, &initial, ApplyPrecondition::InitialActivation) .await .expect("flip scenario initial package activates"); + // Each value seals to an envelope longer than 57 bytes, whose base64 + // PostgreSQL breaks across lines, so the journal capture of every sealed + // chunk decodes a wrapped projection. let secrets = (0..5) - .map(|index| format!("flip-secret-{index}")) + .map(|index| format!("flip-secret-{index}-{}", "long-envelope-".repeat(6))) .collect::>(); seed_flip_rows(&database, &prior, &secrets, true).await; diff --git a/crates/registry-breg/tests/postgres_spatial_migration.rs b/crates/registry-breg/tests/postgres_spatial_migration.rs index 0bac9c1689..3ebadb21bd 100644 --- a/crates/registry-breg/tests/postgres_spatial_migration.rs +++ b/crates/registry-breg/tests/postgres_spatial_migration.rs @@ -29,9 +29,9 @@ use registry_breg::package::{ }; use registry_breg::postgres::{ begin_record_transaction, install_compiled_schema, managed_schema_fingerprint, - provision_postgis_prerequisites, verify_catalog_identity_for_catalog, verify_postgis, - ClaimContext, ExpectedManagedCatalog, ExpectedRegistryIdentity, PostgresFailure, - RegistryLockKey, SqlIdentifier, + provision_postgis_prerequisites, provision_spatial_bbox_role, + verify_catalog_identity_for_catalog, verify_postgis, ClaimContext, ExpectedManagedCatalog, + ExpectedRegistryIdentity, PostgresFailure, PostgresKernelError, RegistryLockKey, SqlIdentifier, }; use registry_breg::startup::{prepare_startup, StartupError}; use registry_breg::CompiledRegistry; @@ -2680,6 +2680,43 @@ async fn cleanup_bbox_role(database: &TestDatabase, bbox_role: &SqlIdentifier) { .expect("test cleanup removes temporary spatial bbox role"); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn postgis_provisioning_keeps_the_sqlstate_of_a_refused_statement() { + let database = TestDatabase::create(1).await; + // The migration role may not create schemas in the database, which is + // what an administrator without `CREATE EXTENSION postgis` rights meets. + let (unprivileged, task) = database.connect_migration().await; + let error = provision_postgis_prerequisites( + &unprivileged, + &database.migration_role, + &database.runtime_role, + ) + .await + .expect_err("a role without schema creation rights cannot provision PostGIS"); + assert_insufficient_privilege(&error); + // Nor may it create roles, so the bbox role batch is refused too. + let error = provision_spatial_bbox_role( + &unprivileged, + &database.migration_role, + &database.runtime_role, + ) + .await + .expect_err("a role without role creation rights cannot provision the bbox role"); + task.abort(); + assert_insufficient_privilege(&error); + database.cleanup().await; +} + +fn assert_insufficient_privilege(error: &PostgresKernelError) { + match error { + PostgresKernelError::Statement(failure) => { + assert_eq!(failure.sqlstate.as_deref(), Some("42501")); + } + other => panic!("expected a statement failure, got {other:?}"), + } + assert!(error.to_string().contains("42501"), "{error}"); +} + async fn cleanup_postgis_prerequisites(database: &TestDatabase, bbox_role: &SqlIdentifier) { cleanup_bbox_role(database, bbox_role).await; database diff --git a/crates/registry-bregctl/src/field_encryption.rs b/crates/registry-bregctl/src/field_encryption.rs index 9e3e479eee..a62d03396d 100644 --- a/crates/registry-bregctl/src/field_encryption.rs +++ b/crates/registry-bregctl/src/field_encryption.rs @@ -29,15 +29,19 @@ pub(crate) struct KeygenOutcome { } /// Value-free generation refusals. -#[derive(Debug, Eq, PartialEq)] +#[derive(Debug, Eq, PartialEq, thiserror::Error)] pub(crate) enum KeygenError { /// A relative path cannot name a deliberate deployment secret location. + #[error("the key path is not absolute")] RelativeOutput, /// The output already exists; key material is never regenerated over it. + #[error("a key file already exists at that path")] OutputExists, /// The platform random source refused. + #[error("the platform random source refused")] RandomSource, /// The filesystem refused a parent directory or the key file itself. + #[error("the filesystem refused a parent directory or the key file")] Write, } diff --git a/crates/registry-bregctl/src/lib.rs b/crates/registry-bregctl/src/lib.rs index 0ab5ac63f3..46af51d1d6 100644 --- a/crates/registry-bregctl/src/lib.rs +++ b/crates/registry-bregctl/src/lib.rs @@ -228,6 +228,8 @@ enum Command { DevSupervisor(dev::SupervisorArgs), #[command(name = "project-dev", hide = true)] ProjectDev(project_dev::Args), + #[command(name = "project-dev-guard", hide = true)] + ProjectDevGuard(project_dev::guard::Args), /// Explain compiled model, access, route, or event inventories. Explain(ExplainArgs), /// Compare an authoring candidate with a rederived closed package. @@ -2908,6 +2910,7 @@ where Command::ProjectDev(args) => { return ExitCode::from(project_dev::run(args, stdout, stderr)); } + Command::ProjectDevGuard(args) => return project_dev::guard::run(args), Command::Init(args) => match (args.from, args.template.as_deref()) { (None, None) => init(&args.destination), (None, Some(id)) => starters::run(&args.destination, id), diff --git a/crates/registry-bregctl/src/project_dev/bindings.rs b/crates/registry-bregctl/src/project_dev/bindings.rs index 4a85b97016..29e00d81d6 100644 --- a/crates/registry-bregctl/src/project_dev/bindings.rs +++ b/crates/registry-bregctl/src/project_dev/bindings.rs @@ -114,14 +114,6 @@ impl Bindings { "Products must place BReg before Casework when bindings are configured.", "Move the BReg product before the Casework product.", )?; - // Each binding requests a token for the other product's audience as - // its OAuth resource, which the products accept only as an absolute URI. - if self.breg_review_authority.is_some() { - require_resource_audience(plan, casework, "bregReviewAuthority")?; - } - if self.casework_context_provider.is_some() { - require_resource_audience(plan, breg, "caseworkContextProvider")?; - } if let Some(binding) = &self.breg_review_authority { plan::require( @@ -173,22 +165,6 @@ fn product_position(plan: &Plan, product: Product) -> Option { .position(|entry| entry.product == product) } -fn require_resource_audience( - plan: &Plan, - position: Option, - binding: &str, -) -> plan::SemanticResult<()> { - let Some(index) = position else { - return Ok(()); - }; - plan::require( - registry_platform_httputil::valid_resource_uri(&plan.products[index].audience), - &format!("/products/{index}/audience"), - format!("The audience must be an absolute URI because /bindings/{binding} requests tokens for it as an OAuth resource."), - "Set audience to an absolute URI without a fragment or credentials, such as `urn:example:review`.", - ) -} - fn require_machine_person<'a>( plan: &'a Plan, alias: &str, @@ -220,42 +196,9 @@ fn require_machine_person<'a>( product.name() ), )?; - let grant = &person.grants[&product]; - plan::require( - !grant.scopes.is_empty(), - field, - format!( - "The binding identity must name a person whose {} grant has a scope.", - product.name() - ), - format!( - "Add a scope to the named person's {} grant or choose another identity.", - product.name() - ), - )?; - // The calling product requests these scopes through the shared OAuth - // client, which refuses the list at startup unless it passes this check. - if let Err(error) = registry_platform_httputil::validate_requested_scopes(&grant.scopes) { - let reason = match error { - registry_platform_httputil::TokenError::Configuration { reason } => reason.to_owned(), - other => other.to_string(), - }; - return plan::refuse( - &format!("/people/{alias}/grants/{}/scopes", product.name()), - &format!("A binding requests this grant's scopes over OAuth, and {reason}."), - "Use only OAuth scope characters (no spaces, quotes, or backslashes), and keep the scopes, joined by spaces, within 4096 bytes.", - ); - } Ok(person) } -struct PreparedBinding { - client_id_ref: String, - client_assertion_key_ref: String, - person: String, - product: Product, -} - struct TokenServer { endpoint: String, shutdown: Option>, @@ -273,11 +216,12 @@ impl Drop for TokenServer { } } +const BREG_REVIEW_CLIENT: &str = "breg-review-client"; +const CASEWORK_CONTEXT_CLIENT: &str = "casework-context-client"; + #[derive(Default)] pub(super) struct RuntimeBindings { server: Option, - breg_review: Option, - casework_context: Option, input: Option>, origins: BTreeMap, // Kept outside the server state so Issuer's Tokio runtime is dropped on @@ -304,39 +248,29 @@ impl RuntimeBindings { ); let mut clients = Vec::new(); - let breg_review = bindings - .breg_review_authority - .as_ref() - .map(|binding| { - prepare_binding( - root, - &input, - &binding.identity, - Product::Casework, - "breg-review-client", - &mut clients, - ) - }) - .transpose()?; - let casework_context = bindings - .casework_context_provider - .as_ref() - .map(|binding| { - prepare_binding( - root, - &input, - &binding.identity, - Product::Breg, - "casework-context-client", - &mut clients, - ) - }) - .transpose()?; + if let Some(binding) = &bindings.breg_review_authority { + prepare_binding( + root, + &input, + &binding.identity, + Product::Casework, + BREG_REVIEW_CLIENT, + &mut clients, + )?; + } + if let Some(binding) = &bindings.casework_context_provider { + prepare_binding( + root, + &input, + &binding.identity, + Product::Breg, + CASEWORK_CONTEXT_CLIENT, + &mut clients, + )?; + } let server = start_token_server(input.clone(), issuer.clone(), clients)?; Ok(Self { server: Some(server), - breg_review, - casework_context, input: Some(input), origins: origins.clone(), _issuer: Some(issuer), @@ -352,55 +286,215 @@ impl RuntimeBindings { .as_ref() .expect("prepared token server") .endpoint; - match product { - Product::Breg => { - let (Some(binding), Some(prepared)) = ( - input.plan.bindings.breg_review_authority.as_ref(), - self.breg_review.as_ref(), - ) else { - return; - }; - let person = &input.plan.people[&prepared.person]; - let grant = &person.grants[&prepared.product]; - config["reviewAuthorities"][binding.id.as_str()] = json!({ - "endpoint": self.origins[&Product::Casework], - "profile": grant.profile, - "privateKeyJwt": { - "tokenEndpoint": endpoint, - "clientIdRef": prepared.client_id_ref, - "clientAssertionKeyRef": prepared.client_assertion_key_ref, - "assertionAudience": input.plan.issuer, - "resource": input.plan.product(Product::Casework).audience, - "scopes": grant.scopes, - }, - "producerId": binding.producer, - "recoveryDays": 7, - }); - } - Product::Casework => { - let (Some(binding), Some(prepared)) = ( - input.plan.bindings.casework_context_provider.as_ref(), - self.casework_context.as_ref(), - ) else { - return; - }; - let person = &input.plan.people[&prepared.person]; - let grant = &person.grants[&prepared.product]; - config["contextProviders"][binding.id.as_str()] = json!({ - "baseUrl": self.origins[&Product::Breg], - "statusProfile": grant.profile, - "tokenEndpoint": endpoint, - "clientAssertionAudience": input.plan.issuer, - "resource": input.plan.product(Product::Breg).audience, + render(&input.plan, product, &self.origins, endpoint, config); + } +} + +/// Writes the binding the plan configures for `product` into its runtime +/// configuration. Only the token endpoint and the product origins belong to +/// the run; every other value comes from the plan. +fn render( + plan: &Plan, + product: Product, + origins: &BTreeMap, + token_endpoint: &str, + config: &mut Value, +) { + match product { + Product::Breg => { + let Some(binding) = plan.bindings.breg_review_authority.as_ref() else { + return; + }; + let grant = &plan.people[&binding.identity].grants[&Product::Casework]; + let (client_id_ref, client_assertion_key_ref) = client_refs(BREG_REVIEW_CLIENT); + config["reviewAuthorities"][binding.id.as_str()] = json!({ + "endpoint": origins[&Product::Casework], + "profile": grant.profile, + "privateKeyJwt": { + "tokenEndpoint": token_endpoint, + "clientIdRef": client_id_ref, + "clientAssertionKeyRef": client_assertion_key_ref, + "assertionAudience": plan.issuer, + "resource": plan.product(Product::Casework).audience, "scopes": grant.scopes, - "clientIdRef": prepared.client_id_ref, - "clientAssertionKeyRef": prepared.client_assertion_key_ref, - }); - } + }, + "producerId": binding.producer, + "recoveryDays": 7, + }); + } + Product::Casework => { + let Some(binding) = plan.bindings.casework_context_provider.as_ref() else { + return; + }; + let grant = &plan.people[&binding.identity].grants[&Product::Breg]; + let (client_id_ref, client_assertion_key_ref) = client_refs(CASEWORK_CONTEXT_CLIENT); + config["contextProviders"][binding.id.as_str()] = json!({ + "baseUrl": origins[&Product::Breg], + "statusProfile": grant.profile, + "tokenEndpoint": token_endpoint, + "clientAssertionAudience": plan.issuer, + "resource": plan.product(Product::Breg).audience, + "scopes": grant.scopes, + "clientIdRef": client_id_ref, + "clientAssertionKeyRef": client_assertion_key_ref, + }); + } + } +} + +/// The secret references of the machine client `filename` names. +fn client_refs(filename: &str) -> (String, String) { + ( + format!("secret:file/{filename}-id"), + format!("secret:file/{filename}-key"), + ) +} + +/// Origin and token endpoint `check` renders with. Nothing listens there: the +/// product checks refuse a binding's shape, not the address it names. +const CHECK_ORIGIN: &str = "http://127.0.0.1:1"; +const CHECK_TOKEN_ENDPOINT: &str = "http://127.0.0.1:1/oauth2/token"; + +/// Writes the binding the plan configures for `product` into `config` as +/// `start` writes it, at the origin and token endpoint `check` renders with. +pub(super) fn render_for_check(plan: &Plan, product: Product, config: &mut Value) { + let origins = BTreeMap::from([ + (Product::Breg, CHECK_ORIGIN.to_owned()), + (Product::Casework, CHECK_ORIGIN.to_owned()), + ]); + render(plan, product, &origins, CHECK_TOKEN_ENDPOINT, config); +} + +/// Renders the review authority binding exactly as `start` writes it and lets +/// BReg's reader refuse what it would refuse at startup, before anything is +/// provisioned. A refusal names the plan field that produced the value, not +/// the generated fragment. Casework checks the context provider binding +/// itself, through `caseworkctl check`. +pub(super) fn check_with_breg_reader(plan: &Plan) -> plan::SemanticResult<()> { + let Some(binding) = &plan.bindings.breg_review_authority else { + return Ok(()); + }; + let mut config = json!({}); + render_for_check(plan, Product::Breg, &mut config); + let grant = grant_field(&binding.identity, Product::Casework); + let audience = audience_field(plan, Product::Casework); + require_reader_accepts( + "BReg", + "/bindings/bregReviewAuthority", + &config["reviewAuthorities"][binding.id.as_str()], + &[ + Source::new( + "/profile", + &format!("{grant}/profile"), + json!("placeholder"), + ), + Source::new( + "/privateKeyJwt/assertionAudience", + "/issuer", + json!(STAND_IN_URI), + ), + Source::new("/privateKeyJwt/resource", &audience, json!(STAND_IN_URI)), + Source::new( + "/privateKeyJwt/scopes", + &format!("{grant}/scopes"), + json!(["placeholder"]), + ), + Source::new( + "/producerId", + "/bindings/bregReviewAuthority/producer", + json!("placeholder"), + ), + ], + |fragment| { + registry_breg::runtime_config::check_review_authority(&binding.id, fragment) + .map_err(|error| error.to_string()) + }, + ) +} + +/// The plan field that produced the member at `member`, a JSON pointer +/// relative to the context provider binding the run writes for Casework. +/// Members only the run supplies land on the binding. +pub(super) fn casework_binding_field( + plan: &Plan, + binding: &CaseworkContextProvider, + member: &str, +) -> String { + let grant = grant_field(&binding.identity, Product::Breg); + match member.split('/').nth(1).unwrap_or_default() { + "statusProfile" => format!("{grant}/profile"), + "clientAssertionAudience" => "/issuer".to_owned(), + "resource" => audience_field(plan, Product::Breg), + "scopes" => format!("{grant}/scopes"), + _ => "/bindings/caseworkContextProvider".to_owned(), + } +} + +const STAND_IN_URI: &str = "urn:example:stand-in"; + +/// One member of a rendered fragment and the plan field that produced it, +/// with a value the reader accepts for that member. +struct Source { + member: &'static str, + field: String, + stand_in: Value, +} + +impl Source { + fn new(member: &'static str, field: &str, stand_in: Value) -> Self { + Self { + member, + field: field.to_owned(), + stand_in, } } } +fn grant_field(alias: &str, product: Product) -> String { + format!( + "/people/{}/grants/{}", + registry_platform_yaml::escape_pointer_segment(alias), + product.name() + ) +} + +fn audience_field(plan: &Plan, product: Product) -> String { + let index = product_position(plan, product).expect("validated product"); + format!("/products/{index}/audience") +} + +/// Refuses when `reader` refuses `fragment`. The reader states a rule without +/// saying which member broke it, so the refusal lands on the plan field of the +/// one member whose replacement by a stand-in makes the reader accept. When no +/// single member explains it, the refusal lands on the binding. +fn require_reader_accepts( + product: &str, + binding_field: &str, + fragment: &Value, + sources: &[Source], + reader: impl Fn(&Value) -> Result<(), String>, +) -> plan::SemanticResult<()> { + let Err(refusal) = reader(fragment) else { + return Ok(()); + }; + let field = sources + .iter() + .find(|source| { + let mut probe = fragment.clone(); + probe + .pointer_mut(source.member) + .map(|member| *member = source.stand_in.clone()) + .is_some() + && reader(&probe).is_ok() + }) + .map_or(binding_field, |source| source.field.as_str()); + plan::refuse( + field, + &format!("{product} refuses the binding configuration the run writes from this value: {refusal}."), + "Correct the value so the product accepts the configuration the run writes for this binding.", + ) +} + fn prepare_binding( root: &Path, input: &RunInput, @@ -408,16 +502,17 @@ fn prepare_binding( product: Product, filename: &str, clients: &mut Vec, -) -> Result { +) -> Result<()> { let person = require_machine_person(&input.plan, person_alias, product, "/bindings")?; let grant = &person.grants[&product]; let key = generate_private_jwk(GeneratedKeyAlgorithm::Es256)?; let public_key = key.public(); - let key_name = format!("{filename}-key"); - let id_name = format!("{filename}-id"); let encoded = encode_private_jwk(&key)?; - write_private(&root.join(&key_name), &encoded)?; - write_private(&root.join(&id_name), person.client.as_bytes())?; + write_private(&root.join(format!("{filename}-key")), &encoded)?; + write_private( + &root.join(format!("{filename}-id")), + person.client.as_bytes(), + )?; clients.push(MachineClient { client_id: person.client.clone(), person: person_alias.to_owned(), @@ -426,12 +521,7 @@ fn prepare_binding( resource: input.plan.product(product).audience.clone(), scope: grant.scopes.join(" "), }); - Ok(PreparedBinding { - client_id_ref: format!("secret:file/{id_name}"), - client_assertion_key_ref: format!("secret:file/{key_name}"), - person: person_alias.to_owned(), - product, - }) + Ok(()) } #[derive(Serialize)] @@ -508,7 +598,8 @@ fn start_token_server( ) -> Result { ensure!(!clients.is_empty(), "a binding token client is required"); let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) - .context("cannot bind the run-local token endpoint")?; + .context("cannot bind the run-local token endpoint") + .map_err(super::operational)?; listener.set_nonblocking(true)?; let origin = format!("http://{}", listener.local_addr()?); let endpoint = format!("{origin}/oauth2/token"); @@ -835,6 +926,7 @@ mod tests { principal: "sub".to_owned(), purpose: false, requires_postgis: false, + encrypts_fields: false, named_clients: BTreeSet::new(), }), casework_principals: BTreeMap::from([("producer".to_owned(), "sub".to_owned())]), @@ -859,26 +951,27 @@ mod tests { } #[test] - fn validation_refuses_a_bound_audience_that_is_not_a_resource_uri() { + fn check_accepts_a_plan_the_breg_reader_accepts() { + check_with_breg_reader(&plan()).unwrap(); + } + + #[test] + fn check_refuses_a_bound_audience_that_is_not_a_resource_uri() { let mut value = plan(); value.products[1].audience = "review".to_owned(); - let error = value.bindings.validate(&value).unwrap_err().to_string(); + let error = check_with_breg_reader(&value).unwrap_err().to_string(); assert!(error.contains("/products/1/audience"), "{error}"); - assert!(error.contains("absolute URI"), "{error}"); + assert!(error.contains("BReg refuses"), "{error}"); + // The context provider resource is Casework's rule, which + // `caseworkctl check` applies to the configuration the run writes. let mut value = plan(); value.products[0].audience = "registry".to_owned(); - let error = value.bindings.validate(&value).unwrap_err().to_string(); - assert!(error.contains("/products/0/audience"), "{error}"); - - let mut value = plan(); - value.bindings.casework_context_provider = None; - value.products[0].audience = "registry".to_owned(); - value.bindings.validate(&value).unwrap(); + check_with_breg_reader(&value).unwrap(); } #[test] - fn validation_refuses_bound_scopes_the_products_would_not_request() { + fn check_refuses_bound_scopes_the_products_would_not_request() { // Both products request a binding's scopes through the shared OAuth // client, which refuses this list before the run could start them. let too_long = (0..17) @@ -894,40 +987,80 @@ mod tests { .unwrap() .scopes .clone_from(&too_long); - let error = value.bindings.validate(&value).unwrap_err().to_string(); + let error = check_with_breg_reader(&value).unwrap_err().to_string(); assert!( error.contains("/people/producer/grants/casework/scopes"), "{error}" ); assert!(error.contains("4096"), "{error}"); + // A person no binding names keeps the plan's own scope rule. let mut value = plan(); + value.bindings.breg_review_authority = None; value .people - .get_mut("context-reader") + .get_mut("producer") .unwrap() .grants - .get_mut(&Product::Breg) + .get_mut(&Product::Casework) .unwrap() - .scopes = vec!["registry:\"quoted\"".to_owned()]; - let error = value.bindings.validate(&value).unwrap_err().to_string(); - assert!( - error.contains("/people/context-reader/grants/breg/scopes"), - "{error}" - ); + .scopes = too_long; + value.bindings.validate(&value).unwrap(); + check_with_breg_reader(&value).unwrap(); + } - // A person no binding names keeps the plan's own scope rule. + #[test] + fn check_refuses_other_binding_values_at_the_plan_field_that_produced_them() { + let mut value = plan(); + value.issuer = "not a uri".to_owned(); + let error = check_with_breg_reader(&value).unwrap_err().to_string(); + assert!(error.contains("/issuer"), "{error}"); + assert!(error.contains("assertion audience"), "{error}"); + + // The profile grammar is a rule only the product readers state: the + // runner restates nothing about it, so BReg's reader catches it and + // the refusal lands on the grant that produced the value. let mut value = plan(); - value.bindings.casework_context_provider = None; value .people - .get_mut("context-reader") + .get_mut("producer") .unwrap() .grants - .get_mut(&Product::Breg) + .get_mut(&Product::Casework) .unwrap() - .scopes = too_long; - value.bindings.validate(&value).unwrap(); + .profile = "not a profile".to_owned(); + let error = check_with_breg_reader(&value).unwrap_err().to_string(); + assert!( + error.contains("/people/producer/grants/casework/profile"), + "{error}" + ); + assert!(error.contains("the profile must be"), "{error}"); + } + + #[test] + fn a_casework_refusal_lands_on_the_plan_field_that_produced_the_member() { + let value = plan(); + let binding = value.bindings.casework_context_provider.as_ref().unwrap(); + for (member, field) in [ + ( + "/statusProfile", + "/people/context-reader/grants/breg/profile", + ), + ("/clientAssertionAudience", "/issuer"), + ("/resource", "/products/0/audience"), + ("/scopes", "/people/context-reader/grants/breg/scopes"), + ("/scopes/0", "/people/context-reader/grants/breg/scopes"), + ("/baseUrl", "/bindings/caseworkContextProvider"), + ("/tokenEndpoint", "/bindings/caseworkContextProvider"), + ("/clientIdRef", "/bindings/caseworkContextProvider"), + ("", "/bindings/caseworkContextProvider"), + ] { + assert_eq!( + casework_binding_field(&value, binding, member), + field, + "{member}" + ); + } } #[test] diff --git a/crates/registry-bregctl/src/project_dev/database.rs b/crates/registry-bregctl/src/project_dev/database.rs index 4fb99a8924..ebdb4283c8 100644 --- a/crates/registry-bregctl/src/project_dev/database.rs +++ b/crates/registry-bregctl/src/project_dev/database.rs @@ -9,6 +9,7 @@ use std::{ use anyhow::{anyhow, bail, Context, Result}; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; use rustls::pki_types::{pem::PemObject as _, CertificateDer}; +use serde::{Deserialize, Serialize}; use tokio::runtime::{Builder, Runtime}; use tokio_postgres::{ config::{Host, SslMode}, @@ -30,6 +31,9 @@ const STATEMENT_TIMEOUT: Duration = Duration::from_secs(10); const OPERATION_TIMEOUT: Duration = Duration::from_secs(15); const CLEANUP_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(30); const KEEPALIVE_IDLE: Duration = Duration::from_secs(30); +const RECOVERY_IDENTITY_QUERY: &str = "SELECT system_identifier, \ + pg_catalog.current_database()::text \ + FROM pg_catalog.pg_control_system()"; /// Lets a role holding CREATEROLE use the roles it creates. PostgreSQL 16 and /// later grant such a creator only ADMIN on them, which is not enough for /// `CREATE SCHEMA ... AUTHORIZATION` or `CREATE DATABASE ... OWNER`. @@ -170,21 +174,82 @@ fn ownership_lock_statement(run_id: &str) -> Result { Ok(format!("SELECT pg_advisory_lock({key})")) } +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(super) struct RecoveryIdentity { + system_identifier: i64, + database: String, +} + +fn require_recovery_identity(expected: &RecoveryIdentity, actual: &RecoveryIdentity) -> Result<()> { + if expected.system_identifier != actual.system_identifier { + bail!( + "this run belongs to a different PostgreSQL cluster; use the administrator URL from startup" + ); + } + if expected.database != actual.database { + bail!( + "this run belongs to a different PostgreSQL maintenance database; use the administrator URL from startup" + ); + } + Ok(()) +} + +fn read_recovery_identity( + runtime: &Runtime, + client: &Client, + deadline: Instant, +) -> Result { + let row = runtime + .block_on(bounded_operation( + "recovery identity check", + client.query_one(RECOVERY_IDENTITY_QUERY, &[]), + remaining_timeout(deadline, OPERATION_TIMEOUT)?, + )) + .map_err(anyhow::Error::new) + .context( + "cannot identify the PostgreSQL cluster; the administrator must be allowed to execute pg_catalog.pg_control_system()", + )?; + Ok(RecoveryIdentity { + system_identifier: row + .try_get(0) + .context("cannot read the PostgreSQL cluster identifier")?, + database: row + .try_get(1) + .context("cannot read the PostgreSQL administrator database name")?, + }) +} + +type AdminConnectionTask = tokio::task::JoinHandle>; + fn open_admin_session( runtime: &Runtime, config: &Config, tls: &MakeRustlsConnect, ownership_lock: &str, + expected_identity: Option<&RecoveryIdentity>, deadline: Instant, -) -> Result<( - Client, - tokio::task::JoinHandle>, -)> { +) -> Result<(Client, AdminConnectionTask, RecoveryIdentity)> { let (client, connection) = runtime.block_on(bounded_connection( config.connect(tls.clone()), remaining_timeout(deadline, CONNECTION_TIMEOUT)?, ))?; let connection_task = runtime.spawn(connection); + // PostgreSQL advisory locks are scoped to one cluster and database. Bind + // both before acquiring the lock, or the same run id at another target + // could appear to be exclusively owned. + let identity = match read_recovery_identity(runtime, &client, deadline) { + Ok(identity) => identity, + Err(error) => { + connection_task.abort(); + return Err(error); + } + }; + if let Some(expected) = expected_identity { + if let Err(error) = require_recovery_identity(expected, &identity) { + connection_task.abort(); + return Err(error); + } + } let locked = runtime.block_on(bounded_operation( "run ownership lock acquisition", client.batch_execute(ownership_lock), @@ -194,7 +259,7 @@ fn open_admin_session( connection_task.abort(); return Err(error).context("cannot acquire the PostgreSQL run ownership lock"); } - Ok((client, connection_task)) + Ok((client, connection_task, identity)) } pub(super) struct DatabaseUrls { @@ -210,11 +275,12 @@ enum Acquisition { pub(super) struct Database { client: Client, - _connection_task: tokio::task::JoinHandle>, + _connection_task: AdminConnectionTask, admin_session_failed: bool, tls: MakeRustlsConnect, admin_config: Config, ownership_lock: String, + recovery_identity: RecoveryIdentity, endpoint: Endpoint, run_id: String, acquisitions: Vec, @@ -257,6 +323,24 @@ impl Endpoint { impl Database { pub(super) fn connect(admin_url: &str, ca_pem: Option<&[u8]>, run_id: &str) -> Result { + Self::connect_checked(admin_url, ca_pem, run_id, None) + } + + pub(super) fn connect_for_recovery( + admin_url: &str, + ca_pem: Option<&[u8]>, + run_id: &str, + expected_identity: &RecoveryIdentity, + ) -> Result { + Self::connect_checked(admin_url, ca_pem, run_id, Some(expected_identity)) + } + + fn connect_checked( + admin_url: &str, + ca_pem: Option<&[u8]>, + run_id: &str, + expected_identity: Option<&RecoveryIdentity>, + ) -> Result { validate_run_id(run_id)?; let (mut admin_config, endpoint) = admin_config(admin_url)?; admin_config.ssl_mode(SslMode::Require); @@ -267,15 +351,19 @@ impl Database { .build() .context("cannot initialize the PostgreSQL executor")?; let ownership_lock = ownership_lock_statement(run_id)?; - let session_deadline = Instant::now() + CONNECTION_TIMEOUT + OPERATION_TIMEOUT; - let (client, connection_task) = open_admin_session( + let session_deadline = Instant::now() + CLEANUP_ATTEMPT_TIMEOUT; + let (client, connection_task, recovery_identity) = open_admin_session( &runtime, &admin_config, &tls, &ownership_lock, + expected_identity, session_deadline, ) - .context("cannot establish the PostgreSQL administrator session over verified TLS")?; + .context("cannot establish the PostgreSQL administrator session over verified TLS") + // The server is unreachable or refused the session: the environment + // failed, not the request. + .map_err(super::operational)?; Ok(Self { client, _connection_task: connection_task, @@ -283,6 +371,7 @@ impl Database { tls, admin_config, ownership_lock, + recovery_identity, endpoint, run_id: run_id.to_owned(), acquisitions: Vec::new(), @@ -290,6 +379,10 @@ impl Database { }) } + pub(super) fn recovery_identity(&self) -> &RecoveryIdentity { + &self.recovery_identity + } + pub(super) fn provision( &mut self, label: &str, @@ -376,11 +469,12 @@ impl Database { // session lock orders this replacement session after an unconfirmed // CREATE has committed or rolled back, so cleanup cannot run too early. self.retire_admin_session(); - let (client, connection_task) = open_admin_session( + let (client, connection_task, _) = open_admin_session( &self.runtime, &self.admin_config, &self.tls, &self.ownership_lock, + Some(&self.recovery_identity), deadline, ) .context("cannot reconnect to the PostgreSQL administrator database")?; @@ -601,6 +695,17 @@ impl Database { self.endpoint.product_url(database, user, password) } + /// Take ownership of everything `provision` can create for one label, so + /// that `cleanup` releases the objects of a run whose supervisor is lost. + pub(super) fn adopt(&mut self, label: &str, spatial: bool) -> Result<()> { + let mut names = run_object_names(&self.run_id, label, spatial)?; + let database = names.pop().context("run has no database name")?; + self.acquisitions + .extend(names.into_iter().map(Acquisition::Role)); + self.acquisitions.push(Acquisition::Database(database)); + Ok(()) + } + /// Names of the databases and roles this run still owns, oldest first. pub(super) fn pending_names(&self) -> Vec { self.acquisitions @@ -917,6 +1022,145 @@ mod tests { ); } + /// A loopback server that accepts one connection and never answers it, as a + /// PostgreSQL server does whose TLS negotiation stalls. The thread ends when + /// the client closes the connection. + fn stalled_server() -> (u16, std::thread::JoinHandle<()>) { + let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + std::io::copy(&mut stream, &mut std::io::sink()).unwrap(); + }); + (port, server) + } + + /// The administrator configuration `Database::connect` builds for `port`. + fn stalled_admin_config(port: u16) -> Config { + let (mut config, _) = admin_config(&format!( + "postgresql://admin:secret@127.0.0.1:{port}/postgres" + )) + .unwrap(); + config.ssl_mode(SslMode::Require); + apply_session_options(&mut config); + config + } + + /// A TLS connector that trusts nothing; a stalled server never reaches TLS. + fn untrusting_tls() -> MakeRustlsConnect { + let _ = rustls::crypto::ring::default_provider().install_default(); + MakeRustlsConnect::new( + rustls::ClientConfig::builder() + .with_root_certificates(rustls::RootCertStore::empty()) + .with_no_client_auth(), + ) + } + + #[test] + fn a_stalled_server_cannot_hold_the_first_administrator_connection() { + let (port, server) = stalled_server(); + let runtime = Builder::new_current_thread().enable_all().build().unwrap(); + let started = Instant::now(); + + let Err(error) = open_admin_session( + &runtime, + &stalled_admin_config(port), + &untrusting_tls(), + "SELECT 1", + None, + Instant::now() + Duration::from_millis(300), + ) else { + panic!("a server that never answers opened a session"); + }; + + assert_eq!( + format!("{error:#}"), + "the PostgreSQL connection handshake timed out" + ); + assert!(started.elapsed() < Duration::from_secs(5)); + // The abandoned attempt closed its connection. + server.join().unwrap(); + } + + /// A plaintext server that completes one PostgreSQL startup without + /// authentication, so a test can hold a real administrator `Client`. + fn trusting_server() -> (u16, std::thread::JoinHandle<()>) { + use std::io::{Read, Write}; + let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut length = [0_u8; 4]; + stream.read_exact(&mut length).unwrap(); + let mut startup = vec![0_u8; u32::from_be_bytes(length) as usize - 4]; + stream.read_exact(&mut startup).unwrap(); + // AuthenticationOk, then ReadyForQuery while idle. + stream + .write_all(b"R\0\0\0\x08\0\0\0\0Z\0\0\0\x05I") + .unwrap(); + std::io::copy(&mut stream, &mut std::io::sink()).unwrap(); + }); + (port, server) + } + + #[test] + fn a_stalled_server_cannot_hold_the_cleanup_reconnection() { + let (trusting_port, trusting) = trusting_server(); + let (stalled_port, stalled) = stalled_server(); + let runtime = Builder::new_current_thread().enable_all().build().unwrap(); + let mut first = Config::new(); + first + .host("127.0.0.1") + .port(trusting_port) + .user("admin") + .dbname("postgres") + .ssl_mode(SslMode::Disable); + let (client, connection) = runtime + .block_on(first.connect(tokio_postgres::NoTls)) + .unwrap(); + let connection_task = runtime.spawn(connection); + // The first session failed mid-operation, so cleanup must reconnect + // first, and the replacement server never answers. + let mut database = Database { + client, + _connection_task: connection_task, + admin_session_failed: true, + tls: untrusting_tls(), + admin_config: stalled_admin_config(stalled_port), + ownership_lock: "SELECT 1".to_owned(), + recovery_identity: RecoveryIdentity { + system_identifier: 23, + database: "postgres".to_owned(), + }, + endpoint: Endpoint { + host: UrlHost::Ipv4(std::net::Ipv4Addr::LOCALHOST), + port: Some(stalled_port), + }, + run_id: RUN_ID.to_owned(), + acquisitions: Vec::new(), + runtime, + }; + let started = Instant::now(); + + let error = database + .execute_cleanup( + "DROP ROLE IF EXISTS rstack_stalled_role", + Instant::now() + Duration::from_millis(300), + ) + .unwrap_err(); + + assert_eq!( + format!("{error:#}"), + "cannot reconnect to the PostgreSQL administrator database: \ + the PostgreSQL connection handshake timed out" + ); + assert!(started.elapsed() < Duration::from_secs(5)); + assert!(database.admin_session_failed); + stalled.join().unwrap(); + drop(database); + trusting.join().unwrap(); + } + #[test] fn post_handshake_operation_deadline_marks_the_effect_unconfirmed() { let runtime = Builder::new_current_thread().enable_time().build().unwrap(); @@ -1005,6 +1249,49 @@ mod tests { assert!(ownership_lock_statement("short").is_err()); } + #[test] + fn recovery_refuses_a_different_postgresql_cluster() { + let expected = RecoveryIdentity { + system_identifier: 23, + database: "postgres".to_owned(), + }; + let actual = RecoveryIdentity { + system_identifier: 42, + database: "postgres".to_owned(), + }; + let error = require_recovery_identity(&expected, &actual).unwrap_err(); + assert_eq!( + error.to_string(), + "this run belongs to a different PostgreSQL cluster; use the administrator URL from startup" + ); + } + + #[test] + fn recovery_refuses_a_different_administrator_database_in_the_same_cluster() { + let expected = RecoveryIdentity { + system_identifier: 23, + database: "postgres".to_owned(), + }; + let actual = RecoveryIdentity { + system_identifier: 23, + database: "template1".to_owned(), + }; + let error = require_recovery_identity(&expected, &actual).unwrap_err(); + assert_eq!( + error.to_string(), + "this run belongs to a different PostgreSQL maintenance database; use the administrator URL from startup" + ); + } + + #[test] + fn recovery_accepts_the_original_postgresql_target() { + let identity = RecoveryIdentity { + system_identifier: 23, + database: "postgres".to_owned(), + }; + require_recovery_identity(&identity, &identity).unwrap(); + } + #[test] fn admin_urls_require_one_explicit_password_authenticated_tcp_database() { let password = random_password().unwrap(); diff --git a/crates/registry-bregctl/src/project_dev/guard.rs b/crates/registry-bregctl/src/project_dev/guard.rs new file mode 100644 index 0000000000..105b6956de --- /dev/null +++ b/crates/registry-bregctl/src/project_dev/guard.rs @@ -0,0 +1,668 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Own one development child process group beyond the worker's lifetime. + +use anyhow::{ensure, Context, Result}; +use rustix::{ + fs::{fcntl_getfl, fcntl_setfl, flock, FlockOperation, OFlags}, + process::{kill_process_group, waitid, Pid, Signal, WaitId, WaitIdOptions}, +}; +use std::{ + ffi::{OsStr, OsString}, + fs::{self, File, OpenOptions}, + io::{self, Read}, + os::{ + fd::AsFd, + unix::{ + fs::{MetadataExt, OpenOptionsExt, PermissionsExt}, + process::CommandExt, + }, + }, + path::{Path, PathBuf}, + process::{self, ChildStdin, ExitCode, ExitStatus, Stdio}, + sync::atomic::{AtomicU64, Ordering}, + thread, + time::{Duration, Instant}, +}; + +use super::{private_directory, write_private, POLL}; + +pub(super) const DIRECTORY: &str = "guards"; +const START_WAIT: Duration = Duration::from_secs(5); +const LAUNCHING: &[u8] = b"launching\n"; +const COMPLETE: &[u8] = b"complete\n"; +static NEXT_GUARD: AtomicU64 = AtomicU64::new(0); + +#[derive(clap::Args, Debug)] +pub(crate) struct Args { + #[arg(long)] + lock: PathBuf, + #[arg(long)] + pid_file: PathBuf, + #[arg(long)] + state_file: PathBuf, + #[arg(last = true, required = true, num_args = 1..)] + command: Vec, +} + +/// The hidden bregctl entry point used by a development worker. +pub(crate) fn run(args: Args) -> ExitCode { + match run_inner(args) { + Ok(status) => exit_code(status), + Err(error) => { + eprintln!("development process guard failed: {error:#}"); + ExitCode::FAILURE + } + } +} + +fn run_inner(args: Args) -> Result { + let (program, arguments) = args + .command + .split_first() + .context("the process guard command is empty")?; + let mut command = process::Command::new(program); + command + .args(arguments) + .env_remove("RSTACK_DEV_DATABASE_URL") + .env_remove("RSTACK_PROJECT_DEV_REQUEST") + .process_group(0) + .stdin(Stdio::null()); + let mut controller = io::stdin(); + supervise( + &args.lock, + &args.pid_file, + &args.state_file, + &mut controller, + command, + ) +} + +/// Take the lock and check the lifetime pipe before spawning. Recovery first +/// excludes the worker and then takes these same locks, so a delayed guard +/// cannot start a workload after its worker has died. +fn supervise( + lock_path: &Path, + pid_file: &Path, + state_file: &Path, + controller: &mut (impl Read + AsFd), + mut command: process::Command, +) -> Result { + let lock = open_lock(lock_path)?; + flock(&lock, FlockOperation::LockExclusive).context("cannot lock the process guard")?; + set_nonblocking(&*controller)?; + if !controller_alive(controller)? { + return Ok(success_status()); + } + + // This receipt precedes spawn. If the guard dies unexpectedly after this + // point, recovery sees an incomplete launch after the lock is released and + // refuses to claim cleanup succeeded. + write_private(state_file, LAUNCHING)?; + let child = match command.spawn() { + Ok(child) => child, + Err(error) => { + write_private(state_file, COMPLETE)?; + return Err(error).context("cannot start guarded command"); + } + }; + let mut workload = Workload::new(child); + let operation = (|| -> Result<()> { + write_private(pid_file, workload.child().id().to_string().as_bytes())?; + loop { + if direct_child_exited(workload.child())? || !controller_alive(controller)? { + return Ok(()); + } + thread::sleep(POLL); + } + })(); + let status = workload.stop()?; + write_private(state_file, COMPLETE)?; + operation?; + Ok(status) +} + +fn complete_state(path: &Path) -> Result<()> { + ensure!( + fs::read(path)? == COMPLETE, + "process guard did not record completed cleanup" + ); + Ok(()) +} + +fn incomplete_state(path: &Path) -> Result { + let state = fs::read(path)?; + ensure!( + state == LAUNCHING || state == COMPLETE, + "process guard has an invalid lifecycle receipt" + ); + Ok(state != COMPLETE) +} + +/// Released guard locks with these receipts cannot establish that the +/// workload group was reaped. Recovery must refuse cleanup for the run. +pub(super) fn incomplete_launches(root: &Path) -> Result> { + let directory = root.join(DIRECTORY); + let mut launches = Vec::new(); + let mut pid_stems = Vec::new(); + for entry in fs::read_dir(&directory).with_context(|| { + format!( + "cannot read process guard directory {}", + directory.display() + ) + })? { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.ends_with(".state") && incomplete_state(&entry.path())? { + launches.push(name.trim_end_matches(".state").to_owned()); + } else if name.ends_with(".pid") { + pid_stems.push(name.trim_end_matches(".pid").to_owned()); + } + } + for stem in pid_stems { + if !directory.join(format!("{stem}.state")).is_file() { + launches.push(stem); + } + } + launches.sort(); + launches.dedup(); + Ok(launches) +} + +/// Keep the lock's scope outside this owner, so every return path stops and +/// reaps a spawned workload before the lock is released. +struct Workload { + child: Option, +} + +impl Workload { + fn new(child: process::Child) -> Self { + Self { child: Some(child) } + } + + fn child(&self) -> &process::Child { + self.child.as_ref().expect("workload is live") + } + + fn stop(&mut self) -> Result { + let mut child = self.child.take().expect("workload is live"); + match stop_direct_child(&mut child) { + Ok(status) => Ok(status), + Err(error) => { + self.child = Some(child); + Err(error) + } + } + } +} + +impl Drop for Workload { + fn drop(&mut self) { + let Some(mut child) = self.child.take() else { + return; + }; + // An error after spawn must not release the guard lock while the group + // remains live. If the structured stop failed, retain the lock while + // this final attempt kills the leader and waits for it. + if stop_direct_child(&mut child).is_err() { + let pid = direct_child_pid(&child).ok(); + if let Some(pid) = pid { + let _ = kill_process_group(pid, Signal::KILL); + } + let _ = child.kill(); + // If signaling failed unexpectedly, waiting here retains the + // guard lock instead of letting recovery overclaim cleanup. + let _ = child.wait(); + } + } +} + +fn open_lock(path: &Path) -> Result { + let file = OpenOptions::new() + .read(true) + .write(true) + .custom_flags(OFlags::NOFOLLOW.bits() as i32) + .open(path) + .with_context(|| format!("cannot open process guard lock {}", path.display()))?; + let metadata = file.metadata()?; + ensure!(metadata.is_file(), "the process guard lock is not a file"); + ensure!( + metadata.permissions().mode() & 0o777 == 0o600, + "the process guard lock is not owner-only" + ); + ensure!( + metadata.uid() == rustix::process::getuid().as_raw(), + "the process guard lock has another owner" + ); + Ok(file) +} + +fn set_nonblocking(fd: impl AsFd) -> Result<()> { + let flags = fcntl_getfl(&fd)?; + fcntl_setfl(fd, flags | OFlags::NONBLOCK)?; + Ok(()) +} + +/// A lifetime pipe carries no protocol data. EOF alone means the worker no +/// longer owns the guarded process and the guard must stop it. +fn controller_alive(controller: &mut impl Read) -> Result { + let mut byte = [0_u8; 1]; + loop { + match controller.read(&mut byte) { + Ok(0) => return Ok(false), + Ok(_) => return Ok(true), + Err(error) if error.kind() == io::ErrorKind::WouldBlock => return Ok(true), + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(error) => return Err(error).context("cannot read the process guard lifetime pipe"), + } + } +} + +pub(super) fn direct_child_pid(child: &process::Child) -> Result { + Pid::from_raw(i32::try_from(child.id())?).context("invalid owned process ID") +} + +pub(super) fn direct_child_exited(child: &process::Child) -> Result { + let options = WaitIdOptions::EXITED | WaitIdOptions::NOHANG | WaitIdOptions::NOWAIT; + loop { + match waitid(WaitId::Pid(direct_child_pid(child)?), options) { + Ok(status) => return Ok(status.is_some()), + Err(rustix::io::Errno::INTR) => continue, + Err(error) => return Err(error.into()), + } + } +} + +fn signal_group(child: &process::Child, signal: Signal) -> Result<()> { + match kill_process_group(direct_child_pid(child)?, signal) { + Ok(()) | Err(rustix::io::Errno::SRCH) => Ok(()), + // Darwin reports EPERM when an exited leader has no signalable group + // members. The leader remains our unreaped child, so this is empty. + Err(rustix::io::Errno::PERM) + if cfg!(target_os = "macos") && direct_child_exited(child)? => + { + Ok(()) + } + Err(error) => Err(error.into()), + } +} + +pub(super) fn stop_direct_child(child: &mut process::Child) -> Result { + if !direct_child_exited(child)? { + signal_group(child, Signal::TERM)?; + let deadline = Instant::now() + Duration::from_secs(2); + while Instant::now() < deadline && !direct_child_exited(child)? { + thread::sleep(POLL); + } + } + // Keep the leader PID reserved until the final group signal, then reap it. + signal_group(child, Signal::KILL)?; + Ok(child.wait()?) +} + +fn success_status() -> ExitStatus { + use std::os::unix::process::ExitStatusExt; + ExitStatus::from_raw(0) +} + +fn exit_code(status: ExitStatus) -> ExitCode { + status + .code() + .and_then(|code| u8::try_from(code).ok()) + .map(ExitCode::from) + .unwrap_or(ExitCode::FAILURE) +} + +/// Builder for a command whose process group is owned by a guard. Output and +/// error streams belong to the guard and are inherited unchanged by the +/// workload, preserving the tool's JSON and log behavior. +pub(super) struct Command { + command: process::Command, + pid_file: PathBuf, + state_file: PathBuf, +} + +impl Command { + pub(super) fn new(root: &Path, name: &str, binary: &Path) -> Result { + let directory = root.join(DIRECTORY); + match private_directory(&directory) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error).context("cannot create process guard directory"), + } + ensure!( + fs::metadata(&directory)?.permissions().mode() & 0o777 == 0o700, + "the process guard directory is not owner-only" + ); + + let sequence = NEXT_GUARD.fetch_add(1, Ordering::Relaxed); + let stem = format!("{sequence:016x}-{name}"); + let lock = directory.join(format!("{stem}.lock")); + let lock_file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&lock) + .with_context(|| format!("cannot create process guard lock {}", lock.display()))?; + lock_file.set_permissions(fs::Permissions::from_mode(0o600))?; + let pid_file = directory.join(format!("{stem}.pid")); + let state_file = directory.join(format!("{stem}.state")); + let mut command = process::Command::new(std::env::current_exe()?); + command + .arg("project-dev-guard") + .arg("--lock") + .arg(lock) + .arg("--pid-file") + .arg(&pid_file) + .arg("--state-file") + .arg(&state_file) + .arg("--") + .arg(binary) + .env_remove("RSTACK_DEV_DATABASE_URL") + .env_remove("RSTACK_PROJECT_DEV_REQUEST") + .process_group(0) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + Ok(Self { + command, + pid_file, + state_file, + }) + } + + pub(super) fn arg(&mut self, argument: impl AsRef) -> &mut Self { + self.command.arg(argument); + self + } + + pub(super) fn args(&mut self, arguments: I) -> &mut Self + where + I: IntoIterator, + S: AsRef, + { + self.command.args(arguments); + self + } + + pub(super) fn stdout(&mut self, stream: impl Into) -> &mut Self { + self.command.stdout(stream); + self + } + + pub(super) fn stderr(&mut self, stream: impl Into) -> &mut Self { + self.command.stderr(stream); + self + } + + pub(super) fn spawn(&mut self) -> Result { + let mut guard = self.command.spawn().context("cannot start process guard")?; + let controller = guard + .stdin + .take() + .context("process guard stdin was not piped")?; + let started: Result = (|| -> Result { + let deadline = Instant::now() + START_WAIT; + loop { + match fs::read_to_string(&self.pid_file) { + Ok(value) => { + let raw: i32 = + value.trim().parse().context("invalid guarded process ID")?; + let pid = Pid::from_raw(raw).context("invalid guarded process ID")?; + return Ok(pid); + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error).context("cannot read guarded process ID"), + } + if guard.try_wait()?.is_some() { + anyhow::bail!("process guard exited before starting its workload"); + } + ensure!( + Instant::now() < deadline, + "process guard did not publish its workload ID" + ); + thread::sleep(Duration::from_millis(10)); + } + })(); + match started { + Ok(pid) => Ok(Child { + guard, + controller: Some(controller), + pid, + state_file: self.state_file.clone(), + }), + Err(error) => { + drop(controller); + let _ = guard.wait(); + Err(error) + } + } + } +} + +/// The worker's handle for a guard and its actual workload process group. +pub(super) struct Child { + guard: process::Child, + controller: Option, + pid: Pid, + state_file: PathBuf, +} + +impl Child { + pub(super) fn id(&self) -> u32 { + self.pid + .as_raw_pid() + .try_into() + .expect("positive process ID") + } + + pub(super) fn exited(&self) -> Result { + direct_child_exited(&self.guard) + } + + pub(super) fn stop(&mut self) -> Result { + drop(self.controller.take()); + let status = self.guard.wait()?; + complete_state(&self.state_file)?; + Ok(status) + } +} + +impl Drop for Child { + fn drop(&mut self) { + if self.controller.is_none() { + return; + } + if let Err(error) = self.stop() { + eprintln!("development process guard cleanup failed: {error:#}"); + } + } +} + +/// The actual runtime groups a run started. Recovery uses this only for its +/// success report after every guard lock is free, never as a signaling source. +pub(super) fn runtime_groups(root: &Path) -> Result> { + let directory = root.join(DIRECTORY); + let mut groups = Vec::new(); + for entry in fs::read_dir(&directory).with_context(|| { + format!( + "cannot read process guard directory {}", + directory.display() + ) + })? { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if !name.contains("-runtime-") || !name.ends_with(".pid") { + continue; + } + complete_state(&entry.path().with_extension("state"))?; + let raw: u32 = fs::read_to_string(entry.path())? + .trim() + .parse() + .context("invalid guarded runtime process ID")?; + groups.push(raw); + } + groups.sort_unstable(); + Ok(groups) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{os::unix::net::UnixStream, sync::mpsc}; + + fn lock_file(root: &Path) -> PathBuf { + let path = root.join("guard.lock"); + OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&path) + .unwrap(); + path + } + + #[test] + fn worker_death_before_pid_publication_spawns_no_workload() { + let scratch = tempfile::tempdir().unwrap(); + let lock = lock_file(scratch.path()); + let pid_file = scratch.path().join("guard.pid"); + let state_file = scratch.path().join("guard.state"); + let marker = scratch.path().join("spawned"); + let (mut controller, worker) = UnixStream::pair().unwrap(); + drop(worker); + let mut command = process::Command::new("/bin/sh"); + command.args(["-c", &format!("touch '{}'", marker.display())]); + + let status = supervise(&lock, &pid_file, &state_file, &mut controller, command).unwrap(); + assert!(status.success()); + assert!(!marker.exists()); + assert!(!pid_file.exists()); + assert!(!state_file.exists()); + } + + #[test] + fn guard_owns_and_reaps_the_workload_process_group() { + let scratch = tempfile::tempdir().unwrap(); + let lock = lock_file(scratch.path()); + let pid_file = scratch.path().join("guard.pid"); + let state_file = scratch.path().join("guard.state"); + let nested_file = scratch.path().join("nested.pid"); + let (mut controller, worker) = UnixStream::pair().unwrap(); + let (finished_tx, finished_rx) = mpsc::sync_channel(1); + let mut command = process::Command::new("/bin/sh"); + command + .args([ + "-c", + r#"sh -c 'trap "" TERM; printf "%s" "$$" > "$1.tmp" && mv "$1.tmp" "$1" && exec sleep 60' sh "$1" & wait"#, + "guard-fixture", + ]) + .arg(&nested_file) + .process_group(0) + .stdin(Stdio::null()); + let guard_lock = lock.clone(); + let guard_pid = pid_file.clone(); + let guard_state = state_file.clone(); + let guard_thread = thread::spawn(move || { + let status = supervise( + &guard_lock, + &guard_pid, + &guard_state, + &mut controller, + command, + ) + .unwrap(); + finished_tx.send(status).unwrap(); + }); + + let leader = wait_pid(&pid_file); + let nested = wait_pid(&nested_file); + assert_eq!(rustix::process::getpgid(Some(leader)).unwrap(), leader); + assert_eq!(rustix::process::getpgid(Some(nested)).unwrap(), leader); + let contender = open_lock(&lock).unwrap(); + assert_eq!( + flock(&contender, FlockOperation::NonBlockingLockExclusive).unwrap_err(), + rustix::io::Errno::WOULDBLOCK + ); + + drop(worker); + finished_rx.recv_timeout(Duration::from_secs(4)).unwrap(); + guard_thread.join().unwrap(); + complete_state(&state_file).unwrap(); + flock(&contender, FlockOperation::NonBlockingLockExclusive).unwrap(); + assert_eq!( + rustix::process::getpgid(Some(leader)).unwrap_err(), + rustix::io::Errno::SRCH + ); + // The guard owns the leader. The OS adopts the nested child when + // the leader exits, and may keep it visible as a zombie after SIGKILL. + let deadline = Instant::now() + Duration::from_secs(5); + let pid = nested.as_raw_pid().to_string(); + loop { + let output = process::Command::new("ps") + .args(["-p", &pid, "-o", "stat="]) + .output() + .unwrap(); + if output.status.code() == Some(1) + && output.stdout.is_empty() + && output.stderr.is_empty() + { + break; + } + assert!( + output.status.success(), + "ps could not read the nested process" + ); + let state = String::from_utf8(output.stdout).unwrap(); + if state.trim_start().starts_with('Z') { + break; + } + assert!( + Instant::now() < deadline, + "the nested process is still live: {state}" + ); + thread::sleep(POLL); + } + } + + #[test] + fn runtime_group_report_excludes_provisioning_tools() { + let scratch = tempfile::tempdir().unwrap(); + let directory = scratch.path().join(DIRECTORY); + fs::create_dir(&directory).unwrap(); + fs::write(directory.join("000-runtime-breg.pid"), "41").unwrap(); + fs::write(directory.join("000-runtime-breg.state"), COMPLETE).unwrap(); + fs::write(directory.join("001-tool-breg-check.pid"), "42").unwrap(); + fs::write(directory.join("001-tool-breg-check.state"), COMPLETE).unwrap(); + fs::write(directory.join("002-runtime-casework.pid"), "43").unwrap(); + fs::write(directory.join("002-runtime-casework.state"), COMPLETE).unwrap(); + assert_eq!(runtime_groups(scratch.path()).unwrap(), vec![41, 43]); + } + + #[test] + fn incomplete_launch_receipt_refuses_recovery() { + let scratch = tempfile::tempdir().unwrap(); + let directory = scratch.path().join(DIRECTORY); + fs::create_dir(&directory).unwrap(); + fs::write(directory.join("000-runtime-breg.state"), LAUNCHING).unwrap(); + fs::write(directory.join("001-tool-breg-check.state"), COMPLETE).unwrap(); + fs::write(directory.join("002-runtime-casework.pid"), "43").unwrap(); + assert_eq!( + incomplete_launches(scratch.path()).unwrap(), + vec!["000-runtime-breg", "002-runtime-casework"] + ); + } + + fn wait_pid(path: &Path) -> Pid { + let deadline = Instant::now() + Duration::from_secs(2); + loop { + if let Ok(value) = fs::read_to_string(path) { + let raw = value.parse().unwrap(); + return Pid::from_raw(raw).unwrap(); + } + assert!(Instant::now() < deadline, "PID was not published"); + thread::sleep(Duration::from_millis(10)); + } + } +} diff --git a/crates/registry-bregctl/src/project_dev/journey.rs b/crates/registry-bregctl/src/project_dev/journey.rs index 0cd7596abf..316a531b4b 100644 --- a/crates/registry-bregctl/src/project_dev/journey.rs +++ b/crates/registry-bregctl/src/project_dev/journey.rs @@ -732,6 +732,7 @@ mod tests { principal: "sub".to_owned(), purpose: false, requires_postgis: false, + encrypts_fields: false, named_clients: Default::default(), }), casework_principals: BTreeMap::new(), diff --git a/crates/registry-bregctl/src/project_dev/mod.rs b/crates/registry-bregctl/src/project_dev/mod.rs index 31b4ac4a24..3d5c0dbea8 100644 --- a/crates/registry-bregctl/src/project_dev/mod.rs +++ b/crates/registry-bregctl/src/project_dev/mod.rs @@ -4,10 +4,12 @@ mod bindings; mod database; +pub(crate) mod guard; mod issuer; mod journey; pub(crate) mod plan; mod products; +mod recovery; use anyhow::{bail, ensure, Context, Result}; use registry_platform_yaml::{Diagnostic, Report}; @@ -36,18 +38,20 @@ use uuid::Uuid; use zeroize::Zeroizing; use self::{ - database::Database, + database::{Database, RecoveryIdentity}, issuer::Issuer, plan::{Plan, Product}, products::{ProductRunner, SchemaPerson}, }; const SCHEMA: &str = "registry.rstack.dev.v1"; +const ERROR_SCHEMA: &str = "registry.rstack.error.v1"; const REQUEST: &str = "RSTACK_PROJECT_DEV_REQUEST"; const DATABASE_URL: &str = "RSTACK_DEV_DATABASE_URL"; const CA_FILE: &str = "RSTACK_DEV_DATABASE_CA"; const REPORT: &str = "report.json"; const PENDING: &str = "cleanup-pending.json"; +const POSTGRES_IDENTITY: &str = "postgres-identity.json"; const CLEANUP_CAP: Duration = Duration::from_secs(30 * 60); const POLL: Duration = Duration::from_millis(100); // A capture can contain the 1 MiB development plan after JSON escaping, @@ -90,6 +94,8 @@ enum Request { Stop { run_id: String, json: bool, + #[serde(default)] + recover: bool, }, } @@ -122,6 +128,9 @@ struct BregAuthority { principal: String, purpose: bool, requires_postgis: bool, + /// The compiled project declares encrypted fields, so the run draws its + /// own development data key. + encrypts_fields: bool, named_clients: BTreeSet, } @@ -134,12 +143,40 @@ struct Binaries { #[derive(Deserialize, Serialize)] #[serde(tag = "outcome", rename_all = "snake_case")] enum WorkerReply { - Ready { report: Value }, - Failed { error: String }, + Ready { + report: Value, + }, + Failed { + error: String, + /// The environment failed rather than the project or plan, so the + /// supervisor reports the operational class. + #[serde(default)] + operational: bool, + }, +} + +impl WorkerReply { + fn failed(error: &anyhow::Error) -> Self { + Self::failure(format!("{error:#}"), error.is::()) + } + + fn failure(error: String, operational: bool) -> Self { + Self::Failed { error, operational } + } +} + +/// The supervisor's error for a failure its worker reported, in the class the +/// worker gave it. +fn restore_failure(error: String, operational: bool) -> anyhow::Error { + if operational { + Operational(error).into() + } else { + anyhow::anyhow!(error) + } } #[derive(Debug, thiserror::Error)] -#[error("run {run_id} still needs cleanup: {detail}")] +#[error("run {run_id} still needs cleanup: {detail}; run rstack dev stop {run_id} --recover to release its resources")] struct CleanupPending { run_id: String, detail: String, @@ -155,13 +192,17 @@ struct RecoveryNeeded { } fn recovery_message(run_id: &str, facts: &Value) -> String { - let mut text = match facts["kind"].as_str() { - Some("cleanup-pending") => { - format!("run {run_id} gave up cleanup; release these resources by hand") + let mut text = if let Some(reason) = facts["reason"].as_str() { + format!("run {run_id} could not be recovered: {reason}; release these resources by hand") + } else { + match facts["kind"].as_str() { + Some("cleanup-pending") => format!( + "run {run_id} gave up cleanup; run rstack dev stop {run_id} --recover or release these resources by hand" + ), + _ => format!( + "run {run_id} lost its supervisor before cleanup; run rstack dev stop {run_id} --recover or release these resources by hand" + ), } - _ => format!( - "run {run_id} lost its supervisor before cleanup; release these resources by hand" - ), }; let mut line = |entry: String| { text.push_str("\n "); @@ -197,6 +238,9 @@ fn recovery_message(run_id: &str, facts: &Value) -> String { { line(format!("database or role {}", name.as_str().unwrap_or("?"))); } + if let Some(key) = facts["field_encryption_key"].as_str() { + line(format!("development field-encryption key {key}")); + } if let Some(reason) = facts["unavailable"].as_str() { line(format!("some facts are unavailable: {reason}")); } @@ -216,9 +260,17 @@ fn lost_supervisor_facts(root: &Path, run_id: &str) -> Value { Ok(objects) => facts["objects"] = json!(objects), Err(error) => facts["unavailable"] = json!(format!("{error:#}")), } + name_field_key(root, &mut facts); facts } +/// Name, never read, the development data key a run left behind. +fn name_field_key(root: &Path, facts: &mut Value) { + if let Some(key) = products::field_key_file(root) { + facts["field_encryption_key"] = json!(key); + } +} + fn recorded_processes(root: &Path) -> Value { read_json::(&root.join("processes.json"), MAXIMUM_RUN_REPORT_BYTES) .ok() @@ -226,7 +278,9 @@ fn recorded_processes(root: &Path) -> Value { .unwrap_or_else(|| json!([])) } -fn provisioned_objects(root: &Path, run_id: &str) -> Result> { +/// The database label and spatial flag of each database the run's plan +/// provisions. +fn provisioned_labels(root: &Path) -> Result> { let input: RunInput = read_json(&root.join("input.json"), MAXIMUM_CAPTURED_INPUT_BYTES) .context("cannot read the captured development plan")?; let spatial = input @@ -243,7 +297,11 @@ fn provisioned_objects(root: &Path, run_id: &str) -> Result> { Product::Casework => labels.push(("casework", false)), } } - labels + Ok(labels) +} + +fn provisioned_objects(root: &Path, run_id: &str) -> Result> { + provisioned_labels(root)? .into_iter() .map(|(label, spatial)| { let mut names = database::run_object_names(run_id, label, spatial)?; @@ -299,12 +357,7 @@ pub(crate) fn run(args: Args, stdout: &mut dyn Write, stderr: &mut dyn Write) -> return match worker(&root, stdout) { Ok(()) => 0, Err(error) => { - let _ = reply( - stdout, - &WorkerReply::Failed { - error: format!("{error:#}"), - }, - ); + let _ = reply(stdout, &WorkerReply::failed(&error)); 3 } }; @@ -330,28 +383,29 @@ pub(crate) fn run(args: Args, stdout: &mut dyn Write, stderr: &mut dyn Write) -> Request::Start { project, ttl, .. } => start(&project, ttl.as_deref()), Request::Status { run_id, .. } => status(&run_id), Request::Token { run_id, person, .. } => token(&run_id, &person), - Request::Stop { run_id, .. } => stop(&run_id), + Request::Stop { + run_id, recover, .. + } => { + if recover { + recover_run(&run_id) + } else { + stop(&run_id) + } + } }; let (result, code) = match outcome { Ok(value) => { let code = u8::from(starting && value["journey"]["result"] == "failed"); (emit(stdout, &value, machine), code) } - Err(error) if machine => { - let mut report = json!({ - "schema_version": SCHEMA, "status": "error", "error": format!("{error:#}") - }); - if let Some(pending) = error.downcast_ref::() { - report["run_id"] = json!(pending.run_id); - report["cleanup_pending"] = json!(true); - } - if let Some(recovery) = error.downcast_ref::() { - report["run_id"] = json!(recovery.run_id); - report["recovery"] = recovery.facts.clone(); + Err(error) => { + let (report, code) = error_report(&error); + if machine { + (writeln!(stdout, "{report}"), code) + } else { + (writeln!(stderr, "rstack dev: {error:#}"), code) } - (writeln!(stdout, "{report}"), 1) } - Err(error) => (writeln!(stderr, "rstack dev: {error:#}"), 1), }; if result.is_ok() { code @@ -360,6 +414,54 @@ pub(crate) fn run(args: Args, stdout: &mut dyn Write, stderr: &mut dyn Write) -> } } +/// The failure of an environment or of a run's resources, as opposed to a +/// request the operator can correct. +#[derive(Debug, thiserror::Error)] +#[error("{0}")] +struct Operational(String); + +/// Classify `error` as an environment failure, such as an unreachable +/// PostgreSQL server, a product binary that cannot be started, or no free +/// loopback port. The message is unchanged. +fn operational(error: anyhow::Error) -> anyhow::Error { + Operational(format!("{error:#}")).into() +} + +/// A malformed argument, refused before any project or run state is read. +#[derive(Debug, thiserror::Error)] +#[error("{0}")] +struct Usage(&'static str); + +/// The error envelope and exit code for a failed command. It is the launcher's +/// `registry.rstack.error.v1` object, plus `run_id` and `recovery` when a run +/// still has resources to release. +fn error_report(error: &anyhow::Error) -> (Value, u8) { + let operational = error.is::() + || error.is::() + || error.is::() + || error.is::(); + let (class, code) = if operational { + ("operational", crate::OPERATIONAL_FAILURE_EXIT) + } else if error.is::() { + ("usage", crate::USAGE_EXIT) + } else { + ("domain", crate::DOMAIN_REFUSAL_EXIT) + }; + let mut report = json!({ + "schema_version": ERROR_SCHEMA, + "status": "refused", + "error": {"class": class, "message": format!("{error:#}")}, + }); + if let Some(pending) = error.downcast_ref::() { + report["run_id"] = json!(pending.run_id); + } + if let Some(recovery) = error.downcast_ref::() { + report["run_id"] = json!(recovery.run_id); + report["recovery"] = recovery.facts.clone(); + } + (report, code) +} + fn emit(output: &mut dyn Write, report: &Value, machine: bool) -> std::io::Result<()> { if machine { return writeln!(output, "{report}"); @@ -373,6 +475,26 @@ fn emit(output: &mut dyn Write, report: &Value, machine: bool) -> std::io::Resul writeln!(output, "{line}")?; } } + if let Some(recovered) = report.get("recovered") { + for group in recovered["stopped_process_groups"] + .as_array() + .into_iter() + .flatten() + { + writeln!(output, " stopped process group {group}")?; + } + for name in recovered["databases_and_roles"] + .as_array() + .into_iter() + .flatten() + { + writeln!( + output, + " released database or role {}", + name.as_str().unwrap_or("?") + )?; + } + } if let Some(products) = report["products"].as_object() { for (product, url) in products { writeln!(output, " {product} {}", url.as_str().expect("product URL"))?; @@ -511,6 +633,7 @@ fn write_check_report( } fn start(project: &Path, ttl: Option<&str>) -> Result { + let ttl_seconds = parse_ttl(ttl.unwrap_or("30m"))?; let project = project .canonicalize() .context("cannot open the development project directory")?; @@ -518,7 +641,6 @@ fn start(project: &Path, ttl: Option<&str>) -> Result { let schema_people = products::schema_people(&project, &plan, true)?; let signals = signals()?; let inspection = products::inspect(&project, &plan, &signals)?; - let ttl_seconds = parse_ttl(ttl.unwrap_or("30m"))?; ensure!( env::var_os(DATABASE_URL).is_some_and(|value| !value.is_empty()), "supply --database-url or {DATABASE_URL} for a TLS PostgreSQL development server" @@ -545,6 +667,8 @@ fn start(project: &Path, ttl: Option<&str>) -> Result { }; let preparation = (|| { write_private(&root.join("input.json"), &capture_input(&input)?)?; + private_directory(&root.join(guard::DIRECTORY))?; + write_private(&root.join(POSTGRES_IDENTITY), b"null")?; let supervisor_log = open_supervisor_log(&root)?; Command::new(env::current_exe()?) .arg("project-dev") @@ -584,8 +708,8 @@ fn start(project: &Path, ttl: Option<&str>) -> Result { loop { match receiver.recv_timeout(POLL) { Ok(Ok(WorkerReply::Ready { report })) => return Ok(report), - Ok(Ok(WorkerReply::Failed { error })) => { - let error = anyhow::anyhow!(error); + Ok(Ok(WorkerReply::Failed { error, operational })) => { + let error = restore_failure(error, operational); let kept_logs = failed_logs_dir(&base, &input.run_id); if root.join("worker.lock").exists() { if let Err(cleanup) = stop(&input.run_id) { @@ -625,13 +749,38 @@ fn start(project: &Path, ttl: Option<&str>) -> Result { } Err(mpsc::RecvTimeoutError::Timeout) => {} } - if signals.load(Ordering::Relaxed) || Instant::now() >= deadline { + if let Some(error) = start_abandoned(&signals, deadline) { stop(&input.run_id).map_err(|error| startup_error(&root, &input.run_id, error))?; - bail!("development run start was interrupted or timed out"); + return Err(error); } } } +/// Why `start` stops waiting for the supervisor's startup report, if it does. +/// An interrupt is the operator's request; a missed deadline means the host or +/// the supervisor failed. +fn start_abandoned(signals: &AtomicBool, deadline: Instant) -> Option { + if signals.load(Ordering::Relaxed) { + Some(anyhow::anyhow!("development run start was interrupted")) + } else if Instant::now() >= deadline { + Some(operational(anyhow::anyhow!( + "development run did not start within six minutes" + ))) + } else { + None + } +} + +/// Runs the supervisor's startup, turning a panic into an operational failure +/// that carries no panic payload. +fn catch_startup(startup: impl FnOnce() -> Result) -> Result { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(startup)).unwrap_or_else(|_| { + Err(operational(anyhow::anyhow!( + "development run startup failed internally" + ))) + }) +} + fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { let name = root .file_name() @@ -659,14 +808,16 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { .mode(0o600) .open(root.join("worker.lock"))?; flock(&lock, FlockOperation::LockExclusive)?; + ensure_private(root, false)?; let mut database = None; let mut expiry = None; - let prepared = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let prepared = catch_startup(|| { products.check()?; let ca = env::var_os(CA_FILE) .map(|path| { let bytes = plan::read(Path::new(&path), 1024 * 1024) - .context("cannot read the PostgreSQL CA file")?; + .context("cannot read the PostgreSQL CA file") + .map_err(operational)?; write_private(&root.join("postgres-ca"), &bytes)?; Ok::<_, anyhow::Error>(bytes) }) @@ -676,8 +827,15 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { .map_err(|_| anyhow::anyhow!("missing PostgreSQL administrator URL"))?, ); let issuer = Arc::new(Issuer::create(root)?); - let database = database.insert(Database::connect(&admin, ca.as_deref(), &input.run_id)?); - let urls = products.start(database, issuer.clone())?; + let connected = Database::connect(&admin, ca.as_deref(), &input.run_id)?; + // The cluster identity must be recorded before anything can create a database or role. + write_private( + &root.join(POSTGRES_IDENTITY), + &serde_json::to_vec(connected.recovery_identity())?, + ) + .context("cannot record the PostgreSQL cluster for recovery") + .map_err(operational)?; + let urls = products.start(database.insert(connected), issuer.clone())?; let journey = products.journey(&issuer, &urls)?; ensure!( products.services_alive()?, @@ -692,14 +850,16 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { }); write_private(&root.join(REPORT), &capture_report(&report)?)?; Ok::<_, anyhow::Error>(report) - })) - .unwrap_or_else(|_| Err(anyhow::anyhow!("development run startup failed internally"))); + }); let mut failure = None; match prepared { Ok(report) => { if let Err(error) = reply(output, &WorkerReply::Ready { report }) { - failure = Some(format!("cannot report development run readiness: {error}")); + failure = Some(WorkerReply::failure( + format!("cannot report development run readiness: {error}"), + true, + )); } else { let deadline = Instant::now() + Duration::from_secs(input.ttl_seconds); while Instant::now() < deadline @@ -713,7 +873,7 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { } } } - Err(error) => failure = Some(format!("{error:#}")), + Err(error) => failure = Some(WorkerReply::failed(&error)), } let stopped = simple_report(&input.run_id, "stopping"); let mark_stopping = write_private(&root.join(REPORT), &serde_json::to_vec(&stopped)?); @@ -728,13 +888,8 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { if cleanup.is_ok() { break; } - if let Some(error) = failure.as_ref().filter(|_| !told_parent) { - let _ = reply( - output, - &WorkerReply::Failed { - error: error.clone(), - }, - ); + if let Some(failed) = failure.as_ref().filter(|_| !told_parent) { + let _ = reply(output, failed); told_parent = true; } let Some(delay) = delays.next() else { @@ -759,12 +914,12 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { Ok(()) } .and_then(|()| remove_run_dir(root)); - if let Some(error) = failure.filter(|_| !told_parent) { + if let Some(WorkerReply::Failed { error, operational }) = failure.filter(|_| !told_parent) { let error = match removed.as_ref() { Ok(()) => error, Err(cleanup) => format!("{error}; run {} {cleanup}", input.run_id), }; - reply(output, &WorkerReply::Failed { error })?; + reply(output, &WorkerReply::failure(error, operational))?; } removed?; // If the root is gone, no stale ready report could have survived. @@ -819,11 +974,12 @@ fn pending_status(root: &Path, run_id: &str, pending: Value) -> Value { "databases_and_roles": pending["databases_and_roles"], "processes": pending["processes"], }); + name_field_key(root, &mut report["recovery"]); report } fn token(run_id: &str, person: &str) -> Result { - ensure!(plan::identifier(person), "invalid test person name"); + ensure!(plan::identifier(person), Usage("invalid test person name")); let report = status(run_id)?; ensure!( report["status"] == "ready", @@ -892,10 +1048,12 @@ fn stop(run_id: &str) -> Result { } let deadline = Instant::now() + Duration::from_secs(60); while root.exists() { - ensure!( - Instant::now() < deadline, - "run {run_id} is still cleaning up or its cleanup has stalled; run rstack dev status {run_id} to see its state" - ); + if Instant::now() >= deadline { + return Err(Operational(format!( + "run {run_id} is still cleaning up or its cleanup has stalled; run rstack dev status {run_id} to see its state" + )) + .into()); + } thread::sleep(POLL); if !root.exists() { break; @@ -919,6 +1077,81 @@ fn stop(run_id: &str) -> Result { Ok(simple_report(run_id, "stopped")) } +/// Wait for the lost supervisor's process guards to finish, then release the +/// databases and roles on the original PostgreSQL cluster and remove the run +/// directory. Hold the local lock across recovery to serialize retries. +fn recover_run(run_id: &str) -> Result { + let root = run_dir(run_id)?; + if !root.exists() { + return Ok(simple_report(run_id, "stopped")); + } + ensure_private(&root, false)?; + ensure!( + root.join("worker.lock").exists(), + "run {run_id} has no supervisor lock, so it is still starting or never started; run rstack dev stop {run_id}, or delete {} if the run never started", + root.display() + ); + let lock = match try_run_lock(&root) { + Ok(lock) => lock, + Err(_) if !root.exists() => return Ok(simple_report(run_id, "stopped")), + Err(error) => return Err(error), + }; + let _lock = lock.with_context(|| { + format!("run {run_id} still has a running supervisor or recovery; use plain stop for a live supervisor") + })?; + let unrecovered = |reason: String| -> anyhow::Error { + let mut facts = lost_supervisor_facts(&root, run_id); + facts["reason"] = json!(reason); + RecoveryNeeded { + run_id: run_id.to_owned(), + facts, + } + .into() + }; + let (stopped_groups, dropped) = (|| -> Result<_> { + // Status can read the last report while recovery holds the worker lock. + write_private( + &root.join(REPORT), + &serde_json::to_vec(&simple_report(run_id, "stopping"))?, + ) + .context("cannot mark the run as stopping")?; + // A null identity means the worker never began provisioning. Missing + // or invalid metadata cannot prove that there is nothing to release. + let identity = read_json::>(&root.join(POSTGRES_IDENTITY), 1024) + .context("cannot read the recorded PostgreSQL cluster for this run")?; + let mut database = identity + .map(|expected| { + let admin = Zeroizing::new(env::var(DATABASE_URL).context( + "no PostgreSQL administrator URL; pass --database-url or set RSTACK_DEV_DATABASE_URL", + )?); + let ca = env::var_os(CA_FILE) + .map(|path| plan::read(Path::new(&path), 1024 * 1024)) + .transpose() + .context("cannot read the PostgreSQL CA file")?; + Database::connect_for_recovery(&admin, ca.as_deref(), run_id, &expected) + }) + .transpose()?; + let stopped_groups = recovery::wait_for_guards(&root)?; + let mut dropped = Vec::new(); + if let Some(database) = database.as_mut() { + for (label, spatial) in provisioned_labels(&root)? { + database.adopt(label, spatial)?; + } + dropped = database.pending_names(); + database.cleanup()?; + } + remove_run_dir(&root)?; + Ok((stopped_groups, dropped)) + })() + .map_err(|error| unrecovered(format!("{error:#}")))?; + let mut report = simple_report(run_id, "stopped"); + report["recovered"] = json!({ + "stopped_process_groups": stopped_groups, + "databases_and_roles": dropped, + }); + Ok(report) +} + fn simple_report(run_id: &str, status: &str) -> Value { json!({"schema_version": SCHEMA, "run_id": run_id, "status": status}) } @@ -939,12 +1172,16 @@ fn supervisor_gone(root: &Path) -> Result { } fn supervisor_running(root: &Path) -> Result { + Ok(try_run_lock(root)?.is_none()) +} + +fn try_run_lock(root: &Path) -> Result> { ensure_private(root, false)?; let lock = File::open(root.join("worker.lock")).context("run supervisor lock is unavailable")?; match flock(&lock, FlockOperation::NonBlockingLockExclusive) { - Ok(()) => Ok(false), - Err(rustix::io::Errno::WOULDBLOCK) => Ok(true), + Ok(()) => Ok(Some(lock)), + Err(rustix::io::Errno::WOULDBLOCK) => Ok(None), Err(error) => Err(error).context("cannot check run supervisor lock"), } } @@ -999,7 +1236,11 @@ fn binaries(product: Product) -> Result { } fn parse_ttl(value: &str) -> Result { - let error = || anyhow::anyhow!("--ttl must be between 1s and 24h, with an s, m, or h suffix"); + let error = || { + anyhow::Error::new(Usage( + "--ttl must be between 1s and 24h, with an s, m, or h suffix", + )) + }; let (number, unit) = value .split_at_checked(value.len().saturating_sub(1)) .ok_or_else(error)?; @@ -1030,7 +1271,7 @@ fn run_dir(run_id: &str) -> Result { && run_id .bytes() .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)), - "invalid run ID; use the ID printed by rstack dev start" + Usage("invalid run ID; use the ID printed by rstack dev start") ); Ok(state_dir().join(run_id)) } @@ -1146,9 +1387,17 @@ fn removing_dir(root: &Path) -> Result { } /// Renames the run directory away before deleting it, so `status` sees either -/// the whole run or no run, never a run whose report is already deleted. +/// the whole run or no run, never a run whose report is already deleted. The +/// run's field-encryption key goes first, so an interrupted deletion never +/// leaves key material behind. fn remove_run_dir(root: &Path) -> Result<()> { let doomed = removing_dir(root)?; + match fs::remove_file(root.join(products::FIELD_KEY)) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => { + return Err(error).context("cannot remove the run's field-encryption key"); + } + _ => {} + } fs::rename(root, &doomed).context("cannot remove run scratch files")?; fs::remove_dir_all(&doomed) .with_context(|| format!("cannot remove run scratch files at {}", doomed.display())) @@ -1156,7 +1405,10 @@ fn remove_run_dir(root: &Path) -> Result<()> { fn note_kept_logs(error: anyhow::Error, kept: Option) -> anyhow::Error { match kept { - Some(path) => anyhow::anyhow!("{error:#}; logs kept at {}", path.display()), + Some(path) => restore_failure( + format!("{error:#}; logs kept at {}", path.display()), + error.is::(), + ), None => error, } } @@ -1202,7 +1454,8 @@ mod tests { "schema_people": [], "breg_authority": { "principal": "p", "purpose": false, - "requires_postgis": postgis, "named_clients": [] + "requires_postgis": postgis, "encrypts_fields": false, + "named_clients": [] }, "casework_principals": {}, "binaries": {} @@ -1281,6 +1534,176 @@ mod tests { ); } + #[test] + fn recovery_names_a_field_key_left_in_the_run_directory() { + let scratch = tempfile::tempdir().unwrap(); + let root = scratch.path(); + fake_run( + root, + json!([{"product": "breg", "project": ".", "audience": "registry"}]), + false, + ); + let facts = lost_supervisor_facts(root, RUN_ID); + assert!(facts.get("field_encryption_key").is_none(), "{facts}"); + let pending = pending_status(root, RUN_ID, pending_report(RUN_ID, vec![], json!([]))); + assert!( + pending["recovery"].get("field_encryption_key").is_none(), + "{pending}" + ); + + let canary = "field-key-canary-value"; + fs::write(root.join(products::FIELD_KEY), canary).unwrap(); + let key = root.join(products::FIELD_KEY); + for facts in [ + lost_supervisor_facts(root, RUN_ID), + pending_status(root, RUN_ID, pending_report(RUN_ID, vec![], json!([])))["recovery"] + .clone(), + ] { + assert_eq!(facts["field_encryption_key"], json!(key), "{facts}"); + assert!(!facts.to_string().contains(canary), "{facts}"); + let text = recovery_message(RUN_ID, &facts); + assert!( + text.contains(&format!( + "development field-encryption key {}", + key.display() + )), + "{text}" + ); + assert!(!text.contains(canary), "{text}"); + } + } + + #[test] + fn errors_map_to_the_envelope_class_and_exit_code() { + let (report, code) = error_report(&anyhow::anyhow!("invalid test person name")); + assert_eq!( + (report["error"]["class"].as_str(), code), + (Some("domain"), 1) + ); + assert_eq!(report["schema_version"], "registry.rstack.error.v1"); + assert_eq!(report["status"], "refused"); + assert!(report.get("run_id").is_none()); + + for malformed in [run_dir("../x").unwrap_err(), parse_ttl("9").unwrap_err()] { + let (report, code) = error_report(&malformed); + assert_eq!( + (report["error"]["class"].as_str(), code), + (Some("usage"), 2), + "{malformed:#}" + ); + } + + let (report, code) = error_report(&anyhow::Error::new(Operational("stalled".into()))); + assert_eq!( + (report["error"]["class"].as_str(), code), + (Some("operational"), 3) + ); + + let pending = anyhow::Error::new(CleanupPending { + run_id: RUN_ID.to_owned(), + detail: "database unreachable".to_owned(), + }) + .context("start failed"); + let (report, code) = error_report(&pending); + assert_eq!( + (report["error"]["class"].as_str(), code), + (Some("operational"), 3) + ); + assert_eq!(report["run_id"], RUN_ID); + let message = report["error"]["message"].as_str().unwrap(); + assert!( + message.contains(&format!("stop {RUN_ID} --recover")), + "{message}" + ); + + let facts = json!({"kind": "lost-supervisor", "run_directory": "/run", "reason": "no URL"}); + let recovery = anyhow::Error::new(RecoveryNeeded { + run_id: RUN_ID.to_owned(), + facts: facts.clone(), + }); + let (report, code) = error_report(&recovery); + assert_eq!( + (report["error"]["class"].as_str(), code), + (Some("operational"), 3) + ); + assert_eq!(report["recovery"], facts); + let message = report["error"]["message"].as_str().unwrap(); + assert!( + message.contains("could not be recovered: no URL"), + "{message}" + ); + } + + #[test] + fn an_interrupted_start_is_refused_and_a_late_or_panicked_start_is_operational() { + let class = |error: &anyhow::Error| error_report(error).0["error"]["class"].clone(); + let later = Instant::now() + Duration::from_secs(60); + assert!(start_abandoned(&AtomicBool::new(false), later).is_none()); + + let interrupted = start_abandoned(&AtomicBool::new(true), later).unwrap(); + assert_eq!(class(&interrupted), "domain"); + assert_eq!( + interrupted.to_string(), + "development run start was interrupted" + ); + + let late = start_abandoned(&AtomicBool::new(false), Instant::now()).unwrap(); + assert_eq!(class(&late), "operational"); + assert_eq!( + late.to_string(), + "development run did not start within six minutes" + ); + + let panicked = catch_startup(|| -> Result<()> { panic!("panic payload") }).unwrap_err(); + assert_eq!(class(&panicked), "operational"); + assert_eq!( + panicked.to_string(), + "development run startup failed internally" + ); + } + + #[test] + fn a_worker_failure_keeps_its_class_through_the_supervisor() { + let class = |error: &anyhow::Error| error_report(error).0["error"]["class"].clone(); + for (failure, expected) in [ + ( + anyhow::Error::new(Operational("server unreachable".into())).context("start"), + "operational", + ), + (anyhow::anyhow!("breg check failed"), "domain"), + ] { + let mut line = Vec::new(); + reply(&mut line, &WorkerReply::failed(&failure)).unwrap(); + let WorkerReply::Failed { error, operational } = serde_json::from_slice(&line).unwrap() + else { + panic!("the worker reported readiness"); + }; + let restored = restore_failure(error, operational); + assert_eq!(restored.to_string(), format!("{failure:#}")); + assert_eq!(class(&restored), expected); + let noted = note_kept_logs(restored, Some(PathBuf::from("/kept/logs"))); + assert!(noted.to_string().ends_with("; logs kept at /kept/logs")); + assert_eq!(class(¬ed), expected); + } + } + + #[test] + fn provisioned_labels_follow_the_plan() { + let scratch = tempfile::tempdir().unwrap(); + fake_run( + scratch.path(), + json!([ + {"product": "breg", "project": ".", "audience": "registry"}, + {"product": "casework", "project": ".", "audience": "casework"} + ]), + true, + ); + assert_eq!( + provisioned_labels(scratch.path()).unwrap(), + [("breg_test", true), ("breg", true), ("casework", false)] + ); + } + #[test] fn lost_supervisor_facts_survive_missing_files() { let scratch = tempfile::tempdir().unwrap(); @@ -1329,6 +1752,25 @@ mod tests { assert_eq!(fs::read_dir(&base).unwrap().count(), 0); } + #[test] + fn a_run_directory_loses_its_field_key_before_it_is_renamed() { + let scratch = tempfile::tempdir().unwrap(); + let base = scratch.path().canonicalize().unwrap(); + let root = base.join(RUN_ID); + private_directory(&root).unwrap(); + fs::write(root.join(products::FIELD_KEY), "key").unwrap(); + fs::write(root.join(REPORT), "{}").unwrap(); + // A non-empty directory at the removal name makes the rename fail, as + // an interrupted deletion would leave the run behind. + let doomed = removing_dir(&root).unwrap(); + private_directory(&doomed).unwrap(); + fs::write(doomed.join("blocker"), "").unwrap(); + + assert!(remove_run_dir(&root).is_err()); + assert!(!root.join(products::FIELD_KEY).exists()); + assert!(root.join(REPORT).exists()); + } + #[test] fn failed_run_logs_move_out_of_the_run_directory() { let scratch = tempfile::tempdir().unwrap(); diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs index 79ce78d7c1..2a9e13ca99 100644 --- a/crates/registry-bregctl/src/project_dev/products.rs +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -3,7 +3,6 @@ use anyhow::{bail, ensure, Context, Result}; use registry_breg::fixtures::journey_step_profiles; -use rustix::process::{kill_process_group, waitid, Pid, Signal, WaitId, WaitIdOptions}; use serde::Serialize; use serde_json::{json, Value}; use std::{ @@ -26,11 +25,27 @@ use super::{ bindings::{Bindings, RuntimeBindings}, clock, database::{Database, DatabaseUrls}, + guard, issuer::Issuer, + operational, plan::{self, Plan, Product}, - private_directory, stopping, write_private, BregAuthority, RunInput, POLL, + private_directory, stopping, write_private, BregAuthority, Operational, RunInput, POLL, }; +/// The run's development field-encryption data key, inside the owner-only run +/// directory and removed with it. +pub(super) const FIELD_KEY: &str = "field-key"; + +/// How long a started product has to answer `/ready`. +const READINESS_WAIT: Duration = Duration::from_secs(30); + +/// The run's field-encryption key file, when the run wrote one, so recovery +/// can name it for removal. +pub(super) fn field_key_file(root: &Path) -> Option { + let path = root.join(FIELD_KEY); + fs::symlink_metadata(&path).is_ok().then_some(path) +} + pub(super) struct ProductRunner<'a> { input: Arc, root: &'a Path, @@ -38,7 +53,7 @@ pub(super) struct ProductRunner<'a> { deadline: Instant, runtime: tokio::runtime::Runtime, http: reqwest::Client, - children: Vec<(Product, Child)>, + children: Vec<(Product, guard::Child)>, bindings: Option, } @@ -97,7 +112,8 @@ impl<'a> ProductRunner<'a> { .iter() .map(|_| TcpListener::bind((Ipv4Addr::LOCALHOST, 0))) .collect::>() - .context("reserve loopback listeners")?; + .context("reserve loopback listeners") + .map_err(operational)?; let urls = self .input .plan @@ -134,7 +150,12 @@ impl<'a> ProductRunner<'a> { } }; let binaries = &self.input.binaries[&product]; - let mut command = child_command(&binaries.runtime); + let mut command = guard::Command::new( + self.root, + &format!("runtime-{}", product.name()), + &binaries.runtime, + ) + .map_err(operational)?; command.arg("--runtime-config").arg(config); if product == Product::Casework { command.arg("serve"); @@ -148,10 +169,11 @@ impl<'a> ProductRunner<'a> { drop(reservation); let child = command .spawn() - .with_context(|| format!("cannot start {}", product.name()))?; + .with_context(|| format!("cannot start {}", product.name())) + .map_err(operational)?; self.children.push((product, child)); self.record_processes()?; - self.ready(product, url)?; + self.ready(product, url, READINESS_WAIT)?; } Ok(urls) } @@ -169,7 +191,12 @@ impl<'a> ProductRunner<'a> { .as_ref() .expect("BReg authority") .requires_postgis; - let rehearsal = database.provision("breg_test", true, requires_postgis)?; + self.write_field_key()?; + // A server that cannot provision, for example without PostGIS or the + // privilege to create it, is an environment failure. + let rehearsal = database + .provision("breg_test", true, requires_postgis) + .map_err(operational)?; let empty_package = self.root.join("breg-test-package"); fs::create_dir(&empty_package)?; let config = self.config( @@ -207,7 +234,9 @@ impl<'a> ProductRunner<'a> { ], )?; let package = build.join("package"); - let urls = database.provision("breg", true, requires_postgis)?; + let urls = database + .provision("breg", true, requires_postgis) + .map_err(operational)?; let config = self.config(Product::Breg, "breg", bind, &package, urls)?; self.tool( Product::Breg, @@ -228,6 +257,32 @@ impl<'a> ProductRunner<'a> { Ok((config, package)) } + /// Draw the run's own data key when the project encrypts fields. BReg's + /// key generator creates the file owner-only from the platform random + /// source; the key is read only by the runtime through its `secret:file` + /// reference. + fn write_field_key(&self) -> Result<()> { + if !self.encrypts_fields() { + return Ok(()); + } + crate::field_encryption::keygen(&self.root.join(FIELD_KEY)) + .map(drop) + .map_err(|error| { + Operational(format!( + "cannot write the development run's field-encryption key into its run \ + directory: {error}" + )) + .into() + }) + } + + fn encrypts_fields(&self) -> bool { + self.input + .breg_authority + .as_ref() + .is_some_and(|authority| authority.encrypts_fields) + } + fn schema_credentials(&self, issuer: &Issuer) -> Result { let mut bindings = Vec::with_capacity(self.input.schema_people.len()); for (index, binding) in self.input.schema_people.iter().enumerate() { @@ -301,7 +356,9 @@ impl<'a> ProductRunner<'a> { package.as_os_str(), ], )?; - let urls = database.provision("casework", false, false)?; + let urls = database + .provision("casework", false, false) + .map_err(operational)?; let config = self.config(Product::Casework, "casework", bind, &package, urls)?; let arguments = ["--runtime-config".as_ref(), config.as_os_str()]; self.tool(Product::Casework, "apply", &arguments)?; @@ -317,45 +374,22 @@ impl<'a> ProductRunner<'a> { package: &Path, urls: DatabaseUrls, ) -> Result { - let mut clients: BTreeSet<_> = self - .input - .plan - .people - .values() - .filter(|person| person.grants.contains_key(&product)) - .map(|person| person.client.as_str()) - .collect(); - if product == Product::Breg { - let authority = self.input.breg_authority.as_ref().expect("BReg authority"); - clients.extend(authority.named_clients.iter().map(String::as_str)); - } - let mut oidc = json!({ - "issuer": self.input.plan.issuer, - "audience": self.input.plan.product(product).audience, - "allowedClients": clients, - "scopeClaim": "scope", - "jwksSource": {"type": "static", "documentRef": "secret:file/jwks"} - }); - let mut database = json!({ - "runtimeUrlRef": self.secret(&format!("{name}-runtime-url"), urls.runtime.as_bytes())?, - "migrationUrlRef": self.secret(&format!("{name}-migration-url"), urls.migration.as_bytes())? - }); - if self.root.join("postgres-ca").is_file() { - database["trustedRootCertificateRef"] = json!("secret:file/postgres-ca"); - } - let audit = json!({ - "hashKeyRef": self.secret(&format!("{name}-audit-key"), random_secret()?.as_bytes())?, - "path": self.root.join(format!("{name}-audit")).join("audit.jsonl"), - }); + let plan = &self.input.plan; + self.secret(&format!("{name}-runtime-url"), urls.runtime.as_bytes())?; + self.secret(&format!("{name}-migration-url"), urls.migration.as_bytes())?; + self.secret(&format!("{name}-audit-key"), random_secret()?.as_bytes())?; let identity = format!("dev-{}-{name}", self.input.run_id); - let mut config = json!({ - "package": {"root": package}, - "secretProviders": {"file": {"root": self.root}}, - "database": database, - "audit": audit, - }); - match product { + let mut config = match product { Product::Breg => { + let authority = self.input.breg_authority.as_ref().expect("BReg authority"); + let (mut config, mut oidc) = shared_config( + plan, + product, + name, + self.root, + package, + &authority.named_clients, + ); oidc["allowedAlgorithm"] = json!("ES256"); oidc["accessTokenType"] = json!("at+jwt"); oidc["scopeSeparator"] = json!(" "); @@ -375,14 +409,21 @@ impl<'a> ProductRunner<'a> { }); config["audit"]["destination"] = json!("file"); config["listener"] = json!({"bind": bind}); - let authority = self.input.breg_authority.as_ref().expect("BReg authority"); if name == "breg" && authority.requires_postgis { config["listener"]["publicOrigin"] = json!(format!("http://{bind}")); } + // A development run initializes a disposable local database, as + // `bregctl dev` does; BReg accepts local-file key custody only + // for that environment. config["identity"] = json!({ - "environment": "development", "instanceId": identity, - "databaseId": identity, "databaseInitializationEnvironment": "development" + "environment": "local", "instanceId": identity, + "databaseId": identity, "databaseInitializationEnvironment": "local" }); + if authority.encrypts_fields { + config["fieldEncryption"] = json!({"provider": { + "type": "local-file", "dekRef": format!("secret:file/{FIELD_KEY}") + }}); + } let mut authority_claims = json!({"principal": authority.principal}); if authority.purpose { authority_claims["purpose"] = json!("registry_purpose"); @@ -400,17 +441,10 @@ impl<'a> ProductRunner<'a> { "recordLockMilliseconds": 5000, "migrationLockMilliseconds": 30000, "migrationStatementMilliseconds": 60000 }); + config } - Product::Casework => { - config["apiVersion"] = - json!("id.registrystack.org/formats/casework/runtime/v1alpha1"); - config["kind"] = json!("CaseworkRuntimeConfig"); - config["listener"] = - json!({"bind": bind, "tlsTermination": "development-loopback"}); - config["identity"] = json!({"databaseId": identity}); - config["authentication"] = json!({"oidc": oidc}); - } - } + Product::Casework => casework_config(plan, name, self.root, package, &identity, bind), + }; if let Some(bindings) = &self.bindings { bindings.configure(product, &mut config); } @@ -421,7 +455,7 @@ impl<'a> ProductRunner<'a> { fn secret(&self, name: &str, bytes: &[u8]) -> Result { write_private(&self.root.join(name), bytes)?; - Ok(format!("secret:file/{name}")) + Ok(secret_ref(name)) } fn tool(&self, product: Product, stage: &str, args: &[&std::ffi::OsStr]) -> Result<()> { @@ -438,8 +472,8 @@ impl<'a> ProductRunner<'a> { ) } - /// Record every live child's process group so a later `stop` can reach - /// processes whose supervisor died. + /// Record every live runtime group for status diagnostics and manual + /// cleanup evidence. Guards, not this record, own process lifetime. fn record_processes(&self) -> Result<()> { let entries: Vec<_> = self .children @@ -449,8 +483,8 @@ impl<'a> ProductRunner<'a> { write_processes(self.root, &entries) } - fn ready(&mut self, product: Product, base: &str) -> Result<()> { - let end = Instant::now() + Duration::from_secs(30); + fn ready(&mut self, product: Product, base: &str, wait: Duration) -> Result<()> { + let end = Instant::now() + wait; let url = format!("{base}/ready"); loop { self.ensure_active()?; @@ -463,12 +497,13 @@ impl<'a> ProductRunner<'a> { self.ensure_alive(product)?; return Ok(()); } - ensure!( - Instant::now() < end, - "{} did not become ready{}", - product.name(), - self.runtime_log_tail(product) - ); + if Instant::now() >= end { + return Err(operational(anyhow::anyhow!( + "{} did not become ready{}", + product.name(), + self.runtime_log_tail(product) + ))); + } thread::sleep(POLL); } } @@ -476,7 +511,7 @@ impl<'a> ProductRunner<'a> { /// Fail while waiting on `waiting`, naming whichever child exited. fn ensure_alive(&self, waiting: Product) -> Result<()> { for (product, child) in &self.children { - if child_exited(child)? { + if child.exited()? { let detail = self.runtime_log_tail(*product); if *product == waiting { bail!("{} exited before readiness{detail}", product.name()); @@ -497,7 +532,7 @@ impl<'a> ProductRunner<'a> { pub(super) fn services_alive(&self) -> Result { for (_, child) in &self.children { - if child_exited(child)? { + if child.exited()? { return Ok(false); } } @@ -509,27 +544,34 @@ impl<'a> ProductRunner<'a> { !stopping(self.root, self.terminate), "development run was stopped" ); - ensure!( - Instant::now() < self.deadline, - "development run did not start within five minutes" - ); + if Instant::now() >= self.deadline { + return Err(operational(anyhow::anyhow!( + "development run did not start within five minutes" + ))); + } Ok(()) } pub(super) fn stop(&mut self) -> Result<()> { let mut failures = Vec::new(); self.children.retain_mut(|(product, child)| { - if stop_child(child).is_err() { + if child.stop().is_err() { failures.push(product.name()); true } else { false } }); - // Drop reaped groups from the record so a later recovery cannot signal - // a reused process group ID. + // Drop reaped groups from the diagnostic record. Recovery waits on + // guard locks and lifecycle receipts rather than signaling these IDs. self.record_processes()?; ensure!(failures.is_empty(), "cannot stop {}", failures.join(", ")); + let incomplete = guard::incomplete_launches(self.root)?; + ensure!( + incomplete.is_empty(), + "process guards did not confirm cleanup for {}", + incomplete.join(", ") + ); drop(self.bindings.take()); Ok(()) } @@ -610,33 +652,152 @@ fn run_logged_tool( ) -> Result<()> { let name = format!("{product}-{stage}"); let mut output = tempfile::tempfile_in(root)?; - let mut child = child_command(binary) + let mut command = guard::Command::new(root, &format!("tool-{product}-{stage}"), binary) + .map_err(operational)?; + let mut child = command .args(args) .stdout(output.try_clone()?) .stderr(open_log(root, &name)?) .spawn() - .with_context(|| format!("cannot start {product} {stage}"))?; - if wait_for_tool(&mut child, active)?.success() { + .with_context(|| format!("cannot start {product} {stage}")) + .map_err(operational)?; + let status = wait_for_guarded_tool(&mut child, active)?; + logged_tool_result(status, &mut output, root, (product, stage)) +} + +fn logged_tool_result( + status: ExitStatus, + output: &mut fs::File, + root: &Path, + (product, stage): (&str, &str), +) -> Result<()> { + if status.success() { return Ok(()); } output.rewind()?; let codes = report_codes(output)?; bail!( "{product} {stage} failed{codes}{}", - failure_detail(&log_path(root, &name)) + failure_detail(&log_path(root, &format!("{product}-{stage}"))) ); } -/// Record each child's process group. Children start as group leaders, so the -/// group ID is the PID. +/// Exercise the log/result boundary in unit tests without routing a shell +/// through the Rust test harness as if it were the bregctl binary. The native +/// lifecycle test covers the real hidden guard dispatch. +#[cfg(test)] +fn run_logged_tool_direct( + binary: &Path, + args: &[&std::ffi::OsStr], + root: &Path, + (product, stage): (&str, &str), +) -> Result<()> { + let name = format!("{product}-{stage}"); + let mut output = tempfile::tempfile_in(root)?; + let mut child = child_command(binary) + .args(args) + .stdout(output.try_clone()?) + .stderr(open_log(root, &name)?) + .spawn()?; + let status = wait_for_tool(&mut child, || Ok(()))?; + logged_tool_result(status, &mut output, root, (product, stage)) +} + +/// Record each runtime process group and the start time of its leader for +/// diagnostics and an operator's manual cleanup decision. Children start as +/// group leaders, so the group ID is the PID. fn write_processes(root: &Path, children: &[(Product, u32)]) -> Result<()> { + let table = super::recovery::process_table()?; let entries: Vec<_> = children .iter() - .map(|(product, pid)| json!({"product": product.name(), "pid": pid, "pgid": pid})) + .map(|(product, pid)| { + let started = table + .iter() + .find(|row| row.pid == *pid) + .map(|row| &row.started); + json!({"product": product.name(), "pid": pid, "pgid": pid, "started": started}) + }) .collect(); write_private(&root.join("processes.json"), &serde_json::to_vec(&entries)?) } +/// The reference to the secret the run writes as `name` in its directory. +fn secret_ref(name: &str) -> String { + format!("secret:file/{name}") +} + +/// The members both products' runtime configurations share, and the OIDC +/// settings each completes. They name the run's secrets by reference only: +/// `config` writes the secrets beside them. +fn shared_config( + plan: &Plan, + product: Product, + name: &str, + root: &Path, + package: &Path, + extra_clients: &BTreeSet, +) -> (Value, Value) { + let clients: BTreeSet<_> = plan + .people + .values() + .filter(|person| person.grants.contains_key(&product)) + .map(|person| person.client.as_str()) + .chain(extra_clients.iter().map(String::as_str)) + .collect(); + let oidc = json!({ + "issuer": plan.issuer, + "audience": plan.product(product).audience, + "allowedClients": clients, + "scopeClaim": "scope", + "jwksSource": {"type": "static", "documentRef": secret_ref("jwks")} + }); + let mut database = json!({ + "runtimeUrlRef": secret_ref(&format!("{name}-runtime-url")), + "migrationUrlRef": secret_ref(&format!("{name}-migration-url")) + }); + if root.join("postgres-ca").is_file() { + database["trustedRootCertificateRef"] = json!(secret_ref("postgres-ca")); + } + let audit = json!({ + "hashKeyRef": secret_ref(&format!("{name}-audit-key")), + "path": root.join(format!("{name}-audit")).join("audit.jsonl"), + }); + let config = json!({ + "package": {"root": package}, + "secretProviders": {"file": {"root": root}}, + "database": database, + "audit": audit, + }); + (config, oidc) +} + +/// The Casework runtime configuration the run writes, before its bindings. +/// `check` renders it with stand-in paths, so the configuration Casework +/// checks is the one `start` writes. +fn casework_config( + plan: &Plan, + name: &str, + root: &Path, + package: &Path, + identity: &str, + bind: &str, +) -> Value { + let (mut config, oidc) = shared_config( + plan, + Product::Casework, + name, + root, + package, + &BTreeSet::new(), + ); + config["apiVersion"] = json!("id.registrystack.org/formats/casework/runtime/v1alpha1"); + config["kind"] = json!("CaseworkRuntimeConfig"); + config["listener"] = json!({"bind": bind, "tlsTermination": "development-loopback"}); + config["identity"] = json!({"databaseId": identity}); + config["authentication"] = json!({"oidc": oidc}); + config +} + fn random_secret() -> Result> { use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; let mut bytes = zeroize::Zeroizing::new([0u8; 32]); @@ -726,6 +887,28 @@ impl InspectionUnavailable { } } +/// The error for an inspection that could not complete. An interrupt is the +/// operator's request and stays as it is. Any other failure is an unavailable +/// inspection whose reason keeps the cause; the cause comes from spawning the +/// tool, its timeout, temporary files and the report parse, and none of those +/// messages carries a configured value. +fn inspection_failed( + field: &str, + action: &str, + error: anyhow::Error, + terminate: &AtomicBool, +) -> anyhow::Error { + if terminate.load(Ordering::Relaxed) { + return error; + } + InspectionUnavailable::new( + field, + &format!("The selected `caseworkctl` helper could not {action}: {error:#}."), + "Install the matching Casework tools or provide RSTACK_CASEWORKCTL_BIN, then run the check again.", + ) + .into() +} + struct SelectedProduct { field: String, root: PathBuf, @@ -736,6 +919,7 @@ pub(super) fn inspect( plan: &Plan, terminate: &AtomicBool, ) -> Result { + super::bindings::check_with_breg_reader(plan)?; let breg_authority = breg_authority(root, plan)?; let casework_principals = casework_principals(root, plan, breg_authority.as_ref(), terminate)?; Ok(ProductInspection { @@ -806,7 +990,6 @@ fn breg_authority(root: &Path, plan: &Plan) -> Result> { "Run `rstack breg check` for authored diagnostics, then correct the BReg project.", ) })?; - refuse_encrypted_fields(®istry, &project.field)?; let review_authorities = registry .review_authorities() .iter() @@ -827,6 +1010,7 @@ fn breg_authority(root: &Path, plan: &Plan) -> Result> { purpose: inventory.purpose_required, named_clients: crate::identity_facts::named_requester_clients(®istry), requires_postgis: breg_requires_postgis(®istry), + encrypts_fields: breg_encrypts_fields(®istry), })) } @@ -886,28 +1070,15 @@ fn validate_breg_binding_ids( Ok(()) } -/// The generated runtime configuration has no field-encryption provider, so a -/// project with encrypted fields would only fail late and obscurely. -fn refuse_encrypted_fields( - registry: ®istry_breg::CompiledRegistry, - product_field: &str, -) -> plan::SemanticResult<()> { - for entity in registry.entities().values() { - for field in entity.fields.values() { - plan::require( - field.encryption.is_none(), - product_field, - format!( - "The BReg project encrypts {}.{}, and rstack dev does not configure field \ - encryption yet.", - entity.id, field.id - ), - "Run this BReg project with `rstack breg` and a field-encryption provider, or \ - select a project without encrypted fields.", - )?; - } - } - Ok(()) +/// The runtime needs a field-encryption provider exactly when the compiled +/// project declares an encrypted field. +fn breg_encrypts_fields(registry: ®istry_breg::CompiledRegistry) -> bool { + registry.entities().values().any(|entity| { + entity + .fields + .values() + .any(|field| field.encryption.is_some()) + }) } /// Every BReg grant must name an access profile the project declares, @@ -969,15 +1140,26 @@ fn casework_principals( "Run `rstack casework check` for authored diagnostics, then correct the Casework project.", ).into()); } - Err(_) => { - return Err(InspectionUnavailable::new( + Err(error) => { + return Err(inspection_failed( &project.field, - "The selected `caseworkctl` helper could not inspect the Casework project.", - "Install the matching Casework tools or provide RSTACK_CASEWORKCTL_BIN, then run the check again.", - ).into()); + "inspect the Casework project", + error, + terminate, + )); } }; - validate_casework_binding_ids(&plan.bindings, &facts)?; + let status_profile = plan + .bindings + .casework_context_provider + .as_ref() + .and_then(|binding| plan.people.get(&binding.identity)) + .and_then(|person| person.grants.get(&Product::Breg)) + .map(|grant| grant.profile.as_str()); + validate_casework_binding_ids(&plan.bindings, &facts, status_profile)?; + if let Some(binding) = &plan.bindings.casework_context_provider { + check_casework_binding(&tool, &project, plan, binding, terminate)?; + } Ok(casework_principals_from_facts(plan, breg, &facts)?) } @@ -1031,15 +1213,33 @@ fn explain_casework_facts( project: &Path, terminate: &AtomicBool, ) -> Result { + let (status, report) = casework_report(tool, "explain", &[project.as_os_str()], terminate)?; + if status.success() { + return casework_report_facts(&report); + } + if status.code() == Some(1) && casework_domain_refusal(&report, "explain") { + return Err(InvalidCaseworkProject.into()); + } + anyhow::bail!("caseworkctl explain did not complete successfully") +} + +/// Runs `caseworkctl --format json ` and returns its +/// exit status with the JSON report it printed. +fn casework_report( + tool: &Path, + command: &str, + arguments: &[&std::ffi::OsStr], + terminate: &AtomicBool, +) -> Result<(ExitStatus, Value)> { const MAX_REPORT: u64 = 4 * 1024 * 1024; let mut child = child_command(tool) - .args(["--format", "json", "explain"]) - .arg(project) + .args(["--format", "json", command]) + .args(arguments) .stdout(Stdio::piped()) .spawn() - .context("cannot start caseworkctl explain")?; - let stdout = child.stdout.take().expect("the explain output was piped"); + .with_context(|| format!("cannot start caseworkctl {command}"))?; + let stdout = child.stdout.take().expect("the report output was piped"); let reader = thread::spawn(move || { let mut bytes = Vec::new(); stdout.take(MAX_REPORT + 1).read_to_end(&mut bytes)?; @@ -1051,31 +1251,25 @@ fn explain_casework_facts( !terminate.load(Ordering::Relaxed), "development product inspection was interrupted" ); - ensure!(Instant::now() < deadline, "caseworkctl explain timed out"); + ensure!(Instant::now() < deadline, "caseworkctl {command} timed out"); Ok(()) })?; let bytes = reader .join() - .map_err(|_| anyhow::anyhow!("cannot collect the Casework authoring report"))??; + .map_err(|_| anyhow::anyhow!("cannot collect the caseworkctl {command} report"))??; ensure!( bytes.len() as u64 <= MAX_REPORT, - "the Casework authoring report exceeds the size limit" + "the caseworkctl {command} report exceeds the size limit" ); - let report: Value = serde_json::from_slice(&bytes) - .map_err(|_| anyhow::anyhow!("caseworkctl explain did not emit a JSON report"))?; - if status.success() { - return casework_report_facts(&report); - } - if status.code() == Some(1) && casework_domain_refusal(&report) { - return Err(InvalidCaseworkProject.into()); - } - anyhow::bail!("caseworkctl explain did not complete successfully") + let report = serde_json::from_slice(&bytes) + .map_err(|_| anyhow::anyhow!("caseworkctl {command} did not emit a JSON report"))?; + Ok((status, report)) } -fn casework_domain_refusal(report: &Value) -> bool { +fn casework_domain_refusal(report: &Value, command: &str) -> bool { report["apiVersion"] == "id.registrystack.org/formats/casework/report/v1alpha2" && report["kind"] == "CaseworkReport" - && report["command"] == "explain" + && report["command"] == command && report["status"] == "domain-refusal" && report["ok"] == false && report["diagnostics"].as_array().is_some_and(|diagnostics| { @@ -1095,6 +1289,137 @@ fn casework_domain_refusal(report: &Value) -> bool { }) } +/// The listener `check` renders the Casework configuration with. Nothing +/// binds it. +const CHECK_BIND: &str = "127.0.0.1:1"; + +/// Renders the Casework runtime configuration the run writes and lets the +/// selected `caseworkctl check` decide whether Casework would accept the +/// context provider binding in it. A refusal lands on the plan field that +/// produced the member Casework names. +fn check_casework_binding( + tool: &Path, + project: &SelectedProduct, + plan: &Plan, + binding: &super::bindings::CaseworkContextProvider, + terminate: &AtomicBool, +) -> Result<()> { + let action = "check the Casework configuration the run writes"; + let checked = (|| { + let directory = tempfile::tempdir()?; + // The runtime loader refuses a file reached through a symbolic link, + // as the default temporary directory is on some systems. + let root = directory.path().canonicalize()?; + let mut config = casework_config( + plan, + "casework", + &root, + &root.join("casework-package"), + "dev-check-casework", + CHECK_BIND, + ); + super::bindings::render_for_check(plan, Product::Casework, &mut config); + let path = root.join("casework-runtime.yaml"); + write_private(&path, &serde_json::to_vec(&config)?)?; + casework_report( + tool, + "check", + &[ + project.root.as_os_str(), + "--runtime-config".as_ref(), + path.as_os_str(), + ], + terminate, + ) + })(); + let (status, report) = match checked { + Ok(checked) => checked, + Err(error) => return Err(inspection_failed(&project.field, action, error, terminate)), + }; + match casework_binding_check(status.code(), &report, &binding.id) { + BindingCheck::Accepted => Ok(()), + BindingCheck::Refused { member, message } => Err(plan::SemanticRefusal::new( + super::bindings::casework_binding_field(plan, binding, &member), + format!( + "Casework refuses the binding configuration the run writes from this value: {}.", + message.trim_end_matches('.') + ), + "Correct the value so the product accepts the configuration the run writes for this binding.", + ) + .into()), + BindingCheck::Unchecked => Err(inspection_failed( + &project.field, + action, + anyhow::anyhow!("caseworkctl check did not report a result for the binding"), + terminate, + )), + } +} + +#[derive(Debug, Eq, PartialEq)] +enum BindingCheck { + Accepted, + /// Casework refused `member`, a JSON pointer relative to the binding. + Refused { + member: String, + message: String, + }, + /// Casework did not read the binding. + Unchecked, +} + +/// What a `caseworkctl check` that exited with `code` and printed `report` +/// decided about the context provider binding `id` in the configuration the +/// run writes. Only refusals at the binding count: the rest of the file names +/// stand-in paths and the project is not yet paired, and `start` checks both +/// as they are. A refusal by the shared loader elsewhere in the file means +/// Casework never read the binding. +fn casework_binding_check(code: Option, report: &Value, id: &str) -> BindingCheck { + if code == Some(0) + && report["apiVersion"] == "id.registrystack.org/formats/casework/report/v1alpha2" + && report["kind"] == "CaseworkReport" + && report["command"] == "check" + && report["status"] == "complete" + && report["ok"] == true + { + return BindingCheck::Accepted; + } + if code != Some(1) || !casework_domain_refusal(report, "check") { + return BindingCheck::Unchecked; + } + let binding = format!( + "/contextProviders/{}", + registry_platform_yaml::escape_pointer_segment(id) + ); + let mut unread = false; + for diagnostic in report["diagnostics"].as_array().into_iter().flatten() { + if diagnostic["severity"] != "error" || diagnostic["artifact"] != "CaseworkRuntimeConfig" { + continue; + } + let (Some(path), Some(code), Some(message)) = ( + diagnostic["path"].as_str(), + diagnostic["code"].as_str(), + diagnostic["message"].as_str(), + ) else { + return BindingCheck::Unchecked; + }; + if let Some(member) = path.strip_prefix(&binding) { + if member.is_empty() || member.starts_with('/') { + return BindingCheck::Refused { + member: member.to_owned(), + message: message.to_owned(), + }; + } + } + unread |= !code.starts_with("casework.runtime."); + } + if unread { + BindingCheck::Unchecked + } else { + BindingCheck::Accepted + } +} + #[derive(Debug, thiserror::Error)] #[error("the Casework project did not pass its owning validation")] struct InvalidCaseworkProject; @@ -1103,7 +1428,9 @@ struct InvalidCaseworkProject; struct CaseworkFacts { profiles: BTreeMap, trusted_producers: BTreeSet, - context_providers: BTreeSet, + /// Each context provider's ID with the profile its policy names for the + /// `service-status` capability, when it names one. + context_providers: BTreeMap>, } fn casework_report_facts(report: &Value) -> Result { @@ -1136,10 +1463,30 @@ fn casework_report_facts(report: &Value) -> Result { Ok(CaseworkFacts { profiles: principals, trusted_producers: casework_report_ids(report, "trustedProducers")?, - context_providers: casework_report_ids(report, "contextProviders")?, + context_providers: casework_context_providers(report)?, }) } +fn casework_context_providers(report: &Value) -> Result>> { + let entries = report["contextProviders"] + .as_array() + .context("caseworkctl explain did not report contextProviders")?; + let mut providers = BTreeMap::new(); + for entry in entries { + let id = entry["id"] + .as_str() + .context("a caseworkctl explain contextProviders entry has no ID")?; + let status_profile = entry["serviceProfiles"]["service-status"] + .as_str() + .map(str::to_owned); + ensure!( + providers.insert(id.to_owned(), status_profile).is_none(), + "caseworkctl explain returned duplicate contextProviders IDs" + ); + } + Ok(providers) +} + fn casework_report_ids(report: &Value, member: &str) -> Result> { let entries = report[member] .as_array() @@ -1160,6 +1507,7 @@ fn casework_report_ids(report: &Value, member: &str) -> Result> fn validate_casework_binding_ids( bindings: &Bindings, facts: &CaseworkFacts, + status_profile: Option<&str>, ) -> plan::SemanticResult<()> { if let Some(binding) = &bindings.breg_review_authority { plan::require( @@ -1171,11 +1519,24 @@ fn validate_casework_binding_ids( } if let Some(binding) = &bindings.casework_context_provider { plan::require( - facts.context_providers.contains(&binding.id), + facts.context_providers.contains_key(&binding.id), "/bindings/caseworkContextProvider/id", "The Casework context provider ID does not name a context provider in the Casework project.", "Set id to a context provider defined by the Casework project.", )?; + // The run writes the identity's BReg grant profile as the provider's + // status profile, and Casework refuses a provider whose binding and + // policy disagree. Refusing here keeps that from surfacing at start. + let policy_profile = facts.context_providers[&binding.id].as_deref(); + plan::require( + policy_profile == status_profile, + &format!( + "/people/{}/grants/breg/profile", + registry_platform_yaml::escape_pointer_segment(&binding.identity) + ), + "The BReg grant profile is not the service-status profile the Casework context provider names.", + "Set the grant profile to the service-status profile in the Casework context provider's serviceProfiles.", + )?; } Ok(()) } @@ -1184,7 +1545,7 @@ impl Drop for ProductRunner<'_> { fn drop(&mut self) { // The normal exit reports stop failures explicitly. for (_, child) in &mut self.children { - let _ = stop_child(child); + let _ = child.stop(); } } } @@ -1200,55 +1561,30 @@ pub(super) fn child_command(binary: &Path) -> Command { command } -fn child_pid(child: &Child) -> Result { - Pid::from_raw(i32::try_from(child.id())?).context("invalid owned process ID") -} - -fn child_exited(child: &Child) -> Result { - let pid = child_pid(child)?; - // Keep the leader's PID until the group has received the final signal. - // Reaping the leader first would allow reuse of the group ID. - let options = WaitIdOptions::EXITED | WaitIdOptions::NOHANG | WaitIdOptions::NOWAIT; +fn wait_for_tool(child: &mut Child, active: impl Fn() -> Result<()>) -> Result { loop { - match waitid(WaitId::Pid(pid), options) { - Ok(status) => return Ok(status.is_some()), - Err(rustix::io::Errno::INTR) => continue, - Err(error) => return Err(error.into()), + if let Err(error) = active() { + guard::stop_direct_child(child)?; + return Err(error); } - } -} - -fn signal_group(child: &Child, signal: Signal) -> Result<()> { - match kill_process_group(child_pid(child)?, signal) { - Ok(()) | Err(rustix::io::Errno::SRCH) => Ok(()), - // Darwin excludes zombies from killpg and reports EPERM when there - // are no signalable members. An owned, same-user group with an - // exited leader can already be empty of live processes. - Err(rustix::io::Errno::PERM) if cfg!(target_os = "macos") && child_exited(child)? => Ok(()), - Err(error) => Err(error.into()), - } -} - -fn stop_child(child: &mut Child) -> Result { - if !child_exited(child)? { - signal_group(child, Signal::TERM)?; - let deadline = Instant::now() + Duration::from_secs(2); - while Instant::now() < deadline && !child_exited(child)? { - thread::sleep(POLL); + if guard::direct_child_exited(child)? { + return guard::stop_direct_child(child); } + thread::sleep(POLL); } - signal_group(child, Signal::KILL)?; - Ok(child.wait()?) } -fn wait_for_tool(child: &mut Child, active: impl Fn() -> Result<()>) -> Result { +fn wait_for_guarded_tool( + child: &mut guard::Child, + active: impl Fn() -> Result<()>, +) -> Result { loop { if let Err(error) = active() { - stop_child(child)?; + child.stop()?; return Err(error); } - if child_exited(child)? { - return stop_child(child); + if child.exited()? { + return child.stop(); } thread::sleep(POLL); } @@ -1599,7 +1935,7 @@ mod tests { let facts = |principal: &str| CaseworkFacts { profiles: BTreeMap::from([("reviewer".to_owned(), principal.to_owned())]), trusted_producers: BTreeSet::new(), - context_providers: BTreeSet::new(), + context_providers: BTreeMap::new(), }; let plan = authority_plan(json!({}), json!({}), false); @@ -1631,6 +1967,7 @@ mod tests { principal: "registry_principal".to_owned(), purpose: false, requires_postgis: false, + encrypts_fields: false, named_clients: BTreeSet::new(), }; let error = casework_principals_from_facts(&plan, Some(&breg), &facts("custom_principal")) @@ -1704,10 +2041,13 @@ mod tests { "reviewPolicies": [], "accessProfiles": [{"id": "reviewer", "principalClaim": "sub"}], "trustedProducers": [{"id": "review-producer"}], - "contextProviders": [{"id": "proposal-context"}] + "contextProviders": [{ + "id": "proposal-context", + "serviceProfiles": {"service-status": "status-reader"} + }] }); let facts = casework_report_facts(&report).unwrap(); - validate_casework_binding_ids(&bindings, &facts).unwrap(); + validate_casework_binding_ids(&bindings, &facts, Some("status-reader")).unwrap(); for (member, field, reason, action) in [ ( @@ -1725,7 +2065,7 @@ mod tests { ] { report[member] = json!([]); let facts = casework_report_facts(&report).unwrap(); - let error = validate_casework_binding_ids(&bindings, &facts) + let error = validate_casework_binding_ids(&bindings, &facts, Some("status-reader")) .unwrap_err() .to_string(); assert!(error.contains(field), "{error}"); @@ -1735,12 +2075,207 @@ mod tests { assert!(!error.contains("proposal-context"), "{error}"); report[member] = match member { "trustedProducers" => json!([{"id": "review-producer"}]), - "contextProviders" => json!([{"id": "proposal-context"}]), + "contextProviders" => json!([{ + "id": "proposal-context", + "serviceProfiles": {"service-status": "status-reader"} + }]), _ => unreachable!(), }; } } + #[test] + fn the_status_profile_the_run_writes_must_match_the_context_provider_policy() { + let bindings = product_bindings(); + let report = |service_profiles: Value| { + json!({ + "apiVersion": "id.registrystack.org/formats/casework/report/v1alpha2", + "kind": "CaseworkReport", "command": "explain", "status": "complete", "ok": true, + "accessProfiles": [], "trustedProducers": [{"id": "review-producer"}], + "contextProviders": [{"id": "proposal-context", "serviceProfiles": service_profiles}] + }) + }; + for (profiles, written) in [ + (json!({"service-status": "other-profile"}), "status-reader"), + (json!({}), "status-reader"), + (json!({"service-status": "status-reader"}), "other-profile"), + ] { + let facts = casework_report_facts(&report(profiles)).unwrap(); + let error = validate_casework_binding_ids(&bindings, &facts, Some(written)) + .unwrap_err() + .to_string(); + assert!( + error.contains("/people/context-reader/grants/breg/profile"), + "{error}" + ); + assert!(error.contains("service-status profile"), "{error}"); + assert!(!error.contains("other-profile"), "{error}"); + } + } + + #[test] + fn only_a_casework_refusal_at_the_binding_refuses_the_binding() { + let report = |status: &str, diagnostics: Value| { + json!({ + "apiVersion": "id.registrystack.org/formats/casework/report/v1alpha2", + "kind": "CaseworkReport", "command": "check", "status": status, + "ok": status == "complete", "diagnostics": diagnostics + }) + }; + let diagnostic = |artifact: &str, code: &str, path: &str| { + json!({ + "severity": "error", "code": code, "artifact": artifact, "path": path, + "message": "the rule", "suggestedAction": "Correct it." + }) + }; + let runtime = "CaseworkRuntimeConfig"; + let refused = |member: &str| BindingCheck::Refused { + member: member.to_owned(), + message: "the rule".to_owned(), + }; + for (code, diagnostics, expected) in [ + (Some(0), json!([]), BindingCheck::Accepted), + ( + Some(1), + json!([diagnostic( + runtime, + "casework.runtime.invalid-context-provider", + "/contextProviders/registry/resource" + )]), + refused("/resource"), + ), + ( + Some(1), + json!([diagnostic( + runtime, + "config.invalid-value", + "/contextProviders/registry/baseUrl" + )]), + refused("/baseUrl"), + ), + ( + Some(1), + json!([diagnostic( + runtime, + "casework.runtime.unknown", + "/contextProviders/registry" + )]), + refused(""), + ), + // Another provider, Casework rules about the stand-in files or + // the unpaired project, and project diagnostics are not this + // binding's. + ( + Some(1), + json!([ + diagnostic( + runtime, + "casework.runtime.invalid-context-provider", + "/contextProviders/registry-other/resource" + ), + diagnostic( + runtime, + "casework.runtime.context-providers-mismatch", + "/contextProviders" + ), + diagnostic( + "CaseworkProject", + "casework.project.invalid", + "/reviewPolicies" + ), + ]), + BindingCheck::Accepted, + ), + // The loader refused the file elsewhere, so Casework never read + // the binding. + ( + Some(1), + json!([diagnostic(runtime, "config.unknown-key", "/listener/port")]), + BindingCheck::Unchecked, + ), + ( + Some(3), + json!([diagnostic( + runtime, + "casework.runtime.unknown", + "/contextProviders/registry" + )]), + BindingCheck::Unchecked, + ), + (None, json!([]), BindingCheck::Unchecked), + ] { + let status = if code == Some(0) { + "complete" + } else { + "domain-refusal" + }; + assert_eq!( + casework_binding_check(code, &report(status, diagnostics.clone()), "registry"), + expected, + "{code:?} {diagnostics}" + ); + } + let explain = json!({ + "apiVersion": "id.registrystack.org/formats/casework/report/v1alpha2", + "kind": "CaseworkReport", "command": "explain", "status": "complete", "ok": true + }); + assert_eq!( + casework_binding_check(Some(0), &explain, "registry"), + BindingCheck::Unchecked + ); + } + + #[test] + fn check_renders_the_casework_configuration_start_writes() { + let scratch = tempfile::tempdir().unwrap(); + let root = scratch.path().join("run"); + super::super::private_directory(&root).unwrap(); + let terminate = AtomicBool::new(false); + let mut input = breg_input(false); + input.plan = serde_json::from_value(json!({ + "issuer": "https://issuer.example.invalid", + "products": [ + {"product": "breg", "project": ".", "audience": "urn:example:registry"}, + {"product": "casework", "project": "casework", "audience": "urn:example:review"} + ], + "people": {"reviewer": { + "subject": "reviewer", "client": "reviewer-client", + "grants": {"casework": {"profile": "reviewer", "scopes": ["casework:review"]}} + }}, + "journey": {"id": "read", "steps": [{ + "id": "read", "product": "casework", "as": "reviewer", "method": "GET", + "path": "/v1/review-tasks", "expect": {"status": 200} + }]} + })) + .unwrap(); + let identity = format!("dev-{}-casework", input.run_id); + let runner = ProductRunner::new(Arc::new(input), &root, &terminate).unwrap(); + let urls = DatabaseUrls { + runtime: "postgresql://runtime:password@127.0.0.1:5432/run".to_owned(), + migration: "postgresql://migration:password@127.0.0.1:5432/run".to_owned(), + }; + let package = root.join("casework-package"); + let path = runner + .config(Product::Casework, "casework", "127.0.0.1:0", &package, urls) + .unwrap(); + let written: Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + + let rendered = casework_config( + &runner.input.plan, + "casework", + &root, + &package, + &identity, + "127.0.0.1:0", + ); + assert_eq!(written, rendered); + assert_eq!( + written["authentication"]["oidc"]["allowedClients"], + json!(["reviewer-client"]) + ); + assert!(!written.to_string().contains("password"), "{written}"); + } + #[test] fn a_report_without_the_access_profile_list_is_refused() { let report = json!({ @@ -1760,12 +2295,11 @@ mod tests { fn a_failed_tool_leaves_its_stderr_in_an_owner_only_log_and_the_error() { let scratch = tempfile::tempdir().unwrap(); let root = scratch.path(); - let error = run_logged_tool( + let error = run_logged_tool_direct( Path::new("/bin/sh"), &["-c".as_ref(), "echo boom >&2; exit 3".as_ref()], root, ("breg", "test"), - || Ok(()), ) .unwrap_err() .to_string(); @@ -1788,12 +2322,11 @@ mod tests { #[test] fn a_successful_tool_leaves_no_failure_text() { let scratch = tempfile::tempdir().unwrap(); - run_logged_tool( + run_logged_tool_direct( Path::new("/bin/sh"), &["-c".as_ref(), "echo fine >&2".as_ref()], scratch.path(), ("breg", "test"), - || Ok(()), ) .unwrap(); } @@ -1828,8 +2361,8 @@ mod tests { assert_eq!( serde_json::from_slice::(&fs::read(&path).unwrap()).unwrap(), json!([ - {"product": "breg", "pid": 4242, "pgid": 4242}, - {"product": "casework", "pid": 4343, "pgid": 4343} + {"product": "breg", "pid": 4242, "pgid": 4242, "started": null}, + {"product": "casework", "pid": 4343, "pgid": 4343, "started": null} ]) ); assert_eq!( @@ -1838,6 +2371,27 @@ mod tests { ); } + #[test] + fn a_live_process_is_recorded_with_its_start_time() { + let scratch = tempfile::tempdir().unwrap(); + let mut child = std::process::Command::new("sleep") + .arg("60") + .spawn() + .unwrap(); + write_processes(scratch.path(), &[(Product::Breg, child.id())]).unwrap(); + let recorded: Value = + serde_json::from_slice(&fs::read(scratch.path().join("processes.json")).unwrap()) + .unwrap(); + let started = recorded[0]["started"].as_str().map(str::to_owned); + assert!(started.is_some(), "{recorded}"); + assert_eq!( + started, + crate::project_dev::recovery::started_at(child.id()) + ); + child.kill().unwrap(); + child.wait().unwrap(); + } + fn minimal_registry(edit: impl FnOnce(&str) -> String) -> registry_breg::CompiledRegistry { let source = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../products/breg/examples/minimal"); @@ -1894,30 +2448,239 @@ mod tests { check_breg_grants(®istry, &people).unwrap(); } + fn encrypt_contact_email(source: &str) -> String { + let source = source.replacen( + "mutationMode: mutable\n display: {label: Contact", + "mutationMode: mutable\n classification: restricted\n display: {label: Contact", + 1, + ); + source.replacen( + "{id: email, type: string, required: true, minimumLength: 1, maximumLength: 254, classification: internal}\n - {id: jurisdiction", + "{id: email, type: string, required: true, minimumLength: 1, maximumLength: 254, classification: restricted, encrypted: true}\n - {id: jurisdiction", + 1, + ) + } + #[test] - fn encrypted_fields_are_refused_before_provisioning() { - let registry = minimal_registry(|source| { - let source = source.replacen( - "mutationMode: mutable\n display: {label: Contact", - "mutationMode: mutable\n classification: restricted\n display: {label: Contact", - 1, + fn the_compiled_project_decides_whether_a_run_encrypts_fields() { + assert!(breg_encrypts_fields(&minimal_registry( + encrypt_contact_email + ))); + assert!(!breg_encrypts_fields(&minimal_registry(str::to_owned))); + } + + fn breg_input(encrypts_fields: bool) -> RunInput { + RunInput { + run_id: "0123456789abcdef0123456789abcdef".to_owned(), + project: PathBuf::from("/tmp/project"), + plan: serde_json::from_value(json!({ + "issuer": "https://issuer.example.invalid", + "products": [{"product": "breg", "project": ".", "audience": "registry"}], + "people": {"reader": { + "subject": "reader", "client": "reader-client", + "grants": {"breg": {"profile": "reader", "scopes": []}} + }}, + "journey": {"id": "read", "steps": [{ + "id": "read", "product": "breg", "as": "reader", "method": "GET", + "path": "/v1/records", "expect": {"status": 200} + }]} + })) + .unwrap(), + ttl_seconds: 300, + schema_people: Vec::new(), + breg_authority: Some(BregAuthority { + principal: "sub".to_owned(), + purpose: false, + requires_postgis: false, + encrypts_fields, + named_clients: BTreeSet::new(), + }), + casework_principals: BTreeMap::new(), + binaries: BTreeMap::new(), + } + } + + fn breg_config(runner: &ProductRunner<'_>, root: &Path, name: &str) -> (Value, String) { + let urls = DatabaseUrls { + runtime: "postgresql://runtime:password@127.0.0.1:5432/run".to_owned(), + migration: "postgresql://migration:password@127.0.0.1:5432/run".to_owned(), + }; + let path = runner + .config( + Product::Breg, + name, + "127.0.0.1:0", + &root.join("package"), + urls, + ) + .unwrap(); + let text = fs::read_to_string(path).unwrap(); + (serde_json::from_str(&text).unwrap(), text) + } + + #[test] + fn an_encrypting_run_configures_local_file_custody_with_an_owner_only_run_key() { + use base64::{engine::general_purpose::STANDARD, Engine}; + let scratch = tempfile::tempdir().unwrap(); + let root = scratch.path().join("run"); + super::super::private_directory(&root).unwrap(); + let terminate = AtomicBool::new(false); + let runner = ProductRunner::new(Arc::new(breg_input(true)), &root, &terminate).unwrap(); + + runner.write_field_key().unwrap(); + let key = root.join(FIELD_KEY); + assert_eq!(field_key_file(&root), Some(key.clone())); + let refused = runner.write_field_key().unwrap_err(); + assert!(refused.is::(), "{refused:#}"); + assert!( + refused + .to_string() + .ends_with("run directory: a key file already exists at that path"), + "{refused:#}" + ); + assert_eq!( + fs::metadata(&key).unwrap().permissions().mode() & 0o777, + 0o600 + ); + let encoded = fs::read_to_string(&key).unwrap(); + assert_eq!(STANDARD.decode(encoded.trim()).unwrap().len(), 32); + + // The rehearsal and the served database read the same run key, by + // reference only; local-file custody requires local initialization. + for name in ["breg-test", "breg"] { + let (config, text) = breg_config(&runner, &root, name); + assert_eq!( + config["fieldEncryption"], + json!({"provider": {"type": "local-file", "dekRef": format!("secret:file/{FIELD_KEY}")}}), + "{name}" ); - source.replacen( - "{id: email, type: string, required: true, minimumLength: 1, maximumLength: 254, classification: internal}\n - {id: jurisdiction", - "{id: email, type: string, required: true, minimumLength: 1, maximumLength: 254, classification: restricted, encrypted: true}\n - {id: jurisdiction", - 1, + assert_eq!(config["secretProviders"]["file"]["root"], json!(root)); + assert_eq!(config["identity"]["environment"], "local"); + assert_eq!( + config["identity"]["databaseInitializationEnvironment"], + "local" + ); + assert!(!text.contains(encoded.trim()), "{name}"); + } + + // Each run draws its own key. + let other = scratch.path().join("other"); + super::super::private_directory(&other).unwrap(); + let second = ProductRunner::new(Arc::new(breg_input(true)), &other, &terminate).unwrap(); + second.write_field_key().unwrap(); + assert_ne!(fs::read_to_string(other.join(FIELD_KEY)).unwrap(), encoded); + } + + #[test] + fn an_unavailable_inspection_is_operational_and_names_its_cause() { + let class = + |error: &anyhow::Error| super::super::error_report(error).0["error"]["class"].clone(); + let unavailable = anyhow::Error::new(InspectionUnavailable::new( + "products[0]", + "The Casework project cannot be inspected.", + "Install the tools.", + )); + assert_eq!(class(&unavailable), "operational"); + + let terminate = AtomicBool::new(false); + let missing = Path::new("/nonexistent/caseworkctl"); + let error = explain_casework_facts(missing, Path::new("."), &terminate).unwrap_err(); + let failed = inspection_failed( + "products[0]", + "inspect the Casework project", + error, + &terminate, + ); + assert_eq!(class(&failed), "operational"); + let reason = failed + .downcast_ref::() + .expect("an unavailable inspection") + .reason(); + assert!( + reason.contains( + "could not inspect the Casework project: cannot start caseworkctl explain" + ), + "{reason}" + ); + } + + #[test] + fn an_interrupted_inspection_stays_the_operators_request() { + let class = + |error: &anyhow::Error| super::super::error_report(error).0["error"]["class"].clone(); + let terminate = AtomicBool::new(true); + let interrupted = anyhow::anyhow!("development product inspection was interrupted"); + let failed = inspection_failed( + "products[0]", + "inspect the Casework project", + interrupted, + &terminate, + ); + assert!(!failed.is::()); + assert_eq!(class(&failed), "domain"); + assert_eq!( + failed.to_string(), + "development product inspection was interrupted" + ); + } + + #[test] + fn missed_readiness_and_start_deadlines_are_operational() { + let class = + |error: &anyhow::Error| super::super::error_report(error).0["error"]["class"].clone(); + let scratch = tempfile::tempdir().unwrap(); + let root = scratch.path().join("run"); + super::super::private_directory(&root).unwrap(); + let terminate = AtomicBool::new(false); + let mut runner = + ProductRunner::new(Arc::new(breg_input(false)), &root, &terminate).unwrap(); + let unused = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .unwrap() + .local_addr() + .unwrap() + .port(); + + let not_ready = runner + .ready( + Product::Breg, + &format!("http://127.0.0.1:{unused}"), + Duration::ZERO, ) - }); + .unwrap_err(); + assert_eq!(class(¬_ready), "operational", "{not_ready:#}"); + assert!( + not_ready + .to_string() + .starts_with("breg did not become ready"), + "{not_ready:#}" + ); + + runner.deadline = Instant::now(); + let late = runner.ensure_active().unwrap_err(); + assert_eq!(class(&late), "operational", "{late:#}"); assert_eq!( - refuse_encrypted_fields(®istry, "/products/0/project") - .unwrap_err() - .to_string(), - "dev.yaml /products/0/project: The BReg project encrypts contact.email, and rstack \ - dev does not configure field encryption yet. Next: Run this BReg project with \ - `rstack breg` and a field-encryption provider, or select a project without \ - encrypted fields." + late.to_string(), + "development run did not start within five minutes" ); - refuse_encrypted_fields(&minimal_registry(str::to_owned), "/products/0/project").unwrap(); + + // A stop requested during startup stays a refusal. + terminate.store(true, Ordering::Relaxed); + let stopped = runner.ensure_active().unwrap_err(); + assert_eq!(class(&stopped), "domain", "{stopped:#}"); + } + + #[test] + fn a_run_without_encrypted_fields_writes_no_key_and_no_provider() { + let scratch = tempfile::tempdir().unwrap(); + let root = scratch.path().join("run"); + super::super::private_directory(&root).unwrap(); + let terminate = AtomicBool::new(false); + let runner = ProductRunner::new(Arc::new(breg_input(false)), &root, &terminate).unwrap(); + runner.write_field_key().unwrap(); + assert_eq!(field_key_file(&root), None); + let (config, _) = breg_config(&runner, &root, "breg"); + assert!(config.get("fieldEncryption").is_none(), "{config}"); + assert_eq!(config["identity"]["environment"], "local"); } #[test] @@ -1937,12 +2700,15 @@ mod tests { .spawn() .unwrap(); wait_for_leader(&child); - assert_eq!(stop_child(&mut child).unwrap().code(), Some(7)); + assert_eq!( + guard::stop_direct_child(&mut child).unwrap().code(), + Some(7) + ); } fn wait_for_leader(child: &Child) { let deadline = Instant::now() + Duration::from_secs(2); - while !child_exited(child).unwrap() { + while !guard::direct_child_exited(child).unwrap() { assert!(Instant::now() < deadline, "the tool leader did not exit"); thread::sleep(POLL); } @@ -1958,7 +2724,7 @@ mod tests { .stdout(Stdio::piped()) .spawn() .unwrap(); - let pid = child_pid(&child).unwrap(); + let pid = guard::direct_child_pid(&child).unwrap(); assert_eq!(rustix::process::getpgid(Some(pid)).unwrap(), pid); let mut pipe = BufReader::new(child.stdout.take().unwrap()); let mut ready = String::new(); @@ -1967,7 +2733,7 @@ mod tests { if leader_exits { wait_for_leader(&child); } - stop_child(&mut child).unwrap(); + guard::stop_direct_child(&mut child).unwrap(); let (sender, receiver) = mpsc::sync_channel(1); let reader = thread::spawn(move || { diff --git a/crates/registry-bregctl/src/project_dev/recovery.rs b/crates/registry-bregctl/src/project_dev/recovery.rs new file mode 100644 index 0000000000..8daef210d0 --- /dev/null +++ b/crates/registry-bregctl/src/project_dev/recovery.rs @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Wait for the guards that own a lost worker's child processes. + +use anyhow::{bail, ensure, Context, Result}; +use rustix::fs::{flock, FlockOperation}; +use std::{ + fs::{self, File}, + path::Path, + process::{Command, Stdio}, + thread, + time::{Duration, Instant}, +}; + +const GUARD_WAIT: Duration = Duration::from_secs(10); +const POLL: Duration = Duration::from_millis(100); + +/// One row of the process table. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct Row { + pub pid: u32, + pub pgid: u32, + pub started: String, +} + +/// Every process with its group and start time. `ps` is the one portable +/// source of start times on Linux and macOS. +pub(super) fn process_table() -> Result> { + let output = ps_command() + .output() + .context("cannot run ps to read process start times")?; + ensure!( + output.status.success(), + "ps could not list processes with their start times" + ); + Ok(parse_table(&String::from_utf8_lossy(&output.stdout))) +} + +/// The `ps` call behind [`process_table`]. +fn ps_command() -> Command { + let mut command = Command::new("ps"); + command + .args(["-A", "-o", "pid=,pgid=,lstart="]) + .env("LC_ALL", "C") + .env("TZ", "UTC") + .stdin(Stdio::null()) + .stderr(Stdio::null()); + command +} + +fn parse_table(text: &str) -> Vec { + text.lines() + .filter_map(|line| { + let mut fields = line.split_whitespace(); + let pid = fields.next()?.parse().ok()?; + let pgid = fields.next()?.parse().ok()?; + // The start time is several words; spacing differs between + // platforms, so it is kept with single spaces. + let started = fields.collect::>().join(" "); + (!started.is_empty()).then_some(Row { pid, pgid, started }) + }) + .collect() +} + +/// The start time of `pid`, or `None` when it cannot be read. +#[cfg(test)] +pub(super) fn started_at(pid: u32) -> Option { + process_table() + .ok()? + .into_iter() + .find(|row| row.pid == pid) + .map(|row| row.started) +} + +/// The worker creates each lock before launching a guard. Once the worker +/// has exited, a late guard cannot start a workload because its control pipe +/// is closed. A guard already running a workload holds its lock through reap. +pub(super) fn wait_for_guards(root: &Path) -> Result> { + wait_for_guards_until(root, Instant::now() + GUARD_WAIT)?; + super::guard::runtime_groups(root) +} + +fn wait_for_guards_until(root: &Path, deadline: Instant) -> Result<()> { + let directory = root.join(super::guard::DIRECTORY); + let entries = fs::read_dir(directory).context("cannot read the run's process guards")?; + for entry in entries { + let path = entry?.path(); + if path.extension().and_then(|value| value.to_str()) != Some("lock") { + continue; + } + let lock = File::open(&path).context("cannot open a run process guard lock")?; + loop { + match flock(&lock, FlockOperation::NonBlockingLockExclusive) { + Ok(()) => break, + Err(rustix::io::Errno::WOULDBLOCK) if Instant::now() < deadline => { + thread::sleep(POLL); + } + Err(rustix::io::Errno::WOULDBLOCK) => { + bail!("a run process guard is still stopping its workload"); + } + Err(error) => return Err(error).context("cannot check a run process guard lock"), + } + } + } + let incomplete = super::guard::incomplete_launches(root)?; + ensure!( + incomplete.is_empty(), + "process guards did not confirm cleanup for {}", + incomplete.join(", ") + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn start_times_are_read_in_one_locale_and_time_zone() { + use std::ffi::OsStr; + let command = ps_command(); + let envs: Vec<_> = command.get_envs().collect(); + for (name, value) in [("LC_ALL", "C"), ("TZ", "UTC")] { + assert!( + envs.contains(&(OsStr::new(name), Some(OsStr::new(value)))), + "{name}: {envs:?}" + ); + } + } + + fn row(pid: u32, pgid: u32, started: &str) -> Row { + Row { + pid, + pgid, + started: started.to_owned(), + } + } + + #[test] + fn the_process_table_parses_spaced_start_times() { + let table = parse_table( + " 1 1 Tue Nov 3 10:00:00 2026\n 4242 4242 Tue Nov 11 12:00:01 2026\nbad line\n", + ); + assert_eq!( + table, + [ + row(1, 1, "Tue Nov 3 10:00:00 2026"), + row(4242, 4242, "Tue Nov 11 12:00:01 2026"), + ] + ); + } + + #[test] + fn a_held_guard_lock_prevents_recovery() { + let root = tempfile::tempdir().unwrap(); + let guards = root.path().join("guards"); + std::fs::create_dir(&guards).unwrap(); + let path = guards.join("test.lock"); + let lock = std::fs::File::create(&path).unwrap(); + rustix::fs::flock(&lock, rustix::fs::FlockOperation::LockExclusive).unwrap(); + + let error = wait_for_guards_until(root.path(), Instant::now()).unwrap_err(); + assert!(error.to_string().contains("process guard"), "{error:#}"); + drop(lock); + wait_for_guards_until(root.path(), Instant::now()).unwrap(); + } + + #[test] + fn a_released_guard_must_have_confirmed_cleanup() { + let root = tempfile::tempdir().unwrap(); + let guards = root.path().join("guards"); + std::fs::create_dir(&guards).unwrap(); + std::fs::File::create(guards.join("test.lock")).unwrap(); + let receipt = guards.join("test.state"); + std::fs::write(&receipt, "launching\n").unwrap(); + + let error = wait_for_guards_until(root.path(), Instant::now()).unwrap_err(); + assert!( + error.to_string().contains("did not confirm cleanup"), + "{error:#}" + ); + std::fs::write(receipt, "complete\n").unwrap(); + wait_for_guards_until(root.path(), Instant::now()).unwrap(); + } + + #[test] + fn a_run_without_guard_metadata_cannot_be_recovered_automatically() { + let root = tempfile::tempdir().unwrap(); + let error = wait_for_guards_until(root.path(), Instant::now()).unwrap_err(); + assert!(error.to_string().contains("process guards"), "{error:#}"); + } +} diff --git a/crates/registry-bregctl/tests/project_dev.rs b/crates/registry-bregctl/tests/project_dev.rs index f895516d1a..5de25de100 100644 --- a/crates/registry-bregctl/tests/project_dev.rs +++ b/crates/registry-bregctl/tests/project_dev.rs @@ -27,6 +27,215 @@ journey:\n id: smoke\n steps:\n - {{id: read, product: breg, as: operator, .unwrap(); } +const RUN_ID: &str = "0123456789abcdef0123456789abcdef"; + +/// Validates a helper failure against the documented error schema and checks +/// its class. +fn assert_error_envelope(report: &serde_json::Value, class: &str) { + let schema: serde_json::Value = serde_json::from_str(include_str!( + "../../../products/platform/schemas/rstack-dev-error.v1.schema.json" + )) + .unwrap(); + let schema = jsonschema::JSONSchema::options() + .with_draft(jsonschema::Draft::Draft202012) + .compile(&schema) + .unwrap(); + if let Err(errors) = schema.validate(report) { + let errors: Vec<_> = errors.map(|error| error.to_string()).collect(); + panic!("{report} violates the error schema: {errors:?}"); + } + assert_eq!(report["error"]["class"], class, "{report}"); +} + +fn helper(request: serde_json::Value, state: &std::path::Path) -> std::process::Output { + Command::new(env!("CARGO_BIN_EXE_bregctl")) + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request.to_string()) + .env("RSTACK_DEV_STATE_DIR", state) + .env_remove("RSTACK_DEV_DATABASE_URL") + .output() + .unwrap() +} + +/// A run directory whose supervisor is gone: its lock file exists, unlocked. +fn lost_run(state: &std::path::Path) -> std::path::PathBuf { + let root = state.join(RUN_ID); + fs::create_dir_all(&root).unwrap(); + fs::set_permissions(state, fs::Permissions::from_mode(0o700)).unwrap(); + fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap(); + fs::write(root.join("worker.lock"), "").unwrap(); + root +} + +#[test] +fn every_helper_failure_prints_the_error_envelope() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + lost_run(&state); + let missing_project = scratch.path().join("missing"); + let other_run = "fedcba9876543210fedcba9876543210"; + let cases = [ + ( + "start of a missing project", + serde_json::json!({"action": "start", "project": missing_project, "json": true}), + "domain", + 1, + ), + ( + "start with a bad ttl", + serde_json::json!({"action": "start", "project": scratch.path(), "ttl": "9", "json": true}), + "usage", + 2, + ), + ( + "status with a malformed run ID", + serde_json::json!({"action": "status", "run_id": "../x", "json": true}), + "usage", + 2, + ), + ( + "token with a malformed person", + serde_json::json!({"action": "token", "run_id": other_run, "person": "../x", "json": true}), + "usage", + 2, + ), + ( + "token for a run that is not ready", + serde_json::json!({"action": "token", "run_id": other_run, "person": "operator", "json": true}), + "domain", + 1, + ), + ( + "stop with a malformed run ID", + serde_json::json!({"action": "stop", "run_id": "../x", "json": true}), + "usage", + 2, + ), + ( + "recover with a malformed run ID", + serde_json::json!({"action": "stop", "run_id": "../x", "json": true, "recover": true}), + "usage", + 2, + ), + ( + "status of a run whose supervisor is lost", + serde_json::json!({"action": "status", "run_id": RUN_ID, "json": true}), + "operational", + 3, + ), + ( + "stop of a run whose supervisor is lost", + serde_json::json!({"action": "stop", "run_id": RUN_ID, "json": true}), + "operational", + 3, + ), + ]; + for (name, request, class, exit) in cases { + let output = helper(request, &state); + assert_eq!(output.status.code(), Some(exit), "{name}: {output:?}"); + assert!(output.stderr.is_empty(), "{name}: {output:?}"); + let report: serde_json::Value = serde_json::from_slice(&output.stdout) + .unwrap_or_else(|error| panic!("{name}: {error}: {output:?}")); + assert_error_envelope(&report, class); + if class == "operational" { + assert_eq!(report["run_id"], RUN_ID, "{name}"); + assert_eq!(report["recovery"]["kind"], "lost-supervisor", "{name}"); + } else { + assert!(report.get("run_id").is_none(), "{name}: {report}"); + } + } +} + +#[test] +fn a_helper_failure_without_json_prints_one_line_and_the_same_exit_class() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + lost_run(&state); + let output = helper( + serde_json::json!({"action": "status", "run_id": RUN_ID, "json": false}), + &state, + ); + assert_eq!(output.status.code(), Some(3), "{output:?}"); + assert!(output.stdout.is_empty(), "{output:?}"); + let text = String::from_utf8(output.stderr).unwrap(); + assert!(text.starts_with("rstack dev: run "), "{text}"); + assert!(text.contains("lost its supervisor"), "{text}"); +} + +fn recover(state: &std::path::Path, run_id: &str) -> std::process::Output { + helper( + serde_json::json!({"action": "stop", "run_id": run_id, "json": true, "recover": true}), + state, + ) +} + +#[test] +fn recover_of_an_absent_run_reports_it_stopped() { + let scratch = tempfile::tempdir().unwrap(); + let output = recover(&scratch.path().join("state"), RUN_ID); + assert_eq!(output.status.code(), Some(0), "{output:?}"); + let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(report["status"], "stopped"); +} + +#[test] +fn recover_refuses_a_run_with_a_running_supervisor() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let root = lost_run(&state); + let lock = fs::File::open(root.join("worker.lock")).unwrap(); + rustix::fs::flock(&lock, rustix::fs::FlockOperation::NonBlockingLockExclusive).unwrap(); + + let output = recover(&state, RUN_ID); + + assert_eq!(output.status.code(), Some(1), "{output:?}"); + let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_error_envelope(&report, "domain"); + let message = report["error"]["message"].as_str().unwrap(); + assert!( + message.contains("still has a running supervisor"), + "{message}" + ); + assert!(root.exists()); +} + +#[test] +fn recover_refuses_a_run_without_a_supervisor_lock() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let root = lost_run(&state); + fs::remove_file(root.join("worker.lock")).unwrap(); + + let output = recover(&state, RUN_ID); + + assert_eq!(output.status.code(), Some(1), "{output:?}"); + let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_error_envelope(&report, "domain"); + assert!(root.exists()); +} + +#[test] +fn recover_that_cannot_proceed_reports_the_facts_and_changes_nothing() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let root = lost_run(&state); + + let output = recover(&state, RUN_ID); + + assert_eq!(output.status.code(), Some(3), "{output:?}"); + let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_error_envelope(&report, "operational"); + assert_eq!(report["run_id"], RUN_ID); + assert_eq!(report["recovery"]["kind"], "lost-supervisor"); + assert!(report["recovery"]["run_directory"] + .as_str() + .unwrap() + .ends_with(RUN_ID)); + let message = report["error"]["message"].as_str().unwrap(); + assert!(message.contains("could not be recovered"), "{message}"); + assert!(root.exists()); +} + fn semantic_project(root: &std::path::Path, grant: &str, step_extra: &str) { fs::write(root.join("registry.yaml"), "fixture\n").unwrap(); fs::write( @@ -130,24 +339,54 @@ fn start_with_casework_tool( } fn fake_casework_tool(root: &std::path::Path) -> std::path::PathBuf { + fake_casework_tool_with_status_profile(root, "operator") +} + +/// A `caseworkctl` whose `proposal-context` provider names `status_profile` +/// for the `service-status` capability. Its `check` refuses a runtime file +/// whose context provider resource is `registry`, as Casework does. +fn fake_casework_tool_with_status_profile( + root: &std::path::Path, + status_profile: &str, +) -> std::path::PathBuf { let tool = root.join("caseworkctl"); fs::write( &tool, concat!( "#!/usr/bin/env bash\n", "set -euo pipefail\n", - "[[ $# -eq 4 ]]\n", - "[[ $1 == --format && $2 == json && $3 == explain ]]\n", - "[[ -f $4/casework.yaml ]]\n", + "[[ $1 == --format && $2 == json && -f $4/casework.yaml ]]\n", + "if [[ $3 == check ]]; then\n", + " [[ $# -eq 6 && $5 == --runtime-config && -f $6 ]]\n", + " if grep -q '\"resource\":\"registry\"' \"$6\"; then\n", + " printf '%s\\n' '", + "{\"apiVersion\":\"id.registrystack.org/formats/casework/report/v1alpha2\",", + "\"kind\":\"CaseworkReport\",\"command\":\"check\",\"status\":\"domain-refusal\",\"ok\":false,", + "\"diagnostics\":[{\"severity\":\"error\",\"code\":\"casework.runtime.invalid-context-provider\",", + "\"artifact\":\"CaseworkRuntimeConfig\",\"path\":\"/contextProviders/proposal-context/resource\",", + "\"message\":\"contextProviders.proposal-context.resource: the resource must be one absolute URI without a fragment or userinfo\",", + "\"suggestedAction\":\"Correct the context provider member so it meets the rule the message names.\"}]}'\n", + " exit 1\n", + " fi\n", + " printf '%s\\n' '", + "{\"apiVersion\":\"id.registrystack.org/formats/casework/report/v1alpha2\",", + "\"kind\":\"CaseworkReport\",\"command\":\"check\",\"status\":\"complete\",\"ok\":true,", + "\"diagnostics\":[]}'\n", + " exit 0\n", + "fi\n", + "[[ $# -eq 4 && $3 == explain ]]\n", "printf '%s\\n' '", "{\"apiVersion\":\"id.registrystack.org/formats/casework/report/v1alpha2\",", "\"kind\":\"CaseworkReport\",\"command\":\"explain\",\"status\":\"complete\",\"ok\":true,", "\"accessProfiles\":[{\"id\":\"reviewer\",\"principalClaim\":\"sub\"}],", "\"reviewPolicies\":[{\"dependencies\":{\"accessProfiles\":[{\"id\":\"reviewer\",\"principalClaim\":\"sub\"}]}}],", "\"trustedProducers\":[{\"id\":\"review-producer\"}],", - "\"contextProviders\":[{\"id\":\"proposal-context\"}]}", - "'\n", - ), + "\"contextProviders\":[{\"id\":\"proposal-context\",", + "\"serviceProfiles\":{\"service-status\":\"", + ) + .to_owned() + + status_profile + + "\"}}]}'\n", ) .unwrap(); let mut permissions = fs::metadata(&tool).unwrap().permissions(); @@ -232,7 +471,8 @@ fn assert_dangling_schema_assignment( assert_eq!(started.status.code(), Some(1), "{started:?}"); let report: serde_json::Value = serde_json::from_slice(&started.stdout).unwrap(); - let error = report["error"].as_str().unwrap(); + assert_error_envelope(&report, "domain"); + let error = report["error"]["message"].as_str().unwrap(); assert!(error.contains(path), "{error}"); assert!(error.contains(message), "{error}"); assert!(error.contains(suggested_action), "{error}"); @@ -258,6 +498,35 @@ fn check_validates_the_closed_plan_without_allocating_run_state() { assert!(!state.exists()); } +#[test] +fn check_accepts_a_project_with_encrypted_fields_without_allocating_state() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + project(scratch.path(), ""); + let registry = scratch.path().join("registry.yaml"); + let source = fs::read_to_string(®istry).unwrap(); + let encrypted = source + .replacen( + "mutationMode: mutable\n display: {label: Record", + "mutationMode: mutable\n classification: restricted\n display: {label: Record", + 1, + ) + .replacen( + "- {id: name, type: string, required: true, minimumLength: 1, maximumLength: 100, classification: internal}\n - {id: owner", + "- {id: name, type: string, required: true, minimumLength: 1, maximumLength: 100, classification: restricted, encrypted: true}\n - {id: owner", + 1, + ); + assert_eq!(encrypted.matches("encrypted: true").count(), 1); + fs::write(®istry, encrypted).unwrap(); + + let result = check(scratch.path(), &state, true); + + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + assert!(result.status.success(), "{report}"); + assert_eq!(report["diagnostics"], serde_json::json!([])); + assert!(!state.exists()); +} + #[test] fn check_and_start_reject_an_unknown_fixture_journey_without_allocating_state() { assert_dangling_schema_assignment( @@ -363,7 +632,8 @@ fn check_and_start_report_an_explicit_assignment_with_no_fixture_file() { assert_eq!(started.status.code(), Some(1), "{started:?}"); let report: serde_json::Value = serde_json::from_slice(&started.stdout).unwrap(); - let error = report["error"].as_str().unwrap(); + assert_error_envelope(&report, "domain"); + let error = report["error"]["message"].as_str().unwrap(); assert!(error.contains("/schemaTestPeople"), "{error}"); assert!( error.contains("fixture journeys file is missing"), @@ -416,7 +686,8 @@ fn check_and_start_refuse_a_fixture_symlink_outside_the_breg_project() { assert_eq!(started.status.code(), Some(1), "{started:?}"); let report: serde_json::Value = serde_json::from_slice(&started.stdout).unwrap(); - let error = report["error"].as_str().unwrap(); + assert_error_envelope(&report, "domain"); + let error = report["error"]["message"].as_str().unwrap(); assert!(error.contains("/products/0/project"), "{error}"); assert!(error.contains("fixture journeys file must be"), "{error}"); assert!(error.contains("without linking it outside"), "{error}"); @@ -678,7 +949,8 @@ fn start_reports_an_actionable_value_free_unstarted_grant() { assert_eq!(result.status.code(), Some(1)); let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); - let error = report["error"].as_str().unwrap(); + assert_error_envelope(&report, "domain"); + let error = report["error"]["message"].as_str().unwrap(); assert!( error.contains("/people/operator/grants/casework"), "{error}" @@ -753,7 +1025,8 @@ journey:\n id: smoke\n steps:\n - {id: read, product: breg, as: editor, met assert_eq!(result.status.code(), Some(1), "{result:?}"); let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); - let error = report["error"].as_str().unwrap(); + assert_error_envelope(&report, "domain"); + let error = report["error"]["message"].as_str().unwrap(); assert!( error.contains("/people/unused/grants/breg/profile"), "{error}" @@ -820,7 +1093,8 @@ fn check_and_start_resolve_casework_profiles_and_bindings_with_the_owning_report assert_eq!(started.status.code(), Some(1), "{started:?}"); let report: serde_json::Value = serde_json::from_slice(&started.stdout).unwrap(); - let error = report["error"].as_str().unwrap(); + assert_error_envelope(&report, "domain"); + let error = report["error"]["message"].as_str().unwrap(); assert!(error.contains(path), "{error}"); assert!(error.contains(message), "{error}"); assert!(error.contains(action), "{error}"); @@ -829,6 +1103,153 @@ fn check_and_start_resolve_casework_profiles_and_bindings_with_the_owning_report } } +#[test] +fn check_refuses_binding_configuration_the_products_would_refuse_at_the_plan_field() { + for (plan_edit, from, to, path, product) in [ + ( + "scope", + "casework: {profile: reviewer, scopes: [casework:review]}", + "casework: {profile: reviewer, scopes: ['casework:\"quoted\"']}", + "/people/reviewer/grants/casework/scopes", + "BReg refuses", + ), + ( + "audience", + "audience: \"urn:example:registry\"", + "audience: registry", + "/products/0/audience", + "Casework refuses", + ), + ] { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let tool = casework_reference_project( + scratch.path(), + "reviewer", + concat!( + "bindings:\n", + " bregReviewAuthority: {id: local-review, producer: review-producer, identity: reviewer}\n", + " caseworkContextProvider: {id: proposal-context, identity: operator, action: review-action}\n", + ), + ); + let dev = scratch.path().join("dev.yaml"); + let text = fs::read_to_string(&dev).unwrap(); + assert!(text.contains(from), "{plan_edit}: {text}"); + fs::write(&dev, text.replace(from, to)).unwrap(); + + let checked = check_with_casework_tool(scratch.path(), &state, &tool); + + assert_eq!(checked.status.code(), Some(1), "{plan_edit}: {checked:?}"); + let report: serde_json::Value = serde_json::from_slice(&checked.stdout).unwrap(); + assert_eq!(report["status"], "domain-refusal", "{plan_edit}"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], path, "{plan_edit}: {diagnostic}"); + assert!( + diagnostic["message"].as_str().unwrap().contains(product), + "{plan_edit}: {diagnostic}" + ); + assert!(!state.exists(), "{plan_edit}"); + } +} + +#[test] +fn check_refuses_a_context_provider_binding_with_the_rule_caseworkctl_check_names() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let tool = casework_reference_project( + scratch.path(), + "reviewer", + "bindings:\n caseworkContextProvider: {id: proposal-context, identity: operator, action: review-action}\n", + ); + let dev = scratch.path().join("dev.yaml"); + let text = fs::read_to_string(&dev).unwrap(); + fs::write( + &dev, + text.replace("audience: \"urn:example:registry\"", "audience: registry"), + ) + .unwrap(); + + let checked = check_with_casework_tool(scratch.path(), &state, &tool); + + assert_eq!(checked.status.code(), Some(1), "{checked:?}"); + let report: serde_json::Value = serde_json::from_slice(&checked.stdout).unwrap(); + assert_eq!(report["status"], "domain-refusal"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/products/0/audience"); + assert_eq!( + diagnostic["message"], + "Casework refuses the binding configuration the run writes from this value: \ +contextProviders.proposal-context.resource: the resource must be one absolute URI without a \ +fragment or userinfo." + ); + assert!(!state.exists()); + + // A helper that cannot check the configuration is unavailable, not a + // refusal of the plan. + fs::rename(&tool, scratch.path().join("caseworkctl-reference")).unwrap(); + fs::write( + &tool, + concat!( + "#!/usr/bin/env bash\n", + "set -euo pipefail\n", + "if [[ $3 == check ]]; then\n", + " printf '%s\\n' '", + "{\"apiVersion\":\"id.registrystack.org/formats/casework/report/v1alpha2\",", + "\"kind\":\"CaseworkReport\",\"command\":\"check\",\"status\":\"domain-refusal\",\"ok\":false,", + "\"diagnostics\":[{\"severity\":\"error\",\"code\":\"config.unknown-key\",", + "\"artifact\":\"CaseworkRuntimeConfig\",\"path\":\"/listener\",", + "\"message\":\"The key is not part of this format.\",", + "\"suggestedAction\":\"Remove the key.\"}]}'\n", + " exit 1\n", + "fi\n", + "exec \"${0%/*}/caseworkctl-reference\" \"$@\"\n", + ), + ) + .unwrap(); + fs::set_permissions(&tool, fs::Permissions::from_mode(0o700)).unwrap(); + + let unavailable = check_with_casework_tool(scratch.path(), &state, &tool); + + assert_eq!(unavailable.status.code(), Some(3), "{unavailable:?}"); + let report: serde_json::Value = serde_json::from_slice(&unavailable.stdout).unwrap(); + assert_eq!(report["status"], "operational-failure"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/products/1/project"); + assert_eq!( + diagnostic["message"], + "The selected `caseworkctl` helper could not check the Casework configuration the run writes: caseworkctl check did not report a result for the binding." + ); + assert!(!state.exists()); +} + +#[test] +fn check_refuses_a_status_profile_the_context_provider_policy_does_not_name() { + let bindings = "bindings:\n caseworkContextProvider: {id: proposal-context, identity: operator, action: review-action}\n"; + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let tool = casework_reference_project(scratch.path(), "reviewer", bindings); + fake_casework_tool_with_status_profile(scratch.path(), "another-status-profile"); + + let checked = check_with_casework_tool(scratch.path(), &state, &tool); + + assert_eq!(checked.status.code(), Some(1), "{checked:?}"); + let report: serde_json::Value = serde_json::from_slice(&checked.stdout).unwrap(); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/people/operator/grants/breg/profile"); + assert!(!diagnostic.to_string().contains("another-status-profile")); + assert!(!state.exists()); + + let started = start_with_casework_tool(scratch.path(), &state, &tool); + assert_eq!(started.status.code(), Some(1), "{started:?}"); + assert!(!state.exists()); + + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let tool = casework_reference_project(scratch.path(), "reviewer", bindings); + let checked = check_with_casework_tool(scratch.path(), &state, &tool); + assert_eq!(checked.status.code(), Some(0), "{checked:?}"); +} + #[test] fn check_distinguishes_an_unavailable_casework_helper_from_an_invalid_project() { let scratch = tempfile::tempdir().unwrap(); @@ -888,7 +1309,7 @@ fn check_distinguishes_an_unavailable_casework_helper_from_an_invalid_project() assert_eq!(diagnostic["path"], "/products/1/project"); assert_eq!( diagnostic["message"], - "The selected `caseworkctl` helper could not inspect the Casework project." + "The selected `caseworkctl` helper could not inspect the Casework project: caseworkctl explain did not complete successfully." ); assert!(!state.exists()); @@ -945,12 +1366,12 @@ fn a_malformed_plan_names_its_field_before_allocating_run_state() { .env_remove("RSTACK_DEV_DATABASE_URL") .output() .unwrap(); - assert!(!result.status.success()); + assert_eq!(result.status.code(), Some(1), "{result:?}"); let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); - assert_eq!(report["schema_version"], "registry.rstack.dev.v1"); + assert_error_envelope(&report, "domain"); assert!(report.get("diagnostics").is_none()); assert!( - report["error"] + report["error"]["message"] .as_str() .unwrap() .contains("/products/0/product"), @@ -973,24 +1394,78 @@ fn a_run_id_cannot_name_a_path() { .unwrap(); assert!(!result.status.success()); let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); - assert!(report["error"].as_str().unwrap().contains("run ID")); + assert!(report["error"]["message"] + .as_str() + .unwrap() + .contains("run ID")); assert_eq!(fs::read_dir(scratch.path()).unwrap().count(), 0); } +/// A system binary such as `true` or `false`, wherever this platform keeps it. +fn system_binary(name: &str) -> std::path::PathBuf { + ["/usr/bin", "/bin"] + .into_iter() + .map(|directory| std::path::Path::new(directory).join(name)) + .find(|path| path.is_file()) + .unwrap() +} + +/// A loopback port nothing listens on: it is reserved and released at once. +fn closed_loopback_port() -> u16 { + std::net::TcpListener::bind(("127.0.0.1", 0)) + .unwrap() + .local_addr() + .unwrap() + .port() +} + +/// Start a BReg run whose product tool and runtime are the given binaries and +/// whose PostgreSQL administrator URL is `database_url`. +fn start_with_breg_binaries( + root: &std::path::Path, + state: &std::path::Path, + (tool, runtime): (&std::path::Path, &std::path::Path), + database_url: &str, +) -> std::process::Output { + let request = serde_json::json!({ + "action": "start", "project": root, "json": true, + }); + Command::new(env!("CARGO_BIN_EXE_bregctl")) + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request.to_string()) + .env("RSTACK_DEV_STATE_DIR", state) + .env("RSTACK_DEV_DATABASE_URL", database_url) + .env_remove("RSTACK_DEV_DATABASE_CA") + .env("RSTACK_BREGCTL_BIN", tool) + .env("RSTACK_BREG_BIN", runtime) + .output() + .unwrap() +} + +/// The run directories `state` still holds; kept logs of failed runs are not +/// runs. +fn live_runs(state: &std::path::Path) -> Vec { + match fs::read_dir(state) { + Ok(entries) => entries + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|name| name != "failed") + .collect(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(), + Err(error) => panic!("cannot read the run state: {error}"), + } +} + #[test] fn worker_restores_a_valid_plan_whose_captured_input_exceeds_512_kib() { let scratch = tempfile::tempdir().unwrap(); let project_root = scratch.path().join("project"); fs::create_dir(&project_root).unwrap(); - fs::write( - project_root.join("registry.yaml"), - include_str!( - "../../../products/platform/conformance/projects/breg-project-dev/registry/registry.yaml" - ), - ) - .unwrap(); + fixture_project(&project_root, ""); let raw = BASE64.encode(vec![b'x'; 400 * 1024]); - let mut plan = serde_json::json!({ + // The captured input carries the plan's base64 body verbatim, so it is + // larger than the body itself. + assert!(raw.len() > 512 * 1024, "raw body: {}", raw.len()); + let plan = serde_json::json!({ "apiVersion": "id.registrystack.org/formats/breg/project-dev/v1alpha1", "kind": "BRegProjectDev", "issuer": "https://issuer.example.invalid", @@ -1019,48 +1494,57 @@ fn worker_restores_a_valid_plan_whose_captured_input_exceeds_512_kib() { "the substantial development plan was refused" ); - let members = plan.as_object_mut().unwrap(); - members.remove("apiVersion"); - members.remove("kind"); - let run_id = "0123456789abcdef0123456789abcdef"; + // The supervisor captures the plan for its worker; a refusing product tool + // proves the worker restored that capture and reached the product check. let state = scratch.path().join("state"); - let root = state.join(run_id); - fs::create_dir_all(&root).unwrap(); - fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap(); - let root = root.canonicalize().unwrap(); - let false_binary = ["/usr/bin/false", "/bin/false"] - .into_iter() - .map(std::path::Path::new) - .find(|path| path.is_file()) - .unwrap() - .canonicalize() - .unwrap(); - let input = serde_json::to_vec(&serde_json::json!({ - "run_id": run_id, - "project": project_root, - "plan": plan, - "ttl_seconds": 1, - "schema_people": [], - "breg_authority": { - "principal": "sub", "purpose": false, "requires_postgis": false, - "named_clients": [] - }, - "casework_principals": {}, - "binaries": {"breg": {"tool": false_binary, "runtime": false_binary}} - })) - .unwrap(); - assert!(input.len() > 512 * 1024, "captured input: {}", input.len()); - fs::write(root.join("input.json"), input).unwrap(); - - let result = Command::new(env!("CARGO_BIN_EXE_bregctl")) - .args(["project-dev", "--worker"]) - .arg(&root) - .env("RSTACK_DEV_STATE_DIR", &state) - .env_remove("RSTACK_DEV_DATABASE_URL") - .output() - .unwrap(); - let reply: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); - let error = reply["error"].as_str().unwrap(); + let false_binary = system_binary("false"); + let result = start_with_breg_binaries( + &project_root, + &state, + (&false_binary, &false_binary), + &format!("postgresql://127.0.0.1:{}/postgres", closed_loopback_port()), + ); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + assert_error_envelope(&report, "domain"); + assert_eq!(result.status.code(), Some(1), "{report}"); + let error = report["error"]["message"].as_str().unwrap(); assert!(error.contains("breg check failed"), "{error}"); assert!(!error.contains("cannot restore the captured development plan")); + assert_eq!(live_runs(&state), Vec::::new()); +} + +#[test] +fn start_reports_an_unreachable_postgresql_server_as_operational() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + fixture_project(scratch.path(), ""); + let canary = "rstack-dev-admin-password-canary"; + + let result = start_with_breg_binaries( + scratch.path(), + &state, + (&system_binary("true"), &system_binary("false")), + &format!( + "postgresql://admin:{canary}@127.0.0.1:{}/postgres", + closed_loopback_port() + ), + ); + + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + assert_error_envelope(&report, "operational"); + assert_eq!(result.status.code(), Some(3), "{report}"); + assert!( + report["error"]["message"] + .as_str() + .unwrap() + .contains("cannot establish the PostgreSQL administrator session"), + "{report}" + ); + // The run released everything itself, so there is nothing to recover. + assert!(report.get("run_id").is_none(), "{report}"); + assert!(report.get("recovery").is_none(), "{report}"); + assert_eq!(live_runs(&state), Vec::::new()); + for output in [&result.stdout, &result.stderr] { + assert!(!String::from_utf8_lossy(output).contains(canary)); + } } diff --git a/crates/registry-casework-breg/src/config.rs b/crates/registry-casework-breg/src/config.rs index 778e7b2a29..edf286ad01 100644 --- a/crates/registry-casework-breg/src/config.rs +++ b/crates/registry-casework-breg/src/config.rs @@ -99,6 +99,152 @@ pub struct BregBinding { #[serde(default, skip_serializing_if = "Option::is_none")] pub delegated_display_exchange: Option, } +/// The member of a context-provider binding that provider construction would +/// refuse, and the rule it breaks. The reason names the rule and never the +/// configured value. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BindingRefusal { + /// JSON pointer to the refused member, relative to the binding. + pub member: String, + pub reason: String, +} + +impl BindingRefusal { + fn new(member: &str, reason: impl Into) -> Self { + Self { + member: member.to_owned(), + reason: reason.into(), + } + } + + fn within(mut self, parent: &str) -> Self { + self.member.insert_str(0, parent); + self + } +} + +const PROFILE_IDENTIFIER_RULE: &str = + "the profile identifier must be 1..=128 bytes of ASCII letters, digits, `-`, `_`, `.`, or `:`"; +const RESOURCE_URI_RULE: &str = + "the resource must be one absolute URI without a fragment or userinfo"; + +impl BregBinding { + /// Checks the binding as provider construction reads it, without + /// resolving a secret or reading a context export. The client identifier + /// is a secret, so the identity check runs with a stand-in; the key and + /// the identifier themselves are checked when the provider is built. + pub fn check_offline(&self) -> Result<(), BindingRefusal> { + use registry_platform_httputil::{valid_resource_uri, validate_requested_scopes}; + if !registry_casework_core::valid_profile_identifier(&self.status_profile) { + return Err(BindingRefusal::new( + "/statusProfile", + PROFILE_IDENTIFIER_RULE, + )); + } + if let Some(profile) = &self.task_grant_profile { + if !registry_casework_core::valid_profile_identifier(profile) { + return Err(BindingRefusal::new( + "/taskGrantProfile", + PROFILE_IDENTIFIER_RULE, + )); + } + if profile == &self.status_profile { + return Err(BindingRefusal::new( + "/taskGrantProfile", + "the task grant profile must differ from the status profile", + )); + } + } + check_client( + &self.token_endpoint, + self.client_assertion_audience.as_deref(), + )?; + if self + .resource + .as_deref() + .is_some_and(|resource| !valid_resource_uri(resource)) + { + return Err(BindingRefusal::new("/resource", RESOURCE_URI_RULE)); + } + if let Some(scopes) = &self.scopes { + validate_requested_scopes(scopes) + .map_err(|error| BindingRefusal::new("/scopes", token_rule(error)))?; + } + let base_url = url::Url::parse(&self.base_url) + .map_err(|_| BindingRefusal::new("/baseUrl", "the service base URL must be a URL"))?; + registry_platform_httputil::ServiceBaseUrl::new(base_url) + .map_err(|error| BindingRefusal::new("/baseUrl", error.to_string()))?; + if let Some(delegated) = &self.delegated_display_exchange { + delegated + .check_offline() + .map_err(|refusal| refusal.within("/delegatedDisplayExchange"))?; + } + Ok(()) + } +} + +/// The identity and transport checks a private-key JWT client runs before it +/// holds a key. The client identifier is a secret, so a stand-in takes its place. +fn check_client(token_endpoint: &str, audience: Option<&str>) -> Result<(), BindingRefusal> { + use registry_platform_httputil::PrivateKeyJwtConfig; + let token_endpoint = url::Url::parse(token_endpoint) + .map_err(|_| BindingRefusal::new("/tokenEndpoint", "the token endpoint must be a URL"))?; + PrivateKeyJwtConfig::check_identity(&token_endpoint, "offline-check", None) + .map_err(|error| BindingRefusal::new("/tokenEndpoint", token_rule(error)))?; + PrivateKeyJwtConfig::check_identity(&token_endpoint, "offline-check", audience) + .map_err(|error| BindingRefusal::new("/clientAssertionAudience", token_rule(error))) +} + +fn token_rule(error: registry_platform_httputil::TokenError) -> String { + match error { + registry_platform_httputil::TokenError::Configuration { reason } => reason.to_owned(), + other => other.to_string(), + } +} + +/// The value rules a delegated display exchange enforces on its resource, +/// scopes and generation before it builds a token client. +pub(crate) fn check_exchange_values( + resource: &str, + scopes: &[String], + generation: &str, +) -> Result<(), BindingRefusal> { + let valid_token = |value: &str, maximum| { + !value.is_empty() + && value.len() <= maximum + && !value + .bytes() + .any(|byte| byte.is_ascii_control() || byte.is_ascii_whitespace()) + }; + if !valid_token(resource, 2048) { + return Err(BindingRefusal::new( + "/resource", + "the resource must be 1..=2048 bytes without control or whitespace characters", + )); + } + if scopes.is_empty() + || scopes.len() > 128 + || scopes.iter().any(|scope| !valid_token(scope, 512)) + || scopes + .iter() + .collect::>() + .len() + != scopes.len() + { + return Err(BindingRefusal::new( + "/scopes", + "the scopes must be 1..=128 distinct values of 1..=512 bytes without control or whitespace characters", + )); + } + if !valid_token(generation, 512) { + return Err(BindingRefusal::new( + "/generation", + "the generation must be 1..=512 bytes without control or whitespace characters", + )); + } + Ok(()) +} + impl std::fmt::Debug for BregBinding { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("BregBinding()") @@ -144,6 +290,24 @@ impl std::fmt::Debug for DelegatedDisplayExchangeBinding { } } +impl DelegatedDisplayExchangeBinding { + /// Checks the exchange as provider construction reads it, without + /// resolving its client identifier or key. + pub fn check_offline(&self) -> Result<(), BindingRefusal> { + use registry_platform_httputil::{valid_resource_uri, validate_requested_scopes}; + check_exchange_values(&self.resource, &self.scopes, &self.generation)?; + check_client( + &self.token_endpoint, + self.client_assertion_audience.as_deref(), + )?; + if !valid_resource_uri(&self.resource) { + return Err(BindingRefusal::new("/resource", RESOURCE_URI_RULE)); + } + validate_requested_scopes(&self.scopes) + .map_err(|error| BindingRefusal::new("/scopes", token_rule(error))) + } +} + pub fn build_provider( binding: &BregBinding, policy: ®istry_casework_core::ReviewContextProviderPolicy, @@ -155,16 +319,12 @@ pub fn build_provider( }; use registry_platform_crypto::PrivateJwk; use std::{sync::Arc, time::Duration}; - if description.is_empty() - || description.len() > 1024 * 1024 - || !registry_casework_core::valid_profile_identifier(&binding.status_profile) - || binding.task_grant_profile.as_ref().is_some_and(|profile| { - !registry_casework_core::valid_profile_identifier(profile) - || profile == &binding.status_profile - }) - { + if description.is_empty() || description.len() > 1024 * 1024 { return Err(ReviewContextError::Invalid); } + binding + .check_offline() + .map_err(|_| ReviewContextError::Invalid)?; let export: BregContextExport = serde_json::from_value( registry_breg_client::decode_exact_json(description) .map_err(|_| ReviewContextError::Invalid)?, @@ -286,6 +446,158 @@ mod tests { use super::*; use serde_json::json; + fn offline_binding() -> serde_json::Value { + json!({ + "baseUrl": "http://127.0.0.1:31001", + "statusProfile": "casework-review-status", + "tokenEndpoint": "http://127.0.0.1:31003/oauth2/token", + "clientIdRef": "secret:file/status-client-id", + "clientAssertionKeyRef": "secret:file/status-client-key", + "clientAssertionAudience": "https://idp.example.test", + "resource": "urn:example:registry", + "scopes": ["registry:read"] + }) + } + + fn offline_check(value: serde_json::Value) -> Result<(), BindingRefusal> { + serde_json::from_value::(value) + .expect("the binding parses") + .check_offline() + } + + fn delegated_exchange() -> serde_json::Value { + json!({ + "tokenEndpoint": "http://127.0.0.1:31003/oauth2/token", + "clientIdRef": "secret:file/source-client-id", + "clientAssertionKeyRef": "secret:file/source-client-key", + "resource": "urn:example:registry", + "scopes": ["registry:proposal:review"], + "generation": "casework-source-v1" + }) + } + + #[test] + fn offline_check_accepts_a_binding_without_resolving_secrets() { + offline_check(offline_binding()).expect("the binding is acceptable"); + let mut binding = offline_binding(); + binding["delegatedDisplayExchange"] = delegated_exchange(); + offline_check(binding).expect("the delegated exchange is acceptable"); + } + + #[test] + fn offline_check_names_the_member_and_rule_provider_construction_refuses() { + for (member, value, rule) in [ + ( + "/statusProfile", + json!("not a profile"), + "profile identifier", + ), + ( + "/taskGrantProfile", + json!("casework-review-status"), + "differ from the status profile", + ), + ( + "/taskGrantProfile", + json!("not a profile"), + "profile identifier", + ), + ("/resource", json!("registry"), "absolute URI"), + ( + "/resource", + json!("urn:example:registry#fragment"), + "absolute URI", + ), + ("/scopes", json!([]), "at least one scope"), + ( + "/scopes", + json!(["registry:read", "registry:read"]), + "repeat", + ), + ("/scopes", json!(["has space"]), "scope-tokens"), + ( + "/clientAssertionAudience", + json!(" "), + "audience must not be empty", + ), + ( + "/tokenEndpoint", + json!("http://idp.example.test/token"), + "HTTPS", + ), + ( + "/tokenEndpoint", + json!("https://idp.example.test/token#top"), + "fragment", + ), + ("/baseUrl", json!("http://breg.example.test"), "HTTPS"), + ( + "/baseUrl", + json!("https://breg.example.test/?view=all"), + "query", + ), + ( + "/baseUrl", + json!("https://breg.example.test/#top"), + "fragment", + ), + ( + "/baseUrl", + json!("https://breg.example.test//api"), + "empty segment", + ), + ( + "/delegatedDisplayExchange/tokenEndpoint", + json!("http://idp.example.test/token"), + "HTTPS", + ), + ( + "/delegatedDisplayExchange/clientAssertionAudience", + json!(""), + "audience must not be empty", + ), + ( + "/delegatedDisplayExchange/resource", + json!("registry"), + "absolute URI", + ), + ( + "/delegatedDisplayExchange/resource", + json!("urn:example:registry two"), + "whitespace", + ), + ("/delegatedDisplayExchange/scopes", json!([]), "1..=128"), + ( + "/delegatedDisplayExchange/scopes", + json!(["registry:\"quoted\""]), + "scope-tokens", + ), + ( + "/delegatedDisplayExchange/generation", + json!(""), + "generation", + ), + ] { + let mut binding = offline_binding(); + binding["delegatedDisplayExchange"] = delegated_exchange(); + let mut target = &mut binding; + for segment in member.split('/').skip(1) { + target = &mut target[segment]; + } + *target = value.clone(); + let refusal = + offline_check(binding).expect_err(&format!("{member} {value} was accepted")); + assert_eq!(refusal.member, member, "{value}: {refusal:?}"); + assert!( + refusal.reason.contains(rule), + "{member} {value}: {refusal:?}" + ); + if let Some(text) = value.as_str().filter(|text| text.trim().len() > 1) { + assert!(!refusal.reason.contains(text), "{member}: {refusal:?}"); + } + } + } + #[test] fn delegated_display_exchange_is_explicit_and_redacted() { let binding: BregBinding = serde_json::from_value(json!({ diff --git a/crates/registry-casework-breg/src/lib.rs b/crates/registry-casework-breg/src/lib.rs index f592eb9d3f..7c3951788a 100644 --- a/crates/registry-casework-breg/src/lib.rs +++ b/crates/registry-casework-breg/src/lib.rs @@ -41,26 +41,8 @@ impl DelegatedDisplayExchange { scopes: Vec, generation: String, ) -> Result { - let valid_token = |value: &str, maximum| { - !value.is_empty() - && value.len() <= maximum - && !value - .bytes() - .any(|byte| byte.is_ascii_control() || byte.is_ascii_whitespace()) - }; - if !valid_token(&resource, 2048) - || scopes.is_empty() - || scopes.len() > 128 - || scopes.iter().any(|scope| !valid_token(scope, 512)) - || scopes - .iter() - .collect::>() - .len() - != scopes.len() - || !valid_token(&generation, 512) - { - return Err(ReviewContextError::Invalid); - } + config::check_exchange_values(&resource, &scopes, &generation) + .map_err(|_| ReviewContextError::Invalid)?; let actor = Arc::new( PrivateKeyJwt::new(Self::client_config( &token_endpoint, diff --git a/crates/registry-casework/src/config.rs b/crates/registry-casework/src/config.rs index bc38974eb7..24797a22ab 100644 --- a/crates/registry-casework/src/config.rs +++ b/crates/registry-casework/src/config.rs @@ -1423,6 +1423,23 @@ impl RuntimeConfig { } } + /// Refuses a context-provider binding that provider construction would + /// refuse, at the member that breaks the rule, without resolving a secret. + fn check_context_providers(&self, findings: &mut Vec) { + for (deployment, binding) in &self.context_providers { + let Err(refusal) = binding.check_offline() else { + continue; + }; + let mut segments = vec!["contextProviders", deployment.as_str()]; + segments.extend(refusal.member.split('/').skip(1)); + findings.push(RuntimeConfigError::InvalidContextProvider { + path: segments.join("."), + pointer: pointer_to(&segments), + reason: refusal.reason, + }); + } + } + fn check_principal_claims( &self, project: &CaseworkProject, @@ -1803,6 +1820,7 @@ const FILE_RULES: &[FileRule] = &[ RuntimeConfig::check_oidc, RuntimeConfig::check_database, RuntimeConfig::check_review_completion_destinations, + RuntimeConfig::check_context_providers, ]; /// The rules a runtime file satisfies against the project it binds. @@ -2698,6 +2716,78 @@ mod tests { .all(|diagnostic| !diagnostic.code.ends_with("-id-collision"))); } + #[test] + fn context_provider_bindings_are_refused_at_the_member_and_rule_provider_construction_refuses() + { + let directory = tempfile::tempdir().unwrap(); + let binding = json!({ + "baseUrl": "https://registry.example.test", + "statusProfile": "review-status", + "tokenEndpoint": "https://identity.example.test/token", + "clientIdRef": "secret:file/client-id", + "clientAssertionKeyRef": "secret:file/client-key", + "resource": "urn:example:registry", + "scopes": ["registry:proposal:review-status"] + }); + let mut value = operator_value(directory.path(), directory.path()); + value["contextProviders"] = json!({ "registry-main": binding.clone() }); + let path = write_operator(&value, directory.path()); + assert!(check_runtime(&path, None, false) + .diagnostics + .iter() + .all(|diagnostic| diagnostic.code != "casework.runtime.invalid-context-provider")); + + for (member, refused, pointer, rule) in [ + ( + "resource", + json!("registry"), + "/contextProviders/registry-main/resource", + "absolute URI", + ), + ( + "baseUrl", + json!("http://registry.example.test"), + "/contextProviders/registry-main/baseUrl", + "HTTPS", + ), + ( + "delegatedDisplayExchange", + json!({ + "tokenEndpoint": "https://identity.example.test/token", + "clientIdRef": "secret:file/source-id", + "clientAssertionKeyRef": "secret:file/source-key", + "resource": "urn:example:registry", + "scopes": ["registry:proposal:review"], + "generation": "" + }), + "/contextProviders/registry-main/delegatedDisplayExchange/generation", + "generation", + ), + ] { + let mut binding = binding.clone(); + binding[member] = refused; + value["contextProviders"] = json!({ "registry-main": binding }); + let config = read_operator(&value, directory.path()); + let error = first(config.findings(None)).expect_err("the binding is refused"); + assert_eq!(error.code(), "casework.runtime.invalid-context-provider"); + assert_eq!(error.pointer(), pointer); + assert!(error.in_file(), "{error}"); + let message = error.to_string(); + assert!(message.contains(rule), "{message}"); + assert!(!message.contains("response"), "{message}"); + let path = write_operator(&value, directory.path()); + let check = check_runtime(&path, None, false); + let diagnostic = check + .diagnostics + .iter() + .find(|diagnostic| diagnostic.code == "casework.runtime.invalid-context-provider") + .expect("the offline check reports the refusal"); + assert_eq!(diagnostic.path, pointer); + assert!(diagnostic.message.contains(rule), "{diagnostic:?}"); + assert!(diagnostic.source.is_some(), "{diagnostic:?}"); + } + } + #[test] fn environment_expressions_substitute_values_but_never_secret_references() { let temp = @@ -2938,6 +3028,12 @@ pub enum RuntimeConfigError { InvalidAuditDestination(#[source] AuditDestinationError), #[error("contextProviders contains a deploymentRef casework.yaml does not declare")] InvalidContextProviders, + #[error("{path}: {reason}")] + InvalidContextProvider { + path: String, + pointer: String, + reason: String, + }, #[error( "{path} must name exactly one completion secret, in a header the runtime does not reserve" )] @@ -2997,6 +3093,7 @@ impl RuntimeConfigError { | Self::BindingSecret { .. } | Self::InvalidReviewCompletionAuth { .. } | Self::InvalidReviewCompletionDestination { .. } + | Self::InvalidContextProvider { .. } | Self::InvalidTaskAuthority { .. } | Self::InvalidDelegatedStaffReader { .. } | Self::MissingIdentity @@ -3020,6 +3117,7 @@ impl RuntimeConfigError { Self::BindingSecret { pointer, .. } | Self::InvalidReviewCompletionAuth { pointer, .. } | Self::InvalidReviewCompletionDestination { pointer, .. } + | Self::InvalidContextProvider { pointer, .. } | Self::InvalidTaskAuthority { pointer, .. } | Self::InvalidDelegatedStaffReader { pointer, .. } => pointer.clone(), Self::MissingIdentity => "/identity".to_owned(), @@ -3066,6 +3164,7 @@ impl RuntimeConfigError { Self::InvalidDatabaseReference => "casework.runtime.invalid-database-reference", Self::InvalidAuditDestination(_) => "casework.runtime.invalid-audit", Self::InvalidContextProviders => "casework.runtime.context-providers-mismatch", + Self::InvalidContextProvider { .. } => "casework.runtime.invalid-context-provider", Self::InvalidReviewCompletionAuth { .. } => { "casework.runtime.invalid-review-completion-auth" } @@ -3141,6 +3240,9 @@ impl RuntimeConfigError { Self::InvalidContextProviders => { "Remove the unrecognized contextProviders key, or declare its deploymentRef in casework.yaml. Omitted providers remain unavailable to their current-source reviews." } + Self::InvalidContextProvider { .. } => { + "Correct the context provider member so it meets the rule the message names." + } Self::InvalidReviewCompletionAuth { .. } => { "Configure exactly one of bearerTokenRef and auth.secretRef, and name in auth.header only a header the runtime does not reserve." } diff --git a/crates/registry-cli-docs/catalog.json b/crates/registry-cli-docs/catalog.json index 8002f85a12..80e5db9aa3 100644 --- a/crates/registry-cli-docs/catalog.json +++ b/crates/registry-cli-docs/catalog.json @@ -25865,7 +25865,7 @@ "name": "dev", "invocation": "rstack dev", "about": "Run and inspect a local Registry Stack development environment", - "long_about": null, + "long_about": "Run and inspect a local Registry Stack development environment. For a walk-through from a project checkout to a running registry and review service, see `https://docs.registrystack.org/configure/run-a-project-with-rstack-dev/`", "usage": "rstack dev [OPTIONS] ", "arguments": [], "options": [ @@ -25912,7 +25912,7 @@ }, { "display": "-h, --help", - "description": "Print help", + "description": "Print help (see a summary with '-h')", "always_required": false, "repeatable": false, "default_values": [], @@ -26300,6 +26300,24 @@ } ], "options": [ + { + "display": "--recover", + "description": "Release the processes, databases and roles of a run whose supervisor was lost", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--database-url ", + "description": "TLS PostgreSQL administrator URL for --recover. Defaults to RSTACK_DEV_DATABASE_URL", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, { "display": "--json", "description": "Emit the runner's JSON report", @@ -26360,7 +26378,15 @@ "environment": null } ], - "constraints": [], + "constraints": [ + { + "kind": "requires-all", + "when": "--database-url ", + "arguments": [ + "--recover" + ] + } + ], "subcommands": [] } ] diff --git a/crates/registry-rstack/Cargo.toml b/crates/registry-rstack/Cargo.toml index 6ee9fdebfa..af0ca1ec4d 100644 --- a/crates/registry-rstack/Cargo.toml +++ b/crates/registry-rstack/Cargo.toml @@ -32,3 +32,4 @@ tempfile.workspace = true ureq.workspace = true [dev-dependencies] +jsonschema.workspace = true diff --git a/crates/registry-rstack/src/cli.rs b/crates/registry-rstack/src/cli.rs index cda7a57406..fa57a6304b 100644 --- a/crates/registry-rstack/src/cli.rs +++ b/crates/registry-rstack/src/cli.rs @@ -155,6 +155,9 @@ enum Command { command: IdentityCommand, }, /// Run and inspect a local Registry Stack development environment. + #[command( + long_about = "Run and inspect a local Registry Stack development environment.\n\nFor a walk-through from a project checkout to a running registry and review service, see `https://docs.registrystack.org/configure/run-a-project-with-rstack-dev/`" + )] Dev { #[command(subcommand)] command: DevCommand, @@ -218,6 +221,14 @@ enum DevCommand { /// Run ID returned by rstack dev start. #[arg(value_name = "RUN_ID")] run_id: String, + /// Release the processes, databases and roles of a run whose supervisor + /// was lost. + #[arg(long)] + recover: bool, + /// TLS PostgreSQL administrator URL for --recover. Defaults to + /// RSTACK_DEV_DATABASE_URL. + #[arg(long, value_name = "URL", requires = "recover")] + database_url: Option, /// Emit the runner's JSON report. #[arg(long)] json: bool, @@ -249,6 +260,8 @@ enum DevRequest<'a> { }, Stop { run_id: &'a str, + #[serde(skip_serializing_if = "std::ops::Not::not")] + recover: bool, json: bool, }, } @@ -1463,12 +1476,18 @@ fn dispatch_dev(cli: &Cli, action: &DevCommand, format: ReportFormat) -> Result< None, Needs::default(), ), - DevCommand::Stop { run_id, json } => ( + DevCommand::Stop { + run_id, + recover, + database_url, + json, + } => ( DevRequest::Stop { run_id, + recover: *recover, json: *json || launcher_json, }, - None, + database_url.as_deref(), Needs::default(), ), }; diff --git a/crates/registry-rstack/tests/dev.rs b/crates/registry-rstack/tests/dev.rs index 360a1b3a8f..70025c2300 100644 --- a/crates/registry-rstack/tests/dev.rs +++ b/crates/registry-rstack/tests/dev.rs @@ -427,6 +427,10 @@ fn dev_actions_encode_the_public_contract_and_launcher_json_format() { vec!["dev", "stop", "run-4", "--json"], serde_json::json!({"action":"stop","run_id":"run-4","json":true}), ), + ( + vec!["dev", "stop", "run-5", "--recover", "--json"], + serde_json::json!({"action":"stop","run_id":"run-5","recover":true,"json":true}), + ), ] { let output = tools .empty_command() @@ -439,6 +443,52 @@ fn dev_actions_encode_the_public_contract_and_launcher_json_format() { } } +#[test] +fn recover_passes_the_administrator_url_through_the_environment_only() { + let tools = SourceTools::new(); + let database_url = "postgresql://example.invalid/dev-secret"; + let output = tools + .empty_command() + .args([ + "dev", + "stop", + "run-6", + "--recover", + "--database-url", + database_url, + ]) + .env("RSTACK_BREGCTL_BIN", &tools.bregctl) + .env("EXPECTED_DATABASE_URL", database_url) + .output() + .unwrap(); + + assert!(output.status.success(), "{output:?}"); + let stdout = String::from_utf8(output.stdout.clone()).unwrap(); + assert!(stdout.contains("database:matched\n"), "{stdout}"); + assert!(!stdout.contains(database_url)); + assert!(!request(&output).to_string().contains(database_url)); +} + +#[test] +fn database_url_is_refused_for_stop_without_recover() { + let tools = SourceTools::new(); + let output = tools + .empty_command() + .args([ + "dev", + "stop", + "run-7", + "--database-url", + "postgresql://example.invalid/x", + "--json", + ]) + .env("RSTACK_BREGCTL_BIN", &tools.bregctl) + .output() + .unwrap(); + + assert_eq!(output.status.code(), Some(2), "{output:?}"); +} + #[test] fn start_requires_the_complete_source_binary_set_without_an_installation() { let tools = SourceTools::new(); @@ -474,6 +524,10 @@ fn root_and_dev_help_need_no_installation_or_binary_overrides() { &["dev", "start", "--help"][..], &["--database-url", "--ttl", "--json"][..], ), + ( + &["dev", "stop", "--help"][..], + &["--recover", "--database-url", "--json"][..], + ), ] { let output = tools.empty_command().args(arguments).output().unwrap(); assert!(output.status.success(), "{arguments:?}: {output:?}"); @@ -502,6 +556,22 @@ fn write_plan(project: &Path, products: &[&str]) { .unwrap(); } +/// Validates a launcher failure against the documented error schema. +fn assert_matches_error_schema(report: &Value) { + let schema: Value = serde_json::from_str(include_str!( + "../../../products/platform/schemas/rstack-dev-error.v1.schema.json" + )) + .unwrap(); + let schema = jsonschema::JSONSchema::options() + .with_draft(jsonschema::Draft::Draft202012) + .compile(&schema) + .unwrap(); + if let Err(errors) = schema.validate(report) { + let errors: Vec<_> = errors.map(|error| error.to_string()).collect(); + panic!("{report} violates the error schema: {errors:?}"); + }; +} + #[test] fn dev_subcommand_json_makes_launcher_failures_json() { let tools = SourceTools::new(); @@ -517,6 +587,7 @@ fn dev_subcommand_json_makes_launcher_failures_json() { assert_eq!(report["schema_version"], "registry.rstack.error.v1"); assert_eq!(report["status"], "refused"); assert_eq!(report["error"]["class"], "domain"); + assert_matches_error_schema(&report); let usage = tools .empty_command() @@ -527,6 +598,7 @@ fn dev_subcommand_json_makes_launcher_failures_json() { let report: Value = serde_json::from_slice(&usage.stdout).unwrap(); assert_eq!(report["schema_version"], "registry.rstack.error.v1"); assert_eq!(report["error"]["class"], "usage"); + assert_matches_error_schema(&report); let human = tools .empty_command() diff --git a/docs/site/astro.config.mjs b/docs/site/astro.config.mjs index 6237c9764f..4ddbefbbf3 100644 --- a/docs/site/astro.config.mjs +++ b/docs/site/astro.config.mjs @@ -910,6 +910,7 @@ export default defineConfig({ { label: 'Overview', slug: 'operate/advanced' }, { label: 'Operator handoff', slug: 'operate' }, { label: 'Call an API with rstack', slug: 'configure/api-connections' }, + { label: 'Run a project with rstack dev', slug: 'configure/run-a-project-with-rstack-dev' }, { label: 'Rotate credentials and trust', slug: 'operate/advanced/rotate-credentials-and-trust' }, { label: 'Connect an OIDC issuer', slug: 'operate/advanced/connect-an-oidc-issuer' }, { label: 'Generate ThunderID configuration', slug: 'operate/advanced/generate-identity-configuration' }, diff --git a/docs/site/src/content/docs/configure/run-a-project-with-rstack-dev.mdx b/docs/site/src/content/docs/configure/run-a-project-with-rstack-dev.mdx new file mode 100644 index 0000000000..e04a8c82bc --- /dev/null +++ b/docs/site/src/content/docs/configure/run-a-project-with-rstack-dev.mdx @@ -0,0 +1,286 @@ +--- +title: Run a project with rstack dev +description: Start a Base Registry Engine and Registry Casework pair from a project checkout on a local TLS PostgreSQL server, call both with development tokens, and recover a failed run. +status: current +owner: registry-docs +source_repos: + - registry-stack +last_reviewed: "2026-11-11" +doc_type: how-to +persona: + - operator +locale: en +standards_referenced: [] +--- + +If you are writing a registry project and want to see Base Registry Engine (BReg) and Registry +Casework serve it together, `rstack dev` starts both from your project directory, runs a short +journey against them, and removes everything when you stop. It does not start PostgreSQL and does +not use Docker: you supply a development PostgreSQL server over TLS. This page takes you from a +checkout to a running pair, using the licensing example that ships with Registry Stack. + +## Before you start + +You need: + +- PostgreSQL 17 or newer command-line tools (`initdb`, `pg_ctl`, `psql`) and `openssl` on your + `PATH`, to run the local server below. A server you already have works too if it serves TLS and + has an administrator role with `LOGIN CREATEROLE CREATEDB`. +- `curl` and `jq`. +- A checkout of Registry Stack at the release you installed, for the example project. + +## Install the components + +Install `rstack` from one verified release, then add the two components the example uses. The +steps are in [Decide your first Base Registry Engine registry](../../tutorials/first-breg/#install-registry-stack); +install `casework` the same way. + +```sh test-skip="uses a downloaded, independently verified release installer" +rstack install breg casework +``` + +Check the selected installation before you continue: + +```sh test-skip="needs the installed release" +rstack components list +``` + +Success is a `breg` line and a `casework` line that each say `installed`. `available` means that the +selected release provides the component but it is not installed, so run the `rstack install` +command again. `unavailable` means that the selected release does not provide the component for +your platform. If your shell cannot find `rstack`, or the command says `No installation selected`, +repeat the verified `rstack` installation steps linked above. + +## Start a local TLS PostgreSQL server + +`rstack dev` always verifies the server certificate and host name, so the server needs a +certificate your machine trusts. This recipe makes a private certificate authority, starts the +server on `127.0.0.1`, and creates a least-privilege administrator. Keep the working directory +outside any repository. If port 5432 is taken, change `PORT`. + +```sh test-skip="needs a PostgreSQL installation and starts a server" +export DEV="$HOME/rstack-dev-postgres" +PORT=5432 +mkdir -p -m 700 "$DEV" +umask 077 +openssl rand -hex 16 > "$DEV/postgres.pass" +openssl rand -hex 16 > "$DEV/devadmin.pass" +initdb -D "$DEV/data" -U postgres --auth=scram-sha-256 --pwfile="$DEV/postgres.pass" -E UTF8 + +openssl req -new -x509 -newkey rsa:2048 -nodes -days 30 -subj /CN=rstack-dev-ca \ + -addext basicConstraints=critical,CA:TRUE -keyout "$DEV/ca.key" -out "$DEV/ca.pem" +openssl req -new -newkey rsa:2048 -nodes -subj /CN=localhost \ + -keyout "$DEV/data/server.key" -out "$DEV/server.csr" +printf 'subjectAltName=DNS:localhost,IP:127.0.0.1\nextendedKeyUsage=serverAuth\n' \ + > "$DEV/server.ext" +openssl x509 -req -in "$DEV/server.csr" -CA "$DEV/ca.pem" -CAkey "$DEV/ca.key" \ + -CAcreateserial -days 30 -extfile "$DEV/server.ext" -out "$DEV/data/server.crt" + +printf "listen_addresses = '127.0.0.1'\nport = %s\nssl = on\n" "$PORT" \ + >> "$DEV/data/postgresql.conf" +pg_ctl -D "$DEV/data" -l "$DEV/postgres.log" -w start +``` + +The passwords are random and stay in owner-only files, so nothing secret reaches your shell +history or a process list. Create the administrator with the password from its file: + +```sh test-skip="needs the running development server" +CONNECTION="host=127.0.0.1 port=$PORT user=postgres dbname=postgres sslmode=verify-full sslrootcert=$DEV/ca.pem" +PGPASSWORD="$(cat "$DEV/postgres.pass")" psql "$CONNECTION" <<'SQL' +\set pw `cat "$DEV/devadmin.pass"` +CREATE ROLE devadmin LOGIN CREATEROLE CREATEDB PASSWORD :'pw'; +SQL +``` + +Success is `CREATE ROLE`. The server must be running before you start a project. Point `rstack dev` +at it: + +```sh test-skip="needs the running development server" +export RSTACK_DEV_DATABASE_CA="$DEV/ca.pem" +export RSTACK_DEV_DATABASE_URL="postgresql://devadmin:$(cat "$DEV/devadmin.pass")@127.0.0.1:$PORT/postgres?sslmode=verify-full" +``` + +Set the URL through an environment variable, not `--database-url`, because a flag is visible to +other local users in the process list. For a spatial project, install PostGIS for this server and +create the administrator as a superuser instead. + +{/* Evidence: products/platform/PROJECT-DEV.md; crates/registry-rstack/src/cli.rs, dispatch_dev. */} + +## Adapt the development plan + +Copy the licensing example from your checkout, then work in the copy: + +```sh test-skip="needs a Registry Stack checkout" +cp -R path/to/registry-stack/products/platform/examples/licensing ./licensing +``` + +The directory holds a BReg project (`registry/`), a Casework project (`casework/`) and the plan +`dev.yaml`. Open `dev.yaml`: `rstack dev` reads only this file to decide what to start. Product +files stay byte-for-byte unchanged, and generated settings, keys and packages live in the run's +owner-only scratch directory. + +To adapt the plan to your own project, change these parts and leave the structure alone: + +- `issuer` names the identity issuer the local test tokens claim. Use a name that is yours. +- `products` lists each product's project directory and token audience, in startup order. +- `people` declares synthetic test people, each with a subject and one grant per product. A grant + selects an access profile of that product's project, its scopes, and extra claims. A BReg grant + must name a profile the BReg project declares. +- `journey` is the sequence of requests the run makes before it reports ready. Each step names a + person, a product, a method, a path starting with `/v1/`, and what it expects back. + +The [`rstack dev` reference](https://github.com/registrystack/registry-stack/blob/main/products/platform/PROJECT-DEV.md) +describes every key, capture type and limit. + +## Check, then start + +`check` validates the plan and resolves its fixtures, grants and references with the installed +authoring tools. It works with the database offline. + +```sh test-skip="needs the installed release and a running development server" +rstack dev check ./licensing +``` + +Success is `Development plan is valid` and exit status 0. Add `--deny-warnings` to refuse any +warning, and `--json` for a machine-readable report. Fix every error before you start: the report +names the plan field that caused it. + +Start the run. The time to live (TTL) begins when the run is ready and defaults to 30 minutes: + +```sh test-skip="needs the installed release and a running development server" +rstack dev start ./licensing --ttl 30m +``` + +The command creates separate databases and logins for the run, starts both products on loopback +ports, runs the journey, and returns while the services keep running: + +```text test-excerpt +Run 2f17770cb47d4502ad0b809a529f4309 ready + breg http://127.0.0.1:64930 + casework http://127.0.0.1:64931 + Journey licensing-approval passed + Logs STATE_DIRECTORY/2f17770cb47d4502ad0b809a529f4309/logs + rstack dev token 2f17770cb47d4502ad0b809a529f4309 --as administrator + rstack dev token 2f17770cb47d4502ad0b809a529f4309 --as applicant + rstack dev token 2f17770cb47d4502ad0b809a529f4309 --as intake + rstack dev token 2f17770cb47d4502ad0b809a529f4309 --as reviewer +``` + +Your run ID and ports differ, and `STATE_DIRECTORY` stands for the state directory your run reports. Copy the run ID: every later command takes it. The +journey has already created an application in BReg and had a reviewer approve it in Casework. +`rstack dev status RUN_ID` repeats this report at any time. + +## Call the products with a token + +`token` writes a short-lived bearer header file for a test person and reports where it is. Tokens +expire five minutes after issue, or with the run if that is sooner, so ask again when one lapses. +Pass the header file to curl with `--header @FILE`: + +```sh test-skip="needs a running development run" +RUN=2f17770cb47d4502ad0b809a529f4309 +HEADERS="$(rstack dev token "$RUN" --as applicant --json | jq -r '.credentials[0].header_file')" +curl -sS --header @"$HEADERS" \ + "http://127.0.0.1:64930/v1/records/applications?accessProfile=applicant" +``` + +Use the BReg URL from your own report. BReg needs the selected profile as the `accessProfile` +query parameter, and the response lists the application the journey created: + +```json test-excerpt +{"items":[{"domainData":{"profession":"Surveyor","reference":"application-001"},"recordIdentifier":"23257c7f-1acd-4fbf-8a3c-4b253238043c","revisionIdentifier":"1"}],"meta":{"datasetIdentifier":"licensing","recordTypeIdentifier":"application","registryIdentifier":"licensing"},"pageInfo":{"nextCursor":null}} +``` + +Casework needs no query parameter, because its header file carries the profile. This asks for the +reviewer's open tasks; the queue is empty because the journey already decided the request: + +```sh test-skip="needs a running development run" +HEADERS="$(rstack dev token "$RUN" --as reviewer --json | jq -r '.credentials[0].header_file')" +curl -sS --header @"$HEADERS" "http://127.0.0.1:64931/v1/review-tasks?queue=applications" +``` + +```json test-excerpt +{"items":[],"status":"complete"} +``` + +A BReg project with encrypted fields runs without extra setup: each run gets its own development +data key, removed when the run ends, so encrypted data does not outlive its run. A step in the +project's `tests/journeys.yaml` that touches an encrypted Record type fails the pre-activation +test, because that test has no key. Exercise those Record types in the plan's `journey` instead. + +{/* Evidence: products/platform/PROJECT-DEV.md; crates/registry-rstack/src/cli.rs, dispatch_dev. */} + +## Stop the run + +```sh test-skip="needs a running development run" +rstack dev stop "$RUN" +``` + +Success is `Run RUN_ID stopped`. Stop, and TTL expiry, terminate both services and remove the run's +databases, roles, keys and files. Stop the PostgreSQL server itself when you finish: + +```sh test-skip="needs the running development server" +pg_ctl -D "$DEV/data" stop +``` + +## When a run fails + +A failed journey is different from a startup failure. `start` exits 1 but prints the normal run +report. With `--json`, its `schema_version` is `registry.rstack.dev.v1`, its `status` is `ready`, +and `journey.result` is `failed`; the journey includes the failed step and its reason. The services +keep running, and `rstack dev status RUN_ID` repeats the report, so use the run ID to inspect the +products and logs before you stop the run. + +Other refusals and failures produce an error report instead. With `--json`, the report is one +object on standard output with `schema_version` `registry.rstack.error.v1` and `status` `refused`. +Its process exit status and `error.class` identify the kind of failure: + +| Exit | `error.class` | Meaning | +| --- | --- | --- | +| 1 | `domain` | The request, the project, the plan or the run's state was refused. Fix the plan or the command. | +| 2 | `usage` | The command line is wrong, such as a malformed run ID, `--ttl` or test person name. | +| 3 | `operational` | The environment failed: PostgreSQL is unreachable or refuses the administrator, a product binary cannot start, no loopback port is free, a product or the run does not start in time, or a run needs its resources released. | + +For example, `rstack dev start ./licensing --json` with the PostgreSQL server stopped (the operating +system error number varies by platform): + +```text test-excerpt +{"error":{"class":"operational","message":"cannot establish the PostgreSQL administrator session over verified TLS: the PostgreSQL connection handshake failed: error connecting to server: Connection refused (os error 61)"},"schema_version":"registry.rstack.error.v1","status":"refused"} +``` + +Start the server and run the command again. Without `--json`, the same message goes to standard +error as `rstack dev: MESSAGE` with the same exit status. + +When a start fails before the run becomes available, the run directory is removed but its logs are +kept at `/failed//logs`, and the error names the path. A live run's logs are in +`/logs`: `supervisor.log` for the supervisor and one file per product stage, such as +`breg-check.log` or `casework-runtime.log`. `rstack dev status RUN_ID` names the logs directory. +Delete kept logs yourself. + +When a run still holds databases, roles or processes, the error carries its `run_id` and a +`recovery` object listing them. This happens if PostgreSQL is unavailable during cleanup for +longer than the supervisor retries (up to 30 minutes), or if the supervisor was lost. Release the +run with: + +```sh test-skip="needs a run that lost its supervisor" +rstack dev stop RUN_ID --recover +``` + +If the supervisor is still running, use plain `stop`. For a lost supervisor, its process guards +stop the products and any product tools still running. Use the original PostgreSQL cluster and +maintenance database for `--recover`. Recovery waits for the guards, drops the databases and +roles, and removes the run directory, including its development key file. If a check or cleanup +step fails, the report explains what to inspect before retrying. The +[reference](https://github.com/registrystack/registry-stack/blob/main/products/platform/PROJECT-DEV.md) +lists the manual steps and the recovery fields. + +{/* Evidence: products/platform/PROJECT-DEV.md; crates/registry-rstack/src/cli.rs, dispatch_dev; + crates/registry-bregctl/src/project_dev/mod.rs; + crates/registry-bregctl/src/project_dev/journey.rs. */} + +## Next + +- [`rstack dev` reference](https://github.com/registrystack/registry-stack/blob/main/products/platform/PROJECT-DEV.md) + for the full plan format, journey steps and recovery behaviour. +- [Call an API with rstack](../api-connections/) to make short requests against the running pair. +- [Test with journeys](../breg-journeys/) to write the BReg project's own journey tests. +- [Encrypt a field](../breg-field-encryption/) for the key custody a deployment uses. diff --git a/products/breg/CHANGELOG.md b/products/breg/CHANGELOG.md index e593ae3bb0..9dea79e345 100644 --- a/products/breg/CHANGELOG.md +++ b/products/breg/CHANGELOG.md @@ -21,6 +21,15 @@ on `geometry.type === "Point"` before reading `coordinates`. In Python the `.pyi` stubs describe the union, so a type checker flags code that assumes a Point. +- Fix reading back an encrypted field on PostgreSQL. A registry that uses + field encryption failed a record, list, snapshot or revision read with + `503 source.unavailable` once a stored value was long enough for its + encrypted envelope to exceed 57 bytes, and the value could not be read + until this fix. A reviewed migration whose history journal capture read + such a value, such as a field-encryption flip that sealed one, failed its + apply, because the capture uses the same decoder. Writes and shorter values + were unaffected. The fault was in the read path since v0.33.0, through + v0.46.0; no stored data changes and nothing needs migrating. ## 0.46.0 - 2026-11-08 diff --git a/products/casework/CHANGELOG.md b/products/casework/CHANGELOG.md index 925bbb49a4..46216c9b87 100644 --- a/products/casework/CHANGELOG.md +++ b/products/casework/CHANGELOG.md @@ -11,6 +11,13 @@ with each access profile's `id` and `principalClaim`, so a caller no longer reads them from a review policy's `dependencies`. The member is additive. +- `caseworkctl check --runtime-config`, and every other command that loads the + runtime configuration, refuses a `contextProviders` binding that startup + would refuse, such as a `resource` that is not an absolute URI + or a `baseUrl` that is not HTTPS, without resolving a secret. The diagnostic + `casework.runtime.invalid-context-provider` sits at the member and names the + rule it breaks. + ## 0.46.0 - 2026-11-08 - Each access profile may carry an optional `label` and `description`. Neither is matched against a caller, a profile that writes neither reads as before, and rewording them leaves a policy's behavioral binding unchanged. Casework keeps both in the review policy description, which any caller who may read that description, requesters included, can see. diff --git a/products/platform/CHANGELOG.md b/products/platform/CHANGELOG.md index ac4a9e0fc6..9c943d5bcc 100644 --- a/products/platform/CHANGELOG.md +++ b/products/platform/CHANGELOG.md @@ -6,6 +6,20 @@ projects together against a PostgreSQL server you supply. See [PROJECT-DEV.md](PROJECT-DEV.md). It is an alpha developer tool and may change. +- `rstack dev stop RUN_ID --recover` releases a run whose supervisor is gone. + It waits for the process guards to finish, drops the run's databases and roles + on the original PostgreSQL cluster, and removes the run directory. You can + retry after a partial failure. +- With `--json`, refusals and startup failures from `rstack dev` print a + `registry.rstack.error.v1` object, published as + `schemas/rstack-dev-error.v1.schema.json`. Its `error.class` is `domain` + (exit 1), `usage` (exit 2) or `operational` (exit 3). If the run still holds + resources, the report includes its `run_id` and any known `recovery` details. + A failed journey exits 1 with the regular run report and leaves the run + available for inspection. +- A BReg project with encrypted fields runs under `rstack dev` without extra + setup. Each run draws its own development data key into its run directory, + readable only by your user, and the key is removed with the run. ## 0.46.0 - 2026-11-08 diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index 8859aed6ab..728752c7cd 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -6,6 +6,10 @@ PostgreSQL server over TLS. `rstack dev` does not start PostgreSQL and does not use Docker: you supply the server. The products' own development modes are unchanged by `rstack dev`. +For a walk-through from a checkout to a running pair, see +[Run a project with rstack dev](https://docs.registrystack.org/configure/run-a-project-with-rstack-dev/). +This file is the reference. + ```sh rstack install breg casework # Set RSTACK_DEV_DATABASE_URL through your shell's secret provider. @@ -23,9 +27,14 @@ product authoring tools. The check works with the database offline. With Use `--deny-warnings` to refuse any unaccepted warning. Start reports the product URLs and the journey result. -A failed journey exits 1 and includes the live run ID so you can inspect the -result and stop the run. Startup failures clean up or report the run ID if -cleanup is pending. The TTL starts at readiness and defaults to 30 minutes. +A failed journey still produces the normal `registry.rstack.dev.v1` run report: +the report has `status` `ready`, `journey.result` `failed`, the failed step and +its reason, and the live run ID. `start` exits 1, but the supervisor and +products keep running. Use `status` to repeat the report, inspect the products +and logs, and then use `stop` to release the run. This differs from a startup +failure, which produces a `registry.rstack.error.v1` error report and cleans up, +or reports the run ID if cleanup is pending. The TTL starts at readiness and +defaults to 30 minutes. Supply one whole-number value from `1s` to `24h` with an `s`, `m`, or `h` suffix. The [licensing example](examples/licensing/dev.yaml) creates an application in @@ -45,6 +54,8 @@ Superuser is not needed for a non-spatial run. The runner sets `createrole_self_grant` on its own sessions, so you do not configure it. The administrator also installs `btree_gist`, which PostgreSQL treats as a trusted extension. +The runner reads `pg_catalog.pg_control_system()` to identify the cluster for +recovery. PostgreSQL allows this function by default. The administrator URL always carries a password. Set it through `RSTACK_DEV_DATABASE_URL`, because `--database-url` is visible to other local @@ -131,12 +142,18 @@ Casework producer, and `bindings.caseworkContextProvider` identifies the BReg context provider and action. `rstack dev check` refuses a binding that names a review authority or action the BReg project does not declare, or a producer or context provider the Casework project does not declare; `start` refuses it -before it provisions anything. A binding requests tokens for the other +before it provisions anything. `check` also writes each binding's product +configuration exactly as `start` does. BReg's review-authority reader and +`caseworkctl check --runtime-config` on the Casework configuration refuse what +each product would refuse at startup, and `check` reports the refusal, with the +rule it broke, at the plan field that produced the value. It refuses a +BReg grant profile that is not the `service-status` profile the Casework context +provider's policy names. A binding requests tokens for the other product's audience as an OAuth resource, so that audience must be an absolute URI, such as `urn:example:review`. It requests the scopes of its identity's grant for that product, so those scopes must be OAuth scope-tokens (no quotes or backslashes) that fit in 4096 bytes when joined by spaces; `check` reports a -violation at the grant's `scopes`. The run pairs the provider on a private copy +violation at the audience or the grant's `scopes`. The run pairs the provider on a private copy before packaging. A reviewer whose Casework context call forwards the bearer to BReg declares both grants, one shared client, and `caseworkContext: true`. Both grants declare the same scopes and additional claims. Casework reads the @@ -204,11 +221,34 @@ Stop and TTL expiry terminate the services and remove the databases, roles, keys, and run files. A spatial BReg run also creates a `NOLOGIN` role whose name starts with `breg_bbox_`; cleanup removes it with the others. +### Encrypted fields + +A BReg project with encrypted fields runs without extra setup. `start` draws a +development data key for the run from the operating system's random source and +writes it as `field-key` in the run directory, readable only by your user. BReg +reads it through its `local-file` key custody, which it accepts only for a +local development database. The key is never printed, never written to the +checkout, and is removed with the run directory on stop, TTL expiry and a +failed start. When a run needs recovery, the report names the key file under +`recovery.field_encryption_key` until `stop --recover` removes it. Every run +gets a new key, so encrypted data does not outlive its run. + +The schema test that `start` runs before activation has no field-encryption +key, so a step in `tests/journeys.yaml` that touches a Record type with an +encrypted field fails it, and `start` reports `breg test failed +(test.step.failed)`. Exercise those Record types in the plan's `journey` +instead. + +This key custody is for development only. A deployment supplies its key as +[BReg field encryption](../../docs/site/src/content/docs/configure/breg-field-encryption.mdx) +describes. + The state directory holds one directory per run. It defaults to `rstack-dev-UID` in the system temporary directory. Set `RSTACK_DEV_STATE_DIR` to use another location. A run directory is renamed to `..removing` before it is deleted, and `status` and `stop` ignore that name. If the deletion -is interrupted, delete that directory yourself. +is interrupted, delete that directory yourself; the run's field-encryption key +is deleted before the rename, so it holds no key material. ## When a run fails @@ -224,30 +264,72 @@ and the error names that path. Delete kept logs yourself. If PostgreSQL becomes unavailable during cleanup, the supervisor retries with backoff for up to 30 minutes. It then writes `cleanup-pending.json` in the run -directory, and `status` reports `cleanup-pending` with the remaining databases, -roles and processes. If the supervisor is lost, `stop` and `status` report the -run directory, the process groups, and the exact database and role names. +directory. If the supervisor is lost, the run's process guards stop the product +runtimes and any product tool that was still running. `stop` and `status` report +the run directory, the recorded process groups, and the database and role names. + +Release a run whose supervisor is gone with: -Release the listed resources by hand, in this order: +```sh +rstack dev stop RUN_ID --recover [--database-url URL] +``` -1. Stop each listed process group, for example `kill -TERM -PGID`. -2. As the administrator, run `DROP DATABASE IF EXISTS "NAME" WITH (FORCE)` for - each listed database. -3. As the administrator, run `DROP ROLE IF EXISTS "NAME"` for each listed role, - including a `breg_bbox_*` role from a spatial run. -4. Remove the run directory. +Use plain `stop` while the supervisor is still running. For recovery, use an +administrator URL for the same PostgreSQL cluster and maintenance database used +at startup, with the same administrator rights. `--database-url` defaults to +`RSTACK_DEV_DATABASE_URL`, as for `start`. Recovery checks the recorded cluster +identifier and maintenance database before dropping any resources. It waits for +the process guards to finish, then drops the databases and roles and removes +the run directory. If any step fails, the report gives the reason and the +resources you can check before retrying. +For runs from an older development build without recovery metadata, use the +manual cleanup steps below. + +A successful recovery reports `status` `stopped` and a `recovered` object with +`stopped_process_groups` and `databases_and_roles`. + +If you need to finish cleanup by hand, do it in this order: + +1. Verify which product processes still belong to the run and stop those + groups. Use the run's configuration paths and process details to establish + ownership before signaling a group. +2. Connect to the original PostgreSQL cluster as the administrator. Run + `DROP DATABASE IF EXISTS "NAME" WITH (FORCE)` for each listed database. +3. Run `DROP ROLE IF EXISTS "NAME"` for each listed role, including a + `breg_bbox_*` role from a spatial run. +4. Remove the run directory. It holds the run's development field-encryption + key when `field_encryption_key` is listed. ### Errors and limitations -With `--json`, a failed `rstack dev` command prints one JSON error object with -`schema_version` (`registry.rstack.dev.v1`), `status` set to `error`, and -`error`. It adds `run_id` when the run needs cleanup and `recovery` with the -resources listed above. Failures that occur before the helper starts, such as a -missing component, print a `registry.rstack.error.v1` object with `status` set -to `refused` and an `error` object holding `class` and `message`. The reports of -`rstack dev check` use `BRegCtlReport` instead. - -`rstack dev` does not configure field encryption yet. `rstack dev check` and -`start` refuse a BReg project with encrypted fields at the product's -`/products/N/project` pointer, and the error names the encrypted -`ENTITY.FIELD`; `start` refuses it before it provisions anything. +With `--json`, a refusal or startup failure from `rstack dev` start, status, +token or stop prints one JSON object on standard output, validated by +`schemas/rstack-dev-error.v1.schema.json`. The launcher prints the same object +for failures before the helper starts, such as a missing component. +When a journey assertion fails, `start` exits 1 and prints the regular +`registry.rstack.dev.v1` run report with `status` `ready` and +`journey.result` `failed`. The run remains available for inspection. + +- `schema_version` is `registry.rstack.error.v1` and `status` is `refused`. +- `error.class` is `domain` (exit 1: the request, the project, the plan or the + run's state was refused), `usage` (exit 2: the command line is wrong, such + as a malformed run ID, `--ttl` or test person name) or `operational` (exit 3: + the environment failed, a cleanup stalled, or a run needs its resources + released). Environment failures include a PostgreSQL server that is + unreachable, refuses the administrator session, or cannot provision a run's + databases and roles, a product binary that cannot be started, no free + loopback port, a product or run that does not start in time, and an internal + failure of the run's supervisor. An interrupted `start` is `domain`. + `error.message` is the human-readable reason. +- `run_id` is present when the failure concerns a run that still has + resources to release. Use it with `stop --recover`. +- `recovery` accompanies `run_id` when the resources are known. It has `kind` + (`lost-supervisor` or `cleanup-pending`), `run_directory`, `processes`, + `objects` or `databases_and_roles`, `field_encryption_key` when the run + directory still holds the run's key file (the path, never the key), and + `reason` when `--recover` could not finish. + +Without `--json`, the message goes to standard error as `rstack dev: MESSAGE` +with the same exit code. The reports of `rstack dev check` use `BRegCtlReport` +instead. A helper started without the launcher's request prints a plain +message and exits 2. diff --git a/products/platform/config-formats.yaml b/products/platform/config-formats.yaml index f64424772b..85daf0a045 100644 --- a/products/platform/config-formats.yaml +++ b/products/platform/config-formats.yaml @@ -4615,6 +4615,8 @@ outOfScope: reason: Coordinator operation input and output contracts, wire payloads governed by each product's API contract. - files: products/breg/contracts/event-record-view/EventRecordView.schema.json reason: Event record view, an inert client projection output governed by the client contract rather than a configuration format. + - files: products/platform/schemas/rstack-dev-error.v1.schema.json + reason: rstack error report printed on stdout with --json, a command output governed by PROJECT-DEV.md rather than a configuration format. - files: products/discovery/profile/schema/*.schema.json reason: Discovery provider description, a published wire resource. - files: products/identifiers/contracts/*.schema.json diff --git a/products/platform/schemas/rstack-dev-error.v1.schema.json b/products/platform/schemas/rstack-dev-error.v1.schema.json new file mode 100644 index 0000000000..2b0c799162 --- /dev/null +++ b/products/platform/schemas/rstack-dev-error.v1.schema.json @@ -0,0 +1,70 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "rstack error report", + "description": "The one JSON object that rstack and rstack dev print on stdout when --json is set and a command is refused or fails.", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "status", "error"], + "properties": { + "schema_version": {"const": "registry.rstack.error.v1"}, + "status": {"const": "refused"}, + "error": { + "type": "object", + "additionalProperties": false, + "required": ["class", "message"], + "properties": { + "class": { + "description": "domain (exit 1): the request or the run's state was refused. usage (exit 2): the command line is wrong. operational (exit 3): the environment failed, for example PostgreSQL was unreachable or refused the connection, a product process could not be started, or no loopback port was available, or resources need cleanup.", + "enum": ["domain", "usage", "operational"] + }, + "message": {"type": "string", "minLength": 1} + } + }, + "run_id": { + "description": "Present when the failure concerns a run that still has resources to release.", + "type": "string", + "pattern": "^[0-9a-f]{32}$" + }, + "recovery": { + "description": "Present with run_id when the resources to release are known.", + "type": "object", + "additionalProperties": false, + "required": ["kind", "run_directory"], + "properties": { + "kind": {"enum": ["lost-supervisor", "cleanup-pending"]}, + "run_directory": {"type": "string"}, + "processes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "product": {"type": "string"}, + "pid": {"type": "integer"}, + "pgid": {"type": "integer"}, + "started": {"type": ["string", "null"]} + } + } + }, + "objects": { + "type": "array", + "items": { + "type": "object", + "required": ["label", "database", "roles"], + "properties": { + "label": {"type": "string"}, + "database": {"type": "string"}, + "roles": {"type": "array", "items": {"type": "string"}} + } + } + }, + "databases_and_roles": {"type": "array", "items": {"type": "string"}}, + "field_encryption_key": { + "description": "Present when the run directory still holds the run's development field-encryption key file. The path only; the key itself is never reported.", + "type": "string" + }, + "unavailable": {"type": "string"}, + "reason": {"type": "string"} + } + } + } +} diff --git a/products/platform/scripts/test-rstack-dev.py b/products/platform/scripts/test-rstack-dev.py index 97dba72c73..8131cdfbf3 100644 --- a/products/platform/scripts/test-rstack-dev.py +++ b/products/platform/scripts/test-rstack-dev.py @@ -6,6 +6,7 @@ import argparse import base64 import concurrent.futures +import fcntl import hashlib import json import os @@ -25,6 +26,7 @@ STACK = Path(__file__).resolve().parents[3] NATIVE_REVIEW = STACK / "products/breg/acceptance/reviewed-evidence-journey" SPATIAL = STACK / "products/breg/acceptance/spatial-service-sites" +MINIMAL = STACK / "products/breg/examples/minimal" HTTP = build_opener(ProxyHandler({})) @@ -184,6 +186,11 @@ def files(root): def dev(bin_dir, env, cwd, *args, expected=0): + return dev_output(bin_dir, env, cwd, *args, expected=expected)[0] + + +def dev_output(bin_dir, env, cwd, *args, expected=0): + """Run `rstack dev` and return its report with everything it printed.""" install_root = Path(env["RSTACK_DEV_STATE_DIR"]).with_name("rstack-install") result = command( bin_dir / "rstack", @@ -206,9 +213,13 @@ def dev(bin_dir, env, cwd, *args, expected=0): raise RuntimeError(f"rstack dev {args[0]} exited {result.returncode}: {report}") if args[0] == "check": assert report["kind"] == "BRegCtlReport" and report["command"] == "dev-check" + elif report["schema_version"] == "registry.rstack.error.v1": + assert report["status"] == "refused", report + assert report["error"]["class"] in ("domain", "usage", "operational"), report + assert report["error"]["message"], report else: assert report["schema_version"] == "registry.rstack.dev.v1" - return report + return report, result.stdout + result.stderr def credential(bin_dir, env, cwd, run, alias): @@ -412,8 +423,10 @@ def exercise(root, bin_dir, pg, project, ca, url, pg_env, spatial): try: exercise_runs(root, bin_dir, pg, checkout, state, env, pg_env) verify_lost_supervisor(bin_dir, pg, checkout, state, env, pg_env, owner_env) + verify_startup_loss(bin_dir, pg, checkout, state, env, pg_env) verify_provisioning_deadline(bin_dir, pg, checkout, state, env, pg_env) verify_spatial_refusal(root, bin_dir, pg, state, env, pg_env) + verify_field_encryption(root, bin_dir, pg, state, env, pg_env) if spatial: verify_spatial( root, bin_dir, pg, {**env, "RSTACK_DEV_DATABASE_URL": url}, pg_env @@ -510,13 +523,14 @@ def exercise_runs(root, bin_dir, pg, checkout, state, env, pg_env): failing.chmod(0o700) failure_env = {**env, "RSTACK_CASEWORK_BIN": str(failing)} report = dev(bin_dir, failure_env, checkout, "start", ".", expected=1) - assert "casework exited before readiness" in report["error"], report - assert "boom" in report["error"], report - logs = kept_logs(state, report["error"]) + assert "casework exited before readiness" in report["error"]["message"], report + assert "boom" in report["error"]["message"], report + logs = kept_logs(state, report["error"]["message"]) assert any("boom" in log.read_text() for log in logs.glob("casework-*.log")), report assert inventory(pg, pg_env) == baseline, report assert live_runs(state) == [], report assert_nothing_listening(bin_dir) + verify_failed_journey(root, bin_dir, checkout, state, env) verify_native_handoff(root, bin_dir, env) assert inventory(pg, pg_env) == baseline assert administrator_locks(pg, pg_env) == baseline_locks @@ -526,6 +540,28 @@ def exercise_runs(root, bin_dir, pg, checkout, state, env, pg_env): ) +def verify_failed_journey(root, bin_dir, checkout, state, env): + project = root / "failed-journey" + shutil.copytree(checkout, project) + plan = project / "dev.yaml" + authored = plan.read_text() + expectation = " expect: {status: 201}" + assert expectation in authored + plan.write_text(authored.replace(expectation, " expect: {status: 202}", 1)) + report = dev(bin_dir, env, project, "start", ".", expected=1) + run = report["run_id"] + try: + assert report["schema_version"] == "registry.rstack.dev.v1", report + assert report["status"] == "ready" and report["journey"]["result"] == "failed" + status = dev(bin_dir, env, project, "status", run) + assert status["status"] == "ready" and status["journey"] == report["journey"] + assert all(accepts_connections(url) for url in report["products"].values()) + finally: + assert dev(bin_dir, env, project, "stop", run)["status"] == "stopped" + assert not (state / run).exists() + print("A failed journey exited 1 with an inspectable run report and live products") + + def verify_provisioning_deadline(bin_dir, pg, checkout, state, env, pg_env): baseline = inventory(pg, pg_env) baseline_locks = administrator_locks(pg, pg_env) @@ -561,8 +597,9 @@ def verify_provisioning_deadline(bin_dir, pg, checkout, state, env, pg_env): assert time.monotonic() < deadline, "the PostgreSQL lock was not acquired" time.sleep(0.1) - report = dev(bin_dir, env, checkout, "start", ".", expected=1) - error = report["error"] + report = dev(bin_dir, env, checkout, "start", ".", expected=3) + assert report["error"]["class"] == "operational", report + error = report["error"]["message"] assert "cannot create an isolated PostgreSQL database" in error, report assert "statement deadline" in error, report assert psql(pg, pg_env, lock_query).strip() == "t" @@ -590,28 +627,7 @@ def listener_pids(url): def recover_lost_supervisor(pg, owner_env, recovery): - """Release a lost run by hand, from the facts `stop` reports. - - Order: stop each process group (TERM, then KILL), drop every database - (WITH (FORCE) ends leftover sessions), drop every role, then remove the - run directory. - """ - for process in recovery["processes"]: - group = process["pgid"] - os.killpg(group, signal.SIGTERM) - - def gone(group=group): - try: - os.killpg(group, 0) - except ProcessLookupError: - return True - return False - - try: - eventually(gone, "a process group ignored SIGTERM", timeout=10) - except AssertionError: - os.killpg(group, signal.SIGKILL) - eventually(gone, "a process group survived SIGKILL", timeout=10) + """Release the PostgreSQL objects and directory from the recovery report.""" objects = recovery["objects"] statements = [ f'DROP DATABASE IF EXISTS "{item["database"]}" WITH (FORCE);' @@ -626,27 +642,29 @@ def verify_lost_supervisor(bin_dir, pg, checkout, state, env, pg_env, owner_env) baseline_locks = administrator_locks(pg, pg_env) report = dev(bin_dir, env, checkout, "start", ".") run = report["run_id"] + recorded = json.loads((state / run / "processes.json").read_text()) + for process in recorded: + assert os.getpgid(process["pid"]) == process["pgid"], process + assert str(process["pid"]) in listener_pids( + report["products"][process["product"]] + ) worker = command( "pgrep", "-f", f"project-dev --worker {re.escape(str(state / run))}" ) [supervisor] = worker.stdout.split() os.kill(int(supervisor), signal.SIGKILL) - status = dev(bin_dir, env, checkout, "status", run, expected=1) - stopped = dev(bin_dir, env, checkout, "stop", run, expected=1) + status = dev(bin_dir, env, checkout, "status", run, expected=3) + stopped = dev(bin_dir, env, checkout, "stop", run, expected=3) for lost in (status, stopped): + assert lost["error"]["class"] == "operational", lost assert lost["recovery"]["kind"] == "lost-supervisor", lost assert lost["run_id"] == run, lost recovery = stopped["recovery"] assert recovery["run_directory"] == str(state / run), recovery processes = recovery["processes"] + assert processes == recorded assert sorted(p["product"] for p in processes) == ["breg", "casework"], recovery - for process in processes: - os.kill(process["pid"], 0) - assert os.getpgid(process["pid"]) == process["pgid"], recovery - listeners = listener_pids(report["products"][process["product"]]) - assert str(process["pid"]) in listeners, recovery - # The pattern behind assert_nothing_listening sees the live products. - assert len(product_pids(bin_dir)) == 2, recovery + assert_nothing_listening(bin_dir, report["products"].values()) databases, roles = inventory(pg, pg_env) assert {item["database"] for item in recovery["objects"]} == ( databases - baseline[0] @@ -664,6 +682,248 @@ def verify_lost_supervisor(bin_dir, pg, checkout, state, env, pg_env, owner_env) assert live_runs(state) == [] assert dev(bin_dir, env, checkout, "status", run)["status"] == "stopped" print("A lost supervisor was recovered by hand from the facts stop reported") + verify_recover(bin_dir, pg, checkout, state, env, pg_env) + + +def lose_supervisor(bin_dir, checkout, state, env): + report = dev(bin_dir, env, checkout, "start", ".") + run = report["run_id"] + worker = command( + "pgrep", "-f", f"project-dev --worker {re.escape(str(state / run))}" + ) + [supervisor] = worker.stdout.split() + os.kill(int(supervisor), signal.SIGKILL) + return report, run + + +def verify_recover(bin_dir, pg, checkout, state, env, pg_env): + baseline = inventory(pg, pg_env) + baseline_locks = administrator_locks(pg, pg_env) + + report, run = lose_supervisor(bin_dir, checkout, state, env) + recovered = dev(bin_dir, env, checkout, "stop", run, "--recover") + assert recovered["status"] == "stopped", recovered + assert len(recovered["recovered"]["stopped_process_groups"]) == 2, recovered + assert_nothing_listening(bin_dir, report["products"].values()) + assert inventory(pg, pg_env) == baseline + eventually( + lambda: administrator_locks(pg, pg_env) == baseline_locks, + "the recovery's administrator lock was not released", + ) + assert live_runs(state) == [] + assert dev(bin_dir, env, checkout, "status", run)["status"] == "stopped" + assert dev(bin_dir, env, checkout, "stop", run, "--recover")["status"] == "stopped" + print("stop --recover released a lost supervisor's processes, databases and roles") + + verify_unrelated_groups(bin_dir, checkout, state, env) + verify_wrong_cluster(bin_dir, pg, checkout, state, env, pg_env) + + # Without an administrator URL nothing is signalled or dropped, and the + # error carries the facts for a hand release. + report, run = lose_supervisor(bin_dir, checkout, state, env) + no_url = {k: v for k, v in env.items() if k != "RSTACK_DEV_DATABASE_URL"} + refused = dev(bin_dir, no_url, checkout, "stop", run, "--recover", expected=3) + assert "no PostgreSQL administrator URL" in refused["error"]["message"], refused + assert refused["run_id"] == run, refused + assert len(refused["recovery"]["processes"]) == 2, refused + assert (state / run).is_dir() and inventory(pg, pg_env) != baseline + dev(bin_dir, env, checkout, "stop", run, "--recover") + assert_nothing_listening(bin_dir, report["products"].values()) + assert inventory(pg, pg_env) == baseline + assert live_runs(state) == [] + print("stop --recover preserves the run when no administrator URL is available") + + +def verify_unrelated_groups(bin_dir, checkout, state, env): + # Recovery must never signal based on a persisted numeric PID, even when + # the current process table appears to confirm it. The test owns both + # unrelated groups and leaves one unreaped until recovery has finished. + with ( + subprocess.Popen(["/bin/sleep", "90"], start_new_session=True) as live, + subprocess.Popen(["/usr/bin/true"], start_new_session=True) as zombie, + ): + try: + + def process_field(pid, field): + return ( + checked( + "ps", + "-p", + str(pid), + "-o", + f"{field}=", + env={**env, "LC_ALL": "C", "TZ": "UTC"}, + ) + .decode() + .strip() + ) + + eventually( + lambda: process_field(zombie.pid, "stat").startswith("Z"), + "the unreaped child did not become a zombie", + ) + report, run = lose_supervisor(bin_dir, checkout, state, env) + processes_file = state / run / "processes.json" + processes = json.loads(processes_file.read_text()) + for child in (live, zombie): + processes.append( + { + "product": "breg", + "pid": child.pid, + "pgid": child.pid, + "started": " ".join(process_field(child.pid, "lstart").split()), + } + ) + processes_file.write_text(json.dumps(processes)) + # Hold a guard long enough to observe what status reports while + # recovery holds worker.lock and the old ready report is stale. + guard_path = state / run / "guards" / "test.lock" + with concurrent.futures.ThreadPoolExecutor(1) as pool: + with guard_path.open("wb") as guard: + fcntl.flock(guard, fcntl.LOCK_EX) + recovering = pool.submit( + dev, bin_dir, env, checkout, "stop", run, "--recover" + ) + report_path = state / run / "report.json" + eventually( + lambda: json.loads(report_path.read_text())["status"] == "stopping", + "recovery did not mark the run as stopping", + ) + assert dev(bin_dir, env, checkout, "status", run)["status"] == "stopping" + recovered = recovering.result(timeout=10) + assert recovered["status"] == "stopped", recovered + assert live.poll() is None, "recovery signaled an unrelated process group" + assert process_field(zombie.pid, "stat").startswith("Z") + assert_nothing_listening(bin_dir, report["products"].values()) + finally: + if live.poll() is None: + live.terminate() + print("Recovery completed with unrelated live and unreaped process groups present") + + +def verify_wrong_cluster(bin_dir, pg, checkout, state, env, pg_env): + other = state.parent / "other-postgres" + other.mkdir() + cluster, ca, url, other_env = prepare_postgres(other, pg) + checked(pg / "pg_ctl", "-D", cluster, "-l", other / "postgres.log", "-w", "start") + try: + wrong_url, _ = create_owner(pg, other_env, url) + other_baseline = inventory(pg, other_env) + baseline = inventory(pg, pg_env) + report, run = lose_supervisor(bin_dir, checkout, state, env) + original_objects = inventory(pg, pg_env) + assert original_objects != baseline + try: + refused = dev( + bin_dir, + { + **env, + "RSTACK_DEV_DATABASE_URL": wrong_url, + "RSTACK_DEV_DATABASE_CA": str(ca), + }, + checkout, + "stop", + run, + "--recover", + expected=3, + ) + assert ( + refused["run_id"] == run and refused["error"]["class"] == "operational" + ) + assert "cluster" in refused["error"]["message"], refused + assert (state / run).is_dir() + assert inventory(pg, pg_env) == original_objects + assert inventory(pg, other_env) == other_baseline + + # PostgreSQL advisory locks are scoped to the maintenance database, + # so even the same cluster must use the database held by startup. + alternate = "devowner_alternate_admin" + psql(pg, pg_env, f"CREATE DATABASE {alternate};") + try: + owner_url = env["RSTACK_DEV_DATABASE_URL"] + wrong_database_url = f"{owner_url.rsplit('/', 1)[0]}/{alternate}" + refused = dev( + bin_dir, + {**env, "RSTACK_DEV_DATABASE_URL": wrong_database_url}, + checkout, + "stop", + run, + "--recover", + expected=3, + ) + assert "maintenance database" in refused["error"]["message"], refused + assert (state / run).is_dir() and inventory( + pg, pg_env + ) == original_objects + finally: + psql(pg, pg_env, f"DROP DATABASE {alternate};") + finally: + if (state / run).is_dir(): + assert ( + dev(bin_dir, env, checkout, "stop", run, "--recover")["status"] + == "stopped" + ) + assert inventory(pg, pg_env) == baseline + assert_nothing_listening(bin_dir, report["products"].values()) + finally: + checked(pg / "pg_ctl", "-D", cluster, "-m", "fast", "-w", "stop") + print( + "Recovery refused a different TLS PostgreSQL cluster and retried on the original" + ) + + +def verify_startup_loss(bin_dir, pg, checkout, state, env, pg_env): + baseline = inventory(pg, pg_env) + shim = state.parent / "slow-bregctl" + ready = state.parent / "slow-bregctl-ready" + shim.write_text("""#!/usr/bin/env python3 +import os +import sys +import time +from pathlib import Path + +if sys.argv[1] == "test": + Path(os.environ["RSTACK_TEST_MARKER"]).touch() + time.sleep(90) + raise SystemExit(1) +os.execv(os.environ["RSTACK_TEST_BREGCTL"], sys.argv) +""") + shim.chmod(0o700) + slow_env = { + **env, + "RSTACK_BREGCTL_BIN": str(shim), + "RSTACK_TEST_MARKER": str(ready), + "RSTACK_TEST_BREGCTL": str(bin_dir / "bregctl"), + } + with concurrent.futures.ThreadPoolExecutor(1) as pool: + starting = pool.submit( + dev, bin_dir, slow_env, checkout, "start", ".", expected=3 + ) + eventually(ready.is_file, "the provisioning tool did not start", timeout=30) + [run_root] = live_runs(state) + assert not (run_root / "processes.json").exists() + assert inventory(pg, pg_env) != baseline + worker = command( + "pgrep", "-f", f"project-dev --worker {re.escape(str(run_root))}" + ) + [supervisor] = worker.stdout.split() + os.kill(int(supervisor), signal.SIGKILL) + assert starting.result(timeout=10)["run_id"] == run_root.name + assert ( + dev(bin_dir, env, checkout, "stop", run_root.name, "--recover")["status"] + == "stopped" + ) + eventually( + lambda: ( + command("pgrep", "-f", f"{re.escape(str(shim))} test").returncode == 1 + ), + "the worker left an unrecorded provisioning tool running", + ) + assert live_runs(state) == [] + assert inventory(pg, pg_env) == baseline + print( + "Recovery drained a guarded provisioning tool before any runtime PID was recorded" + ) def verify_native_handoff(root, bin_dir, env): @@ -765,16 +1025,264 @@ def verify_spatial_refusal(root, bin_dir, pg, state, env, pg_env): "SELECT default_version FROM pg_available_extensions WHERE name = 'postgis';", ).strip() supported = bool(offered) and tuple(map(int, offered.split(".")[:2])) >= (3, 5) - report = dev(bin_dir, env, checkout, "start", ".", expected=1) + report = dev(bin_dir, env, checkout, "start", ".", expected=3) + assert report["error"]["class"] == "operational", report if supported: - assert "must be allowed to CREATE EXTENSION postgis" in report["error"], report + assert "must be allowed to CREATE EXTENSION postgis" in report["error"]["message"], report else: - assert "PostGIS 3.5 or later installed" in report["error"], report + assert "PostGIS 3.5 or later installed" in report["error"]["message"], report assert inventory(pg, pg_env) == baseline, report assert live_runs(state) == [], report print("A spatial start without PostGIS rights was refused and left nothing behind") +def encrypted_project(root, name): + """Copy the minimal registry with a note Record type whose body is + encrypted, and write a plan that creates a note and reads it back. + + The fixture journeys leave notes alone: `bregctl test` runs them without + field-encryption key state, so a fixture step that wrote an encrypted + field would be refused there before the run reached its own key.""" + checkout = root / name + registry = checkout / "registry" + shutil.copytree(MINIMAL, registry) + source = (registry / "registry.yaml").read_text() + for anchor, addition in [ + ( + "\nactions:\n", + " - id: note\n" + " route: notes\n" + " primaryDataset: directory\n" + " mutationMode: mutable\n" + " classification: restricted\n" + " display: {label: Note, description: A note whose body is encrypted.}\n" + " fields:\n" + " - {id: title, type: string, required: true, minimumLength: 1," + " maximumLength: 160, classification: internal}\n" + " - {id: body, type: string, required: true, minimumLength: 1," + " maximumLength: 2000, classification: restricted, encrypted: true}\n" + " - {id: jurisdiction, type: string, required: true, minimumLength: 1," + " maximumLength: 40, classification: internal}\n", + ), + ( + " actions:\n - action: update-contact-email\n", + " - recordType: note\n" + " operations: [create, get]\n" + " readableFields: [title, body, jurisdiction]\n" + " writableFields: [title, body, jurisdiction]\n" + " rowBoundaries:\n" + " - {field: jurisdiction, claim: jurisdiction, operator: equals}\n", + ), + ]: + # Each addition goes in front of its anchor: the note after the + # contact Record type, and its grant after the contact grant. + assert source.count(anchor) == 1, anchor + source = source.replace(anchor, addition + anchor) + (registry / "registry.yaml").write_text(source) + # Both fixture principals hold the same grant, so each fixture step names + # the person whose jurisdiction it expects. + people = { + "synthetic-directory-editor": ("editor", "zone-a"), + "synthetic-foreign-editor": ("foreign-editor", "zone-b"), + } + schema_test_people = {} + steps = 0 + for line in (registry / "tests/journeys.yaml").read_text().splitlines(): + if match := re.fullmatch(r" - id: (\S+)", line): + journey = schema_test_people.setdefault(match[1], {}) + elif match := re.fullmatch(r" - id: (\S+)", line): + step = match[1] + steps += 1 + elif match := re.fullmatch(r" +principal: (\S+)", line): + journey[step] = people[match[1]][0] + assert steps and sum(map(len, schema_test_people.values())) == steps, ( + schema_test_people + ) + body = f"encrypted-note-{secrets.token_hex(8)}" + plan = { + "apiVersion": "id.registrystack.org/formats/breg/project-dev/v1alpha1", + "kind": "BRegProjectDev", + "issuer": "https://directory-smoke.example.invalid", + "products": [ + {"product": "breg", "project": "registry", "audience": "directory"} + ], + "people": { + alias: { + "subject": principal, + "client": "directory-smoke", + "grants": { + "breg": { + "profile": "directory-editor", + "scopes": ["registry:directory:write"], + "claims": { + "jurisdiction": jurisdiction, + "registry_actor_kind": "human", + "registry_purpose": "directory-administration", + }, + } + }, + } + for principal, (alias, jurisdiction) in people.items() + }, + "schemaTestPeople": schema_test_people, + "journey": { + "id": "encrypted-note", + "steps": [ + { + "id": "create-note", + "product": "breg", + "as": "editor", + "method": "POST", + "path": "/v1/records/notes", + "headers": {"Idempotency-Key": "encrypted-note"}, + "body": { + "data": { + "title": "Encrypted note", + "body": body, + "jurisdiction": "zone-a", + } + }, + "expect": {"status": 201}, + "capture": { + "note": { + "type": "pointer", + "pointer": "/data/recordIdentifier", + } + }, + }, + { + "id": "read-note", + "product": "breg", + "as": "editor", + "method": "GET", + "path": "/v1/records/notes/{{note}}", + "expect": { + "status": 200, + "json": {"/data/domainData/body": body}, + }, + }, + ], + }, + } + (checkout / "dev.yaml").write_text(json.dumps(plan)) + return checkout, body + + +def field_key(state, run): + """The run's key, after checking its custody: an owner-only file in the + owner-only run directory holding 32 random bytes in base64.""" + key = state / run / "field-key" + assert key.parent.stat().st_mode & 0o777 == 0o700, key + assert key.stat().st_mode & 0o777 == 0o600, key + encoded = key.read_bytes() + assert len(base64.b64decode(encoded, validate=True)) == 32, key + return encoded + + +def tree_bytes(root, skip=()): + return [ + path.read_bytes() + for path in root.rglob("*") + if path.is_file() and path not in skip + ] + + +def without_key(encoded, outputs): + """No output carries the key, as base64 or as raw bytes. The assertion + names no value so a failure cannot print the key.""" + raw = base64.b64decode(encoded) + for output in outputs: + assert encoded not in output and raw not in output, "the field key leaked" + + +def stored_values(pg, pg_env, run): + """Every row of the run's BReg database, as text with byte columns hex + encoded, so plaintext is found whether it sits in text or bytes.""" + dump = checked( + pg / "pg_dump", + "--data-only", + env={**pg_env, "PGDATABASE": f"rstack_{run}_breg"}, + ) + return lambda value: value.encode() in dump or value.encode().hex().encode() in dump + + +def verify_field_encryption(root, bin_dir, pg, state, env, pg_env): + """A project with encrypted fields runs on a per-run key that stays in the + run directory and is removed on every cleanup path.""" + checkout, body = encrypted_project(root, "encrypted") + before = files(checkout) + baseline = inventory(pg, pg_env) + + report, printed = dev_output(bin_dir, env, checkout, "start", ".") + run = report["run_id"] + try: + assert report["journey"]["result"] == "passed", report + key = field_key(state, run) + credential(bin_dir, env, checkout, run, "editor") + _, token = dev_output(bin_dir, env, checkout, "token", run, "--as", "editor") + _, status = dev_output(bin_dir, env, checkout, "status", run) + without_key( + key, + [printed, token, status] + + tree_bytes(state / run, skip={state / run / "field-key"}) + + tree_bytes(checkout), + ) + stored = stored_values(pg, pg_env, run) + assert stored("Encrypted note") and stored("__bregEncryptedV1") + assert not stored(body), "the encrypted body is stored in plaintext" + finally: + stopped, printed = dev_output(bin_dir, env, checkout, "stop", run) + assert stopped["status"] == "stopped", stopped + assert not (state / run).exists() + without_key(key, [printed]) + assert inventory(pg, pg_env) == baseline + print("An encrypted field round-tripped on an owner-only run key that stop removed") + + short = dev(bin_dir, env, checkout, "start", ".", "--ttl", "5s") + field_key(state, short["run_id"]) + wait_for_stop(bin_dir, env, checkout, short["run_id"]) + assert not (state / short["run_id"]).exists() + assert live_runs(state) == [] + + # The runtime fixture copies the key it was given so the test can look + # for it in everything the failed start left behind. + seen = root / "failed-run-key" + failing = root / "failing-breg" + failing.write_text( + "#!/bin/sh\numask 077\n" + f"cp \"$(dirname \"$2\")/field-key\" '{seen}'\n" + "printf 'boom\\n' >&2\nexit 1\n" + ) + failing.chmod(0o700) + failure_env = {**env, "RSTACK_BREG_BIN": str(failing)} + report, printed = dev_output( + bin_dir, failure_env, checkout, "start", ".", expected=1 + ) + assert "breg exited before readiness" in report["error"]["message"], report + logs = kept_logs(state, report["error"]["message"]) + key = seen.read_bytes() + assert len(base64.b64decode(key, validate=True)) == 32 + assert not list(state.rglob("field-key")), "a failed start kept its key" + without_key(key, [printed] + tree_bytes(logs.parent)) + assert live_runs(state) == [] + assert inventory(pg, pg_env) == baseline + + report, run = lose_supervisor(bin_dir, checkout, state, env) + lost, printed = dev_output(bin_dir, env, checkout, "stop", run, expected=3) + key = field_key(state, run) + assert lost["recovery"]["field_encryption_key"] == str(state / run / "field-key") + without_key(key, [printed]) + recovered, printed = dev_output(bin_dir, env, checkout, "stop", run, "--recover") + assert recovered["status"] == "stopped", recovered + assert not (state / run).exists() + without_key(key, [printed]) + assert_nothing_listening(bin_dir, report["products"].values()) + assert inventory(pg, pg_env) == baseline + assert live_runs(state) == [] + assert files(checkout) == before + print("TTL expiry, a failed start and stop --recover removed the run's key") + + def verify_spatial(root, bin_dir, pg, env, pg_env): checkout, registry, bbox = spatial_project(root, "spatial") before = files(registry) @@ -868,6 +1376,24 @@ def verify_offline_plan(root, bin_dir, project): assert diagnostic["suggestedAction"] assert unknown not in diagnostic["message"] + diagnostic["suggestedAction"] assert not state.exists() + + # Casework's own check refuses the context provider binding the run would + # write, and the refusal lands on the plan field that produced it. + binding = root / "offline-context-binding" + binding.mkdir() + for product in ("registry", "casework"): + shutil.copytree(NATIVE_REVIEW / product, binding / product) + authored = (NATIVE_REVIEW / "dev.yaml").read_text() + audience = 'audience: "urn:journey:breg"' + assert authored.count(audience) == 1 + (binding / "dev.yaml").write_text(authored.replace(audience, "audience: registry")) + refused = dev(bin_dir, check_env, binding, "check", ".", expected=1) + [diagnostic] = refused["diagnostics"] + assert refused["status"] == "domain-refusal" + assert diagnostic["path"] == "/products/0/audience" + assert diagnostic["message"].startswith("Casework refuses"), diagnostic + assert "contextProviders.registry.resource: " in diagnostic["message"], diagnostic + assert not state.exists() print("Offline checks resolve product profiles with only the authoring tools")