From 1744fd37cd0fb4e8bfbf0950383c8c434056b770 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 23:09:12 +0700 Subject: [PATCH 01/34] feat: run BReg and Casework together for development Signed-off-by: Jeremi Joslin --- .github/scripts/ci_changes.py | 5 +- .github/scripts/test_ci_changes.py | 2 + .github/workflows/ci.yml | 10 + Cargo.lock | 2 + Cargo.toml | 1 + crates/registry-bregctl/Cargo.toml | 8 +- crates/registry-bregctl/src/identity_facts.rs | 52 +- crates/registry-bregctl/src/lib.rs | 6 + .../src/project_dev/bindings.rs | 913 +++++++++++++++++ .../src/project_dev/database.rs | 608 +++++++++++ .../src/project_dev/issuer.rs | 125 +++ .../src/project_dev/journey.rs | 581 +++++++++++ .../registry-bregctl/src/project_dev/mod.rs | 733 +++++++++++++ .../registry-bregctl/src/project_dev/plan.rs | 962 ++++++++++++++++++ .../src/project_dev/products.rs | 883 ++++++++++++++++ crates/registry-bregctl/src/tool_schema.rs | 10 +- crates/registry-bregctl/tests/project_dev.rs | 53 + crates/registry-bregctl/tests/tool_schema.rs | 7 +- crates/registry-cli-docs/catalog.json | 414 ++++++++ crates/registry-rstack/README.md | 6 + crates/registry-rstack/src/cli.rs | 243 ++++- crates/registry-rstack/tests/dev.rs | 467 +++++++++ products/breg/SECURITY-MODEL.md | 6 + .../reviewed-evidence-journey/dev.yaml | 176 ++++ .../tools/project-dev.v1alpha1.schema.json | 464 +++++++++ products/casework/SECURITY-MODEL.md | 7 + .../identifiers/generated/catalog.v1.json | 32 +- products/platform/PROJECT-DEV.md | 80 ++ .../examples/licensing/casework/casework.yaml | 59 ++ products/platform/examples/licensing/dev.yaml | 120 +++ .../examples/licensing/registry/registry.yaml | 47 + .../licensing/registry/tests/journeys.yaml | 19 + products/platform/scripts/test-rstack-dev.py | 453 +++++++++ 33 files changed, 7516 insertions(+), 38 deletions(-) create mode 100644 crates/registry-bregctl/src/project_dev/bindings.rs create mode 100644 crates/registry-bregctl/src/project_dev/database.rs create mode 100644 crates/registry-bregctl/src/project_dev/issuer.rs create mode 100644 crates/registry-bregctl/src/project_dev/journey.rs create mode 100644 crates/registry-bregctl/src/project_dev/mod.rs create mode 100644 crates/registry-bregctl/src/project_dev/plan.rs create mode 100644 crates/registry-bregctl/src/project_dev/products.rs create mode 100644 crates/registry-bregctl/tests/project_dev.rs create mode 100644 crates/registry-rstack/tests/dev.rs create mode 100644 products/breg/acceptance/reviewed-evidence-journey/dev.yaml create mode 100644 products/breg/generated/tools/project-dev.v1alpha1.schema.json create mode 100644 products/platform/PROJECT-DEV.md create mode 100644 products/platform/examples/licensing/casework/casework.yaml create mode 100644 products/platform/examples/licensing/dev.yaml create mode 100644 products/platform/examples/licensing/registry/registry.yaml create mode 100644 products/platform/examples/licensing/registry/tests/journeys.yaml create mode 100644 products/platform/scripts/test-rstack-dev.py diff --git a/.github/scripts/ci_changes.py b/.github/scripts/ci_changes.py index 6f728ce6da..4a9aaaa0d8 100644 --- a/.github/scripts/ci_changes.py +++ b/.github/scripts/ci_changes.py @@ -382,6 +382,9 @@ "docs/site/src/content/docs/configure/casework.mdx", "docs/site/src/content/docs/operate/change-a-casework-package.mdx", "docs/site/src/content/docs/tutorials/first-casework.mdx", + "products/breg/acceptance/reviewed-evidence-journey/**", + "products/platform/examples/licensing/**", + "products/platform/scripts/test-rstack-dev.py", ) # Every input the Registry Messaging tutorial gate replays or is built from. @@ -573,7 +576,7 @@ # reader makes carries a token that session issued. The clients in the Casework # shard are not on the replayed path. CASEWORK_TUTORIAL_PACKAGES = frozenset( - {"registry-casework", "registry-caseworkctl", "registry-breg", "registry-bregctl", "registry-thunderid-tooling"} + {"registry-casework", "registry-caseworkctl", "registry-breg", "registry-bregctl", "registry-rstack", "registry-thunderid-tooling"} ) # The gate builds and runs exactly messagingctl, which links the Messaging diff --git a/.github/scripts/test_ci_changes.py b/.github/scripts/test_ci_changes.py index 21ce80ec5b..1b3bd9f751 100644 --- a/.github/scripts/test_ci_changes.py +++ b/.github/scripts/test_ci_changes.py @@ -1945,6 +1945,8 @@ def test_casework_tutorial_routing(self) -> None: "docs/site/scripts/tutorial-runner/toolsets.mjs", "docs/site/src/content/docs/tutorials/first-casework.mdx", "docs/site/package.json", + "products/platform/examples/licensing/dev.yaml", + "products/platform/scripts/test-rstack-dev.py", ) for path in infrastructure: with self.subTest(path=path): diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7de868a5ce..ea3fe80443 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2697,6 +2697,16 @@ jobs: -p registry-breg --features registry-breg/runtime \ -p registry-bregctl -p registry-rstack --bins + - name: Verify the combined licensing development run with native PostgreSQL + shell: bash + run: | + set -euo pipefail + cargo build --locked -p registry-rstack + python3 products/platform/scripts/test-rstack-dev.py \ + --bin-dir "${GITHUB_WORKSPACE}/target/debug" \ + --postgres-bin-dir "$(pg_config --bindir)" \ + --scratch-parent "${RUNNER_TEMP}" + - name: Execute the Registry Casework tutorials from a reader directory shell: bash env: diff --git a/Cargo.lock b/Cargo.lock index 0b2f2c4d79..750b810ea5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6327,9 +6327,11 @@ dependencies = [ "inquire", "jsonschema", "p256", + "postgres-protocol", "rcgen", "registry-breg", "registry-breg-client", + "registry-casework-core", "registry-cli-api", "registry-cli-reference", "registry-cli-report", diff --git a/Cargo.toml b/Cargo.toml index 4536e3a3bb..50d855842b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -331,6 +331,7 @@ tempfile = { version = "3" } thiserror = { version = "2" } time = { version = "0.3", features = ["serde", "formatting", "parsing", "macros"] } tokio = { version = "1", features = ["fs", "macros", "net", "rt", "rt-multi-thread", "signal", "sync", "time"] } +postgres-protocol = "=0.6.12" tokio-postgres = { version = "=0.7.18", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } tokio-postgres-rustls = { version = "=0.14.0", default-features = false, features = ["ring", "native-certs"] } tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } diff --git a/crates/registry-bregctl/Cargo.toml b/crates/registry-bregctl/Cargo.toml index e4190a4ebf..09609c6c4b 100644 --- a/crates/registry-bregctl/Cargo.toml +++ b/crates/registry-bregctl/Cargo.toml @@ -52,6 +52,7 @@ hmac.workspace = true inquire.workspace = true jsonschema.workspace = true p256.workspace = true +postgres-protocol.workspace = true rcgen.workspace = true registry-platform-canonical-json.workspace = true registry-platform-config.workspace = true @@ -64,11 +65,13 @@ registry-cli-api.workspace = true registry-cli-reference.workspace = true registry-breg = { workspace = true, features = ["runtime", "tooling"] } registry-breg-client.workspace = true +registry-casework-core.workspace = true registry-evidence-verifier.workspace = true registry-linkml.workspace = true registry-manifest-core.workspace = true reqwest.workspace = true rustix.workspace = true +rustls.workspace = true schemars = { workspace = true, optional = true } serde.workspace = true serde_json.workspace = true @@ -79,6 +82,8 @@ tempfile.workspace = true thiserror.workspace = true time.workspace = true tokio = { workspace = true, features = ["rt"] } +tokio-postgres.workspace = true +tokio-postgres-rustls.workspace = true uuid = { workspace = true, features = ["serde"] } zeroize.workspace = true registry-platform-crypto.workspace = true @@ -87,7 +92,4 @@ url.workspace = true [dev-dependencies] registry-breg.workspace = true registry-platform-testing = { workspace = true, features = ["test-utils"] } -rustls.workspace = true -tokio-postgres.workspace = true -tokio-postgres-rustls.workspace = true wat = "1.259.0" diff --git a/crates/registry-bregctl/src/identity_facts.rs b/crates/registry-bregctl/src/identity_facts.rs index 8e30fac601..64370df215 100644 --- a/crates/registry-bregctl/src/identity_facts.rs +++ b/crates/registry-bregctl/src/identity_facts.rs @@ -197,6 +197,37 @@ impl<'a> Grant<'a> { } } +fn compiled_grants(compiled: &CompiledRegistry) -> impl Iterator> { + let entity_grants = compiled + .entities() + .values() + .flat_map(|entity| entity.access_profiles.values().map(Grant::profile)); + let dataset_grants = compiled + .statistical_datasets() + .values() + .flat_map(|dataset| dataset.access_profiles.values().map(Grant::profile)); + let action_grants = compiled.actions().actions.iter().flat_map(|action| { + let proposal = match &action.execution { + CompiledActionExecution::Proposal { proposal } => proposal.permissions.as_slice(), + CompiledActionExecution::Immediate => &[], + }; + action + .permissions + .iter() + .map(Grant::action) + .chain(proposal.iter().map(Grant::proposal)) + }); + entity_grants.chain(dataset_grants).chain(action_grants) +} + +/// Clients the runtime must list even if no fixture mints tokens for them. +pub(crate) fn named_requester_clients(compiled: &CompiledRegistry) -> BTreeSet { + compiled_grants(compiled) + .flat_map(|grant| grant.requester_clients) + .chain(compiled.recipients().clients().map(str::to_owned)) + .collect() +} + /// The claims BReg refuses a token for a grant without, apart from the /// subject every token carries and the reserved consent claims the engine /// fills from the verified client. @@ -286,26 +317,7 @@ pub(crate) fn facts( let mut profiles: BTreeMap = BTreeMap::new(); let mut omitted_agent_profiles = BTreeSet::new(); - let entity_grants = compiled - .entities() - .values() - .flat_map(|entity| entity.access_profiles.values().map(Grant::profile)); - let dataset_grants = compiled - .statistical_datasets() - .values() - .flat_map(|dataset| dataset.access_profiles.values().map(Grant::profile)); - let action_grants = compiled.actions().actions.iter().flat_map(|action| { - let proposal = match &action.execution { - CompiledActionExecution::Proposal { proposal } => proposal.permissions.as_slice(), - CompiledActionExecution::Immediate => &[], - }; - action - .permissions - .iter() - .map(Grant::action) - .chain(proposal.iter().map(Grant::proposal)) - }); - for grant in entity_grants.chain(dataset_grants).chain(action_grants) { + for grant in compiled_grants(compiled) { if grant.actor_kind == Some(ActorKindSource::Agent) { omitted_agent_profiles.insert(grant.id.to_owned()); continue; diff --git a/crates/registry-bregctl/src/lib.rs b/crates/registry-bregctl/src/lib.rs index a3c0166809..36b4de7272 100644 --- a/crates/registry-bregctl/src/lib.rs +++ b/crates/registry-bregctl/src/lib.rs @@ -62,6 +62,7 @@ mod instance_claim_lifecycle; mod module_lock_patch; mod package_inspection; mod package_lifecycle; +mod project_dev; mod proposal_retention; mod reconcile_lifecycle; mod report; @@ -225,6 +226,8 @@ enum Command { Examples(dev::examples::ExamplesArgs), #[command(name = "__dev-supervisor", hide = true)] DevSupervisor(dev::SupervisorArgs), + #[command(name = "project-dev", hide = true)] + ProjectDev(project_dev::Args), /// Explain compiled model, access, route, or event inventories. Explain(ExplainArgs), /// Compare an authoring candidate with a rederived closed package. @@ -2902,6 +2905,9 @@ where } }; } + Command::ProjectDev(args) => { + return ExitCode::from(project_dev::run(args, stdout, stderr)); + } Command::Init(args) => match (args.from, args.template.as_deref()) { (None, None) => init(&args.destination), (None, Some(id)) => starters::run(&args.destination, id), diff --git a/crates/registry-bregctl/src/project_dev/bindings.rs b/crates/registry-bregctl/src/project_dev/bindings.rs new file mode 100644 index 0000000000..b89c7c978f --- /dev/null +++ b/crates/registry-bregctl/src/project_dev/bindings.rs @@ -0,0 +1,913 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Run-local bindings between BReg and Casework. +//! +//! The token endpoint exists only for one development run. It accepts the two +//! machine identities the plan explicitly binds, verifies their private-key +//! JWT assertions, and mints the same bounded product token a named test person +//! would receive. Keys, tokens, and assertion identifiers never enter reports +//! or logs. + +use anyhow::{ensure, Context, Result}; +use axum::{ + body::Bytes, + extract::{DefaultBodyLimit, State}, + http::{header, HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + routing::post, + Json, Router, +}; +use registry_platform_crypto::{ + generate_private_jwk, + jws::{parse_compact_jws, JwsLimits}, + parse_json_strict, GeneratedKeyAlgorithm, PrivateJwk, PublicJwk, SigningAlgorithm, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::{ + collections::{BTreeMap, BTreeSet}, + net::{Ipv4Addr, TcpListener}, + path::Path, + sync::{Arc, Mutex, Weak}, + thread, + time::Duration, +}; +use zeroize::Zeroizing; + +use super::{ + clock, + issuer::Issuer, + plan::{self, Person, Plan, Product}, + write_private, RunInput, +}; + +const ASSERTION_TYPE: &str = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer"; +const FORM_LIMIT: usize = 16 * 1024; +const ASSERTION_LIMIT: usize = 12 * 1024; +const ASSERTION_LIFETIME_SECONDS: u64 = 300; +const TOKEN_LIFETIME_SECONDS: u64 = 300; +const CLOCK_LEEWAY_SECONDS: u64 = 5; +const MAXIMUM_REPLAY_ENTRIES: usize = 256; + +#[derive(Default, Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct Bindings { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(super) breg_review_authority: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(super) casework_context_provider: Option, +} + +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct BregReviewAuthority { + pub(super) id: String, + pub(super) producer: String, + pub(super) person: String, +} + +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct CaseworkContextProvider { + pub(super) id: String, + pub(super) person: String, + pub(super) action: String, +} + +impl Bindings { + pub(super) fn validate(&self, plan: &Plan) -> Result<()> { + if self.breg_review_authority.is_none() && self.casework_context_provider.is_none() { + return Ok(()); + } + let breg = product_position(plan, Product::Breg); + let casework = product_position(plan, Product::Casework); + ensure!( + breg.is_some() && casework.is_some(), + "dev.yaml /bindings requires both BReg and Casework products" + ); + ensure!( + breg < casework, + "dev.yaml /products must place BReg before Casework when /bindings is configured" + ); + + if let Some(binding) = &self.breg_review_authority { + ensure!( + plan::identifier(&binding.id), + "dev.yaml /bindings/bregReviewAuthority/id must be a local identifier" + ); + ensure!( + plan::identifier(&binding.producer), + "dev.yaml /bindings/bregReviewAuthority/producer must be a local identifier" + ); + require_machine_person( + plan, + &binding.person, + Product::Casework, + "/bindings/bregReviewAuthority/person", + )?; + } + if let Some(binding) = &self.casework_context_provider { + ensure!( + plan::identifier(&binding.id), + "dev.yaml /bindings/caseworkContextProvider/id must be a local identifier" + ); + ensure!( + plan::identifier(&binding.action), + "dev.yaml /bindings/caseworkContextProvider/action must be a local identifier" + ); + require_machine_person( + plan, + &binding.person, + Product::Breg, + "/bindings/caseworkContextProvider/person", + )?; + } + Ok(()) + } +} + +fn product_position(plan: &Plan, product: Product) -> Option { + plan.products + .iter() + .position(|entry| entry.product == product) +} + +fn require_machine_person<'a>( + plan: &'a Plan, + alias: &str, + product: Product, + field: &str, +) -> Result<&'a Person> { + ensure!( + plan::identifier(alias), + "dev.yaml {field} must be a test person alias" + ); + let person = plan + .people + .get(alias) + .with_context(|| format!("dev.yaml {field} must name a test person"))?; + let grant = person.grants.get(&product).with_context(|| { + format!( + "dev.yaml {field} must name a person with a {} grant", + product.name() + ) + })?; + ensure!( + !grant.scopes.is_empty(), + "dev.yaml {field} must name a person whose {} grant has a scope", + product.name() + ); + Ok(person) +} + +struct PreparedBinding { + client_id_ref: String, + client_assertion_key_ref: String, + person: String, + product: Product, +} + +struct TokenServer { + endpoint: String, + shutdown: Option>, + thread: Option>, +} + +impl Drop for TokenServer { + fn drop(&mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +#[derive(Default)] +pub(super) struct RuntimeBindings { + server: Option, + breg_review: Option, + casework_context: Option, + input: Option>, + origins: BTreeMap, + // Kept outside the server state so Issuer's Tokio runtime is dropped on + // the supervisor thread, never from inside the token server's runtime. + _issuer: Option>, +} + +impl RuntimeBindings { + pub(super) fn prepare( + root: &Path, + input: Arc, + issuer: Arc, + origins: &BTreeMap, + ) -> Result { + let bindings = &input.plan.bindings; + if bindings.breg_review_authority.is_none() && bindings.casework_context_provider.is_none() + { + return Ok(Self::default()); + } + bindings.validate(&input.plan)?; + ensure!( + origins.contains_key(&Product::Breg) && origins.contains_key(&Product::Casework), + "run-local bindings require reserved BReg and Casework origins" + ); + + let mut clients = Vec::new(); + let breg_review = bindings + .breg_review_authority + .as_ref() + .map(|binding| { + prepare_binding( + root, + &input, + &binding.person, + Product::Casework, + "breg-review-client", + &mut clients, + ) + }) + .transpose()?; + let casework_context = bindings + .casework_context_provider + .as_ref() + .map(|binding| { + prepare_binding( + root, + &input, + &binding.person, + Product::Breg, + "casework-context-client", + &mut clients, + ) + }) + .transpose()?; + let server = start_token_server(input.clone(), issuer.clone(), clients)?; + Ok(Self { + server: Some(server), + breg_review, + casework_context, + input: Some(input), + origins: origins.clone(), + _issuer: Some(issuer), + }) + } + + pub(super) fn configure(&self, product: Product, config: &mut Value) { + let Some(input) = self.input.as_ref() else { + return; + }; + let endpoint = &self + .server + .as_ref() + .expect("prepared token server") + .endpoint; + match product { + Product::Breg => { + let (Some(binding), Some(prepared)) = ( + input.plan.bindings.breg_review_authority.as_ref(), + self.breg_review.as_ref(), + ) else { + return; + }; + let person = &input.plan.people[&prepared.person]; + let grant = &person.grants[&prepared.product]; + config["reviewAuthorities"][binding.id.as_str()] = json!({ + "endpoint": self.origins[&Product::Casework], + "profile": grant.profile, + "privateKeyJwt": { + "tokenEndpoint": endpoint, + "clientIdRef": prepared.client_id_ref, + "clientAssertionKeyRef": prepared.client_assertion_key_ref, + "assertionAudience": input.plan.issuer, + "resource": input.plan.product(Product::Casework).audience, + "scopes": grant.scopes, + }, + "producerId": binding.producer, + "recoveryDays": 7, + }); + } + Product::Casework => { + let (Some(binding), Some(prepared)) = ( + input.plan.bindings.casework_context_provider.as_ref(), + self.casework_context.as_ref(), + ) else { + return; + }; + let person = &input.plan.people[&prepared.person]; + let grant = &person.grants[&prepared.product]; + config["contextProviders"][binding.id.as_str()] = json!({ + "baseUrl": self.origins[&Product::Breg], + "statusProfile": grant.profile, + "tokenEndpoint": endpoint, + "clientAssertionAudience": input.plan.issuer, + "resource": input.plan.product(Product::Breg).audience, + "scopes": grant.scopes, + "clientIdRef": prepared.client_id_ref, + "clientAssertionKeyRef": prepared.client_assertion_key_ref, + }); + } + } + } +} + +fn prepare_binding( + root: &Path, + input: &RunInput, + person_alias: &str, + product: Product, + filename: &str, + clients: &mut Vec, +) -> Result { + let person = require_machine_person(&input.plan, person_alias, product, "/bindings")?; + let grant = &person.grants[&product]; + let key = generate_private_jwk(GeneratedKeyAlgorithm::Es256)?; + let public_key = key.public(); + let key_name = format!("{filename}-key"); + let id_name = format!("{filename}-id"); + let encoded = encode_private_jwk(&key)?; + write_private(&root.join(&key_name), &encoded)?; + write_private(&root.join(&id_name), person.client.as_bytes())?; + clients.push(MachineClient { + client_id: person.client.clone(), + person: person_alias.to_owned(), + product, + public_key, + resource: input.plan.product(product).audience.clone(), + scope: grant.scopes.join(" "), + }); + Ok(PreparedBinding { + client_id_ref: format!("secret:file/{id_name}"), + client_assertion_key_ref: format!("secret:file/{key_name}"), + person: person_alias.to_owned(), + product, + }) +} + +#[derive(Serialize)] +struct PrivateJwkDocument<'a> { + kty: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + kid: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + alg: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + crv: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + d: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + x: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + y: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + n: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + e: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + p: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + q: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + dp: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + dq: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + qi: Option<&'a str>, +} + +pub(super) fn encode_private_jwk(key: &PrivateJwk) -> Result>> { + Ok(Zeroizing::new(serde_json::to_vec(&PrivateJwkDocument { + kty: &key.kty, + kid: key.kid.as_deref(), + alg: key.alg.as_deref(), + crv: key.crv.as_deref(), + d: key.d.as_deref(), + x: key.x.as_deref(), + y: key.y.as_deref(), + n: key.n.as_deref(), + e: key.e.as_deref(), + p: key.p.as_deref(), + q: key.q.as_deref(), + dp: key.dp.as_deref(), + dq: key.dq.as_deref(), + qi: key.qi.as_deref(), + })?)) +} + +struct MachineClient { + client_id: String, + person: String, + product: Product, + public_key: PublicJwk, + resource: String, + scope: String, +} + +struct TokenState { + input: Arc, + issuer: Weak, + clients: Vec, + audience: String, + replay: Mutex>, +} + +fn start_token_server( + input: Arc, + issuer: Arc, + clients: Vec, +) -> Result { + ensure!(!clients.is_empty(), "a binding token client is required"); + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .context("cannot bind the run-local token endpoint")?; + listener.set_nonblocking(true)?; + let origin = format!("http://{}", listener.local_addr()?); + let endpoint = format!("{origin}/oauth2/token"); + let state = Arc::new(TokenState { + audience: input.plan.issuer.clone(), + input, + issuer: Arc::downgrade(&issuer), + clients, + replay: Mutex::new(BTreeMap::new()), + }); + let app = Router::new() + .route("/oauth2/token", post(token)) + .layer(DefaultBodyLimit::max(FORM_LIMIT)) + .with_state(state); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let server_thread = thread::Builder::new() + .name("rstack-dev-token".to_owned()) + .spawn(move || { + let Ok(runtime) = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + else { + return; + }; + runtime.block_on(async move { + let Ok(listener) = tokio::net::TcpListener::from_std(listener) else { + return; + }; + let _ = axum::serve(listener, app) + .with_graceful_shutdown(async move { + let _ = shutdown_rx.await; + }) + .await; + }); + })?; + Ok(TokenServer { + endpoint, + shutdown: Some(shutdown_tx), + thread: Some(server_thread), + }) +} + +async fn token(State(state): State>, headers: HeaderMap, body: Bytes) -> Response { + match tokio::time::timeout(Duration::from_secs(5), issue_token(state, headers, body)).await { + Ok(response) => response, + Err(_) => oauth_error(StatusCode::SERVICE_UNAVAILABLE, "temporarily_unavailable"), + } +} + +async fn issue_token(state: Arc, headers: HeaderMap, body: Bytes) -> Response { + if !headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value + .split(';') + .next() + .is_some_and(|media| media.trim() == "application/x-www-form-urlencoded") + }) + || body.len() > FORM_LIMIT + || !body.is_ascii() + { + return oauth_error(StatusCode::BAD_REQUEST, "invalid_request"); + } + let Some(form) = parse_form(&body) else { + return oauth_error(StatusCode::BAD_REQUEST, "invalid_request"); + }; + if form.get("grant_type").map(String::as_str) != Some("client_credentials") { + return oauth_error(StatusCode::BAD_REQUEST, "unsupported_grant_type"); + } + if form.get("client_assertion_type").map(String::as_str) != Some(ASSERTION_TYPE) { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + } + let (Some(client_id), Some(assertion), Some(resource), Some(scope)) = ( + form.get("client_id"), + form.get("client_assertion"), + form.get("resource"), + form.get("scope"), + ) else { + return oauth_error(StatusCode::BAD_REQUEST, "invalid_request"); + }; + if assertion.len() > ASSERTION_LIMIT { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + } + + let limits = JwsLimits { + max_compact_bytes: ASSERTION_LIMIT, + max_header_bytes: 1024, + max_payload_bytes: 2048, + }; + let Ok(parsed) = parse_compact_jws(assertion, &limits) else { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + }; + let Some(kid) = parsed.header().get("kid").and_then(Value::as_str) else { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + }; + let Some(client) = state.clients.iter().find(|candidate| { + candidate.client_id == client_id.as_str() + && candidate.public_key.kid.as_deref() == Some(kid) + && candidate.resource == resource.as_str() + && candidate.scope == scope.as_str() + }) else { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + }; + if parsed.header().len() != 3 + || parsed.header().get("typ").and_then(Value::as_str) != Some("JWT") + || parsed.header().get("alg").and_then(Value::as_str) != Some("ES256") + { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + } + let Ok(payload) = parsed.verify(&client.public_key, &[SigningAlgorithm::Es256]) else { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + }; + let Ok(payload) = parse_json_strict(payload) else { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + }; + let Ok(claims) = serde_json::from_value::(payload) else { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + }; + let Ok(now) = clock() else { + return oauth_error(StatusCode::SERVICE_UNAVAILABLE, "temporarily_unavailable"); + }; + if claims.iss != client.client_id + || claims.sub != client.client_id + || claims.aud != state.audience + || claims.jti.is_empty() + || claims.jti.len() > 128 + || claims + .jti + .chars() + .any(|value| value.is_control() || value.is_whitespace()) + || claims.iat > now.saturating_add(CLOCK_LEEWAY_SECONDS) + || claims.exp <= now + || !(1..=ASSERTION_LIFETIME_SECONDS).contains(&claims.exp.saturating_sub(claims.iat)) + { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + } + { + let Ok(mut replay) = state.replay.lock() else { + return oauth_error(StatusCode::SERVICE_UNAVAILABLE, "temporarily_unavailable"); + }; + replay.retain(|_, expiry| *expiry > now); + if replay.contains_key(&claims.jti) || replay.len() >= MAXIMUM_REPLAY_ENTRIES { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + } + replay.insert(claims.jti, claims.exp); + } + let Some(person) = state.input.plan.people.get(&client.person) else { + return oauth_error(StatusCode::SERVICE_UNAVAILABLE, "temporarily_unavailable"); + }; + let Some(issuer) = state.issuer.upgrade() else { + return oauth_error(StatusCode::SERVICE_UNAVAILABLE, "temporarily_unavailable"); + }; + let Ok(access_token) = issuer + .sign( + &state.input, + person, + client.product, + now.saturating_add(TOKEN_LIFETIME_SECONDS), + ) + .await + else { + return oauth_error(StatusCode::SERVICE_UNAVAILABLE, "temporarily_unavailable"); + }; + ( + StatusCode::OK, + [ + (header::CACHE_CONTROL, "no-store"), + (header::PRAGMA, "no-cache"), + ], + Json(json!({ + "access_token": &*access_token, + "token_type": "Bearer", + "expires_in": TOKEN_LIFETIME_SECONDS, + })), + ) + .into_response() +} + +fn parse_form(body: &[u8]) -> Option> { + let mut values = BTreeMap::new(); + let allowed = BTreeSet::from([ + "grant_type", + "client_id", + "client_assertion_type", + "client_assertion", + "resource", + "scope", + ]); + for (name, value) in url::form_urlencoded::parse(body) { + if !allowed.contains(name.as_ref()) + || values + .insert(name.into_owned(), value.into_owned()) + .is_some() + { + return None; + } + } + (values.len() == allowed.len()).then_some(values) +} + +fn oauth_error(status: StatusCode, error: &'static str) -> Response { + ( + status, + [(header::CACHE_CONTROL, "no-store")], + Json(json!({"error": error})), + ) + .into_response() +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct AssertionClaims { + iss: String, + sub: String, + aud: String, + iat: u64, + exp: u64, + jti: String, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::project_dev::{ + plan::{Grant, Journey, ProductPlan}, + BregAuthority, + }; + use registry_platform_crypto::{jws::sign_compact_jws, LocalJwkSigner, PrivateJwk}; + use registry_platform_httputil::{PrivateKeyJwt, PrivateKeyJwtConfig, TokenProvider}; + use serde_json::{Map, Value}; + use std::{collections::BTreeMap, path::PathBuf}; + + fn grant(profile: &str, scope: &str) -> Grant { + Grant { + profile: profile.to_owned(), + scopes: vec![scope.to_owned()], + claims: Map::new(), + } + } + + fn plan() -> Plan { + Plan { + issuer: "https://identity.example.test/realms/dev".to_owned(), + products: vec![ + ProductPlan { + product: Product::Breg, + project: PathBuf::from("registry"), + audience: "urn:dev:breg".to_owned(), + }, + ProductPlan { + product: Product::Casework, + project: PathBuf::from("casework"), + audience: "urn:dev:casework".to_owned(), + }, + ], + people: BTreeMap::from([ + ( + "producer".to_owned(), + Person { + subject: "review-producer".to_owned(), + client: "review-producer-client".to_owned(), + grants: BTreeMap::from([( + Product::Casework, + grant("producer", "casework:review:produce"), + )]), + casework_context: false, + }, + ), + ( + "context-reader".to_owned(), + Person { + subject: "context-reader".to_owned(), + client: "context-reader-client".to_owned(), + grants: BTreeMap::from([( + Product::Breg, + grant("casework-review-status", "registry:proposal:review-status"), + )]), + casework_context: false, + }, + ), + ]), + bindings: Bindings { + breg_review_authority: Some(BregReviewAuthority { + id: "casework".to_owned(), + producer: "registry".to_owned(), + person: "producer".to_owned(), + }), + casework_context_provider: Some(CaseworkContextProvider { + id: "registry".to_owned(), + person: "context-reader".to_owned(), + action: "reviewed-change".to_owned(), + }), + }, + journey: Journey { + id: "binding".to_owned(), + steps: Vec::new(), + }, + schema_test_people: BTreeMap::new(), + } + } + + fn input(plan: Plan) -> RunInput { + RunInput { + run_id: "0123456789abcdef0123456789abcdef".to_owned(), + project: PathBuf::from("/tmp/project"), + plan, + ttl_seconds: 300, + schema_people: Vec::new(), + breg_authority: Some(BregAuthority { + principal: "sub".to_owned(), + purpose: false, + named_clients: BTreeSet::new(), + }), + casework_principals: BTreeMap::from([("producer".to_owned(), "sub".to_owned())]), + binaries: BTreeMap::new(), + } + } + + #[test] + fn validation_refuses_missing_grants_and_wrong_product_order() { + let mut value = plan(); + value.products.reverse(); + let error = value.bindings.validate(&value).unwrap_err().to_string(); + assert!(error.contains("/products"), "{error}"); + + let mut value = plan(); + value.people.get_mut("producer").unwrap().grants.clear(); + let error = value.bindings.validate(&value).unwrap_err().to_string(); + assert!( + error.contains("/bindings/bregReviewAuthority/person"), + "{error}" + ); + } + + async fn sign_assertion( + key: &PrivateJwk, + client: &str, + audience: &str, + issued_at: u64, + jti: &str, + ) -> String { + let signer = LocalJwkSigner::new(key.clone()).unwrap(); + let claims = serde_json::to_vec(&json!({ + "iss": client, "sub": client, "aud": audience, + "iat": issued_at, "exp": issued_at + 60, "jti": jti, + })) + .unwrap(); + sign_compact_jws( + Map::from_iter([("typ".to_owned(), Value::String("JWT".to_owned()))]), + &claims, + &signer, + ) + .await + .unwrap() + } + + #[test] + fn endpoint_accepts_one_assertion_and_refuses_replay_tampering_and_wrong_resource() { + let root = tempfile::tempdir().unwrap(); + let input = Arc::new(input(plan())); + let issuer = Arc::new(Issuer::create(root.path()).unwrap()); + let origins = BTreeMap::from([ + (Product::Breg, "http://127.0.0.1:31001".to_owned()), + (Product::Casework, "http://127.0.0.1:31002".to_owned()), + ]); + let runtime = + RuntimeBindings::prepare(root.path(), input.clone(), issuer, &origins).unwrap(); + let endpoint = runtime.server.as_ref().unwrap().endpoint.clone(); + let encoded = + Zeroizing::new(std::fs::read(root.path().join("casework-context-client-key")).unwrap()); + let key = PrivateJwk::parse(std::str::from_utf8(&encoded).unwrap()).unwrap(); + let now = clock().unwrap(); + let request_runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let provider = PrivateKeyJwt::new( + PrivateKeyJwtConfig::new( + endpoint.parse().unwrap(), + "context-reader-client", + key.clone(), + ) + .with_audience(input.plan.issuer.clone()) + .with_resource("urn:dev:breg") + .with_scopes(vec!["registry:proposal:review-status".to_owned()]), + ) + .unwrap(); + request_runtime + .block_on(provider.bearer_token()) + .expect("the maintained private-key JWT client obtains a token"); + + let assertion = request_runtime.block_on(sign_assertion( + &key, + "context-reader-client", + &input.plan.issuer, + now, + "assertion-once", + )); + let client = reqwest::Client::builder().no_proxy().build().unwrap(); + let form = [ + ("grant_type", "client_credentials"), + ("client_id", "context-reader-client"), + ("client_assertion_type", ASSERTION_TYPE), + ("client_assertion", assertion.as_str()), + ("resource", "urn:dev:breg"), + ("scope", "registry:proposal:review-status"), + ]; + let response = request_runtime + .block_on(client.post(&endpoint).form(&form).send()) + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let replay = request_runtime + .block_on(client.post(&endpoint).form(&form).send()) + .unwrap(); + assert_eq!(replay.status(), StatusCode::UNAUTHORIZED); + + let mut tampered = assertion.clone().into_bytes(); + let last = tampered.last_mut().unwrap(); + *last = if *last == b'A' { b'B' } else { b'A' }; + let tampered = String::from_utf8(tampered).unwrap(); + let invalid = [ + ("grant_type", "client_credentials"), + ("client_id", "context-reader-client"), + ("client_assertion_type", ASSERTION_TYPE), + ("client_assertion", tampered.as_str()), + ("resource", "urn:dev:breg"), + ("scope", "registry:proposal:review-status"), + ]; + let response = request_runtime + .block_on(client.post(&endpoint).form(&invalid).send()) + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + + let second = request_runtime.block_on(sign_assertion( + &key, + "context-reader-client", + &input.plan.issuer, + now, + "wrong-resource", + )); + let wrong = [ + ("grant_type", "client_credentials"), + ("client_id", "context-reader-client"), + ("client_assertion_type", ASSERTION_TYPE), + ("client_assertion", second.as_str()), + ("resource", "urn:dev:casework"), + ("scope", "registry:proposal:review-status"), + ]; + let response = request_runtime + .block_on(client.post(&endpoint).form(&wrong).send()) + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + + #[test] + fn configure_writes_native_product_bindings() { + let root = tempfile::tempdir().unwrap(); + let input = Arc::new(input(plan())); + let issuer = Arc::new(Issuer::create(root.path()).unwrap()); + let origins = BTreeMap::from([ + (Product::Breg, "http://127.0.0.1:31001".to_owned()), + (Product::Casework, "http://127.0.0.1:31002".to_owned()), + ]); + let runtime = RuntimeBindings::prepare(root.path(), input, issuer, &origins).unwrap(); + let mut breg = json!({}); + runtime.configure(Product::Breg, &mut breg); + assert_eq!( + breg["reviewAuthorities"]["casework"]["endpoint"], + "http://127.0.0.1:31002" + ); + assert_eq!( + breg["reviewAuthorities"]["casework"]["privateKeyJwt"]["resource"], + "urn:dev:casework" + ); + assert!(breg["reviewAuthorities"]["casework"]["tokenRef"].is_null()); + + let mut casework = json!({}); + runtime.configure(Product::Casework, &mut casework); + assert_eq!( + casework["contextProviders"]["registry"]["baseUrl"], + "http://127.0.0.1:31001" + ); + assert_eq!( + casework["contextProviders"]["registry"]["statusProfile"], + "casework-review-status" + ); + } +} diff --git a/crates/registry-bregctl/src/project_dev/database.rs b/crates/registry-bregctl/src/project_dev/database.rs new file mode 100644 index 0000000000..c5c53059f2 --- /dev/null +++ b/crates/registry-bregctl/src/project_dev/database.rs @@ -0,0 +1,608 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::str::FromStr; + +use anyhow::{anyhow, bail, Context, Result}; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; +use rustls::pki_types::{pem::PemObject as _, CertificateDer}; +use tokio::runtime::{Builder, Runtime}; +use tokio_postgres::{ + config::{Host, SslMode}, + Client, Config, +}; +use tokio_postgres_rustls::MakeRustlsConnect; +use url::{Host as UrlHost, Url}; +use zeroize::Zeroizing; + +const BREG_SCHEMAS: [&str; 5] = [ + "registry_internal", + "registry_data", + "registry_source", + "registry_derived", + "registry_context", +]; + +pub(super) struct DatabaseUrls { + pub runtime: String, + pub migration: String, +} + +#[derive(Clone)] +enum Acquisition { + Database(String), + Role(String), +} + +pub(super) struct Database { + client: Client, + _connection_task: tokio::task::JoinHandle>, + tls: MakeRustlsConnect, + admin_config: Config, + endpoint: Endpoint, + run_id: String, + acquisitions: Vec, + runtime: Runtime, +} + +#[derive(Clone)] +struct Endpoint { + host: String, + port: Option, +} + +impl Database { + pub(super) fn connect(admin_url: &str, ca_pem: Option<&[u8]>, run_id: &str) -> Result { + validate_run_id(run_id)?; + let (mut admin_config, endpoint) = admin_config(admin_url)?; + admin_config.ssl_mode(SslMode::Require); + let tls = tls_connector(ca_pem)?; + let runtime = Builder::new_current_thread() + .enable_all() + .build() + .context("cannot initialize the PostgreSQL executor")?; + let (client, connection) = runtime + .block_on(admin_config.connect(tls.clone())) + .context("cannot connect to the PostgreSQL administrator database over verified TLS")?; + let connection_task = runtime.spawn(connection); + Ok(Self { + client, + _connection_task: connection_task, + tls, + admin_config, + endpoint, + run_id: run_id.to_owned(), + acquisitions: Vec::new(), + runtime, + }) + } + + pub(super) fn provision(&mut self, label: &str, breg: bool) -> Result { + validate_label(label)?; + let names = Names::new(&self.run_id, label); + if self.acquisitions.iter().any(|acquisition| { + matches!(acquisition, Acquisition::Database(database) if database == &names.database) + }) { + bail!("the {label} database was already provisioned for this run"); + } + + let migration_password = random_password()?; + let runtime_password = random_password()?; + self.create_role(&names.migration_role, &migration_password)?; + self.create_role(&names.runtime_role, &runtime_password)?; + self.create_database(&names)?; + self.configure_database(&names, breg)?; + + Ok(DatabaseUrls { + runtime: self.product_url(&names.database, &names.runtime_role, &runtime_password)?, + migration: self.product_url( + &names.database, + &names.migration_role, + &migration_password, + )?, + }) + } + + pub(super) fn cleanup(&mut self) -> Result<()> { + let mut failures = Vec::new(); + let mut retained = Vec::new(); + let acquisitions = self.acquisitions.clone(); + for acquisition in acquisitions.into_iter().rev() { + let result = match &acquisition { + Acquisition::Database(name) => self.execute_cleanup(&format!( + "DROP DATABASE IF EXISTS {} WITH (FORCE)", + quote_identifier(name) + )), + Acquisition::Role(name) => { + self.execute_cleanup(&format!("DROP ROLE IF EXISTS {}", quote_identifier(name))) + } + }; + if let Err(error) = result { + failures.push(match &acquisition { + Acquisition::Database(_) => format!("database: {error:#}"), + Acquisition::Role(_) => format!("role: {error:#}"), + }); + retained.push(acquisition); + } + } + retained.reverse(); + self.acquisitions = retained; + if failures.is_empty() { + Ok(()) + } else { + bail!( + "PostgreSQL cleanup is incomplete for {} resource(s): {}", + failures.len(), + failures.join("; ") + ) + } + } + + fn reconnect_admin(&mut self) -> Result<()> { + let (client, connection) = self + .runtime + .block_on(self.admin_config.connect(self.tls.clone())) + .context("cannot reconnect to the PostgreSQL administrator database")?; + self._connection_task.abort(); + self.client = client; + self._connection_task = self.runtime.spawn(connection); + Ok(()) + } + + fn ensure_admin_connection(&mut self) -> Result<()> { + if self.client.is_closed() { + self.reconnect_admin()?; + } + Ok(()) + } + + fn execute_cleanup(&mut self, statement: &str) -> Result<()> { + self.ensure_admin_connection()?; + match self.runtime.block_on(self.client.batch_execute(statement)) { + Ok(()) => Ok(()), + Err(first) if self.client.is_closed() => { + self.reconnect_admin()?; + self.runtime + .block_on(self.client.batch_execute(statement)) + .with_context(|| { + format!( + "the PostgreSQL cleanup operation failed after reconnecting ({first})" + ) + }) + } + Err(error) => Err(error).context("the PostgreSQL cleanup operation failed"), + } + } + + fn create_role(&mut self, role: &str, password: &str) -> Result<()> { + let verifier = Zeroizing::new(postgres_protocol::password::scram_sha_256( + password.as_bytes(), + )); + let statement = Zeroizing::new(create_role_sql(role, &verifier)); + let created = self.runtime.block_on(self.client.batch_execute(&statement)); + if created.is_ok() || created.as_ref().is_err_and(|error| error.is_closed()) { + // A closed connection cannot tell us whether PostgreSQL committed + // the statement. Retaining the exact run-owned name lets cleanup's + // idempotent DROP cover both outcomes without replaying CREATE. + self.acquisitions.push(Acquisition::Role(role.to_owned())); + } + created.context("cannot create a least-privilege PostgreSQL role") + } + + fn create_database(&mut self, names: &Names) -> Result<()> { + let created = self.runtime.block_on(self.client.batch_execute(&format!( + "CREATE DATABASE {} WITH ALLOW_CONNECTIONS false", + quote_identifier(&names.database) + ))); + if created.is_ok() || created.as_ref().is_err_and(|error| error.is_closed()) { + self.acquisitions + .push(Acquisition::Database(names.database.clone())); + } + created.context("cannot create an isolated PostgreSQL database")?; + + self.execute_admin(&database_access_sql(names)) + .context("cannot restrict the PostgreSQL database to this run's roles") + } + + fn configure_database(&self, names: &Names, breg: bool) -> Result<()> { + let statement = database_setup_sql(names, breg); + + let mut target = self.admin_config.clone(); + target.dbname(&names.database).ssl_mode(SslMode::Require); + self.runtime + .block_on(async { + let (client, connection) = target + .connect(self.tls.clone()) + .await + .context("cannot connect to the provisioned PostgreSQL database")?; + let connection_task = tokio::spawn(connection); + let executed = client.batch_execute(&statement).await; + drop(client); + let closed = connection_task + .await + .context("the PostgreSQL connection task stopped unexpectedly")?; + executed.context("cannot configure the provisioned PostgreSQL database")?; + closed.context("the PostgreSQL configuration connection failed")?; + Result::<()>::Ok(()) + }) + .context("cannot finish PostgreSQL database provisioning") + } + + fn execute_admin(&self, statement: &str) -> Result<()> { + self.runtime + .block_on(self.client.batch_execute(statement)) + .context("the PostgreSQL administrator operation failed") + } + + fn product_url(&self, database: &str, user: &str, password: &str) -> Result { + let mut url = + Url::parse("postgresql://localhost/").expect("the fixed PostgreSQL URL base is valid"); + url.set_host(Some(&self.endpoint.host)) + .map_err(|_| anyhow!("the PostgreSQL host cannot be represented in a product URL"))?; + url.set_port(self.endpoint.port) + .map_err(|_| anyhow!("the PostgreSQL port cannot be represented in a product URL"))?; + url.set_username(user) + .map_err(|_| anyhow!("the generated PostgreSQL role cannot be represented in a URL"))?; + url.set_password(Some(password)).map_err(|_| { + anyhow!("the generated PostgreSQL password cannot be represented in a URL") + })?; + url.set_path(&format!("/{database}")); + url.query_pairs_mut().append_pair("sslmode", "require"); + Ok(url.into()) + } +} + +struct Names { + database: String, + migration_role: String, + runtime_role: String, +} + +impl Names { + fn new(run_id: &str, label: &str) -> Self { + let database = format!("rstack_{run_id}_{label}"); + Self { + migration_role: format!("{database}_migration"), + runtime_role: format!("{database}_runtime"), + database, + } + } +} + +fn validate_run_id(run_id: &str) -> Result<()> { + if run_id.len() == 32 + && run_id + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) + { + Ok(()) + } else { + bail!("the database run id must be exactly 32 lowercase hexadecimal characters") + } +} + +fn validate_label(label: &str) -> Result<()> { + if matches!(label, "breg" | "breg_test" | "casework") { + Ok(()) + } else { + bail!("the database label must be breg, breg_test, or casework") + } +} + +fn database_access_sql(names: &Names) -> String { + format!( + "REVOKE ALL ON DATABASE {database} FROM PUBLIC; \ + GRANT CONNECT ON DATABASE {database} TO {migration}, {runtime}; \ + ALTER DATABASE {database} ALLOW_CONNECTIONS true", + database = quote_identifier(&names.database), + migration = quote_identifier(&names.migration_role), + runtime = quote_identifier(&names.runtime_role), + ) +} + +fn create_role_sql(role: &str, verifier: &str) -> String { + format!( + "CREATE ROLE {} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOBYPASSRLS PASSWORD {}", + quote_identifier(role), + quote_literal(verifier) + ) +} + +fn database_setup_sql(names: &Names, breg: bool) -> String { + if breg { + let schemas = BREG_SCHEMAS + .iter() + .map(|schema| { + format!( + "CREATE SCHEMA {schema} AUTHORIZATION {owner}; \ + REVOKE ALL ON SCHEMA {schema} FROM PUBLIC", + schema = quote_identifier(schema), + owner = quote_identifier(&names.migration_role), + ) + }) + .collect::>() + .join("; "); + format!("CREATE EXTENSION btree_gist; {schemas}") + } else { + format!( + "REVOKE CREATE ON SCHEMA public FROM PUBLIC; \ + GRANT USAGE, CREATE ON SCHEMA public TO {}", + quote_identifier(&names.migration_role) + ) + } +} + +fn admin_config(admin_url: &str) -> Result<(Config, Endpoint)> { + let parsed = + Url::parse(admin_url).map_err(|_| anyhow!("the PostgreSQL admin URL is invalid"))?; + if !matches!(parsed.scheme(), "postgres" | "postgresql") + || parsed.cannot_be_a_base() + || parsed.fragment().is_some() + { + bail!("the PostgreSQL admin URL must be an absolute postgres:// or postgresql:// URL"); + } + if parsed.username().is_empty() + || parsed.password().is_none_or(str::is_empty) + || parsed.host().is_none() + { + bail!("the PostgreSQL admin URL must include an explicit host, user, and password"); + } + let database = parsed.path().strip_prefix('/').unwrap_or_default(); + if database.is_empty() || database.contains('/') { + bail!("the PostgreSQL admin URL must include one explicit database name"); + } + if parsed.query_pairs().any(|(key, _)| { + matches!( + key.as_ref(), + "host" | "hostaddr" | "port" | "user" | "password" | "dbname" + ) + }) { + bail!( + "the PostgreSQL admin URL must not override its endpoint or credentials in the query" + ); + } + + let config = + Config::from_str(admin_url).map_err(|_| anyhow!("the PostgreSQL admin URL is invalid"))?; + if config.get_hosts().len() != 1 + || !matches!(config.get_hosts().first(), Some(Host::Tcp(_))) + || !config.get_hostaddrs().is_empty() + || config.get_user().is_none_or(str::is_empty) + || config.get_password().is_none_or(<[u8]>::is_empty) + || config.get_dbname().is_none_or(str::is_empty) + { + bail!("the PostgreSQL admin URL must name one TCP host, user, password, and database"); + } + + let host = match parsed.host().expect("the host was checked") { + UrlHost::Domain(value) => value.to_owned(), + UrlHost::Ipv4(value) => value.to_string(), + UrlHost::Ipv6(value) => value.to_string(), + }; + Ok(( + config, + Endpoint { + host, + port: parsed.port(), + }, + )) +} + +fn tls_connector(ca_pem: Option<&[u8]>) -> Result { + let provider = rustls::crypto::ring::default_provider(); + if provider.install_default().is_err() + && rustls::crypto::CryptoProvider::get_default().is_none() + { + bail!("cannot initialize the PostgreSQL TLS provider"); + } + + if let Some(pem) = ca_pem { + let mut roots = rustls::RootCertStore::empty(); + let mut count = 0usize; + for certificate in CertificateDer::pem_slice_iter(pem) { + roots + .add(certificate.context("the PostgreSQL CA PEM is invalid")?) + .context("the PostgreSQL CA certificate is invalid")?; + count += 1; + } + if count == 0 { + bail!("the PostgreSQL CA PEM contains no certificates"); + } + let config = rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + Ok(MakeRustlsConnect::new(config)) + } else { + MakeRustlsConnect::with_native_certs() + .map(|value| value.0) + .map_err(|_| anyhow!("cannot load native roots for PostgreSQL TLS")) + } +} + +fn random_password() -> Result> { + let mut bytes = Zeroizing::new([0_u8; 32]); + getrandom::fill(bytes.as_mut()).context("cannot generate a PostgreSQL credential")?; + Ok(Zeroizing::new(URL_SAFE_NO_PAD.encode(bytes.as_ref()))) +} + +fn quote_identifier(value: &str) -> String { + format!("\"{}\"", value.replace('"', "\"\"")) +} + +fn quote_literal(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} + +#[cfg(test)] +mod tests { + use super::*; + + const RUN_ID: &str = "0123456789abcdef0123456789abcdef"; + + #[test] + fn run_ids_are_exact_lowercase_hex() { + assert!(validate_run_id(RUN_ID).is_ok()); + for invalid in [ + "0123456789abcdef0123456789abcde", + "0123456789abcdef0123456789abcdef0", + "0123456789ABCDEF0123456789ABCDEF", + "0123456789abcdef0123456789abcdeg", + ] { + assert!(validate_run_id(invalid).is_err(), "accepted {invalid}"); + } + } + + #[test] + fn only_owned_database_labels_are_accepted() { + for label in ["breg", "breg_test", "casework"] { + assert!(validate_label(label).is_ok(), "refused {label}"); + } + for invalid in ["", "breg-test", "casework_test", "postgres"] { + assert!(validate_label(invalid).is_err(), "accepted {invalid}"); + } + } + + #[test] + fn generated_names_are_run_scoped_and_fit_postgresql() { + let names = Names::new(RUN_ID, "breg_test"); + assert_eq!( + names.database, + "rstack_0123456789abcdef0123456789abcdef_breg_test" + ); + assert!(names.migration_role.ends_with("_breg_test_migration")); + assert!(names.runtime_role.ends_with("_breg_test_runtime")); + assert!(names.database.len() <= 63); + assert!(names.migration_role.len() <= 63); + assert!(names.runtime_role.len() <= 63); + } + + #[test] + fn admin_urls_require_one_explicit_password_authenticated_tcp_database() { + assert!( + admin_config("postgresql://admin:secret@db.example:5432/postgres?sslmode=require") + .is_ok() + ); + assert!(admin_config("postgres://admin:secret@[::1]:5432/postgres").is_ok()); + for invalid in [ + "http://admin:secret@db.example/postgres", + "postgresql://db.example/postgres", + "postgresql://admin@db.example/postgres", + "postgresql://admin:@db.example/postgres", + "postgresql://admin:secret@db.example", + "postgresql://admin:secret@db.example/one/two", + "postgresql://admin:secret@db.example/postgres?host=other.example", + "postgresql://admin:secret@db.example/postgres?dbname=other", + "postgresql://admin:secret@db.example/postgres?hostaddr=127.0.0.1", + "postgresql:///postgres", + ] { + assert!(admin_config(invalid).is_err(), "accepted {invalid}"); + } + } + + #[test] + fn generated_product_urls_contain_only_generated_credentials_and_required_tls() { + let endpoint = Endpoint { + host: "db.example".to_owned(), + port: Some(5543), + }; + let url = product_url_for_test(&endpoint, "owned", "runtime", "generated-secret"); + assert_eq!( + url, + "postgresql://runtime:generated-secret@db.example:5543/owned?sslmode=require" + ); + assert!(!url.contains("admin")); + } + + #[test] + fn database_access_is_closed_to_public_before_connections_are_enabled() { + let names = Names::new(RUN_ID, "casework"); + let sql = database_access_sql(&names); + let revoke = sql.find("REVOKE ALL ON DATABASE").unwrap(); + let grant = sql.find("GRANT CONNECT ON DATABASE").unwrap(); + let enable = sql.find("ALLOW_CONNECTIONS true").unwrap(); + assert!(revoke < grant && grant < enable, "{sql}"); + assert!(sql.contains("e_identifier(&names.migration_role))); + assert!(sql.contains("e_identifier(&names.runtime_role))); + } + + #[test] + fn breg_setup_installs_extension_and_all_owned_schemas() { + let names = Names::new(RUN_ID, "breg"); + let sql = database_setup_sql(&names, true); + assert!(sql.starts_with("CREATE EXTENSION btree_gist;"), "{sql}"); + for schema in BREG_SCHEMAS { + assert!( + sql.contains(&format!( + "CREATE SCHEMA {} AUTHORIZATION {}", + quote_identifier(schema), + quote_identifier(&names.migration_role) + )), + "{sql}" + ); + assert!( + sql.contains(&format!( + "REVOKE ALL ON SCHEMA {} FROM PUBLIC", + quote_identifier(schema) + )), + "{sql}" + ); + } + } + + #[test] + fn casework_setup_gives_only_migration_role_public_schema_creation() { + let names = Names::new(RUN_ID, "casework"); + let sql = database_setup_sql(&names, false); + assert!( + sql.starts_with("REVOKE CREATE ON SCHEMA public FROM PUBLIC;"), + "{sql}" + ); + assert!(sql.contains(&format!( + "GRANT USAGE, CREATE ON SCHEMA public TO {}", + quote_identifier(&names.migration_role) + ))); + assert!(!sql.contains(&names.runtime_role)); + } + + #[test] + fn identifiers_and_passwords_are_quoted_for_postgresql() { + assert_eq!(quote_identifier("a\"b"), "\"a\"\"b\""); + assert_eq!(quote_literal("a'b"), "'a''b'"); + } + + #[test] + fn role_statement_carries_a_postgresql_scram_verifier_not_cleartext() { + // postgres-protocol's fixed-salt vector for password `secret`. Its + // production API generates a fresh random salt for every verifier. + let verifier = "SCRAM-SHA-256$4096:AQIDBAUGBwgJCgsMDQ4PEA==$8rrDg00OqaiWXJ7p+sCgHEIaBSHY89ZJl3mfIsf32oY=:05L1f+yZbiN8O0AnO40Og85NNRhvzTS57naKRWCcsIA="; + let sql = create_role_sql("rstack_role", verifier); + assert_eq!( + sql, + format!( + "CREATE ROLE \"rstack_role\" LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOBYPASSRLS PASSWORD '{verifier}'" + ) + ); + assert!(!sql.contains("PASSWORD 'secret'")); + } + + #[test] + fn empty_or_malformed_custom_root_sets_are_refused() { + assert!(tls_connector(Some(b"")).is_err()); + assert!(tls_connector(Some(b"not a certificate")).is_err()); + } + + fn product_url_for_test( + endpoint: &Endpoint, + database: &str, + user: &str, + password: &str, + ) -> String { + let mut url = Url::parse("postgresql://localhost/").unwrap(); + url.set_host(Some(&endpoint.host)).unwrap(); + url.set_port(endpoint.port).unwrap(); + url.set_username(user).unwrap(); + url.set_password(Some(password)).unwrap(); + url.set_path(&format!("/{database}")); + url.query_pairs_mut().append_pair("sslmode", "require"); + url.into() + } +} diff --git a/crates/registry-bregctl/src/project_dev/issuer.rs b/crates/registry-bregctl/src/project_dev/issuer.rs new file mode 100644 index 0000000000..d03c2309b1 --- /dev/null +++ b/crates/registry-bregctl/src/project_dev/issuer.rs @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Local access-token issuer. Signing is available only to the owner through the CLI. + +use anyhow::{ensure, Context, Result}; +use registry_platform_crypto::{ + generate_private_jwk, jws::sign_compact_jws, GeneratedKeyAlgorithm, LocalJwkSigner, PrivateJwk, +}; +use serde_json::{json, Value}; +use std::{path::Path, time::Duration}; +use zeroize::Zeroizing; + +use super::{ + clock, + plan::{self, Person, Product}, + write_private, RunInput, +}; + +pub(super) struct Issuer { + signer: LocalJwkSigner, + runtime: tokio::runtime::Runtime, +} + +impl Issuer { + pub(super) fn create(root: &Path) -> Result { + let key = generate_private_jwk(GeneratedKeyAlgorithm::Es256)?; + let encoded = super::bindings::encode_private_jwk(&key)?; + write_private(&root.join("issuer-key"), &encoded)?; + write_private( + &root.join("jwks"), + &serde_json::to_vec(&json!({ + "keys": [key.public()] + }))?, + )?; + Self::from_key(key) + } + + pub(super) fn load(root: &Path) -> Result { + let encoded = Zeroizing::new(plan::read(&root.join("issuer-key"), 16 * 1024)?); + let text = std::str::from_utf8(&encoded).context("invalid local issuer key encoding")?; + let key = + PrivateJwk::parse(text).map_err(|_| anyhow::anyhow!("invalid local issuer key"))?; + Self::from_key(key) + } + + fn from_key(key: PrivateJwk) -> Result { + Ok(Self { + signer: LocalJwkSigner::new(key)?, + runtime: tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?, + }) + } + + pub(super) fn token( + &self, + input: &RunInput, + person: &Person, + product: Product, + expires: u64, + ) -> Result> { + self.runtime + .block_on(self.sign(input, person, product, expires)) + } + + pub(super) async fn sign( + &self, + input: &RunInput, + person: &Person, + product: Product, + expires: u64, + ) -> Result> { + let now = clock()?; + let expires = expires.min(now + Duration::from_secs(300).as_secs()); + ensure!(expires > now, "the development run has expired"); + let plan = &input.plan; + let grant = &person.grants[&product]; + let forward_context = product == Product::Casework && person.casework_context; + let mut claims = grant.claims.clone(); + let scopes = grant + .scopes + .iter() + .map(String::as_str) + .collect::>(); + let audience = if forward_context { + json!([ + plan.product(product).audience, + plan.product(Product::Breg).audience + ]) + } else { + json!(plan.product(product).audience) + }; + if product == Product::Breg || forward_context { + let authority = input.breg_authority.as_ref().expect("BReg authority"); + claims.insert(authority.principal.clone(), json!(person.subject)); + } + if product == Product::Casework { + let principal = &input.casework_principals[&grant.profile]; + if principal != "sub" { + claims.insert(principal.clone(), json!(person.subject)); + } + } + claims.extend([ + ("iss".to_owned(), json!(plan.issuer)), + ("aud".to_owned(), audience), + ("sub".to_owned(), json!(person.subject)), + ("client_id".to_owned(), json!(person.client)), + ( + "scope".to_owned(), + json!(scopes.into_iter().collect::>().join(" ")), + ), + ("iat".to_owned(), json!(now)), + ("exp".to_owned(), json!(expires)), + ("jti".to_owned(), json!(uuid::Uuid::new_v4())), + ]); + let header = json!({"typ": "at+jwt"}); + let bytes = serde_json::to_vec(&Value::Object(claims))?; + let token = sign_compact_jws( + header.as_object().expect("header object").clone(), + &bytes, + &self.signer, + ) + .await?; + Ok(Zeroizing::new(token)) + } +} diff --git a/crates/registry-bregctl/src/project_dev/journey.rs b/crates/registry-bregctl/src/project_dev/journey.rs new file mode 100644 index 0000000000..fefa083817 --- /dev/null +++ b/crates/registry-bregctl/src/project_dev/journey.rs @@ -0,0 +1,581 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Execute the bounded HTTP journey declared by a development plan. + +use anyhow::{ensure, Context, Result}; +use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; +use reqwest::header::{HeaderMap, ETAG}; +use serde::Serialize; +use serde_json::Value; +use std::{ + collections::BTreeMap, + thread, + time::{Duration, Instant}, +}; + +use super::{ + clock, + issuer::Issuer, + plan::{self, Capture, CapturedHeader, Check, Product, Step}, + RunInput, +}; + +const RESPONSE_LIMIT: usize = 1024 * 1024; +const JOURNEY_TRANSFER_LIMIT: usize = 8 * 1024 * 1024; +const JOURNEY_ATTEMPT_LIMIT: usize = 512; + +#[derive(Serialize)] +pub(super) struct JourneyReport { + pub id: String, + pub result: &'static str, + pub steps: Vec, +} + +#[derive(Serialize)] +pub(super) struct StepReport { + id: String, + status: u16, + expected_status: u16, + result: &'static str, +} + +struct ResponseData { + status: u16, + headers: HeaderMap, + body: Value, +} + +pub(super) fn run( + input: &RunInput, + issuer: &Issuer, + urls: &BTreeMap, + active: impl Fn() -> Result<()>, +) -> Result { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + let http = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(5)) + .build()?; + let journey = &input.plan.journey; + let mut report = JourneyReport { + id: journey.id.clone(), + result: "passed", + steps: Vec::new(), + }; + let mut captures = BTreeMap::::new(); + let mut transferred = 0usize; + let mut attempts = 0usize; + + for step in &journey.steps { + active()?; + let outcome = execute_step( + input, + issuer, + urls, + step, + &captures, + &http, + &runtime, + &active, + &mut transferred, + &mut attempts, + ); + let (status, passed, updates) = outcome.unwrap_or_else(|_| (0, false, BTreeMap::new())); + if passed { + captures.extend(updates); + } + report.steps.push(StepReport { + id: step.id.clone(), + status, + expected_status: step.expect.status.first(), + result: if passed { "passed" } else { "failed" }, + }); + if !passed { + report.result = "failed"; + break; + } + } + Ok(report) +} + +#[allow(clippy::too_many_arguments)] +fn execute_step( + input: &RunInput, + issuer: &Issuer, + urls: &BTreeMap, + step: &Step, + captures: &BTreeMap, + http: &reqwest::Client, + runtime: &tokio::runtime::Runtime, + active: &impl Fn() -> Result<()>, + transferred: &mut usize, + attempts: &mut usize, +) -> Result<(u16, bool, BTreeMap)> { + let lookup = |name: &str| { + captures + .get(name) + .cloned() + .context("missing journey capture") + }; + let path = expand_text(&step.path, lookup)?; + let mut url = request_url(&urls[&step.product], &path)?; + let person = &input.plan.people[&step.person]; + let grant = &person.grants[&step.product]; + if step.product == Product::Breg { + ensure!( + !url.query_pairs().any(|(name, _)| name == "accessProfile"), + "the plan's grant selects the BReg profile" + ); + url.query_pairs_mut() + .append_pair("accessProfile", &grant.profile); + } + + let headers = step + .headers + .iter() + .map(|(name, value)| Ok((name.clone(), expand_text(value, lookup)?))) + .collect::>>()?; + let json_body = step + .body + .as_ref() + .map(|body| expand_json(body, lookup)) + .transpose()?; + let raw_body = step + .raw + .as_ref() + .map(|body| BASE64.decode(&body.base64)) + .transpose()?; + let request_bytes = json_body + .as_ref() + .map(serde_json::to_vec) + .transpose()? + .map_or_else(|| raw_body.as_ref().map_or(0, Vec::len), |body| body.len()); + let poll_deadline = step + .poll + .as_ref() + .map(|poll| Instant::now() + Duration::from_millis(poll.timeout_ms)); + + loop { + active()?; + add_attempt(attempts)?; + add_transfer(transferred, request_bytes)?; + let token = issuer.token(input, person, step.product, clock()? + 300)?; + let mut request = http + .request((&step.method).into(), url.clone()) + .bearer_auth(&*token); + if step.product == Product::Casework { + request = request + .header("Registry-Casework-Profile", &grant.profile) + .header("Registry-Review-Profile", &grant.profile); + } + for (name, value) in &headers { + request = request.header(name, value); + } + if let Some(body) = &json_body { + request = request.json(body); + } else if let (Some(spec), Some(body)) = (&step.raw, &raw_body) { + request = request + .header(reqwest::header::CONTENT_TYPE, &spec.content_type) + .body(body.clone()); + } + let (response, response_bytes) = runtime.block_on(read_response(request))?; + add_transfer(transferred, response_bytes)?; + let captures = matches_expectation(step, &response, lookup)? + .then(|| capture_updates(step, &response, lookup)) + .and_then(Result::ok); + if let Some(updates) = captures { + return Ok((response.status, true, updates)); + } + let Some((poll, deadline)) = step.poll.as_ref().zip(poll_deadline) else { + return Ok((response.status, false, BTreeMap::new())); + }; + let now = Instant::now(); + if now >= deadline { + return Ok((response.status, false, BTreeMap::new())); + } + thread::sleep(Duration::from_millis(poll.interval_ms).min(deadline - now)); + } +} + +fn request_url(base: &str, path: &str) -> Result { + ensure!(plan::valid_path(path), "invalid journey path"); + let base = url::Url::parse(base)?; + ensure!( + base.scheme() == "http" + && base.username().is_empty() + && base.password().is_none() + && base.query().is_none() + && base.fragment().is_none() + && base.path() == "/" + && matches!(base.host(), Some(url::Host::Ipv4(address)) if address.is_loopback()), + "journey product URL is not a numeric loopback HTTP origin" + ); + let url = base.join(path)?; + ensure!( + url.scheme() == base.scheme() + && url.host() == base.host() + && url.port_or_known_default() == base.port_or_known_default() + && url.username().is_empty() + && url.password().is_none() + && url.fragment().is_none() + && url.path().starts_with("/v1/"), + "journey path leaves the product API" + ); + Ok(url) +} + +fn matches_expectation( + step: &Step, + response: &ResponseData, + lookup: impl Fn(&str) -> Result + Copy, +) -> Result { + if !step.expect.status.contains(response.status) { + return Ok(false); + } + for (pointer, expected) in &step.expect.json { + if response.body.pointer(pointer) != Some(&expand_json(expected, lookup)?) { + return Ok(false); + } + } + if let Some(code) = &step.expect.code { + if response.body.pointer("/code").and_then(Value::as_str) + != Some(expand_text(code, lookup)?.as_str()) + { + return Ok(false); + } + } + for check in &step.expect.checks { + if !matches_check(check, &response.body, lookup)? { + return Ok(false); + } + } + Ok(true) +} + +fn matches_check( + check: &Check, + body: &Value, + lookup: impl Fn(&str) -> Result + Copy, +) -> Result { + let Some(actual) = body.pointer(&check.pointer) else { + return Ok(false); + }; + if let Some(expected) = &check.equals { + return Ok(actual == &expand_json(expected, lookup)?); + } + if let Some(expected) = check.length { + let length = match actual { + Value::Array(value) => value.len(), + Value::Object(value) => value.len(), + Value::String(value) => value.chars().count(), + _ => return Ok(false), + }; + return Ok(length == expected); + } + if let Some(expected) = &check.contains { + let expected = expand_json(expected, lookup)?; + return Ok(match (actual, expected) { + (Value::Array(values), expected) => values.contains(&expected), + (Value::String(value), Value::String(expected)) => value.contains(&expected), + (Value::Object(values), Value::String(expected)) => values.contains_key(&expected), + _ => false, + }); + } + Ok(false) +} + +fn capture_updates( + step: &Step, + response: &ResponseData, + lookup: impl Fn(&str) -> Result + Copy, +) -> Result> { + step.capture + .iter() + .map(|(name, capture)| { + Ok(( + name.clone(), + scalar_capture(capture_value(capture, response, lookup)?)?, + )) + }) + .collect() +} + +fn capture_value( + capture: &Capture, + response: &ResponseData, + lookup: impl Fn(&str) -> Result + Copy, +) -> Result { + match capture { + Capture::Pointer(pointer) => response + .body + .pointer(pointer) + .cloned() + .context("journey capture pointer was not present"), + Capture::Header(header) => match header.header { + CapturedHeader::Etag => Ok(Value::String( + response + .headers + .get(ETAG) + .context("journey response had no ETag")? + .to_str() + .context("journey response ETag was not text")? + .to_owned(), + )), + }, + Capture::Value(capture) => { + let value = response + .body + .pointer(&capture.pointer) + .cloned() + .context("journey capture pointer was not present")?; + if let Some(query) = &capture.query { + let text = value + .as_str() + .context("journey query capture did not select text")?; + let url = url::Url::parse(text) + .or_else(|_| url::Url::parse("http://127.0.0.1/")?.join(text))?; + let value = url + .query_pairs() + .find_map(|(name, value)| (name == query.as_str()).then(|| value.into_owned())) + .context("journey capture query parameter was not present")?; + Ok(Value::String(value)) + } else { + Ok(value) + } + } + Capture::Find(capture) => { + let expected = expand_json(&capture.r#where.equals, lookup)?; + let items = response + .body + .pointer(&capture.find) + .and_then(Value::as_array) + .context("journey find capture did not select an array")?; + let mut matches = items + .iter() + .filter(|item| item.pointer(&capture.r#where.pointer) == Some(&expected)); + let item = matches + .next() + .context("journey find capture found no item")?; + ensure!( + matches.next().is_none(), + "journey find capture matched more than one item" + ); + item.pointer(&capture.pointer) + .cloned() + .context("journey find capture pointer was not present") + } + } +} + +fn scalar_capture(value: Value) -> Result { + ensure!( + matches!(value, Value::String(_) | Value::Number(_) | Value::Bool(_)) + && serde_json::to_vec(&value).is_ok_and(|bytes| bytes.len() <= 4096), + "journey captures must be bounded scalar values" + ); + Ok(value) +} + +fn expand_json(value: &Value, lookup: impl Fn(&str) -> Result + Copy) -> Result { + match value { + Value::String(value) => { + if let Some(name) = exact_reference(value) { + lookup(name) + } else { + Ok(Value::String(expand_text(value, lookup)?)) + } + } + Value::Array(values) => values + .iter() + .map(|value| expand_json(value, lookup)) + .collect(), + Value::Object(values) => values + .iter() + .map(|(key, value)| Ok((key.clone(), expand_json(value, lookup)?))) + .collect(), + value => Ok(value.clone()), + } +} + +fn expand_text(value: &str, lookup: impl Fn(&str) -> Result + Copy) -> Result { + plan::expand(value, |name| scalar_text(&lookup(name)?)) +} + +fn exact_reference(value: &str) -> Option<&str> { + let name = value.strip_prefix("{{")?.strip_suffix("}}")?; + (!name.is_empty() && !name.contains(['{', '}'])).then_some(name) +} + +fn scalar_text(value: &Value) -> Result { + match value { + Value::String(value) => Ok(value.clone()), + Value::Number(value) => Ok(value.to_string()), + Value::Bool(value) => Ok(value.to_string()), + _ => anyhow::bail!("journey interpolation requires a scalar capture"), + } +} + +fn add_transfer(transferred: &mut usize, bytes: usize) -> Result<()> { + *transferred = transferred + .checked_add(bytes) + .context("journey transfer bound overflowed")?; + ensure!( + *transferred <= JOURNEY_TRANSFER_LIMIT, + "journey transferred too much data" + ); + Ok(()) +} + +fn add_attempt(attempts: &mut usize) -> Result<()> { + *attempts = attempts + .checked_add(1) + .context("journey attempt bound overflowed")?; + ensure!( + *attempts <= JOURNEY_ATTEMPT_LIMIT, + "journey made too many requests" + ); + Ok(()) +} + +async fn read_response(request: reqwest::RequestBuilder) -> Result<(ResponseData, usize)> { + let mut response = request.send().await?; + let status = response.status().as_u16(); + let headers = response.headers().clone(); + let mut bytes = Vec::new(); + while let Some(chunk) = response.chunk().await? { + ensure!( + bytes.len() + chunk.len() <= RESPONSE_LIMIT, + "journey response is too large" + ); + bytes.extend_from_slice(&chunk); + } + let length = bytes.len(); + let body = serde_json::from_slice(&bytes).unwrap_or(Value::Null); + Ok(( + ResponseData { + status, + headers, + body, + }, + length, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::project_dev::plan::ExpectedStatus; + use serde_json::json; + + fn no_capture(_: &str) -> Result { + anyhow::bail!("unexpected capture") + } + + #[test] + fn exact_capture_reference_preserves_scalar_type() { + let captures = BTreeMap::from([("revision", json!(7))]); + let expanded = expand_json( + &json!({ + "revision": "{{revision}}", "label": "revision-{{revision}}" + }), + |name| Ok(captures[name].clone()), + ) + .unwrap(); + assert_eq!(expanded, json!({"revision": 7, "label": "revision-7"})); + } + + #[test] + fn request_url_stays_on_the_numeric_loopback_v1_origin() { + let url = request_url("http://127.0.0.1:31415/", "/v1/records?id=one").unwrap(); + assert_eq!(url.as_str(), "http://127.0.0.1:31415/v1/records?id=one"); + assert!(request_url("http://localhost:31415/", "/v1/records").is_err()); + assert!(request_url("http://127.0.0.1:31415/", "/v1/../admin").is_err()); + } + + #[test] + fn checks_compare_typed_values_lengths_and_membership() { + let body = json!({"state": true, "items": [1, 2], "label": "ready-now"}); + for check in [ + Check { + pointer: "/state".into(), + equals: Some(json!(true)), + length: None, + contains: None, + }, + Check { + pointer: "/items".into(), + equals: None, + length: Some(2), + contains: None, + }, + Check { + pointer: "/items".into(), + equals: None, + length: None, + contains: Some(json!(2)), + }, + Check { + pointer: "/label".into(), + equals: None, + length: None, + contains: Some(json!("now")), + }, + ] { + assert!(matches_check(&check, &body, no_capture).unwrap()); + } + } + + #[test] + fn expectation_accepts_any_declared_status_and_problem_code() { + let step: Step = serde_json::from_value(json!({ + "id": "read", "product": "breg", "as": "person", "method": "GET", + "path": "/v1/records", "expect": {"status": [404, 409], "code": "missing"} + })) + .unwrap(); + let response = ResponseData { + status: 409, + headers: HeaderMap::new(), + body: json!({"code": "missing"}), + }; + assert!(matches_expectation(&step, &response, no_capture).unwrap()); + assert_eq!(step.expect.status.first(), 404); + } + + #[test] + fn find_capture_requires_one_matching_item() { + let capture: Capture = serde_json::from_value(json!({ + "find": "/items", "where": {"pointer": "/id", "equals": "two"}, + "pointer": "/revision" + })) + .unwrap(); + let response = ResponseData { + status: 200, + headers: HeaderMap::new(), + body: json!({"items": [ + {"id": "one", "revision": 1}, {"id": "two", "revision": 2} + ]}), + }; + assert_eq!( + capture_value(&capture, &response, no_capture).unwrap(), + json!(2) + ); + } + + #[test] + fn expected_status_helpers_keep_the_first_for_reports() { + let status = ExpectedStatus::Many(vec![202, 200]); + assert_eq!(status.first(), 202); + assert!(status.contains(200)); + } + + #[test] + fn total_request_work_is_bounded_across_steps() { + let mut attempts = 0; + for _ in 0..JOURNEY_ATTEMPT_LIMIT { + add_attempt(&mut attempts).unwrap(); + } + assert!(add_attempt(&mut attempts).is_err()); + } +} diff --git a/crates/registry-bregctl/src/project_dev/mod.rs b/crates/registry-bregctl/src/project_dev/mod.rs new file mode 100644 index 0000000000..e149b47e38 --- /dev/null +++ b/crates/registry-bregctl/src/project_dev/mod.rs @@ -0,0 +1,733 @@ +// SPDX-License-Identifier: Apache-2.0 +//! The private runner behind the rstack launcher. The bootstrap binary does +//! not link the database, HTTP server, or signing libraries. + +mod bindings; +mod database; +mod issuer; +mod journey; +pub(crate) mod plan; +mod products; + +use anyhow::{bail, ensure, Context, Result}; +use rustix::fs::{flock, FlockOperation}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::{ + collections::{BTreeMap, BTreeSet}, + env, + fs::{self, File, OpenOptions}, + io::{BufRead, BufReader, Read, Write}, + os::unix::{ + fs::{DirBuilderExt, MetadataExt, OpenOptionsExt}, + process::CommandExt, + }, + path::{Path, PathBuf}, + process::{Command, Stdio}, + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc, Arc, + }, + thread, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; +use uuid::Uuid; +use zeroize::Zeroizing; + +use self::{ + database::Database, + issuer::Issuer, + plan::{Plan, Product}, + products::{ProductRunner, SchemaPerson}, +}; + +const SCHEMA: &str = "registry.rstack.dev.v1"; +const REQUEST: &str = "RSTACK_PROJECT_DEV_REQUEST"; +const DATABASE_URL: &str = "RSTACK_DEV_DATABASE_URL"; +const CA_FILE: &str = "RSTACK_DEV_DATABASE_CA"; +const REPORT: &str = "report.json"; +const POLL: Duration = Duration::from_millis(100); + +#[derive(clap::Args, Debug)] +pub(crate) struct Args { + #[arg(long, hide = true)] + worker: Option, +} + +#[derive(Deserialize)] +#[serde(tag = "action", rename_all = "snake_case", deny_unknown_fields)] +enum Request { + Start { + project: PathBuf, + ttl: Option, + json: bool, + }, + Status { + run_id: String, + json: bool, + }, + Token { + run_id: String, + person: String, + json: bool, + }, + Stop { + run_id: String, + json: bool, + }, +} + +impl Request { + fn json(&self) -> bool { + match self { + Self::Start { json, .. } + | Self::Status { json, .. } + | Self::Token { json, .. } + | Self::Stop { json, .. } => *json, + } + } +} + +#[derive(Deserialize, Serialize)] +struct RunInput { + run_id: String, + project: PathBuf, + plan: Plan, + ttl_seconds: u64, + schema_people: Vec, + breg_authority: Option, + casework_principals: BTreeMap, + binaries: BTreeMap, +} + +#[derive(Deserialize, Serialize)] +struct BregAuthority { + principal: String, + purpose: bool, + named_clients: BTreeSet, +} + +#[derive(Deserialize, Serialize)] +struct Binaries { + tool: PathBuf, + runtime: PathBuf, +} + +#[derive(Deserialize, Serialize)] +#[serde(tag = "outcome", rename_all = "snake_case")] +enum WorkerReply { + Ready { report: Value }, + Failed { error: String }, +} + +#[derive(Debug, thiserror::Error)] +#[error("run {run_id} still needs cleanup: {detail}")] +struct CleanupPending { + run_id: String, + detail: String, +} + +fn startup_error(root: &Path, run_id: &str, error: anyhow::Error) -> anyhow::Error { + if root.exists() { + CleanupPending { + run_id: run_id.to_owned(), + detail: format!("{error:#}"), + } + .into() + } else { + error + } +} + +pub(crate) fn run(args: Args, stdout: &mut dyn Write, stderr: &mut dyn Write) -> u8 { + if let Some(root) = args.worker { + // The CLI panic hook exits immediately. In the supervisor, unwind so + // its startup boundary can release the run without exposing payloads. + std::panic::set_hook(Box::new(|_| {})); + return match worker(&root, stdout) { + Ok(()) => 0, + Err(error) => { + let _ = reply( + stdout, + &WorkerReply::Failed { + error: format!("{error:#}"), + }, + ); + 3 + } + }; + } + let request = env::var(REQUEST) + .ok() + .and_then(|text| serde_json::from_str::(&text).ok()); + let Some(request) = request else { + let _ = writeln!( + stderr, + "run project development commands through rstack dev" + ); + return 2; + }; + let machine = request.json(); + let starting = matches!(&request, Request::Start { .. }); + let outcome = match request { + Request::Start { project, ttl, .. } => start(&project, ttl.as_deref()), + Request::Status { run_id, .. } => status(&run_id), + Request::Token { run_id, person, .. } => token(&run_id, &person), + Request::Stop { run_id, .. } => stop(&run_id), + }; + let (result, code) = match outcome { + Ok(value) => { + let code = u8::from(starting && value["journey"]["result"] == "failed"); + (emit(stdout, &value, machine), code) + } + Err(error) if machine => { + let mut report = + json!({"schema_version": SCHEMA, "status": "error", "error": format!("{error:#}")}); + if let Some(pending) = error.downcast_ref::() { + report["run_id"] = json!(pending.run_id); + report["cleanup_pending"] = json!(true); + } + (writeln!(stdout, "{report}"), 1) + } + Err(error) => (writeln!(stderr, "rstack dev: {error:#}"), 1), + }; + if result.is_ok() { + code + } else { + 3 + } +} + +fn emit(output: &mut dyn Write, report: &Value, machine: bool) -> std::io::Result<()> { + if machine { + return writeln!(output, "{report}"); + } + let run = report["run_id"].as_str().expect("run report"); + let status = report["status"].as_str().expect("run status"); + writeln!(output, "Run {run} {status}")?; + if let Some(products) = report["products"].as_object() { + for (product, url) in products { + writeln!(output, " {product} {}", url.as_str().expect("product URL"))?; + } + } + if let Some(result) = report["journey"]["result"].as_str() { + writeln!( + output, + " Journey {} {result}", + report["journey"]["id"].as_str().expect("journey ID") + )?; + if let Some(steps) = report["journey"]["steps"].as_array() { + for step in steps.iter().filter(|step| step["result"] == "failed") { + writeln!( + output, + " {} expected HTTP {}, observed {}", + step["id"], step["expected_status"], step["status"] + )?; + } + } + } + if let Some(people) = report["people"].as_array() { + for name in people { + writeln!( + output, + " rstack dev token {run} --as {}", + name.as_str().expect("person name") + )?; + } + } + if let Some(credentials) = report["credentials"].as_array() { + for credential in credentials { + writeln!( + output, + " {} headers {}", + credential["product"].as_str().expect("product"), + credential["header_file"].as_str().expect("header path") + )?; + } + } + Ok(()) +} + +fn start(project: &Path, ttl: Option<&str>) -> Result { + let project = project + .canonicalize() + .context("cannot open the development project directory")?; + let plan = plan::load(&project)?; + let schema_people = products::schema_people(&project, &plan)?; + let breg_authority = products::breg_authority(&project, &plan)?; + let casework_principals = + products::casework_principals(&project, &plan, breg_authority.as_ref())?; + let ttl_seconds = parse_ttl(ttl.unwrap_or("30m"))?; + ensure!( + env::var_os(DATABASE_URL).is_some_and(|value| !value.is_empty()), + "supply --database-url or {DATABASE_URL} for a TLS PostgreSQL development server" + ); + let binaries = plan + .products + .iter() + .map(|entry| Ok((entry.product, binaries(entry.product)?))) + .collect::>()?; + let signals = signals()?; + let run_id = Uuid::new_v4().simple().to_string(); + let base = state_dir(); + ensure_private(&base, true)?; + let root = base.canonicalize()?.join(&run_id); + private_directory(&root)?; + let input = RunInput { + run_id, + project, + plan, + ttl_seconds, + schema_people, + breg_authority, + casework_principals, + binaries, + }; + let preparation = (|| { + write_private(&root.join("input.json"), &serde_json::to_vec(&input)?)?; + Command::new(env::current_exe()?) + .arg("project-dev") + .arg("--worker") + .arg(&root) + .env_remove(REQUEST) + .process_group(0) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .context("cannot start the development run supervisor") + })(); + let mut child = match preparation { + Ok(child) => child, + Err(error) => { + fs::remove_dir_all(&root).context("cannot remove unstarted run directory")?; + return Err(error); + } + }; + let stream = child.stdout.take().expect("startup channel"); + let (sender, receiver) = mpsc::sync_channel(1); + thread::spawn(move || { + let mut line = String::new(); + let result = BufReader::new(stream) + .take(512 * 1024) + .read_line(&mut line) + .context("cannot read startup report") + .and_then(|_| { + serde_json::from_str::(&line) + .context("missing or invalid startup report") + }); + let _ = sender.send(result); + }); + let deadline = Instant::now() + Duration::from_secs(360); + loop { + match receiver.recv_timeout(POLL) { + Ok(Ok(WorkerReply::Ready { report })) => return Ok(report), + Ok(Ok(WorkerReply::Failed { error })) => { + let error = anyhow::anyhow!(error); + if root.join("worker.lock").exists() { + if let Err(cleanup) = stop(&input.run_id) { + return Err(startup_error( + &root, + &input.run_id, + error.context(format!( + "automatic cleanup could not finish: {cleanup:#}" + )), + )); + } + } else if root.exists() { + fs::remove_dir_all(&root).context("cannot remove unstarted run directory")?; + } + return Err(error); + } + Ok(Err(error)) => { + if child.try_wait()?.is_some() + && root.exists() + && !root.join("worker.lock").exists() + { + fs::remove_dir_all(&root).context("cannot remove unstarted run directory")?; + } + return Err(startup_error(&root, &input.run_id, error)); + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + return Err(startup_error( + &root, + &input.run_id, + anyhow::anyhow!("the supervisor lost its startup report"), + )); + } + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + if signals.load(Ordering::Relaxed) || Instant::now() >= deadline { + stop(&input.run_id).map_err(|error| startup_error(&root, &input.run_id, error))?; + bail!("development run start was interrupted or timed out"); + } + } +} + +fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { + let name = root + .file_name() + .and_then(|value| value.to_str()) + .context("invalid worker run directory")?; + let expected = run_dir(name)?; + ensure!( + expected.canonicalize()? == root, + "worker run directory differs from the allocated run" + ); + ensure_private(root, false)?; + let input = Arc::new( + read_json::(&root.join("input.json")) + .context("cannot restore the captured development plan")?, + ); + ensure!( + input.run_id == name, + "captured development plan has a different run ID" + ); + let signals = signals()?; + let mut products = ProductRunner::new(input.clone(), root, &signals)?; + let lock = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(root.join("worker.lock"))?; + flock(&lock, FlockOperation::LockExclusive)?; + let mut database = None; + let prepared = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + products.check()?; + let ca = env::var_os(CA_FILE) + .map(|path| { + let bytes = plan::read(Path::new(&path), 1024 * 1024) + .context("cannot read the PostgreSQL CA file")?; + write_private(&root.join("postgres-ca"), &bytes)?; + Ok::<_, anyhow::Error>(bytes) + }) + .transpose()?; + let admin = Zeroizing::new( + env::var(DATABASE_URL) + .map_err(|_| anyhow::anyhow!("missing PostgreSQL administrator URL"))?, + ); + let issuer = Arc::new(Issuer::create(root)?); + let database = database.insert(Database::connect(&admin, ca.as_deref(), &input.run_id)?); + let urls = products.start(database, issuer.clone())?; + let journey = products.journey(&issuer, &urls)?; + ensure!( + products.services_alive()?, + "a product exited during the journey" + ); + let expires_at = clock()? + input.ttl_seconds; + let report = json!({ + "schema_version": SCHEMA, "run_id": input.run_id, "status": "ready", + "products": urls, "people": input.plan.people.keys().collect::>(), + "journey": journey, "expires_at": expires_at, + }); + write_private(&root.join(REPORT), &serde_json::to_vec(&report)?)?; + Ok::<_, anyhow::Error>(report) + })) + .unwrap_or_else(|_| Err(anyhow::anyhow!("development run startup failed internally"))); + + let mut failure = None; + match prepared { + Ok(report) => { + if let Err(error) = reply(output, &WorkerReply::Ready { report }) { + failure = Some(format!("cannot report development run readiness: {error}")); + } else { + let deadline = Instant::now() + Duration::from_secs(input.ttl_seconds); + while Instant::now() < deadline && !stopping(root, &signals) { + if !products.services_alive().unwrap_or(false) { + break; + } + thread::sleep(POLL); + } + } + } + Err(error) => failure = Some(format!("{error:#}")), + } + let stopped = simple_report(&input.run_id, "stopping"); + let mark_stopping = write_private(&root.join(REPORT), &serde_json::to_vec(&stopped)?); + let mut told_parent = false; + loop { + // Keep the exact ownership list until both the supervised processes + // and their database resources have been released. + let cleanup = products + .stop() + .and_then(|()| database.as_mut().map_or(Ok(()), Database::cleanup)); + if cleanup.is_ok() { + break; + } + if let Some(error) = failure.as_ref().filter(|_| !told_parent) { + let _ = reply( + output, + &WorkerReply::Failed { + error: error.clone(), + }, + ); + told_parent = true; + } + thread::sleep(Duration::from_secs(2)); + } + let removed = fs::remove_dir_all(root).context("cannot remove run scratch files"); + if let Some(error) = failure.filter(|_| !told_parent) { + let error = match removed.as_ref() { + Ok(()) => error, + Err(cleanup) => format!("{error}; run {} {cleanup}", input.run_id), + }; + reply(output, &WorkerReply::Failed { error })?; + } + removed?; + // If the root is gone, no stale ready report could have survived. + mark_stopping?; + Ok(()) +} + +fn reply(output: &mut dyn Write, message: &WorkerReply) -> Result<()> { + serde_json::to_writer(&mut *output, message)?; + writeln!(output)?; + output.flush()?; + Ok(()) +} + +fn status(run_id: &str) -> Result { + let root = run_dir(run_id)?; + if !root.exists() { + return Ok(simple_report(run_id, "stopped")); + } + let observed = (|| { + ensure!( + supervisor_running(&root)?, + "run {run_id} lost its supervisor before cleanup" + ); + let report: Value = read_json(&root.join(REPORT)) + .context("the development run has not reported readiness")?; + if report["expires_at"] + .as_u64() + .is_some_and(|expiry| clock().is_ok_and(|now| now >= expiry)) + { + return Ok(simple_report(run_id, "stopping")); + } + Ok(report) + })(); + if observed.is_err() && !root.exists() { + return Ok(simple_report(run_id, "stopped")); + } + observed +} + +fn token(run_id: &str, person: &str) -> Result { + ensure!(plan::identifier(person), "invalid test person name"); + let report = status(run_id)?; + ensure!( + report["status"] == "ready", + "development run {run_id} is not ready" + ); + let root = run_dir(run_id)?; + let input: RunInput = read_json(&root.join("input.json"))?; + let signer = Issuer::load(&root)?; + let expires = report["expires_at"] + .as_u64() + .context("run expiry is missing")?; + let subject = input + .plan + .people + .get(person) + .context("test person is not in the run's plan")?; + let mut credentials = Vec::new(); + for (product, grant) in &subject.grants { + let token = signer.token(&input, subject, *product, expires)?; + let path = root.join(format!( + "headers-{}-{person}-{}", + product.name(), + Uuid::new_v4().simple() + )); + let headers = match product { + Product::Breg => format!("Authorization: Bearer {}\n", &*token), + Product::Casework => format!("Authorization: Bearer {}\nRegistry-Casework-Profile: {}\nRegistry-Review-Profile: {}\n", + &*token, grant.profile, grant.profile), + }; + write_private(&path, Zeroizing::new(headers).as_bytes())?; + credentials + .push(json!({"product": product, "profile": grant.profile, "header_file": path})); + } + Ok(json!({ + "schema_version": SCHEMA, "run_id": run_id, "status": "token", + "person": person, "credentials": credentials, + })) +} + +fn stop(run_id: &str) -> Result { + let root = run_dir(run_id)?; + if !root.exists() { + return Ok(simple_report(run_id, "stopped")); + } + ensure_private(&root, false)?; + match OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(root.join("stop")) + { + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound && !root.exists() => { + return Ok(simple_report(run_id, "stopped")); + } + Err(error) => return Err(error).context("cannot signal the run to stop"), + } + let deadline = Instant::now() + Duration::from_secs(60); + while root.exists() { + ensure!( + Instant::now() < deadline, + "run {run_id} has not completed cleanup; check the PostgreSQL server and retry stop" + ); + thread::sleep(POLL); + if root.exists() && root.join(REPORT).exists() { + let alive = supervisor_running(&root); + if root.exists() { + ensure!(alive?, "run {run_id} lost its supervisor before cleanup"); + } + } + } + Ok(simple_report(run_id, "stopped")) +} + +fn simple_report(run_id: &str, status: &str) -> Value { + json!({"schema_version": SCHEMA, "run_id": run_id, "status": status}) +} + +fn supervisor_running(root: &Path) -> Result { + ensure_private(root, false)?; + let lock = + File::open(root.join("worker.lock")).context("run supervisor lock is unavailable")?; + match flock(&lock, FlockOperation::NonBlockingLockExclusive) { + Ok(()) => Ok(false), + Err(rustix::io::Errno::WOULDBLOCK) => Ok(true), + Err(error) => Err(error).context("cannot check run supervisor lock"), + } +} + +fn signals() -> Result> { + let terminate = Arc::new(AtomicBool::new(false)); + for signal in [signal_hook::consts::SIGTERM, signal_hook::consts::SIGINT] { + signal_hook::flag::register(signal, terminate.clone())?; + } + Ok(terminate) +} + +fn stopping(root: &Path, signals: &AtomicBool) -> bool { + signals.load(Ordering::Relaxed) || root.join("stop").exists() +} + +fn binaries(product: Product) -> Result { + let get = |name: &str| -> Result { + let path = env::var_os(name) + .map(PathBuf::from) + .with_context(|| format!("{name} is required for the development run"))?; + let path = path + .canonicalize() + .with_context(|| format!("cannot resolve {name}"))?; + ensure!(path.is_file(), "{name} must be a binary file"); + Ok(path) + }; + Ok(match product { + Product::Breg => Binaries { + tool: get("RSTACK_BREGCTL_BIN")?, + runtime: get("RSTACK_BREG_BIN")?, + }, + Product::Casework => Binaries { + tool: get("RSTACK_CASEWORKCTL_BIN")?, + runtime: get("RSTACK_CASEWORK_BIN")?, + }, + }) +} + +fn parse_ttl(value: &str) -> Result { + let error = || anyhow::anyhow!("--ttl must be between 1s and 24h, with an s, m, or h suffix"); + let (number, unit) = value + .split_at_checked(value.len().saturating_sub(1)) + .ok_or_else(error)?; + let seconds = match unit { + "s" => 1, + "m" => 60, + "h" => 3600, + _ => return Err(error()), + }; + number + .parse::() + .ok() + .and_then(|value| value.checked_mul(seconds)) + .filter(|seconds| (1..=86400).contains(seconds)) + .ok_or_else(error) +} + +fn state_dir() -> PathBuf { + env::var_os("RSTACK_DEV_STATE_DIR").map_or_else( + || env::temp_dir().join(format!("rstack-dev-{}", rustix::process::getuid().as_raw())), + PathBuf::from, + ) +} + +fn run_dir(run_id: &str) -> Result { + ensure!( + run_id.len() == 32 + && run_id + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)), + "invalid run ID; use the ID printed by rstack dev start" + ); + Ok(state_dir().join(run_id)) +} + +fn private_directory(path: &Path) -> std::io::Result<()> { + fs::DirBuilder::new().mode(0o700).create(path) +} + +fn ensure_private(path: &Path, create: bool) -> Result<()> { + if create { + match private_directory(path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => { + return Err(error).context("cannot create the development state directory") + } + } + } + let info = + fs::symlink_metadata(path).context("cannot inspect the development state directory")?; + ensure!( + info.is_dir() + && info.uid() == rustix::process::getuid().as_raw() + && info.mode() & 0o077 == 0, + "the development state directory must be owner-only and must not be a symlink" + ); + Ok(()) +} + +fn read_json(path: &Path) -> Result { + Ok(serde_json::from_slice(&plan::read(path, 512 * 1024)?)?) +} + +fn write_private(path: &Path, bytes: &[u8]) -> Result<()> { + let mut file = + tempfile::NamedTempFile::new_in(path.parent().context("file has no directory")?)?; + file.write_all(bytes)?; + file.persist(path).map_err(|error| error.error)?; + Ok(()) +} + +fn clock() -> Result { + Ok(SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs()) +} + +#[cfg(test)] +mod tests { + use super::parse_ttl; + + #[test] + fn ttl_is_bounded_and_requires_a_unit() { + assert_eq!(parse_ttl("1s").unwrap(), 1); + assert_eq!(parse_ttl("30m").unwrap(), 1800); + assert_eq!(parse_ttl("24h").unwrap(), 86400); + for invalid in ["", "0s", "3600", "25h", "18446744073709551615h", "é"] { + assert!(parse_ttl(invalid).is_err(), "{invalid}"); + } + } +} diff --git a/crates/registry-bregctl/src/project_dev/plan.rs b/crates/registry-bregctl/src/project_dev/plan.rs new file mode 100644 index 0000000000..ad20279eb8 --- /dev/null +++ b/crates/registry-bregctl/src/project_dev/plan.rs @@ -0,0 +1,962 @@ +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::{bail, Context, Result}; +use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; +use registry_platform_yaml::{ApiVersion, EnvelopeRule, Expect, FormatSpec, Reader}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + io::Read, + path::{Component, Path, PathBuf}, +}; +use url::Url; + +pub(crate) const FORMAT: FormatSpec<'static> = FormatSpec { + kind: "ProjectDevPlan", + envelope: EnvelopeRule::ApiVersionKind { + api_versions: &[ApiVersion::current( + "id.registrystack.org/formats/project-dev/v1alpha1", + )], + retired_api_versions: &[], + }, + removed_keys: &[], +}; + +#[cfg(feature = "schema")] +pub(crate) fn schema() -> schemars::Schema { + schemars::schema_for!(Plan) +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "lowercase")] +pub(super) enum Product { + Breg, + Casework, +} + +impl Product { + pub(super) fn name(self) -> &'static str { + match self { + Self::Breg => "breg", + Self::Casework => "casework", + } + } +} + +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct Plan { + pub issuer: String, + pub products: Vec, + pub people: BTreeMap, + pub journey: Journey, + #[serde(default)] + pub bindings: super::bindings::Bindings, + #[serde(default)] + pub schema_test_people: BTreeMap>, +} + +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct ProductPlan { + pub product: Product, + pub project: PathBuf, + pub audience: String, +} + +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct Person { + pub subject: String, + pub client: String, + pub grants: BTreeMap, + /// Forward the same authority to both products. Both grants must declare the same scopes and additional claims. + #[serde(default, rename = "caseworkContext")] + pub casework_context: bool, +} + +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct Grant { + pub profile: String, + pub scopes: Vec, + #[serde(default)] + pub claims: Map, +} + +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct Journey { + pub id: String, + pub steps: Vec, +} + +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct Step { + pub id: String, + pub product: Product, + #[serde(rename = "as")] + pub person: String, + pub method: Method, + pub path: String, + pub body: Option, + pub raw: Option, + #[serde(default)] + pub headers: BTreeMap, + pub expect: Expected, + #[serde(default)] + pub capture: BTreeMap, + pub poll: Option, +} + +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "UPPERCASE")] +pub(super) enum Method { + Get, + Post, + Put, + Patch, + Delete, +} + +impl From<&Method> for reqwest::Method { + fn from(method: &Method) -> Self { + match method { + Method::Get => Self::GET, + Method::Post => Self::POST, + Method::Put => Self::PUT, + Method::Patch => Self::PATCH, + Method::Delete => Self::DELETE, + } + } +} + +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct Expected { + pub status: ExpectedStatus, + #[serde(default)] + pub json: BTreeMap, + #[serde(default)] + pub checks: Vec, + pub code: Option, +} + +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(untagged)] +pub(super) enum ExpectedStatus { + One(u16), + Many(Vec), +} + +impl ExpectedStatus { + pub(super) fn first(&self) -> u16 { + match self { + Self::One(status) => *status, + Self::Many(statuses) => statuses[0], + } + } + + pub(super) fn contains(&self, status: u16) -> bool { + match self { + Self::One(expected) => *expected == status, + Self::Many(expected) => expected.contains(&status), + } + } + + fn values(&self) -> Vec { + match self { + Self::One(status) => vec![*status], + Self::Many(statuses) => statuses.clone(), + } + } +} + +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct Check { + pub pointer: String, + pub equals: Option, + pub length: Option, + pub contains: Option, +} + +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(untagged)] +pub(super) enum Capture { + Pointer(String), + Header(HeaderCapture), + Find(FindCapture), + Value(ValueCapture), +} + +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct HeaderCapture { + pub header: CapturedHeader, +} + +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "lowercase")] +pub(super) enum CapturedHeader { + Etag, +} + +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct ValueCapture { + pub pointer: String, + pub query: Option, +} + +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct FindCapture { + pub find: String, + pub r#where: FindWhere, + pub pointer: String, +} + +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct FindWhere { + pub pointer: String, + pub equals: Value, +} + +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct Poll { + pub timeout_ms: u64, + pub interval_ms: u64, +} + +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct RawBody { + pub content_type: String, + pub base64: String, +} + +pub(super) fn identifier(value: &str) -> bool { + (1..=64).contains(&value.len()) + && value + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') +} + +fn bounded_atom(value: &str) -> bool { + (1..=256).contains(&value.len()) && !value.chars().any(|c| c.is_control() || c.is_whitespace()) +} + +pub(super) fn reserved_claim(claim: &str) -> bool { + matches!( + claim, + "iss" + | "aud" + | "sub" + | "client_id" + | "azp" + | "exp" + | "iat" + | "nbf" + | "jti" + | "scope" + | "registry_principal" + | "registry_assertion_issuer" + ) || claim.starts_with("registry_grant_") +} + +fn require(condition: bool, field: &str, message: &str) -> Result<()> { + if !condition { + bail!("dev.yaml {field} {message}"); + } + Ok(()) +} + +pub(super) fn read(path: &Path, limit: u64) -> Result> { + let mut bytes = Vec::new(); + fs::File::open(path)? + .take(limit + 1) + .read_to_end(&mut bytes)?; + anyhow::ensure!(bytes.len() as u64 <= limit, "file exceeds the size limit"); + Ok(bytes) +} + +pub(super) fn load(root: &Path) -> Result { + let file = root.join("dev.yaml"); + let bytes = read(&file, 256 * 1024).context("cannot read dev.yaml")?; + let plan = Reader::new("dev.yaml") + .decode::(&bytes, &Expect::one(&FORMAT))? + .value; + plan.validate(root)?; + Ok(plan) +} + +impl Plan { + pub(super) fn product(&self, product: Product) -> &ProductPlan { + self.products + .iter() + .find(|entry| entry.product == product) + .expect("validated product") + } + + fn validate(&self, root: &Path) -> Result<()> { + require( + Url::parse(&self.issuer).is_ok_and(|url| { + url.scheme() == "https" + && url.host().is_some() + && url.username().is_empty() + && url.password().is_none() + && url.query().is_none() + && url.fragment().is_none() + && self.issuer.len() <= 512 + }), + "/issuer", + "must be the project's HTTPS issuer URL", + )?; + require( + !self.products.is_empty(), + "/products", + "must name at least one product", + )?; + let mut products = BTreeSet::new(); + let mut audiences = BTreeSet::new(); + for (index, entry) in self.products.iter().enumerate() { + let field = format!("/products/{index}"); + require( + products.insert(entry.product), + &format!("{field}/product"), + "is repeated", + )?; + require( + bounded_atom(&entry.audience) && audiences.insert(&entry.audience), + &format!("{field}/audience"), + "must be a distinct, bounded audience", + )?; + require( + entry.project == Path::new(".") + || (!entry.project.as_os_str().is_empty() + && entry + .project + .components() + .all(|part| matches!(part, Component::Normal(_)))), + &format!("{field}/project"), + "must be a path inside the project directory", + )?; + let manifest = match entry.product { + Product::Breg => "registry.yaml", + Product::Casework => "casework.yaml", + }; + require( + root.join(&entry.project) + .join(manifest) + .canonicalize() + .is_ok_and(|path| path.starts_with(root) && path.is_file()), + &format!("{field}/project"), + &format!("must contain {manifest} inside the project directory"), + )?; + } + require( + (1..=32).contains(&self.people.len()), + "/people", + "must define 1 to 32 people", + )?; + for (name, person) in &self.people { + require( + identifier(name), + "/people", + "contains an invalid person name", + )?; + let field = format!("/people/{name}"); + require( + bounded_atom(&person.subject), + &format!("{field}/subject"), + "must be a bounded subject", + )?; + require( + bounded_atom(&person.client), + &format!("{field}/client"), + "must be a bounded client ID", + )?; + require( + !person.grants.is_empty(), + &format!("{field}/grants"), + "must name a product grant", + )?; + for (product, grant) in &person.grants { + let field = format!("{field}/grants/{}", product.name()); + require( + products.contains(product), + &field, + "must name a started product", + )?; + require( + identifier(&grant.profile), + &format!("{field}/profile"), + "must name a profile", + )?; + let scopes = grant.scopes.iter().collect::>(); + require( + scopes.len() == grant.scopes.len() + && scopes.len() <= 32 + && scopes.iter().all(|scope| bounded_atom(scope)), + &format!("{field}/scopes"), + "must contain up to 32 distinct scopes", + )?; + for claim in grant.claims.keys() { + require( + bounded_atom(claim) && !reserved_claim(claim), + &format!("{field}/claims"), + "contains a reserved or invalid claim", + )?; + } + } + if person.casework_context { + require( + self.bindings.casework_context_provider.is_some(), + &format!("{field}/caseworkContext"), + "requires /bindings/caseworkContextProvider", + )?; + let breg = person.grants.get(&Product::Breg); + let casework = person.grants.get(&Product::Casework); + require( + breg.is_some() && casework.is_some(), + &format!("{field}/caseworkContext"), + "requires both BReg and Casework grants", + )?; + let (breg, casework) = ( + breg.expect("checked grant"), + casework.expect("checked grant"), + ); + require( + breg.claims == casework.claims + && breg.scopes.iter().collect::>() + == casework.scopes.iter().collect(), + &format!("{field}/caseworkContext"), + "requires the same scopes and additional claims in both grants", + )?; + } + } + for (journey, steps) in &self.schema_test_people { + for (step, person) in steps { + require( + self.people + .get(person) + .is_some_and(|person| person.grants.contains_key(&Product::Breg)), + &format!("/schemaTestPeople/{journey}/{step}"), + "must name a person with a BReg grant", + )?; + } + } + for product in products { + require( + self.people + .values() + .any(|person| person.grants.contains_key(&product)), + "/people", + &format!("must define at least one grant for {}", product.name()), + )?; + } + self.bindings.validate(self)?; + self.journey.validate(self) + } +} + +impl Journey { + fn validate(&self, plan: &Plan) -> Result<()> { + require( + identifier(&self.id), + "/journey/id", + "must be a local identifier", + )?; + require( + (1..=32).contains(&self.steps.len()), + "/journey/steps", + "must contain 1 to 32 steps", + )?; + let mut ids = BTreeSet::new(); + let mut captures = BTreeSet::new(); + for (index, step) in self.steps.iter().enumerate() { + let field = format!("/journey/steps/{index}"); + require( + step_identifier(&step.id) && ids.insert(&step.id), + &format!("{field}/id"), + "must be a unique local identifier", + )?; + require( + plan.people + .get(&step.person) + .is_some_and(|person| person.grants.contains_key(&step.product)), + &format!("{field}/as"), + "must name a person with a grant for the step's product", + )?; + require( + valid_path(&step.path), + &format!("{field}/path"), + "must be a relative product API path starting with /v1/", + )?; + let statuses = step.expect.status.values(); + require( + (1..=16).contains(&statuses.len()) + && statuses.iter().all(|status| (200..=599).contains(status)) + && statuses.iter().collect::>().len() == statuses.len(), + &format!("{field}/expect/status"), + "must contain 1 to 16 distinct HTTP statuses from 200 to 599", + )?; + require( + step.body.is_none() || step.raw.is_none(), + &field, + "must not contain both body and raw", + )?; + require( + step.raw.is_none() || !matches!(step.method, Method::Get), + &format!("{field}/raw"), + "is supported only for non-GET requests", + )?; + if let Some(body) = &step.body { + require( + serde_json::to_vec(body).is_ok_and(|bytes| bytes.len() <= 1024 * 1024), + &format!("{field}/body"), + "must encode to at most 1 MiB", + )?; + } + if let Some(raw) = &step.raw { + require( + (1..=256).contains(&raw.content_type.len()) + && raw.content_type.contains('/') + && raw + .content_type + .bytes() + .all(|byte| byte.is_ascii() && byte != b'\r' && byte != b'\n'), + &format!("{field}/raw/contentType"), + "must be a bounded HTTP media type", + )?; + require( + raw.base64.len() <= 1_398_104 + && BASE64 + .decode(&raw.base64) + .is_ok_and(|bytes| bytes.len() <= 1024 * 1024), + &format!("{field}/raw/base64"), + "must be valid base64 encoding at most 1 MiB", + )?; + } + if let Some(poll) = &step.poll { + require( + matches!(step.method, Method::Get), + &format!("{field}/poll"), + "is supported only for GET requests", + )?; + require( + (1..=60_000).contains(&poll.timeout_ms) + && (50..=5_000).contains(&poll.interval_ms) + && poll.interval_ms <= poll.timeout_ms + && poll.timeout_ms.div_ceil(poll.interval_ms) < 250, + &format!("{field}/poll"), + "must use a 1 to 60000 ms timeout, a 50 to 5000 ms interval, and at most 250 attempts", + )?; + } + for header in step.headers.keys() { + require( + matches!( + header.as_str(), + "If-Match" | "Idempotency-Key" | "Registry-Source-Profile" + ), + &format!("{field}/headers"), + "contains a header outside the supported list", + )?; + } + let verify_reference = |value: &str| { + captures + .contains(value) + .then(|| value.to_owned()) + .context("must refer to an earlier capture") + }; + expand(&step.path, verify_reference) + .with_context(|| format!("dev.yaml {field}/path"))?; + for value in step.headers.values() { + expand(value, verify_reference) + .with_context(|| format!("dev.yaml {field}/headers"))?; + } + if let Some(body) = &step.body { + expand_json(body, verify_reference) + .with_context(|| format!("dev.yaml {field}/body"))?; + } + require( + step.expect.json.keys().all(|value| pointer(value)), + &format!("{field}/expect/json"), + "must use JSON pointers", + )?; + for value in step.expect.json.values() { + expand_json(value, verify_reference) + .with_context(|| format!("dev.yaml {field}/expect/json"))?; + } + require( + step.expect.checks.len() <= 32, + &format!("{field}/expect/checks"), + "must contain at most 32 checks", + )?; + for check in &step.expect.checks { + require( + pointer(&check.pointer), + &format!("{field}/expect/checks"), + "must use JSON pointers", + )?; + require( + usize::from(check.equals.is_some()) + + usize::from(check.length.is_some()) + + usize::from(check.contains.is_some()) + == 1, + &format!("{field}/expect/checks"), + "must contain exactly one of equals, length, or contains", + )?; + require( + check.length.is_none_or(|length| length <= 1024 * 1024), + &format!("{field}/expect/checks"), + "contains an excessive length", + )?; + for value in check.equals.iter().chain(&check.contains) { + expand_json(value, verify_reference) + .with_context(|| format!("dev.yaml {field}/expect/checks"))?; + } + } + if let Some(code) = &step.expect.code { + require( + bounded_atom(code), + &format!("{field}/expect/code"), + "must be a bounded problem code", + )?; + expand(code, verify_reference) + .with_context(|| format!("dev.yaml {field}/expect/code"))?; + } + for (name, capture) in &step.capture { + require( + capture_identifier(name) && !captures.contains(name.as_str()), + &format!("{field}/capture"), + "must use unique bounded capture names", + )?; + match capture { + Capture::Pointer(pointer_value) => require( + pointer(pointer_value), + &format!("{field}/capture/{name}"), + "must use a JSON pointer", + )?, + Capture::Header(_) => {} + Capture::Value(capture) => { + require( + pointer(&capture.pointer), + &format!("{field}/capture/{name}/pointer"), + "must be a JSON pointer", + )?; + if let Some(query) = &capture.query { + require( + bounded_atom(query), + &format!("{field}/capture/{name}/query"), + "must name a bounded query parameter", + )?; + } + } + Capture::Find(capture) => { + require( + pointer(&capture.find) + && pointer(&capture.r#where.pointer) + && pointer(&capture.pointer), + &format!("{field}/capture/{name}"), + "must use JSON pointers", + )?; + expand_json(&capture.r#where.equals, |value| { + captures + .contains(value) + .then(|| value.to_owned()) + .context("must refer to an earlier capture") + }) + .with_context(|| format!("dev.yaml {field}/capture/{name}/find"))?; + } + } + captures.insert(name.clone()); + } + } + Ok(()) + } +} + +fn step_identifier(value: &str) -> bool { + (1..=64).contains(&value.len()) + && value + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"-.".contains(&byte)) + && !matches!(value.as_bytes().first(), Some(b'-' | b'.')) + && !value.ends_with('.') + && !value.contains("..") +} + +fn capture_identifier(value: &str) -> bool { + (1..=64).contains(&value.len()) + && value.as_bytes()[0].is_ascii_alphabetic() + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') +} + +fn pointer(value: &str) -> bool { + value.is_empty() || value.starts_with('/') +} + +pub(super) fn valid_path(value: &str) -> bool { + value.len() <= 2048 + && value.starts_with("/v1/") + && !value.contains(['\\', '#']) + && !value.chars().any(|c| c.is_control() || c.is_whitespace()) +} + +pub(super) fn expand(value: &str, lookup: impl Fn(&str) -> Result) -> Result { + let mut result = String::new(); + let mut remainder = value; + while let Some((prefix, after)) = remainder.split_once("{{") { + result.push_str(prefix); + let (name, tail) = after + .split_once("}}") + .context("has an unclosed capture reference")?; + result.push_str(&lookup(name)?); + remainder = tail; + } + result.push_str(remainder); + Ok(result) +} + +pub(super) fn expand_json( + value: &Value, + lookup: impl Fn(&str) -> Result + Copy, +) -> Result { + match value { + Value::String(value) => Ok(Value::String(expand(value, lookup)?)), + Value::Array(values) => values + .iter() + .map(|item| expand_json(item, lookup)) + .collect(), + Value::Object(values) => values + .iter() + .map(|(key, item)| Ok((key.clone(), expand_json(item, lookup)?))) + .collect(), + value => Ok(value.clone()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn plan(step: Value) -> (tempfile::TempDir, Plan) { + let root = tempfile::tempdir().unwrap(); + fs::write(root.path().join("registry.yaml"), "fixture").unwrap(); + let plan = serde_json::from_value(json!({ + "issuer": "https://issuer.example.invalid", + "products": [{ + "product": "breg", "project": ".", "audience": "registry" + }], + "people": {"operator": { + "subject": "operator", "client": "operator-client", + "grants": {"breg": {"profile": "operator", "scopes": []}} + }}, + "journey": {"id": "smoke", "steps": [step]} + })) + .unwrap(); + (root, plan) + } + + fn canonical(root: &tempfile::TempDir) -> PathBuf { + root.path().canonicalize().unwrap() + } + + #[test] + fn studio_journey_wire_variants_deserialize() { + let (_, plan) = plan(json!({ + "id": "create.v1", "product": "breg", "as": "operator", + "method": "POST", "path": "/v1/records", + "raw": {"contentType": "application/octet-stream", "base64": "YQ=="}, + "expect": { + "status": [200, 201], "code": "request.accepted", + "checks": [ + {"pointer": "/items", "length": 1}, + {"pointer": "/state", "equals": "ready"}, + {"pointer": "/tags", "contains": "open"} + ] + }, + "capture": { + "requestId": {"pointer": "/requestId", "query": "id"}, + "etag": {"header": "etag"}, + "selected-item": { + "find": "/items", "where": {"pointer": "/id", "equals": "one"}, + "pointer": "/value" + } + } + })); + assert_eq!(plan.journey.steps[0].expect.status.first(), 200); + assert!(plan.journey.steps[0].expect.status.contains(201)); + assert!(matches!( + plan.journey.steps[0].capture["etag"], + Capture::Header(_) + )); + assert!(!plan.people["operator"].casework_context); + } + + #[test] + fn validation_accepts_dotted_steps_and_camel_case_captures() { + let (root, plan) = plan(json!({ + "id": "read.v1", "product": "breg", "as": "operator", + "method": "GET", "path": "/v1/records", + "expect": {"status": [200, 202]}, + "capture": {"recordId": "/recordIdentifier"}, + "poll": {"timeoutMs": 500, "intervalMs": 50} + })); + plan.validate(&canonical(&root)).unwrap(); + } + + #[test] + fn validation_refuses_polling_a_mutation() { + let (root, plan) = plan(json!({ + "id": "create", "product": "breg", "as": "operator", + "method": "POST", "path": "/v1/records", "body": {}, + "expect": {"status": 201}, + "poll": {"timeoutMs": 500, "intervalMs": 50} + })); + let error = plan.validate(&canonical(&root)).unwrap_err().to_string(); + assert!(error.contains("supported only for GET"), "{error}"); + } + + #[test] + fn validation_refuses_forward_capture_references() { + let (root, plan) = plan(json!({ + "id": "read", "product": "breg", "as": "operator", + "method": "GET", "path": "/v1/records/{{later}}", + "expect": {"status": 200}, + "capture": {"later": "/recordIdentifier"} + })); + let error = format!("{:#}", plan.validate(&canonical(&root)).unwrap_err()); + assert!(error.contains("earlier capture"), "{error}"); + } + + #[test] + fn validation_refuses_unbounded_poll_attempts() { + let (root, plan) = plan(json!({ + "id": "read", "product": "breg", "as": "operator", + "method": "GET", "path": "/v1/records", + "expect": {"status": 200}, + "poll": {"timeoutMs": 30000, "intervalMs": 50} + })); + let error = plan.validate(&canonical(&root)).unwrap_err().to_string(); + assert!(error.contains("at most 250 attempts"), "{error}"); + } + + #[test] + fn validation_accepts_the_studio_review_poll() { + let (root, plan) = plan(json!({ + "id": "read", "product": "breg", "as": "operator", + "method": "GET", "path": "/v1/records", "expect": {"status": 200}, + "poll": {"timeoutMs": 60000, "intervalMs": 250} + })); + plan.validate(&canonical(&root)).unwrap(); + } + + #[test] + fn casework_context_requires_the_binding_and_both_grants() { + let (root, plan) = plan(json!({ + "id": "read", "product": "breg", "as": "operator", + "method": "GET", "path": "/v1/records", "expect": {"status": 200} + })); + let mut value = serde_json::to_value(plan).unwrap(); + value["people"]["operator"]["caseworkContext"] = json!(true); + let plan: Plan = serde_json::from_value(value).unwrap(); + let error = plan.validate(&canonical(&root)).unwrap_err().to_string(); + assert!( + error.contains("requires /bindings/caseworkContextProvider"), + "{error}" + ); + } + + #[test] + fn casework_context_requires_both_grants_to_declare_all_forwarded_authority() { + let root = tempfile::tempdir().unwrap(); + fs::write(root.path().join("registry.yaml"), "fixture").unwrap(); + fs::write(root.path().join("casework.yaml"), "fixture").unwrap(); + let value = |casework_jurisdiction: &str, casework_scopes: &[&str]| { + json!({ + "issuer": "https://issuer.example.invalid", + "products": [ + {"product": "breg", "project": ".", "audience": "registry"}, + {"product": "casework", "project": ".", "audience": "review"} + ], + "people": { + "context-reader": { + "subject": "context-reader", "client": "context-client", + "grants": {"breg": { + "profile": "context-reader", "scopes": ["registry:read"] + }} + }, + "reviewer": { + "subject": "reviewer", "client": "reviewer-client", + "caseworkContext": true, + "grants": { + "breg": { + "profile": "reviewer", "scopes": ["registry:read", "casework:review"], + "claims": {"jurisdiction": "north"} + }, + "casework": { + "profile": "reviewer", "scopes": casework_scopes, + "claims": {"jurisdiction": casework_jurisdiction} + } + } + } + }, + "bindings": {"caseworkContextProvider": { + "id": "registry", "person": "context-reader", "action": "review-record" + }}, + "journey": {"id": "review", "steps": [{ + "id": "read", "product": "breg", "as": "reviewer", + "method": "GET", "path": "/v1/records", "expect": {"status": 200} + }]} + }) + }; + let shared_scopes = ["casework:review", "registry:read"]; + let valid: Plan = serde_json::from_value(value("north", &shared_scopes)).unwrap(); + valid.validate(&canonical(&root)).unwrap(); + + let mut one_sided_claim = value("north", &shared_scopes); + one_sided_claim["people"]["reviewer"]["grants"]["breg"]["claims"]["extra"] = json!(true); + for invalid in [ + value("south", &shared_scopes), + value("north", &["casework:review"]), + one_sided_claim, + ] { + let invalid: Plan = serde_json::from_value(invalid).unwrap(); + let error = invalid.validate(&canonical(&root)).unwrap_err().to_string(); + assert!( + error.contains("same scopes and additional claims"), + "{error}" + ); + } + } +} diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs new file mode 100644 index 0000000000..8635cb7377 --- /dev/null +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -0,0 +1,883 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Activate ordinary packages and serve them on the run's private bindings. + +use anyhow::{bail, ensure, Context, Result}; +use registry_breg::fixtures::journey_step_profiles; +use rustix::process::{kill_process_group, waitid, Pid, Signal, WaitId, WaitIdOptions}; +use serde::Serialize; +use serde_json::{json, Value}; +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + io::{Read, Seek}, + net::{Ipv4Addr, TcpListener}, + os::unix::process::CommandExt, + path::{Path, PathBuf}, + process::{Child, Command, ExitStatus, Stdio}, + sync::{atomic::AtomicBool, Arc}, + thread, + time::{Duration, Instant}, +}; + +use super::{ + bindings::RuntimeBindings, + clock, + database::{Database, DatabaseUrls}, + issuer::Issuer, + plan::{self, Plan, Product}, + stopping, write_private, BregAuthority, RunInput, POLL, +}; + +pub(super) struct ProductRunner<'a> { + input: Arc, + root: &'a Path, + terminate: &'a AtomicBool, + deadline: Instant, + runtime: tokio::runtime::Runtime, + http: reqwest::Client, + children: Vec<(Product, Child)>, + bindings: Option, +} + +impl<'a> ProductRunner<'a> { + pub(super) fn new( + input: Arc, + root: &'a Path, + terminate: &'a AtomicBool, + ) -> Result { + Ok(Self { + input, + root, + terminate, + deadline: Instant::now() + Duration::from_secs(300), + runtime: tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?, + http: reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(5)) + .build()?, + children: Vec::new(), + bindings: None, + }) + } + + pub(super) fn check(&self) -> Result<()> { + for entry in &self.input.plan.products { + if entry.product == Product::Casework + && self.input.plan.bindings.casework_context_provider.is_some() + { + // The owning tool validates the paired provider export before + // packaging; the published source is left intact. + continue; + } + self.tool( + entry.product, + "check", + &[self.input.project.join(&entry.project).as_os_str()], + )?; + } + Ok(()) + } + + pub(super) fn start( + &mut self, + database: &mut Database, + issuer: Arc, + ) -> Result> { + // Keep the reservations for later products until each is ready to bind. + let ports: Vec<_> = self + .input + .plan + .products + .iter() + .map(|_| TcpListener::bind((Ipv4Addr::LOCALHOST, 0))) + .collect::>() + .context("reserve loopback listeners")?; + let urls = self + .input + .plan + .products + .iter() + .zip(&ports) + .map(|(entry, listener)| { + Ok((entry.product, format!("http://{}", listener.local_addr()?))) + }) + .collect::>>()?; + self.bindings = Some(RuntimeBindings::prepare( + self.root, + self.input.clone(), + issuer.clone(), + &urls, + )?); + let mut breg_package = None; + let input = self.input.clone(); + for (entry, reservation) in input.plan.products.iter().zip(ports) { + self.ensure_active()?; + let product = entry.product; + let project = self.input.project.join(&entry.project); + let url = &urls[&product]; + let bind = url.strip_prefix("http://").expect("local URL"); + let config = match product { + Product::Breg => { + let (config, package) = + self.activate_breg(database, &issuer, &project, bind)?; + breg_package = Some(package); + config + } + Product::Casework => { + self.activate_casework(database, &project, bind, breg_package.as_deref())? + } + }; + let binaries = &self.input.binaries[&product]; + let mut command = child_command(&binaries.runtime); + command.arg("--runtime-config").arg(config); + if product == Product::Casework { + command.arg("serve"); + } + // Released runtimes own their listeners. If a competing process + // takes this port in the handoff, startup fails and is cleaned up. + drop(reservation); + let child = command + .stdout(Stdio::null()) + .spawn() + .with_context(|| format!("cannot start {}", product.name()))?; + self.children.push((product, child)); + self.ready(product, url)?; + } + Ok(urls) + } + + fn activate_breg( + &self, + database: &mut Database, + issuer: &Issuer, + project: &Path, + bind: &str, + ) -> Result<(PathBuf, PathBuf)> { + let rehearsal = database.provision("breg_test", true)?; + let empty_package = self.root.join("breg-test-package"); + fs::create_dir(&empty_package)?; + let config = self.config( + Product::Breg, + "breg-test", + "127.0.0.1:0", + &empty_package, + rehearsal, + )?; + let credentials = self.schema_credentials(issuer)?; + let receipt = self.root.join("schema-test-receipt.json"); + self.tool( + Product::Breg, + "test", + &[ + project.as_os_str(), + "--runtime-config".as_ref(), + config.as_os_str(), + "--credentials".as_ref(), + credentials.as_os_str(), + "--output".as_ref(), + receipt.as_os_str(), + ], + )?; + let build = self.root.join("breg-build"); + self.tool( + Product::Breg, + "package", + &[ + project.as_os_str(), + "--test-receipt".as_ref(), + receipt.as_os_str(), + "--output".as_ref(), + build.as_os_str(), + ], + )?; + let package = build.join("package"); + let urls = database.provision("breg", true)?; + let config = self.config(Product::Breg, "breg", bind, &package, urls)?; + self.tool( + Product::Breg, + "apply", + &[ + "--runtime-config".as_ref(), + config.as_os_str(), + "--package".as_ref(), + package.as_os_str(), + "--initial".as_ref(), + ], + )?; + self.tool( + Product::Breg, + "verify", + &["--runtime-config".as_ref(), config.as_os_str()], + )?; + Ok((config, package)) + } + + fn schema_credentials(&self, issuer: &Issuer) -> Result { + let mut bindings = Vec::with_capacity(self.input.schema_people.len()); + for (index, binding) in self.input.schema_people.iter().enumerate() { + let token = issuer.token( + &self.input, + &self.input.plan.people[&binding.person], + Product::Breg, + clock()? + 300, + )?; + let filename = format!("breg-schema-token-{index}"); + write_private(&self.root.join(&filename), token.as_bytes())?; + bindings.push(json!({ + "journeyId": binding.journey, "stepId": binding.step, + "credential": {"type": "bearer", "tokenRef": format!("secret:file/{filename}")} + })); + } + let path = self.root.join("schema-test-credentials.json"); + write_private( + &path, + &serde_json::to_vec(&json!({ + "apiVersion": crate::test_lifecycle::CREDENTIALS_API_VERSION, + "kind": crate::test_lifecycle::CREDENTIALS_KIND, + "bindings": bindings + }))?, + )?; + Ok(path) + } + + fn activate_casework( + &self, + database: &mut Database, + project: &Path, + bind: &str, + breg_package: Option<&Path>, + ) -> Result { + let paired; + let project = if let Some(binding) = &self.input.plan.bindings.casework_context_provider { + let source = + breg_package.context("the Casework context provider needs the BReg package")?; + paired = self.root.join("casework-paired"); + copy_project(project, &paired)?; + self.tool( + Product::Casework, + "provider", + &[ + "pair".as_ref(), + paired.as_os_str(), + "--provider-id".as_ref(), + binding.id.as_ref(), + "--breg-package".as_ref(), + source.as_os_str(), + "--action-id".as_ref(), + binding.action.as_ref(), + "--bregctl-bin".as_ref(), + self.input.binaries[&Product::Breg].tool.as_os_str(), + "--apply".as_ref(), + ], + )?; + self.tool(Product::Casework, "check", &[paired.as_os_str()])?; + &paired + } else { + project + }; + let package = self.root.join("casework-package"); + self.tool( + Product::Casework, + "package", + &[ + project.as_os_str(), + "--output".as_ref(), + package.as_os_str(), + ], + )?; + let urls = database.provision("casework", false)?; + let config = self.config(Product::Casework, "casework", bind, &package, urls)?; + let arguments = ["--runtime-config".as_ref(), config.as_os_str()]; + self.tool(Product::Casework, "apply", &arguments)?; + self.tool(Product::Casework, "doctor", &arguments)?; + Ok(config) + } + + fn config( + &self, + product: Product, + name: &str, + bind: &str, + package: &Path, + urls: DatabaseUrls, + ) -> Result { + let mut clients: BTreeSet<_> = self + .input + .plan + .people + .values() + .filter(|person| person.grants.contains_key(&product)) + .map(|person| person.client.as_str()) + .collect(); + if product == Product::Breg { + let authority = self.input.breg_authority.as_ref().expect("BReg authority"); + clients.extend(authority.named_clients.iter().map(String::as_str)); + } + let mut oidc = json!({ + "issuer": self.input.plan.issuer, + "audience": self.input.plan.product(product).audience, + "allowedClients": clients, + "scopeClaim": "scope", + "jwksSource": {"type": "static", "documentRef": "secret:file/jwks"} + }); + let mut database = json!({ + "runtimeUrlRef": self.secret(&format!("{name}-runtime-url"), urls.runtime.as_bytes())?, + "migrationUrlRef": self.secret(&format!("{name}-migration-url"), urls.migration.as_bytes())? + }); + if self.root.join("postgres-ca").is_file() { + database["trustedRootCertificateRef"] = json!("secret:file/postgres-ca"); + } + let audit = json!({ + "hashKeyRef": self.secret(&format!("{name}-audit-key"), random_secret()?.as_bytes())?, + "path": self.root.join(format!("{name}-audit")).join("audit.jsonl"), + }); + let identity = format!("dev-{}-{name}", self.input.run_id); + let mut config = json!({ + "package": {"root": package}, + "secretProviders": {"file": {"root": self.root}}, + "database": database, + "audit": audit, + }); + match product { + Product::Breg => { + oidc["allowedAlgorithm"] = json!("ES256"); + oidc["accessTokenType"] = json!("at+jwt"); + oidc["scopeSeparator"] = json!(" "); + oidc["maximumTokenLifetimeSeconds"] = json!(300); + oidc["leewayMilliseconds"] = json!(1000); + oidc["jwksCache"] = json!({ + "cacheTtlSeconds": 600, "negativeCacheTtlSeconds": 60, + "refreshCooldownSeconds": 30, "maximumDocumentBytes": 65536, + "attemptTimeoutMilliseconds": 5000, "outageToleranceSeconds": 0 + }); + config["apiVersion"] = json!("id.registrystack.org/formats/breg/runtime/v1alpha1"); + config["kind"] = json!("BRegRuntimeConfig"); + config["database"]["pool"] = json!({"maximumConnections": 4}); + config["database"]["roles"] = json!({ + "runtime": url::Url::parse(&urls.runtime)?.username(), + "migration": url::Url::parse(&urls.migration)?.username(), + }); + config["audit"]["destination"] = json!("file"); + config["listener"] = json!({"bind": bind}); + config["identity"] = json!({ + "environment": "development", "instanceId": identity, + "databaseId": identity, "databaseInitializationEnvironment": "development" + }); + let authority = self.input.breg_authority.as_ref().expect("BReg authority"); + let mut authority_claims = json!({"principal": authority.principal}); + if authority.purpose { + authority_claims["purpose"] = json!("registry_purpose"); + } + config["authentication"] = json!({ + "oidc": oidc, + "authorityClaims": authority_claims + }); + config["cursor"] = json!({ + "secretRef": self.secret(&format!("{name}-cursor-key"), random_secret()?.as_bytes())?, "maximumAgeSeconds": 300 + }); + config["destinations"] = json!({"hooks": {}}); + config["operationalTimeouts"] = json!({ + "httpRequestMilliseconds": 10000, "shutdownGraceMilliseconds": 1000, + "recordLockMilliseconds": 5000, "migrationLockMilliseconds": 30000, + "migrationStatementMilliseconds": 60000 + }); + } + Product::Casework => { + config["apiVersion"] = + json!("id.registrystack.org/formats/casework/runtime/v1alpha1"); + config["kind"] = json!("CaseworkRuntimeConfig"); + config["listener"] = + json!({"bind": bind, "tlsTermination": "development-loopback"}); + config["identity"] = json!({"databaseId": identity}); + config["authentication"] = json!({"oidc": oidc}); + } + } + if let Some(bindings) = &self.bindings { + bindings.configure(product, &mut config); + } + let path = self.root.join(format!("{name}-runtime.yaml")); + write_private(&path, &serde_json::to_vec(&config)?)?; + Ok(path) + } + + fn secret(&self, name: &str, bytes: &[u8]) -> Result { + write_private(&self.root.join(name), bytes)?; + Ok(format!("secret:file/{name}")) + } + + fn tool(&self, product: Product, stage: &str, args: &[&std::ffi::OsStr]) -> Result<()> { + self.ensure_active()?; + let mut output = tempfile::tempfile_in(self.root)?; + let mut child = child_command(&self.input.binaries[&product].tool) + .arg(stage) + .args(args) + .args(["--format", "json"]) + .stdout(output.try_clone()?) + .spawn() + .with_context(|| format!("cannot start {} {stage}", product.name()))?; + loop { + if let Err(error) = self.ensure_active() { + stop_child(&mut child)?; + return Err(error); + } + if child_exited(&child)? { + let status = stop_child(&mut child)?; + if status.success() { + return Ok(()); + } + output.rewind()?; + let codes = report_codes(output)?; + bail!("{} {stage} failed{codes}", product.name()); + } + thread::sleep(POLL); + } + } + + fn ready(&mut self, product: Product, base: &str) -> Result<()> { + let end = Instant::now() + Duration::from_secs(30); + let url = format!("{base}/ready"); + loop { + self.ensure_active()?; + ensure!( + self.services_alive()?, + "{} exited before readiness", + product.name() + ); + if self + .runtime + .block_on(async { self.http.get(&url).send().await }) + .is_ok_and(|response| response.status().is_success()) + { + ensure!( + self.services_alive()?, + "{} exited before readiness", + product.name() + ); + return Ok(()); + } + ensure!( + Instant::now() < end, + "{} did not become ready", + product.name() + ); + thread::sleep(POLL); + } + } + + pub(super) fn services_alive(&self) -> Result { + for (_, child) in &self.children { + if child_exited(child)? { + return Ok(false); + } + } + Ok(true) + } + + pub(super) fn ensure_active(&self) -> Result<()> { + ensure!( + !stopping(self.root, self.terminate), + "development run was stopped" + ); + ensure!( + Instant::now() < self.deadline, + "development run did not start within five minutes" + ); + Ok(()) + } + + pub(super) fn stop(&mut self) -> Result<()> { + let mut failures = Vec::new(); + self.children.retain_mut(|(product, child)| { + if stop_child(child).is_err() { + failures.push(product.name()); + true + } else { + false + } + }); + ensure!(failures.is_empty(), "cannot stop {}", failures.join(", ")); + drop(self.bindings.take()); + Ok(()) + } + + pub(super) fn journey( + &self, + issuer: &Issuer, + urls: &BTreeMap, + ) -> Result { + super::journey::run(&self.input, issuer, urls, || self.ensure_active()) + } +} + +fn random_secret() -> Result> { + use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; + let mut bytes = zeroize::Zeroizing::new([0u8; 32]); + getrandom::fill(bytes.as_mut()).map_err(|_| anyhow::anyhow!("cannot generate a local key"))?; + let mut encoded = zeroize::Zeroizing::new(String::new()); + URL_SAFE_NO_PAD.encode_string(bytes.as_slice(), &mut encoded); + Ok(encoded) +} + +fn copy_project(source: &Path, target: &Path) -> Result<()> { + fn copy_dir( + source: &Path, + target: &Path, + depth: usize, + count: &mut usize, + bytes: &mut u64, + ) -> Result<()> { + ensure!( + depth <= 16, + "Casework project directories exceed the depth limit" + ); + fs::create_dir(target)?; + for entry in fs::read_dir(source)? { + let entry = entry?; + if entry.file_name() == ".git" { + continue; + } + *count += 1; + ensure!( + *count <= 256, + "Casework project exceeds the file count limit" + ); + let info = fs::symlink_metadata(entry.path())?; + let destination = target.join(entry.file_name()); + if info.is_dir() { + copy_dir(&entry.path(), &destination, depth + 1, count, bytes)?; + } else { + ensure!( + info.is_file(), + "Casework project contains a symlink or non-file entry" + ); + *bytes = bytes.saturating_add(info.len()); + ensure!( + *bytes <= 32 * 1024 * 1024, + "Casework project exceeds the copy size limit" + ); + fs::copy(entry.path(), destination)?; + } + } + Ok(()) + } + copy_dir(source, target, 0, &mut 0, &mut 0) +} + +pub(super) fn breg_authority(root: &Path, plan: &Plan) -> Result> { + let Some(entry) = plan + .products + .iter() + .find(|entry| entry.product == Product::Breg) + else { + return Ok(None); + }; + let registry = crate::compile( + &root.join(&entry.project), + crate::ProfileArg::Production, + "project-dev", + ) + .map_err(|_| { + anyhow::anyhow!( + "BReg project check failed; run rstack breg check for the authored diagnostics" + ) + })?; + let inventory = registry_breg::authority::authority_inventory(®istry)?; + ensure!( + inventory.principal_claims.len() <= 1, + "the BReg project's access profiles must select one principal claim" + ); + let principal = inventory + .principal_claims + .into_iter() + .next() + .unwrap_or_else(|| "sub".to_owned()); + for (name, person) in &plan.people { + if let Some(grant) = person.grants.get(&Product::Breg) { + ensure!(!grant.claims.contains_key(&principal), "dev.yaml /people/{name}/grants/breg/claims cannot replace the BReg principal claim"); + } + } + Ok(Some(BregAuthority { + principal, + purpose: inventory.purpose_required, + named_clients: crate::identity_facts::named_requester_clients(®istry), + })) +} + +pub(super) fn casework_principals( + root: &Path, + plan: &Plan, + breg: Option<&BregAuthority>, +) -> Result> { + let Some(entry) = plan + .products + .iter() + .find(|entry| entry.product == Product::Casework) + else { + return Ok(BTreeMap::new()); + }; + let path = root.join(&entry.project).join("casework.yaml"); + let bytes = plan::read(&path, 1024 * 1024).context("cannot read casework.yaml")?; + let Ok(project) = + registry_casework_core::CaseworkProject::read(&path.display().to_string(), &bytes) + else { + bail!( + "Casework project check failed. Run rstack casework check for the authored diagnostics" + ); + }; + let mut principals = BTreeMap::new(); + for (name, person) in &plan.people { + let Some(grant) = person.grants.get(&Product::Casework) else { + continue; + }; + let profile = project.value.access_profiles.iter().find(|profile| profile.id == grant.profile) + .with_context(|| format!("dev.yaml /people/{name}/grants/casework/profile must name a Casework access profile"))?; + let claim = profile.principal_claim.as_str(); + ensure!( + matches!(claim, "sub" | "registry_principal") + || (!plan::reserved_claim(claim) + && !matches!(claim, "registry_actor_kind" | "registry_purpose")), + "dev.yaml /people/{name}/grants/casework/profile names an incompatible principal claim" + ); + ensure!( + !grant.claims.contains_key(claim), + "dev.yaml /people/{name}/grants/casework/claims cannot replace the Casework principal claim" + ); + if person.casework_context { + ensure!( + claim == "sub" || breg.is_some_and(|authority| authority.principal == claim), + "dev.yaml /people/{name}/caseworkContext requires Casework to read sub or the BReg principal claim" + ); + } + principals.insert(profile.id.clone(), profile.principal_claim.clone()); + } + Ok(principals) +} + +impl Drop for ProductRunner<'_> { + fn drop(&mut self) { + // The normal exit reports stop failures explicitly. + for (_, child) in &mut self.children { + let _ = stop_child(child); + } + } +} + +pub(super) fn child_command(binary: &Path) -> Command { + let mut command = Command::new(binary); + command + .env_remove("RSTACK_DEV_DATABASE_URL") + .env_remove("RSTACK_PROJECT_DEV_REQUEST") + .process_group(0) + .stdin(Stdio::null()) + .stderr(Stdio::null()); + command +} + +fn child_pid(child: &Child) -> Result { + Pid::from_raw(i32::try_from(child.id())?).context("invalid owned process ID") +} + +fn child_exited(child: &Child) -> Result { + let pid = child_pid(child)?; + // Keep the leader's PID until the group has received the final signal. + // Reaping the leader first would allow reuse of the group ID. + let options = WaitIdOptions::EXITED | WaitIdOptions::NOHANG | WaitIdOptions::NOWAIT; + loop { + match waitid(WaitId::Pid(pid), options) { + Ok(status) => return Ok(status.is_some()), + Err(rustix::io::Errno::INTR) => continue, + Err(error) => return Err(error.into()), + } + } +} + +fn signal_group(pid: Pid, signal: Signal) -> Result<()> { + match kill_process_group(pid, signal) { + Ok(()) | Err(rustix::io::Errno::SRCH) => Ok(()), + Err(error) => Err(error.into()), + } +} + +fn stop_child(child: &mut Child) -> Result { + let pid = child_pid(child)?; + if !child_exited(child)? { + signal_group(pid, Signal::TERM)?; + let deadline = Instant::now() + Duration::from_secs(2); + while Instant::now() < deadline && !child_exited(child)? { + thread::sleep(POLL); + } + } + signal_group(pid, Signal::KILL)?; + Ok(child.wait()?) +} + +fn report_codes(output: impl Read) -> Result { + let mut bytes = Vec::new(); + output.take(256 * 1024).read_to_end(&mut bytes)?; + let Ok(value) = serde_json::from_slice::(&bytes) else { + return Ok(String::new()); + }; + let codes: BTreeSet<_> = value["diagnostics"] + .as_array() + .into_iter() + .flatten() + .filter_map(|item| { + let code = item["code"].as_str().filter(|code| { + code.len() <= 128 + && code + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"-_.".contains(&byte)) + })?; + let path = item["path"] + .as_str() + .or_else(|| item["pointer"].as_str()) + .filter(|path| { + !path.is_empty() + && path.len() <= 128 + && path + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"-_./~".contains(&byte)) + }); + Some(path.map_or_else(|| code.to_owned(), |path| format!("{code} at {path}"))) + }) + .take(6) + .collect(); + if codes.is_empty() { + Ok(String::new()) + } else { + Ok(format!( + " ({})", + codes.into_iter().collect::>().join(", ") + )) + } +} + +#[derive(serde::Deserialize, Serialize)] +pub(super) struct SchemaPerson { + journey: String, + step: String, + person: String, +} + +pub(super) fn schema_people(root: &Path, plan: &Plan) -> Result> { + let Some(entry) = plan + .products + .iter() + .find(|entry| entry.product == Product::Breg) + else { + ensure!( + plan.schema_test_people.is_empty(), + "dev.yaml /schemaTestPeople requires a BReg product" + ); + return Ok(Vec::new()); + }; + let path = root.join(&entry.project).join("tests/journeys.yaml"); + let bytes = plan::read(&path, 1024 * 1024).context("cannot read BReg tests/journeys.yaml")?; + let steps = + journey_step_profiles(&bytes).context("cannot validate BReg tests/journeys.yaml")?; + let mut used = BTreeSet::new(); + let mut bindings = Vec::new(); + for step in steps { + let eligible = |person: &plan::Person| { + person.grants.get(&Product::Breg).is_some_and(|grant| { + grant.profile == step.access_profile + && step.scopes.as_ref().is_none_or(|scopes| { + scopes.iter().collect::>() == grant.scopes.iter().collect() + }) + && step.purpose.as_ref().is_none_or(|purpose| { + grant.claims.get("registry_purpose").and_then(Value::as_str) + == Some(purpose) + }) + }) + }; + let explicit = plan + .schema_test_people + .get(&step.journey_id) + .and_then(|steps| steps.get(&step.step_id)); + let aliases: Vec<_> = plan + .people + .iter() + .filter(|(name, person)| { + explicit.is_none_or(|selected| selected == *name) && eligible(person) + }) + .map(|(name, _)| name) + .collect(); + let [person] = aliases.as_slice() else { + bail!("dev.yaml /schemaTestPeople/{}/{} must select one person with matching BReg profile, scope and purpose", step.journey_id, step.step_id); + }; + used.insert((step.journey_id.clone(), step.step_id.clone())); + bindings.push(SchemaPerson { + journey: step.journey_id, + step: step.step_id, + person: (*person).clone(), + }); + } + for (journey, steps) in &plan.schema_test_people { + for step in steps.keys() { + ensure!( + used.contains(&(journey.clone(), step.clone())), + "dev.yaml /schemaTestPeople/{journey}/{step} does not name a BReg fixture step" + ); + } + } + Ok(bindings) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{ + io::{BufRead, BufReader}, + sync::mpsc, + }; + + #[test] + fn cancellation_closes_the_pipe_of_a_nested_tool() { + closes_nested_tool_pipe(false); + } + + #[test] + fn cancellation_cleans_up_a_nested_tool_after_its_leader_exits() { + closes_nested_tool_pipe(true); + } + + fn closes_nested_tool_pipe(leader_exits: bool) { + let leader_end = if leader_exits { "exit 0" } else { "wait" }; + let mut child = child_command(Path::new("/bin/sh")) + .arg("-c") + .arg(format!( + "sh -c 'trap \"\" TERM; printf \"ready\\n\"; exec sleep 60' & {leader_end}" + )) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let pid = child_pid(&child).unwrap(); + assert_eq!(rustix::process::getpgid(Some(pid)).unwrap(), pid); + let mut pipe = BufReader::new(child.stdout.take().unwrap()); + let mut ready = String::new(); + pipe.read_line(&mut ready).unwrap(); + assert_eq!(ready, "ready\n"); + let deadline = Instant::now() + Duration::from_secs(2); + if leader_exits { + while !child_exited(&child).unwrap() { + assert!(Instant::now() < deadline, "the tool leader did not exit"); + thread::sleep(POLL); + } + } + stop_child(&mut child).unwrap(); + + let (sender, receiver) = mpsc::sync_channel(1); + let reader = thread::spawn(move || { + let _ = sender.send(pipe.read(&mut [0])); + }); + let eof = receiver.recv_timeout(Duration::from_secs(3)); + if eof.is_err() { + let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL); + } + reader.join().unwrap(); + assert_eq!(eof.unwrap().unwrap(), 0); + } +} diff --git a/crates/registry-bregctl/src/tool_schema.rs b/crates/registry-bregctl/src/tool_schema.rs index 731a8fa938..803c044193 100644 --- a/crates/registry-bregctl/src/tool_schema.rs +++ b/crates/registry-bregctl/src/tool_schema.rs @@ -13,7 +13,7 @@ use registry_breg::{fixtures, migration_plan}; use registry_platform_yaml::{EnvelopeRule, FormatSpec, VersionStatus}; use serde_json::{json, Map, Value}; -use crate::{dev, init_from_model, test_lifecycle}; +use crate::{dev, init_from_model, project_dev, test_lifecycle}; const SCHEMA_DIALECT: &str = "https://json-schema.org/draft/2020-12/schema"; const SCHEMA_ID_BASE: &str = "https://id.registrystack.org/schemas/"; @@ -27,7 +27,7 @@ struct Tool { members: fn() -> schemars::Schema, } -fn tools() -> [Tool; 7] { +fn tools() -> [Tool; 8] { [ Tool { format: "journeys", @@ -53,6 +53,12 @@ fn tools() -> [Tool; 7] { spec: dev::config::DEV_CLIENTS_FORMAT, members: dev::config::clients_schema, }, + Tool { + format: "project-dev", + title: "Registry Stack project development plan", + spec: project_dev::plan::FORMAT, + members: project_dev::plan::schema, + }, Tool { format: "example-scenarios", title: "Base Registry Engine example scenarios", diff --git a/crates/registry-bregctl/tests/project_dev.rs b/crates/registry-bregctl/tests/project_dev.rs new file mode 100644 index 0000000000..bbe6c8d81d --- /dev/null +++ b/crates/registry-bregctl/tests/project_dev.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::{fs, process::Command}; + +#[test] +fn a_malformed_plan_names_its_field_before_allocating_run_state() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + fs::write( + scratch.path().join("dev.yaml"), + "apiVersion: id.registrystack.org/formats/project-dev/v1alpha1\nkind: ProjectDevPlan\nissuer: https://example.test\nproducts: [{product: coordinator, project: registry, audience: test}]\npeople: {}\njourney: {id: licensing, steps: []}\n", + ) + .unwrap(); + let request = serde_json::json!({ + "action": "start", "project": scratch.path(), "json": true, + }); + let result = Command::new(env!("CARGO_BIN_EXE_bregctl")) + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request.to_string()) + .env("RSTACK_DEV_STATE_DIR", &state) + .env_remove("RSTACK_DEV_DATABASE_URL") + .output() + .unwrap(); + assert!(!result.status.success()); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + assert_eq!(report["schema_version"], "registry.rstack.dev.v1"); + assert!( + report["error"] + .as_str() + .unwrap() + .contains("/products/0/product"), + "{report}" + ); + assert!(!state.exists()); +} + +#[test] +fn a_run_id_cannot_name_a_path() { + let scratch = tempfile::tempdir().unwrap(); + let request = serde_json::json!({ + "action": "stop", "run_id": "../other-run", "json": true, + }); + let result = Command::new(env!("CARGO_BIN_EXE_bregctl")) + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request.to_string()) + .env("RSTACK_DEV_STATE_DIR", scratch.path()) + .output() + .unwrap(); + assert!(!result.status.success()); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + assert!(report["error"].as_str().unwrap().contains("run ID")); + assert_eq!(fs::read_dir(scratch.path()).unwrap().count(), 0); +} diff --git a/crates/registry-bregctl/tests/tool_schema.rs b/crates/registry-bregctl/tests/tool_schema.rs index b4802bebfa..5f30c77ace 100644 --- a/crates/registry-bregctl/tests/tool_schema.rs +++ b/crates/registry-bregctl/tests/tool_schema.rs @@ -31,6 +31,10 @@ const EXAMPLES: &[(&str, &str)] = &[ "dev-clients.v1alpha1.schema.json", "products/breg/acceptance/facility/dev-clients.yaml", ), + ( + "project-dev.v1alpha1.schema.json", + "products/platform/examples/licensing/dev.yaml", + ), ( "example-scenarios.v1alpha1.schema.json", "products/breg/examples/minimal/examples/scenarios.json", @@ -401,13 +405,14 @@ fn a_wrong_header_a_missing_header_and_an_explicit_null_are_refused() { } /// The identifiers the catalog records, in file-name order. -const IDENTIFIERS: [&str; 7] = [ +const IDENTIFIERS: [&str; 8] = [ "https://id.registrystack.org/schemas/breg/backup-binding/backup-binding.v1alpha1.schema.json", "https://id.registrystack.org/schemas/breg/dev-clients/dev-clients.v1alpha1.schema.json", "https://id.registrystack.org/schemas/breg/example-scenarios/example-scenarios.v1alpha1.schema.json", "https://id.registrystack.org/schemas/breg/journeys/journeys.v1.schema.json", "https://id.registrystack.org/schemas/breg/migration-descriptor/migration-descriptor.v1alpha1.schema.json", "https://id.registrystack.org/schemas/breg/model-selection/model-selection.v1alpha1.schema.json", + "https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json", "https://id.registrystack.org/schemas/breg/schema-test-credentials/schema-test-credentials.v1.schema.json", ]; diff --git a/crates/registry-cli-docs/catalog.json b/crates/registry-cli-docs/catalog.json index 34fbd4c741..df8852d306 100644 --- a/crates/registry-cli-docs/catalog.json +++ b/crates/registry-cli-docs/catalog.json @@ -25860,6 +25860,420 @@ "subcommands": [] } ] + }, + { + "name": "dev", + "invocation": "rstack dev", + "about": "Run and inspect a local Registry Stack development environment", + "long_about": null, + "usage": "rstack dev [OPTIONS] ", + "arguments": [], + "options": [ + { + "display": "--format ", + "description": "Report encoding. Defaults to human even when stdout is piped. Evidence test and fixtures run also accept junit", + "always_required": false, + "repeatable": true, + "default_values": [ + "human" + ], + "possible_values": [ + "human", + "json" + ], + "environment": null + }, + { + "display": "--install-root ", + "description": "Installation generation storage directory", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--bin-dir ", + "description": "Directory for the single public rstack executable link", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--asset-dir ", + "description": "Directory of already downloaded, verified release assets", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "-h, --help", + "description": "Print help", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "constraints": [], + "subcommands": [ + { + "name": "start", + "invocation": "rstack dev start", + "about": "Start a development environment for a registry project", + "long_about": null, + "usage": "rstack dev start [OPTIONS] ", + "arguments": [ + { + "display": "", + "description": "Project directory containing dev.yaml", + "always_required": true, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "options": [ + { + "display": "--database-url ", + "description": "TLS PostgreSQL administrator URL. Defaults to RSTACK_DEV_DATABASE_URL", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--ttl ", + "description": "Lifetime after readiness, from 1s to 24h. Defaults to 30m", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--json", + "description": "Emit the runner's JSON report", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--format ", + "description": "Report encoding. Defaults to human even when stdout is piped. Evidence test and fixtures run also accept junit", + "always_required": false, + "repeatable": true, + "default_values": [ + "human" + ], + "possible_values": [ + "human", + "json" + ], + "environment": null + }, + { + "display": "--install-root ", + "description": "Installation generation storage directory", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--bin-dir ", + "description": "Directory for the single public rstack executable link", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--asset-dir ", + "description": "Directory of already downloaded, verified release assets", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "-h, --help", + "description": "Print help", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "constraints": [], + "subcommands": [] + }, + { + "name": "status", + "invocation": "rstack dev status", + "about": "Report the state of a development environment", + "long_about": null, + "usage": "rstack dev status [OPTIONS] ", + "arguments": [ + { + "display": "", + "description": "Run ID returned by rstack dev start", + "always_required": true, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "options": [ + { + "display": "--json", + "description": "Emit the runner's JSON report", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--format ", + "description": "Report encoding. Defaults to human even when stdout is piped. Evidence test and fixtures run also accept junit", + "always_required": false, + "repeatable": true, + "default_values": [ + "human" + ], + "possible_values": [ + "human", + "json" + ], + "environment": null + }, + { + "display": "--install-root ", + "description": "Installation generation storage directory", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--bin-dir ", + "description": "Directory for the single public rstack executable link", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--asset-dir ", + "description": "Directory of already downloaded, verified release assets", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "-h, --help", + "description": "Print help", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "constraints": [], + "subcommands": [] + }, + { + "name": "token", + "invocation": "rstack dev token", + "about": "Issue a development token for one named synthetic person", + "long_about": null, + "usage": "rstack dev token [OPTIONS] --as ", + "arguments": [ + { + "display": "", + "description": "Run ID returned by rstack dev start", + "always_required": true, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "options": [ + { + "display": "--as ", + "description": "Synthetic person configured by the project", + "always_required": true, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--json", + "description": "Emit the runner's JSON report", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--format ", + "description": "Report encoding. Defaults to human even when stdout is piped. Evidence test and fixtures run also accept junit", + "always_required": false, + "repeatable": true, + "default_values": [ + "human" + ], + "possible_values": [ + "human", + "json" + ], + "environment": null + }, + { + "display": "--install-root ", + "description": "Installation generation storage directory", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--bin-dir ", + "description": "Directory for the single public rstack executable link", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--asset-dir ", + "description": "Directory of already downloaded, verified release assets", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "-h, --help", + "description": "Print help", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "constraints": [], + "subcommands": [] + }, + { + "name": "stop", + "invocation": "rstack dev stop", + "about": "Stop a development environment", + "long_about": null, + "usage": "rstack dev stop [OPTIONS] ", + "arguments": [ + { + "display": "", + "description": "Run ID returned by rstack dev start", + "always_required": true, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "options": [ + { + "display": "--json", + "description": "Emit the runner's JSON report", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--format ", + "description": "Report encoding. Defaults to human even when stdout is piped. Evidence test and fixtures run also accept junit", + "always_required": false, + "repeatable": true, + "default_values": [ + "human" + ], + "possible_values": [ + "human", + "json" + ], + "environment": null + }, + { + "display": "--install-root ", + "description": "Installation generation storage directory", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--bin-dir ", + "description": "Directory for the single public rstack executable link", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--asset-dir ", + "description": "Directory of already downloaded, verified release assets", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "-h, --help", + "description": "Print help", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "constraints": [], + "subcommands": [] + } + ] } ] }, diff --git a/crates/registry-rstack/README.md b/crates/registry-rstack/README.md index a4034dd569..2c471ce436 100644 --- a/crates/registry-rstack/README.md +++ b/crates/registry-rstack/README.md @@ -100,6 +100,12 @@ from the products' own configuration and one operator deployment file, and `rstack identity check FILE` checks that file or a product's identity facts offline. See [Generate ThunderID configuration](../../docs/site/src/content/docs/operate/advanced/generate-identity-configuration.mdx). +## Develop a composed project + +Run BReg and Casework together with `rstack dev start PROJECT` against a +dedicated TLS PostgreSQL server. The project's `dev.yaml` declares the +products, test people and journey. See [Run a project for development](../../products/platform/PROJECT-DEV.md). + ## Command migration | Previous local command | Managed command | diff --git a/crates/registry-rstack/src/cli.rs b/crates/registry-rstack/src/cli.rs index ba3f526af0..fc3c3b31bc 100644 --- a/crates/registry-rstack/src/cli.rs +++ b/crates/registry-rstack/src/cli.rs @@ -5,6 +5,7 @@ use crate::install::{self, BootstrapOptions}; use crate::store::{GenerationStore, InvocationLease}; use crate::ReleaseManifest; use clap::{ArgAction, CommandFactory, Parser, Subcommand, ValueEnum}; +use serde::Serialize; use serde_json::{json, Value}; use std::ffi::OsString; use std::fs; @@ -153,10 +154,88 @@ enum Command { #[command(subcommand)] command: IdentityCommand, }, + /// Run and inspect a local Registry Stack development environment. + Dev { + #[command(subcommand)] + command: DevCommand, + }, #[command(external_subcommand)] Product(Vec), } +#[derive(Subcommand)] +enum DevCommand { + /// Start a development environment for a registry project. + Start { + /// Project directory containing dev.yaml. + #[arg(value_name = "PROJECT")] + project: PathBuf, + /// TLS PostgreSQL administrator URL. Defaults to RSTACK_DEV_DATABASE_URL. + #[arg(long, value_name = "URL")] + database_url: Option, + /// Lifetime after readiness, from 1s to 24h. Defaults to 30m. + #[arg(long, value_name = "DURATION")] + ttl: Option, + /// Emit the runner's JSON report. + #[arg(long)] + json: bool, + }, + /// Report the state of a development environment. + Status { + /// Run ID returned by rstack dev start. + #[arg(value_name = "RUN_ID")] + run_id: String, + /// Emit the runner's JSON report. + #[arg(long)] + json: bool, + }, + /// Issue a development token for one named synthetic person. + Token { + /// Run ID returned by rstack dev start. + #[arg(value_name = "RUN_ID")] + run_id: String, + /// Synthetic person configured by the project. + #[arg(long = "as", value_name = "NAME")] + person: String, + /// Emit the runner's JSON report. + #[arg(long)] + json: bool, + }, + /// Stop a development environment. + Stop { + /// Run ID returned by rstack dev start. + #[arg(value_name = "RUN_ID")] + run_id: String, + /// Emit the runner's JSON report. + #[arg(long)] + json: bool, + }, +} + +#[derive(Serialize)] +#[serde(tag = "action", rename_all = "snake_case")] +enum DevRequest<'a> { + Start { + project: &'a std::path::Path, + #[serde(skip_serializing_if = "Option::is_none")] + ttl: Option<&'a str>, + json: bool, + }, + Status { + run_id: &'a str, + json: bool, + }, + Token { + run_id: &'a str, + person: &'a str, + json: bool, + }, + Stop { + run_id: &'a str, + json: bool, + }, +} + #[derive(Subcommand)] enum IdentityCommand { /// Generate identity-provider configuration from product identity facts. @@ -548,6 +627,7 @@ fn execute(cli: Cli, format_supplied: bool, evidence_junit: bool) -> Result<(), ReportFormat::Human => { reportln!("rstack [options]"); reportln!("Use rstack help for offline product help."); + reportln!("Use rstack dev --help for local development commands."); reportln!("Use rstack components list to inspect installation availability."); reportln!("Products: {}", PRODUCTS.join(", ")); } @@ -691,6 +771,7 @@ fn execute(cli: Cli, format_supplied: bool, evidence_junit: bool) -> Result<(), PRODUCTS.join(" ") ); } + Some(Command::Dev { command }) => dispatch_dev(&cli, command, format)?, Some(Command::Identity { command }) => match command { IdentityCommand::Generate { target: IdentityTarget::Thunderid(arguments), @@ -1239,6 +1320,155 @@ fn first_path_rstack() -> Option { None } +fn dispatch_dev(cli: &Cli, action: &DevCommand, format: ReportFormat) -> Result<(), CliFailure> { + let launcher_json = matches!(format, ReportFormat::Json); + let (request, database_url, needs_services) = match action { + DevCommand::Start { + project, + database_url, + ttl, + json, + } => ( + DevRequest::Start { + project, + ttl: ttl.as_deref(), + json: *json || launcher_json, + }, + database_url.as_deref(), + true, + ), + DevCommand::Status { run_id, json } => ( + DevRequest::Status { + run_id, + json: *json || launcher_json, + }, + None, + false, + ), + DevCommand::Token { + run_id, + person, + json, + } => ( + DevRequest::Token { + run_id, + person, + json: *json || launcher_json, + }, + None, + false, + ), + DevCommand::Stop { run_id, json } => ( + DevRequest::Stop { + run_id, + json: *json || launcher_json, + }, + None, + false, + ), + }; + let request = serde_json::to_string(&request).map_err(|error| { + CliFailure::operational(format!("cannot encode development request: {error}")) + })?; + + let selected = selected_state(cli)?; + let (mut command, lease) = if let Some((lease, selection, manifest)) = selected { + let required_components = if needs_services { + &["breg", "casework"][..] + } else { + &["breg"][..] + }; + for component in required_components { + if !selection.components.iter().any(|name| name == component) { + return Err(CliFailure::domain(format!( + "component {component} is missing; run rstack install {component}" + ))); + } + } + let executable = lease.path().join("components/breg/bregctl/bregctl"); + if !executable.is_file() { + return Err(CliFailure::operational( + "selected BReg helper is missing".into(), + )); + } + let mut command = ProcessCommand::new(executable); + set_selected_generation_environment(&mut command, lease.path(), &selection, &manifest)?; + let required_variables = if needs_services { + &[ + "RSTACK_BREGCTL_BIN", + "RSTACK_BREG_BIN", + "RSTACK_CASEWORKCTL_BIN", + "RSTACK_CASEWORK_BIN", + ][..] + } else { + &["RSTACK_BREGCTL_BIN"][..] + }; + let missing: Vec<_> = required_variables + .iter() + .filter(|variable| { + !command.get_envs().any(|(name, value)| { + name == std::ffi::OsStr::new(**variable) && value.is_some() + }) + }) + .copied() + .collect(); + if !missing.is_empty() { + return Err(CliFailure::operational(format!( + "selected release lacks development binaries for {}", + missing.join(", ") + ))); + } + (command, Some(lease)) + } else { + let variables = if needs_services { + &[ + "RSTACK_BREGCTL_BIN", + "RSTACK_BREG_BIN", + "RSTACK_CASEWORKCTL_BIN", + "RSTACK_CASEWORK_BIN", + ][..] + } else { + &["RSTACK_BREGCTL_BIN"][..] + }; + let mut paths = Vec::with_capacity(variables.len()); + let mut missing = Vec::new(); + for variable in variables { + let path = std::env::var_os(variable) + .filter(|value| !value.is_empty()) + .map(PathBuf::from); + match path { + Some(path) if path.is_file() => paths.push((*variable, path)), + _ => missing.push(*variable), + } + } + if !missing.is_empty() { + return Err(CliFailure::domain(format!( + "without a managed installation, rstack dev requires existing source binary overrides: {}", + missing.join(", ") + ))); + } + let mut command = ProcessCommand::new(&paths[0].1); + for (variable, path) in paths { + command.env(variable, path); + } + (command, None) + }; + + command + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request); + if let Some(database_url) = database_url { + command.env("RSTACK_DEV_DATABASE_URL", database_url); + } + if let Some(lease) = &lease { + inherit_invocation_lease(lease)?; + } + Err(CliFailure::operational(format!( + "cannot launch development helper: {}", + command.exec() + ))) +} + fn dispatch_product( cli: &Cli, product: &str, @@ -1312,6 +1542,14 @@ fn dispatch_product( } else { command.args(arguments); } + inherit_invocation_lease(&lease)?; + Err(CliFailure::operational(format!( + "cannot launch {product}: {}", + command.exec() + ))) +} + +fn inherit_invocation_lease(lease: &InvocationLease) -> Result<(), CliFailure> { let mut flags = rustix::io::fcntl_getfd(lease.lease_file()).map_err(|error| { CliFailure::operational(format!("cannot inspect invocation lease: {error}")) })?; @@ -1319,10 +1557,7 @@ fn dispatch_product( rustix::io::fcntl_setfd(lease.lease_file(), flags).map_err(|error| { CliFailure::operational(format!("cannot inherit invocation lease: {error}")) })?; - Err(CliFailure::operational(format!( - "cannot launch {product}: {}", - command.exec() - ))) + Ok(()) } /// Put a launcher report format before the first native Evidence runtime token diff --git a/crates/registry-rstack/tests/dev.rs b/crates/registry-rstack/tests/dev.rs new file mode 100644 index 0000000000..d4ae903b1b --- /dev/null +++ b/crates/registry-rstack/tests/dev.rs @@ -0,0 +1,467 @@ +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +const CURRENT_VERSION: &str = env!("CARGO_PKG_VERSION"); + +fn scratch() -> tempfile::TempDir { + if let Ok(target) = std::env::var("CARGO_TARGET_DIR") { + tempfile::tempdir_in(target).unwrap() + } else { + tempfile::tempdir().unwrap() + } +} + +fn executable(path: &Path, body: &str) { + fs::write(path, body).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); +} + +fn digest(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) +} + +fn platform() -> &'static str { + match (std::env::consts::OS, std::env::consts::ARCH) { + ("linux", "x86_64") => "linux-amd64", + ("linux", "aarch64") => "linux-arm64", + ("macos", "aarch64") => "macos-arm64", + other => panic!("unsupported test platform: {other:?}"), + } +} + +fn release_asset(directory: &Path, binary: &str, body: &str, fips: bool) -> (String, Vec) { + let release = format!("v{CURRENT_VERSION}"); + let platform = platform(); + let archived = platform == "macos-arm64"; + let asset_name = format!( + "{binary}-{release}-{platform}{}", + if archived { ".tar.gz" } else { "" } + ); + let asset = directory.join(&asset_name); + if archived { + let contents = directory.join(format!("contents-{binary}")); + fs::create_dir(&contents).unwrap(); + let archived_binary = asset_name.trim_end_matches(".tar.gz"); + executable(&contents.join(archived_binary), body); + fs::write(contents.join("THIRD_PARTY_NOTICES"), "fixture\n").unwrap(); + let mut entries = vec![archived_binary, "THIRD_PARTY_NOTICES"]; + if fips { + fs::write(contents.join("libaws_lc_fips_test.dylib"), "fixture\n").unwrap(); + entries.push("libaws_lc_fips_test.dylib"); + } + let output = Command::new("tar") + .arg("-czf") + .arg(&asset) + .arg("-C") + .arg(&contents) + .args(entries) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + } else { + executable(&asset, body); + } + let bytes = fs::read(asset).unwrap(); + (asset_name, bytes) +} + +fn versioned_tool(binary: &str, body: &str) -> String { + format!( + "#!/usr/bin/env bash\nset -euo pipefail\nif [[ \"${{1:-}}\" == '--version' ]]; then printf '{binary} {CURRENT_VERSION}\\n'; exit 0; fi\n{body}" + ) +} + +struct ManagedInstall { + _scratch: tempfile::TempDir, + root: PathBuf, + bin: PathBuf, + project: PathBuf, +} + +fn managed_install() -> ManagedInstall { + let scratch = scratch(); + let assets = scratch.path().join("assets"); + fs::create_dir(&assets).unwrap(); + let executable = env!("CARGO_BIN_EXE_rstack").replace('\'', "'\"'\"'"); + let launcher_body = format!( + "#!/usr/bin/env bash\nset -euo pipefail\nif [[ \"${{1:-}}\" == '--version' ]]; then printf 'rstack {CURRENT_VERSION}\\n'; exit 0; fi\nif [[ -n \"${{REGISTRY_CARGO_RUNTIME_LIBRARY_PATH:-}}\" ]]; then export DYLD_FALLBACK_LIBRARY_PATH=\"$REGISTRY_CARGO_RUNTIME_LIBRARY_PATH\"; fi\nexec '{executable}' \"$@\"\n" + ); + let helper_body = versioned_tool( + "bregctl", + "printf 'arg:%s\\n' \"$@\"\nprintf 'request:%s\\n' \"${RSTACK_PROJECT_DEV_REQUEST:?}\"\n[[ \"${RSTACK_DEV_DATABASE_URL:-}\" == \"${EXPECTED_DATABASE_URL:?}\" ]]\nprintf 'database:matched\\n'\nprintf 'generation:%s\\n' \"${RSTACK_GENERATION_DIR:?}\"\nprintf 'release:%s\\n' \"${RSTACK_RELEASE_TAG:?}\"\nprintf 'bregctl:%s\\n' \"${RSTACK_BREGCTL_BIN:?}\"\nprintf 'breg:%s\\n' \"${RSTACK_BREG_BIN:?}\"\nprintf 'caseworkctl:%s\\n' \"${RSTACK_CASEWORKCTL_BIN:?}\"\nprintf 'casework:%s\\n' \"${RSTACK_CASEWORK_BIN:?}\"\n", + ); + let (launcher_name, launcher_bytes) = release_asset(&assets, "rstack", &launcher_body, false); + let (bregctl_name, bregctl_bytes) = release_asset(&assets, "bregctl", &helper_body, true); + let (breg_name, breg_bytes) = + release_asset(&assets, "breg", &versioned_tool("breg", "exit 0\n"), true); + let (caseworkctl_name, caseworkctl_bytes) = release_asset( + &assets, + "caseworkctl", + &versioned_tool("caseworkctl", "exit 0\n"), + true, + ); + let (casework_name, casework_bytes) = release_asset( + &assets, + "casework", + &versioned_tool("casework", "exit 0\n"), + true, + ); + let release = format!("v{CURRENT_VERSION}"); + let platform = platform(); + let payload = |name: &str, bytes: &[u8]| { + serde_json::json!({ + "name":name, + "kind":"binary", + "size":bytes.len(), + "sha256":digest(bytes) + }) + }; + let component_asset = |name: &str, bytes: &[u8]| serde_json::json!({"name":name,"size":bytes.len(),"sha256":digest(bytes)}); + let manifest_name = format!("registry-stack-{release}-release-manifest.json"); + let manifest_path = assets.join(&manifest_name); + let manifest = serde_json::json!({ + "schema_version":"registry-stack.release-candidate.v2", + "repository":"registrystack/registry-stack", + "release":{"tag":release.as_str(),"version":CURRENT_VERSION,"source_sha":"a".repeat(40),"release_id":"test"}, + "payloads":[ + payload(&launcher_name, &launcher_bytes), + payload(&bregctl_name, &bregctl_bytes), + payload(&breg_name, &breg_bytes), + payload(&caseworkctl_name, &caseworkctl_bytes), + payload(&casework_name, &casework_bytes) + ], + "components":{ + "schema_version":"registry-stack.components.v1", + "release":release.as_str(), + "launcher":{(platform):component_asset(&launcher_name, &launcher_bytes)}, + "components":{ + "breg":{ + "entrypoint":"bregctl", + "dependencies":[], + "platforms":{(platform):{ + "assets":[ + component_asset(&bregctl_name, &bregctl_bytes), + component_asset(&breg_name, &breg_bytes) + ], + "capabilities":{"tooling":true,"runtime":true} + }} + }, + "casework":{ + "entrypoint":"caseworkctl", + "dependencies":["breg"], + "platforms":{(platform):{ + "assets":[ + component_asset(&caseworkctl_name, &caseworkctl_bytes), + component_asset(&casework_name, &casework_bytes) + ], + "capabilities":{"tooling":true,"runtime":true} + }} + } + } + } + }); + let manifest_bytes = serde_json::to_vec(&manifest).unwrap(); + fs::write(&manifest_path, &manifest_bytes).unwrap(); + let checksums = assets.join("SHA256SUMS"); + let mut checksum_text = String::new(); + for (name, bytes) in [ + (&launcher_name, &launcher_bytes), + (&bregctl_name, &bregctl_bytes), + (&breg_name, &breg_bytes), + (&caseworkctl_name, &caseworkctl_bytes), + (&casework_name, &casework_bytes), + ] { + checksum_text.push_str(&format!("{} {name}\n", digest(bytes))); + } + checksum_text.push_str(&format!("{} {manifest_name}\n", digest(&manifest_bytes))); + fs::write(&checksums, checksum_text).unwrap(); + + let root = scratch.path().join("lib"); + let bin = scratch.path().join("bin"); + let installed = Command::new(env!("CARGO_BIN_EXE_rstack")) + .args(["bootstrap", "--release-manifest"]) + .arg(&manifest_path) + .arg("--checksum-file") + .arg(&checksums) + .arg("--launcher-asset") + .arg(assets.join(&launcher_name)) + .arg("--asset-dir") + .arg(&assets) + .arg("--install-root") + .arg(&root) + .arg("--bin-dir") + .arg(&bin) + .args(["breg", "casework"]) + .output() + .unwrap(); + assert!(installed.status.success(), "{installed:?}"); + let project = scratch.path().join("project"); + fs::create_dir(&project).unwrap(); + ManagedInstall { + _scratch: scratch, + root, + bin, + project, + } +} + +struct SourceTools { + _scratch: tempfile::TempDir, + project: PathBuf, + install_root: PathBuf, + bin_dir: PathBuf, + bregctl: PathBuf, + breg: PathBuf, + caseworkctl: PathBuf, + casework: PathBuf, +} + +impl SourceTools { + fn new() -> Self { + let scratch = scratch(); + let project = scratch.path().join("registry project"); + fs::create_dir(&project).unwrap(); + let install_root = scratch.path().join("uninstalled"); + let bin_dir = scratch.path().join("bin"); + let bregctl = scratch.path().join("bregctl"); + executable( + &bregctl, + "#!/usr/bin/env bash\nset -euo pipefail\nprintf 'arg:%s\\n' \"$@\"\nprintf 'request:%s\\n' \"${RSTACK_PROJECT_DEV_REQUEST:?}\"\nif [[ -n \"${EXPECTED_DATABASE_URL:-}\" ]]; then\n [[ \"${RSTACK_DEV_DATABASE_URL:-}\" == \"$EXPECTED_DATABASE_URL\" ]]\n printf 'database:matched\\n'\nelse\n printf 'database:%s\\n' \"${RSTACK_DEV_DATABASE_URL:+set}\"\nfi\nprintf 'bregctl:%s\\n' \"${RSTACK_BREGCTL_BIN:-}\"\nprintf 'breg:%s\\n' \"${RSTACK_BREG_BIN:-}\"\nprintf 'caseworkctl:%s\\n' \"${RSTACK_CASEWORKCTL_BIN:-}\"\nprintf 'casework:%s\\n' \"${RSTACK_CASEWORK_BIN:-}\"\n", + ); + let breg = scratch.path().join("breg"); + let caseworkctl = scratch.path().join("caseworkctl"); + let casework = scratch.path().join("casework"); + for path in [&breg, &caseworkctl, &casework] { + executable(path, "#!/usr/bin/env bash\nset -euo pipefail\nexit 0\n"); + } + Self { + _scratch: scratch, + project, + install_root, + bin_dir, + bregctl, + breg, + caseworkctl, + casework, + } + } + + fn empty_command(&self) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_rstack")); + command + .args(["--install-root", self.install_root.to_str().unwrap()]) + .args(["--bin-dir", self.bin_dir.to_str().unwrap()]) + .env_remove("RSTACK_DEV_DATABASE_URL") + .env_remove("RSTACK_BREGCTL_BIN") + .env_remove("RSTACK_BREG_BIN") + .env_remove("RSTACK_CASEWORKCTL_BIN") + .env_remove("RSTACK_CASEWORK_BIN"); + command + } + + fn command(&self) -> Command { + let mut command = self.empty_command(); + command + .env("RSTACK_BREGCTL_BIN", &self.bregctl) + .env("RSTACK_BREG_BIN", &self.breg) + .env("RSTACK_CASEWORKCTL_BIN", &self.caseworkctl) + .env("RSTACK_CASEWORK_BIN", &self.casework); + command + } +} + +fn request(output: &Output) -> Value { + let stdout = String::from_utf8(output.stdout.clone()).unwrap(); + let request = stdout + .lines() + .find_map(|line| line.strip_prefix("request:")) + .unwrap_or_else(|| panic!("missing request in {stdout:?}")); + serde_json::from_str(request).unwrap() +} + +fn output_field<'a>(output: &'a str, name: &str) -> &'a str { + output + .lines() + .find_map(|line| line.strip_prefix(&format!("{name}:"))) + .unwrap_or_else(|| panic!("missing {name} in {output}")) +} + +#[test] +fn managed_start_uses_one_release_generation_on_supported_platforms() { + let install = managed_install(); + let database_url = "postgresql://example.invalid/managed-secret"; + let output = Command::new(install.bin.join("rstack")) + .arg("--install-root") + .arg(&install.root) + .arg("--bin-dir") + .arg(&install.bin) + .args(["dev", "start"]) + .arg(&install.project) + .args(["--database-url", database_url]) + .env("EXPECTED_DATABASE_URL", database_url) + .output() + .unwrap(); + + assert!(output.status.success(), "{output:?}"); + assert!(output.stderr.is_empty(), "{output:?}"); + let stdout = String::from_utf8(output.stdout.clone()).unwrap(); + assert!(!stdout.contains(database_url), "{stdout}"); + assert_eq!( + stdout + .lines() + .filter(|line| line.starts_with("arg:")) + .collect::>(), + ["arg:project-dev"] + ); + assert_eq!( + request(&output), + serde_json::json!({ + "action":"start", + "project":install.project, + "json":false + }) + ); + let generation = PathBuf::from(output_field(&stdout, "generation")); + assert_eq!( + output_field(&stdout, "release"), + format!("v{CURRENT_VERSION}") + ); + for (name, relative) in [ + ("bregctl", "components/breg/bregctl/bregctl"), + ("breg", "components/breg/breg/breg"), + ("caseworkctl", "components/casework/caseworkctl/caseworkctl"), + ("casework", "components/casework/casework/casework"), + ] { + assert_eq!( + PathBuf::from(output_field(&stdout, name)), + generation.join(relative) + ); + } +} + +#[test] +fn source_start_sends_one_json_request_and_binary_set() { + let tools = SourceTools::new(); + let database_url = "postgresql://example.invalid/dev-secret"; + let output = tools + .command() + .args(["dev", "start"]) + .arg(&tools.project) + .args(["--ttl", "45m", "--database-url", database_url, "--json"]) + .env("EXPECTED_DATABASE_URL", database_url) + .output() + .unwrap(); + + assert!(output.status.success(), "{output:?}"); + assert!(output.stderr.is_empty(), "{output:?}"); + let stdout = String::from_utf8(output.stdout.clone()).unwrap(); + assert_eq!( + stdout + .lines() + .filter(|line| line.starts_with("arg:")) + .collect::>(), + ["arg:project-dev"] + ); + assert_eq!( + request(&output), + serde_json::json!({ + "action":"start", + "project":tools.project, + "ttl":"45m", + "json":true + }) + ); + assert!(stdout.contains("database:matched\n")); + assert!(stdout.contains(&format!("bregctl:{}\n", tools.bregctl.display()))); + assert!(stdout.contains(&format!("breg:{}\n", tools.breg.display()))); + assert!(stdout.contains(&format!("caseworkctl:{}\n", tools.caseworkctl.display()))); + assert!(stdout.contains(&format!("casework:{}\n", tools.casework.display()))); + assert!(!stdout + .lines() + .any(|line| line == format!("arg:{database_url}"))); + assert!(!request(&output).to_string().contains(database_url)); + assert!(!stdout.contains(database_url)); +} + +#[test] +fn dev_actions_encode_the_public_contract_and_launcher_json_format() { + let tools = SourceTools::new(); + for (arguments, expected) in [ + ( + vec!["--format", "json", "dev", "status", "run-1"], + serde_json::json!({"action":"status","run_id":"run-1","json":true}), + ), + ( + vec!["dev", "token", "run-2", "--as", "reviewer", "--json"], + serde_json::json!({ + "action":"token", + "run_id":"run-2", + "person":"reviewer", + "json":true + }), + ), + ( + vec!["dev", "stop", "run-3"], + serde_json::json!({"action":"stop","run_id":"run-3","json":false}), + ), + ( + vec!["dev", "stop", "run-4", "--json"], + serde_json::json!({"action":"stop","run_id":"run-4","json":true}), + ), + ] { + let output = tools + .empty_command() + .args(arguments) + .env("RSTACK_BREGCTL_BIN", &tools.bregctl) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + assert_eq!(request(&output), expected); + } +} + +#[test] +fn start_requires_the_complete_source_binary_set_without_an_installation() { + let tools = SourceTools::new(); + let output = tools + .empty_command() + .args(["dev", "start"]) + .arg(&tools.project) + .env("RSTACK_BREGCTL_BIN", &tools.bregctl) + .output() + .unwrap(); + + assert_eq!(output.status.code(), Some(1), "{output:?}"); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(stderr.contains("RSTACK_BREG_BIN"), "{stderr}"); + assert!(stderr.contains("RSTACK_CASEWORKCTL_BIN"), "{stderr}"); + assert!(stderr.contains("RSTACK_CASEWORK_BIN"), "{stderr}"); +} + +#[test] +fn root_and_dev_help_need_no_installation_or_binary_overrides() { + let tools = SourceTools::new(); + for (arguments, expected) in [ + (&["--help"][..], &["dev"][..]), + ( + &["dev", "--help"][..], + &["start", "status", "token", "stop"][..], + ), + ( + &["dev", "start", "--help"][..], + &["--database-url", "--ttl", "--json"][..], + ), + ] { + let output = tools.empty_command().args(arguments).output().unwrap(); + assert!(output.status.success(), "{arguments:?}: {output:?}"); + assert!(output.stderr.is_empty(), "{arguments:?}: {output:?}"); + let stdout = String::from_utf8(output.stdout).unwrap(); + for text in expected { + assert!(stdout.contains(text), "{arguments:?}: {stdout}"); + } + } +} diff --git a/products/breg/SECURITY-MODEL.md b/products/breg/SECURITY-MODEL.md index 63e29e8dfd..13500fc101 100644 --- a/products/breg/SECURITY-MODEL.md +++ b/products/breg/SECURITY-MODEL.md @@ -461,6 +461,12 @@ mitigation, and related matrix IDs. - `database.testOnlyPlaintext` exists only in test builds; a release binary refuses it with `breg.runtime.plaintext-database`. +`rstack dev` can instead activate BReg as part of an isolated project run. +It uses the normal package and database activation checks, distinct TLS +PostgreSQL credentials, and a loopback listener. The runtime trusts the +run's static signing key under the declared issuer name. See +[Run isolation](../platform/PROJECT-DEV.md#run-isolation). + ## 5. Reporting Report vulnerabilities privately as described in [SECURITY.md](../../SECURITY.md). diff --git a/products/breg/acceptance/reviewed-evidence-journey/dev.yaml b/products/breg/acceptance/reviewed-evidence-journey/dev.yaml new file mode 100644 index 0000000000..48b8477422 --- /dev/null +++ b/products/breg/acceptance/reviewed-evidence-journey/dev.yaml @@ -0,0 +1,176 @@ +# yaml-language-server: $schema=https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json +apiVersion: id.registrystack.org/formats/project-dev/v1alpha1 +kind: ProjectDevPlan +issuer: https://issuer.example.test +products: + - {product: breg, project: registry, audience: "urn:journey:breg"} + - {product: casework, project: casework, audience: "urn:journey:casework"} +bindings: + bregReviewAuthority: {id: casework, producer: registry, person: producer} + caseworkContextProvider: {id: registry, person: context-status, action: correct-status} +people: + operator: + subject: generic-registry-operator + client: operator + grants: + breg: + profile: operator + scopes: ["registry:generic:operate"] + claims: {registry_actor_kind: human, registry_purpose: registry-operations} + source: + subject: evidence-source + client: source + grants: + breg: + profile: evidence-source + scopes: ["registry:evidence:lookup"] + claims: {registry_actor_kind: service, registry_purpose: evidence-source-read} + applier: + subject: applier + client: applier + grants: + breg: + profile: applier + scopes: ["registry:apply"] + claims: {registry_actor_kind: human, registry_purpose: registry-operations} + reviewer: + subject: staff + client: staff + caseworkContext: true + grants: + breg: + profile: reviewer + scopes: ["registry:review", "casework:staff"] + claims: {registry_actor_kind: human, registry_purpose: registry-operations} + casework: + profile: staff + scopes: ["registry:review", "casework:staff"] + claims: {registry_actor_kind: human, registry_purpose: registry-operations} + administrator: + subject: administrator + client: administrator + grants: + casework: + profile: administrator + scopes: ["casework:admin"] + claims: {registry_actor_kind: human} + producer: + subject: producer + client: producer + grants: + casework: + profile: requester + scopes: ["casework:request"] + claims: {registry_actor_kind: service} + context-status: + subject: context-status + client: context-status + grants: + breg: + profile: review-status + scopes: ["registry:review-status"] + claims: {registry_actor_kind: service, registry_purpose: registry-operations} +journey: + id: native-review-handoff + steps: + - id: staff-queue + product: casework + as: administrator + method: POST + path: /v1/directory/bootstrap + headers: {If-Match: '"0"', Idempotency-Key: staff-queue} + body: + teamId: reviewers + staff: [{issuer: https://issuer.example.test, subject: staff}] + supervisors: [] + queueId: reviews + expect: {status: 200} + - id: create-record + product: breg + as: operator + method: POST + path: /v1/records/records + headers: {Idempotency-Key: create-record} + body: + data: {code: DEV-PROOF, registrationNumber: DEV-REG, status: retired, label: Synthetic record} + expect: {status: 201} + capture: {record: /data/recordIdentifier} + - id: start-proposal + product: breg + as: operator + method: POST + path: /v1/actions/correct-status/proposals + headers: {Idempotency-Key: start-proposal} + body: {input: {subjectId: "{{record}}", status: active}} + expect: {status: 201} + capture: {proposal: /id, draftEtag: {header: etag}} + - id: submit-proposal + product: breg + as: operator + method: POST + path: "/v1/proposals/{{proposal}}/submit" + headers: {If-Match: "{{draftEtag}}", Idempotency-Key: submit-proposal} + body: {} + expect: {status: 200, json: {/state: submitted}} + capture: {submissionVersion: /submissionVersion, effectDigest: /effectDigest} + - id: find-task + product: casework + as: reviewer + method: GET + path: /v1/review-tasks?queue=reviews + headers: {Registry-Source-Profile: reviewer} + expect: {status: 200} + poll: {timeoutMs: 60000, intervalMs: 500} + capture: + task: {find: /items, where: {pointer: /subject/id, equals: "{{proposal}}"}, pointer: /taskId} + request: {find: /items, where: {pointer: /subject/id, equals: "{{proposal}}"}, pointer: /requestId} + taskRevision: {find: /items, where: {pointer: /subject/id, equals: "{{proposal}}"}, pointer: /revision} + - id: read-context-from-breg + product: casework + as: reviewer + method: GET + path: "/v1/review-tasks/{{task}}/context" + headers: {Registry-Source-Profile: reviewer} + expect: + status: 200 + json: + /context/projection/display/currentStatus: retired + /context/projection/display/proposedStatus: active + - id: claim-task + product: casework + as: reviewer + method: POST + path: "/v1/review-tasks/{{task}}/claim" + headers: {If-Match: '"{{taskRevision}}"', Idempotency-Key: claim-task, Registry-Source-Profile: reviewer} + expect: {status: 200} + capture: {claimRevision: /revision} + - id: approve-correction + product: casework + as: reviewer + method: POST + path: "/v1/review-tasks/{{task}}/decisions" + headers: {If-Match: '"{{claimRevision}}"', Idempotency-Key: approve-correction, Registry-Source-Profile: reviewer} + body: {decision: {type: approve, outcome: approved}} + expect: {status: 204} + - id: read-review-result + product: casework + as: producer + method: GET + path: "/v1/review-requests/{{request}}/result" + expect: {status: 200, json: {/subject/id: "{{proposal}}", /status: approved}} + - id: await-source-decision + product: breg + as: applier + method: GET + path: "/v1/proposals/{{proposal}}" + expect: {status: 200, json: {/decision/status: approved}} + poll: {timeoutMs: 60000, intervalMs: 500} + capture: {approvalEtag: {header: etag}} + - id: apply-correction + product: breg + as: applier + method: POST + path: "/v1/proposals/{{proposal}}/apply" + headers: {If-Match: "{{approvalEtag}}", Idempotency-Key: apply-correction} + body: {submissionVersion: "{{submissionVersion}}", effectDigest: "{{effectDigest}}"} + expect: {status: 200, json: {/state: applied}} diff --git a/products/breg/generated/tools/project-dev.v1alpha1.schema.json b/products/breg/generated/tools/project-dev.v1alpha1.schema.json new file mode 100644 index 0000000000..592b434cd6 --- /dev/null +++ b/products/breg/generated/tools/project-dev.v1alpha1.schema.json @@ -0,0 +1,464 @@ +{ + "$defs": { + "Bindings": { + "additionalProperties": false, + "properties": { + "bregReviewAuthority": { + "$ref": "#/$defs/BregReviewAuthority" + }, + "caseworkContextProvider": { + "$ref": "#/$defs/CaseworkContextProvider" + } + }, + "type": "object" + }, + "BregReviewAuthority": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "person": { + "type": "string" + }, + "producer": { + "type": "string" + } + }, + "required": [ + "id", + "producer", + "person" + ], + "type": "object" + }, + "Capture": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/HeaderCapture" + }, + { + "$ref": "#/$defs/FindCapture" + }, + { + "$ref": "#/$defs/ValueCapture" + } + ] + }, + "CapturedHeader": { + "enum": [ + "etag" + ], + "type": "string" + }, + "CaseworkContextProvider": { + "additionalProperties": false, + "properties": { + "action": { + "type": "string" + }, + "id": { + "type": "string" + }, + "person": { + "type": "string" + } + }, + "required": [ + "id", + "person", + "action" + ], + "type": "object" + }, + "Check": { + "additionalProperties": false, + "properties": { + "contains": true, + "equals": true, + "length": { + "format": "uint", + "minimum": 0, + "type": "integer" + }, + "pointer": { + "type": "string" + } + }, + "required": [ + "pointer" + ], + "type": "object" + }, + "Expected": { + "additionalProperties": false, + "properties": { + "checks": { + "default": [], + "items": { + "$ref": "#/$defs/Check" + }, + "type": "array" + }, + "code": { + "type": "string" + }, + "json": { + "additionalProperties": true, + "default": {}, + "type": "object" + }, + "status": { + "$ref": "#/$defs/ExpectedStatus" + } + }, + "required": [ + "status" + ], + "type": "object" + }, + "ExpectedStatus": { + "anyOf": [ + { + "format": "uint16", + "maximum": 65535, + "minimum": 0, + "type": "integer" + }, + { + "items": { + "format": "uint16", + "maximum": 65535, + "minimum": 0, + "type": "integer" + }, + "type": "array" + } + ] + }, + "FindCapture": { + "additionalProperties": false, + "properties": { + "find": { + "type": "string" + }, + "pointer": { + "type": "string" + }, + "where": { + "$ref": "#/$defs/FindWhere" + } + }, + "required": [ + "find", + "where", + "pointer" + ], + "type": "object" + }, + "FindWhere": { + "additionalProperties": false, + "properties": { + "equals": true, + "pointer": { + "type": "string" + } + }, + "required": [ + "pointer", + "equals" + ], + "type": "object" + }, + "Grant": { + "additionalProperties": false, + "properties": { + "claims": { + "additionalProperties": true, + "default": {}, + "type": "object" + }, + "profile": { + "type": "string" + }, + "scopes": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "profile", + "scopes" + ], + "type": "object" + }, + "HeaderCapture": { + "additionalProperties": false, + "properties": { + "header": { + "$ref": "#/$defs/CapturedHeader" + } + }, + "required": [ + "header" + ], + "type": "object" + }, + "Journey": { + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "steps": { + "items": { + "$ref": "#/$defs/Step" + }, + "type": "array" + } + }, + "required": [ + "id", + "steps" + ], + "type": "object" + }, + "Method": { + "enum": [ + "GET", + "POST", + "PUT", + "PATCH", + "DELETE" + ], + "type": "string" + }, + "Person": { + "additionalProperties": false, + "properties": { + "caseworkContext": { + "default": false, + "description": "Forward the same authority to both products. Both grants must declare the same scopes and additional claims.", + "type": "boolean" + }, + "client": { + "type": "string" + }, + "grants": { + "additionalProperties": false, + "properties": { + "breg": { + "$ref": "#/$defs/Grant" + }, + "casework": { + "$ref": "#/$defs/Grant" + } + }, + "type": "object" + }, + "subject": { + "type": "string" + } + }, + "required": [ + "subject", + "client", + "grants" + ], + "type": "object" + }, + "Poll": { + "additionalProperties": false, + "properties": { + "intervalMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "timeoutMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "timeoutMs", + "intervalMs" + ], + "type": "object" + }, + "Product": { + "enum": [ + "breg", + "casework" + ], + "type": "string" + }, + "ProductPlan": { + "additionalProperties": false, + "properties": { + "audience": { + "type": "string" + }, + "product": { + "$ref": "#/$defs/Product" + }, + "project": { + "type": "string" + } + }, + "required": [ + "product", + "project", + "audience" + ], + "type": "object" + }, + "RawBody": { + "additionalProperties": false, + "properties": { + "base64": { + "type": "string" + }, + "contentType": { + "type": "string" + } + }, + "required": [ + "contentType", + "base64" + ], + "type": "object" + }, + "Step": { + "additionalProperties": false, + "properties": { + "as": { + "type": "string" + }, + "body": true, + "capture": { + "additionalProperties": { + "$ref": "#/$defs/Capture" + }, + "default": {}, + "type": "object" + }, + "expect": { + "$ref": "#/$defs/Expected" + }, + "headers": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "type": "object" + }, + "id": { + "type": "string" + }, + "method": { + "$ref": "#/$defs/Method" + }, + "path": { + "type": "string" + }, + "poll": { + "$ref": "#/$defs/Poll" + }, + "product": { + "$ref": "#/$defs/Product" + }, + "raw": { + "$ref": "#/$defs/RawBody" + } + }, + "required": [ + "id", + "product", + "as", + "method", + "path", + "expect" + ], + "type": "object" + }, + "ValueCapture": { + "additionalProperties": false, + "properties": { + "pointer": { + "type": "string" + }, + "query": { + "type": "string" + } + }, + "required": [ + "pointer" + ], + "type": "object" + } + }, + "$id": "https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "apiVersion": { + "const": "id.registrystack.org/formats/project-dev/v1alpha1", + "description": "The format and version this file is written in.", + "type": "string" + }, + "bindings": { + "$ref": "#/$defs/Bindings", + "default": {} + }, + "issuer": { + "type": "string" + }, + "journey": { + "$ref": "#/$defs/Journey" + }, + "kind": { + "const": "ProjectDevPlan", + "description": "The kind of document this file is.", + "type": "string" + }, + "people": { + "additionalProperties": { + "$ref": "#/$defs/Person" + }, + "type": "object" + }, + "products": { + "items": { + "$ref": "#/$defs/ProductPlan" + }, + "type": "array" + }, + "schemaTestPeople": { + "additionalProperties": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "default": {}, + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "issuer", + "products", + "people", + "journey" + ], + "title": "Registry Stack project development plan", + "type": "object" +} diff --git a/products/casework/SECURITY-MODEL.md b/products/casework/SECURITY-MODEL.md index 57d6c6b913..ff0fac46a2 100644 --- a/products/casework/SECURITY-MODEL.md +++ b/products/casework/SECURITY-MODEL.md @@ -192,6 +192,13 @@ Deployment behavior is in the [README](README.md), keys with a static `jwksSource`. - **Related:** CASEWORK-SEC-01. +`rstack dev` uses a loopback listener and a private static key pin for each +development run. The shared issuer gives a named synthetic person only +the scopes and additional claims declared in their Casework grant. A +reviewer token forwarded to BReg uses authority explicitly declared in both +grants. Casework applies its usual client, profile, and human-identity checks. See +[Run isolation](../platform/PROJECT-DEV.md#run-isolation). + ### CASEWORK-RES-09: key refusal is by `kid`, locally, at startup - **Condition:** `authentication.oidc.revokedKeyIds` lists a compromised or diff --git a/products/identifiers/generated/catalog.v1.json b/products/identifiers/generated/catalog.v1.json index 8c6a02f161..1aa55fbee6 100644 --- a/products/identifiers/generated/catalog.v1.json +++ b/products/identifiers/generated/catalog.v1.json @@ -4776,7 +4776,7 @@ "description": "Base Registry Engine tool-file JSON Schemas at version v1alpha1.", "source": { "path": "crates/registry-bregctl/src/tool_schema.rs", - "sha256": "28567d54120dd29d734ddcd3e10c4c6ee128eb7b2d195e4c77bd1cd6c9bc5cfd" + "sha256": "8197938479291503ba0975ae3c595b580a2f686bed6ae3cf4fe57cf01318c94d" }, "artifact": { "path": "products/breg/generated/tools/backup-binding.v1alpha1.schema.json", @@ -4794,7 +4794,7 @@ "description": "Base Registry Engine tool-file JSON Schemas at version v1alpha1.", "source": { "path": "crates/registry-bregctl/src/tool_schema.rs", - "sha256": "28567d54120dd29d734ddcd3e10c4c6ee128eb7b2d195e4c77bd1cd6c9bc5cfd" + "sha256": "8197938479291503ba0975ae3c595b580a2f686bed6ae3cf4fe57cf01318c94d" }, "artifact": { "path": "products/breg/generated/tools/dev-clients.v1alpha1.schema.json", @@ -4848,7 +4848,7 @@ "description": "Base Registry Engine tool-file JSON Schemas at version v1alpha1.", "source": { "path": "crates/registry-bregctl/src/tool_schema.rs", - "sha256": "28567d54120dd29d734ddcd3e10c4c6ee128eb7b2d195e4c77bd1cd6c9bc5cfd" + "sha256": "8197938479291503ba0975ae3c595b580a2f686bed6ae3cf4fe57cf01318c94d" }, "artifact": { "path": "products/breg/generated/tools/example-scenarios.v1alpha1.schema.json", @@ -4866,7 +4866,7 @@ "description": "Base Registry Engine tool-file JSON Schemas at version v1.", "source": { "path": "crates/registry-bregctl/src/tool_schema.rs", - "sha256": "28567d54120dd29d734ddcd3e10c4c6ee128eb7b2d195e4c77bd1cd6c9bc5cfd" + "sha256": "8197938479291503ba0975ae3c595b580a2f686bed6ae3cf4fe57cf01318c94d" }, "artifact": { "path": "products/breg/generated/tools/journeys.v1.schema.json", @@ -4920,7 +4920,7 @@ "description": "Base Registry Engine tool-file JSON Schemas at version v1alpha1.", "source": { "path": "crates/registry-bregctl/src/tool_schema.rs", - "sha256": "28567d54120dd29d734ddcd3e10c4c6ee128eb7b2d195e4c77bd1cd6c9bc5cfd" + "sha256": "8197938479291503ba0975ae3c595b580a2f686bed6ae3cf4fe57cf01318c94d" }, "artifact": { "path": "products/breg/generated/tools/migration-descriptor.v1alpha1.schema.json", @@ -4956,7 +4956,7 @@ "description": "Base Registry Engine tool-file JSON Schemas at version v1alpha1.", "source": { "path": "crates/registry-bregctl/src/tool_schema.rs", - "sha256": "28567d54120dd29d734ddcd3e10c4c6ee128eb7b2d195e4c77bd1cd6c9bc5cfd" + "sha256": "8197938479291503ba0975ae3c595b580a2f686bed6ae3cf4fe57cf01318c94d" }, "artifact": { "path": "products/breg/generated/tools/model-selection.v1alpha1.schema.json", @@ -4982,6 +4982,24 @@ "mediaType": "application/schema+json" } }, + { + "uri": "https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json", + "kind": "schema", + "status": "active", + "compatibilityLine": "v1alpha1", + "owner": "breg", + "title": "Registry Stack project development plan", + "description": "Base Registry Engine tool-file JSON Schemas at version v1alpha1.", + "source": { + "path": "crates/registry-bregctl/src/tool_schema.rs", + "sha256": "8197938479291503ba0975ae3c595b580a2f686bed6ae3cf4fe57cf01318c94d" + }, + "artifact": { + "path": "products/breg/generated/tools/project-dev.v1alpha1.schema.json", + "sha256": "cb996ac1008a1e9cac70f9766f4c30e20dc793f91dffb648bf26335c7742e233", + "mediaType": "application/schema+json" + } + }, { "uri": "https://id.registrystack.org/schemas/breg/project/project.v1alpha1.schema.json", "kind": "schema", @@ -5100,7 +5118,7 @@ "description": "Base Registry Engine tool-file JSON Schemas at version v1.", "source": { "path": "crates/registry-bregctl/src/tool_schema.rs", - "sha256": "28567d54120dd29d734ddcd3e10c4c6ee128eb7b2d195e4c77bd1cd6c9bc5cfd" + "sha256": "8197938479291503ba0975ae3c595b580a2f686bed6ae3cf4fe57cf01318c94d" }, "artifact": { "path": "products/breg/generated/tools/schema-test-credentials.v1.schema.json", diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md new file mode 100644 index 0000000000..b48d877e1b --- /dev/null +++ b/products/platform/PROJECT-DEV.md @@ -0,0 +1,80 @@ +# Run a project for development + +Use `rstack dev` to start a checkout's BReg and Casework projects together. +Install matching BReg and Casework components and supply a dedicated development +PostgreSQL server over TLS. The database administrator needs to create and +remove roles and databases and install `btree_gist`. Use the platform's trusted +roots or set `RSTACK_DEV_DATABASE_CA` to your server's CA bundle in PEM format. + +```sh +rstack install breg casework +# Set RSTACK_DEV_DATABASE_URL through your shell's secret provider. +rstack dev start ./licensing --ttl 30m --json +rstack dev status RUN_ID +rstack dev token RUN_ID --as applicant --json +rstack dev stop RUN_ID +``` + +You can also pass `--database-url`. Use the environment variable when the URL +includes a password. Start reports the product URLs and the journey result. +A failed journey exits 1 and includes the live run ID so you can inspect the +result and stop the run. Startup failures clean up or report the run ID if +cleanup is pending. The TTL starts at readiness and defaults to 30 minutes. +Supply one whole-number value from `1s` to `24h` with an `s`, `m`, or `h` suffix. + +The [licensing example](examples/licensing/dev.yaml) creates an application in +BReg, requests approval of the captured application in Casework, and checks the +reviewer's decision. Point `start` at the directory containing `dev.yaml`. +Product files remain byte-for-byte unchanged. Generated runtime settings, keys, +and packages live in the run's owner-only scratch directory. + +## Write the development plan + +Use the [versioned plan schema](../breg/generated/tools/project-dev.v1alpha1.schema.json). +List the project's issuer, product directories and audiences in startup order, +and test people with synthetic subjects and exact product grants. A grant selects +the access profile, scopes, and additional claims. Each selected profile gets +the test person's subject in its principal claim. The products enforce their +ordinary permissions. If several people match a BReg schema-test step, select +its alias under `schemaTestPeople..`. The run obtains the +normal schema-test receipt before packaging. + +For a native review handoff, `bindings.bregReviewAuthority` identifies the +Casework producer, and `bindings.caseworkContextProvider` identifies the BReg +context provider and action. The run pairs the provider on a private copy +before packaging. A reviewer whose Casework context call forwards the bearer +to BReg declares both grants, one shared client, and `caseworkContext: true`. +Both grants declare the same scopes and additional claims. Casework reads the +token subject or the BReg principal claim. The token carries both audiences +and the authority explicitly shared by both grants. +See the [native review example](../breg/acceptance/reviewed-evidence-journey/dev.yaml). + +Journey steps call relative product API paths and assert the HTTP status and +optional JSON-pointer values. Capture a string or number by JSON pointer and +refer to it as `{{capture-name}}` in a later path, request body, header, or +expected value. Use URI-safe characters for values in a path. Each step names +the test person it runs as. + +Use a bounded GET poll for asynchronous delivery or result retrieval. Each +poll can wait up to 60 seconds. A journey can make at most 512 requests and +transfer at most 8 MiB of request and response bodies in total. + +`token` writes a short-lived bearer header file for each grant. Its JSON report +gives the file path and the selected profile. Pass the file to curl with +`--header @HEADER_FILE`. With BReg, include the selected profile as the +`accessProfile` query parameter. With Casework, the header file carries the profile. + +## Run isolation + +Each run listens on loopback, creates separate databases and migration and +runtime logins, and pins its own public signing key in the runtime files. The +local issuer signs under the project's issuer name with a key unique to the run. +Use synthetic test people and keep the issuer's private directory under your +user account. + +Stop and TTL expiry terminate the services and remove the databases, roles, +keys, and run files. If PostgreSQL temporarily becomes unavailable, the +supervisor retries cleanup and `stop` reports pending work. Keep the supervisor +alive to finish cleanup when PostgreSQL returns. After an uncatchable +supervisor exit or machine failure, identify the exact databases and roles by +run ID before removing them. diff --git a/products/platform/examples/licensing/casework/casework.yaml b/products/platform/examples/licensing/casework/casework.yaml new file mode 100644 index 0000000000..a3b3bccdef --- /dev/null +++ b/products/platform/examples/licensing/casework/casework.yaml @@ -0,0 +1,59 @@ +apiVersion: id.registrystack.org/formats/casework/project/v1alpha1 +kind: CaseworkProject +project: {id: licensing-review, version: "1"} +accessProfiles: + administrator: + principalClaim: registry_principal + requiredScopes: ["casework:admin"] + actsAs: administrator + reviewer: + principalClaim: sub + requiredScopes: ["casework:review"] + actsAs: staff + supervisor: + principalClaim: sub + requiredScopes: ["casework:supervise"] + actsAs: supervisor + intake: + principalClaim: producer_identity + requiredScopes: ["casework:intake"] + actsAs: requester +queues: + applications: + label: Applications awaiting review +reviewPolicies: + licensing-approval: + version: "1" + purpose: approval + context: {mode: submitted} + stages: + - id: review + queue: applications + decidingProfiles: [reviewer] + requiredApprovals: 1 + excludeInitiator: true + retention: + terminalRetentionDays: 30 + accountabilityRetentionDays: 90 + displaySchema: + type: object + additionalProperties: false + required: [reference, profession] + properties: + reference: {type: string, maxLength: 80} + profession: {type: string, maxLength: 80} + outcomes: + approved: + label: Approve application + settlement: approved + reasonRequired: false + publicReason: optional +trustedProducers: + licensing-intake: + profile: intake + issuer: https://identity.licensing.example.test + subject: intake + trustedInitiatorIssuer: https://identity.licensing.example.test + sourceNamespaces: [licensing] + reviewPolicies: [licensing-approval] + retention: {receiptRetentionDays: 7} diff --git a/products/platform/examples/licensing/dev.yaml b/products/platform/examples/licensing/dev.yaml new file mode 100644 index 0000000000..2869a9cf40 --- /dev/null +++ b/products/platform/examples/licensing/dev.yaml @@ -0,0 +1,120 @@ +# yaml-language-server: $schema=https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json +apiVersion: id.registrystack.org/formats/project-dev/v1alpha1 +kind: ProjectDevPlan +issuer: https://identity.licensing.example.test +products: + - {product: breg, project: registry, audience: "urn:example:licensing:registry"} + - {product: casework, project: casework, audience: "urn:example:licensing:review"} +people: + applicant: + subject: applicant + client: applicant-portal + grants: + breg: + profile: applicant + scopes: ["licensing:apply"] + claims: {application_reference: application-001, registry_actor_kind: human} + intake: + subject: intake + client: licensing-intake + grants: + casework: + profile: intake + scopes: ["casework:intake"] + claims: {registry_actor_kind: service} + administrator: + subject: administrator + client: licensing-workbench + grants: + casework: + profile: administrator + scopes: ["casework:admin"] + claims: {registry_actor_kind: human} + reviewer: + subject: reviewer + client: licensing-workbench + grants: + casework: + profile: reviewer + scopes: ["casework:review"] + claims: {registry_actor_kind: human} +journey: + id: licensing-approval + steps: + - id: create-application + product: breg + as: applicant + method: POST + path: /v1/records/applications + headers: {Idempotency-Key: create-application} + body: {data: {reference: application-001, profession: Surveyor}} + expect: {status: 201} + capture: {application: /data/recordIdentifier} + - id: staff-review-queue + product: casework + as: administrator + method: POST + path: /v1/directory/bootstrap + headers: {If-Match: '"0"', Idempotency-Key: staff-review-queue} + body: + teamId: licensing-reviewers + staff: [{issuer: https://identity.licensing.example.test, subject: reviewer}] + supervisors: [] + queueId: applications + expect: {status: 200} + - id: request-approval + product: casework + as: intake + method: POST + path: /v1/review-requests + headers: {Idempotency-Key: request-approval} + body: + policy: {id: licensing-approval} + subject: + source: licensing + type: application + id: "{{application}}" + version: "1" + # SHA-256 of the synthetic submitted snapshot's canonical JSON. + digest: "sha256:847c1042e1013e5b80dc8bdb7c021b584be3a7b778a4f242e42777b07539b9e5" + requesterReference: application-001 + initiator: {issuer: https://identity.licensing.example.test, subject: applicant} + context: + mode: submitted + snapshot: {reference: application-001, profession: Surveyor} + expect: {status: 201} + capture: {request: /requestId} + - id: find-review-task + product: casework + as: reviewer + method: GET + path: /v1/review-tasks?queue=applications + expect: {status: 200, json: {/items/0/requestId: "{{request}}"}} + capture: {task: /items/0/taskId, task-revision: /items/0/revision} + - id: claim-review + product: casework + as: reviewer + method: POST + path: "/v1/review-tasks/{{task}}/claim" + headers: {If-Match: '"{{task-revision}}"', Idempotency-Key: claim-review} + expect: {status: 200} + capture: {claim-revision: /revision} + - id: approve-application + product: casework + as: reviewer + method: POST + path: "/v1/review-tasks/{{task}}/decisions" + headers: {If-Match: '"{{claim-revision}}"', Idempotency-Key: approve-application} + body: {decision: {type: approve, outcome: approved}} + expect: {status: 204} + - id: read-approval + product: casework + as: intake + method: GET + path: "/v1/review-requests/{{request}}/result" + expect: + status: 200 + json: + /subject/id: "{{application}}" + /status: approved + /outcome: approved diff --git a/products/platform/examples/licensing/registry/registry.yaml b/products/platform/examples/licensing/registry/registry.yaml new file mode 100644 index 0000000000..f53edc63ca --- /dev/null +++ b/products/platform/examples/licensing/registry/registry.yaml @@ -0,0 +1,47 @@ +apiVersion: id.registrystack.org/formats/breg/project/v1alpha1 +kind: BRegProject +project: + id: licensing + canonicalBaseIri: https://licensing.example.test + version: 0.1.0 + defaultLanguage: en + display: {label: Licensing registry, description: Synthetic professional licence applications.} +package: + sourceRevision: licensing-example-0.1.0 +datasets: + - id: licensing + display: {label: Licensing, description: Applications awaiting an independent review.} +recordTypes: + - id: application + route: applications + primaryDataset: licensing + mutationMode: mutable + display: {label: Application, description: One licence application.} + fields: + - {id: reference, type: string, required: true, minimumLength: 1, maximumLength: 80, classification: internal} + - {id: profession, type: string, required: true, minimumLength: 1, maximumLength: 80, classification: internal} +accessProfiles: + - id: applicant + default: true + principalClaim: registry_principal + requiredScopes: ["licensing:apply"] + permissions: + records: + - recordType: application + operations: [create, get, list] + readableFields: [reference, profession] + writableFields: [reference, profession] + rowBoundaries: + - {field: reference, claim: application_reference, operator: equals} + - id: auditor + principalClaim: registry_principal + actorKind: human + requesterClients: [licensing-auditor] + requiredScopes: ["licensing:audit"] + permissions: + records: + - recordType: application + operations: [get] + readableFields: [reference] + rowBoundaries: + - {field: reference, claim: application_reference, operator: equals} diff --git a/products/platform/examples/licensing/registry/tests/journeys.yaml b/products/platform/examples/licensing/registry/tests/journeys.yaml new file mode 100644 index 0000000000..2b3b358f78 --- /dev/null +++ b/products/platform/examples/licensing/registry/tests/journeys.yaml @@ -0,0 +1,19 @@ +apiVersion: id.registrystack.org/formats/breg/journeys/v1 +kind: BRegJourneys +journeys: + - id: submit-application + steps: + - id: submit + recordType: application + accessProfile: applicant + claims: + principal: applicant + scopes: ["licensing:apply"] + directClaims: {application_reference: application-001} + request: + type: create + data: {reference: application-001, profession: Surveyor} + expect: + outcome: success + status: 201 + fields: {reference: application-001, profession: Surveyor} diff --git a/products/platform/scripts/test-rstack-dev.py b/products/platform/scripts/test-rstack-dev.py new file mode 100644 index 0000000000..f9414ed8c9 --- /dev/null +++ b/products/platform/scripts/test-rstack-dev.py @@ -0,0 +1,453 @@ +#!/usr/bin/env python3 +"""Exercise rstack dev with real BReg, Casework and a disposable TLS PostgreSQL.""" + +from __future__ import annotations + +import argparse +import base64 +import concurrent.futures +import hashlib +import json +import os +import secrets +import shutil +import socket +import subprocess +import tempfile +import time +from pathlib import Path +from urllib.error import HTTPError +from urllib.parse import unquote, urlsplit +from urllib.request import ProxyHandler, Request, build_opener + +STACK = Path(__file__).resolve().parents[3] +NATIVE_REVIEW = STACK / "products/breg/acceptance/reviewed-evidence-journey" +HTTP = build_opener(ProxyHandler({})) + + +def command(*args, env=None, data=None, cwd=None, timeout=120): + return subprocess.run( + args, + env=env, + input=data, + cwd=cwd, + capture_output=True, + timeout=timeout, + check=False, + ) + + +def checked(*args, **kwargs): + result = command(*args, **kwargs) + if result.returncode: + raise RuntimeError(f"{Path(args[0]).name} failed with exit {result.returncode}") + return result.stdout + + +def port(): + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + return listener.getsockname()[1] + + +def prepare_postgres(root, pg): + password = secrets.token_urlsafe(32) + pwfile = root / "init-password" + pwfile.touch(mode=0o600) + pwfile.write_text(password) + cluster = root / "postgres" + checked( + pg / "initdb", + "-D", + cluster, + "-U", + "devadmin", + "--pwfile", + pwfile, + "--auth-local=trust", + "--auth-host=scram-sha-256", + "--no-locale", + "-E", + "UTF8", + ) + pwfile.unlink() + ca = root / "ca.pem" + checked( + "openssl", + "req", + "-new", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-subj", + "/CN=rstack-dev-test", + "-days", + "1", + "-keyout", + root / "ca.key", + "-out", + ca, + "-addext", + "basicConstraints=critical,CA:TRUE", + ) + checked( + "openssl", + "req", + "-new", + "-newkey", + "rsa:2048", + "-nodes", + "-subj", + "/CN=127.0.0.1", + "-keyout", + cluster / "server.key", + "-out", + root / "server.csr", + ) + (cluster / "server.key").chmod(0o600) + extensions = root / "server.ext" + extensions.write_text("subjectAltName=IP:127.0.0.1\nextendedKeyUsage=serverAuth\n") + checked( + "openssl", + "x509", + "-req", + "-in", + root / "server.csr", + "-CA", + ca, + "-CAkey", + root / "ca.key", + "-CAcreateserial", + "-days", + "1", + "-extfile", + extensions, + "-out", + cluster / "server.crt", + ) + db_port = port() + with (cluster / "postgresql.conf").open("a") as config: + config.write( + f"\nlisten_addresses = '127.0.0.1'\nport = {db_port}\n" + "unix_socket_directories = ''\nssl = on\nlog_min_error_statement = panic\n" + ) + pg_env = { + **os.environ, + "PGHOST": "127.0.0.1", + "PGPORT": str(db_port), + "PGDATABASE": "postgres", + "PGUSER": "devadmin", + "PGPASSWORD": password, + "PGSSLMODE": "verify-full", + "PGSSLROOTCERT": str(ca), + } + url = f"postgresql://devadmin:{password}@127.0.0.1:{db_port}/postgres" + return cluster, ca, url, pg_env + + +def psql(pg, env, query): + return checked( + pg / "psql", "-XAtq", "-v", "ON_ERROR_STOP=1", env=env, data=query.encode() + ).decode() + + +def inventory(pg, env): + databases = psql( + pg, env, "SELECT datname FROM pg_database WHERE datname LIKE 'rstack%';" + ) + roles = psql(pg, env, "SELECT rolname FROM pg_roles WHERE rolname LIKE 'rstack%';") + return set(databases.splitlines()), set(roles.splitlines()) + + +def files(root): + return { + path.relative_to(root): hashlib.sha256(path.read_bytes()).digest() + for path in root.rglob("*") + if path.is_file() + } + + +def dev(bin_dir, env, cwd, *args, expected=0): + result = command( + bin_dir / "rstack", "dev", *args, "--json", env=env, cwd=cwd, timeout=390 + ) + try: + report = json.loads(result.stdout) + except ValueError: + raise RuntimeError( + f"rstack dev {args[0]} did not return JSON (exit {result.returncode})" + ) from None + if result.returncode != expected: + raise RuntimeError(f"rstack dev {args[0]} exited {result.returncode}: {report}") + assert report["schema_version"] == "registry.rstack.dev.v1" + return report + + +def credential(bin_dir, env, cwd, run, alias): + report = dev(bin_dir, env, cwd, "token", run, "--as", alias) + [entry] = report["credentials"] + header_file = Path(entry["header_file"]) + assert header_file.stat().st_mode & 0o077 == 0 + return dict(line.split(": ", 1) for line in header_file.read_text().splitlines()) + + +def get(url, headers): + try: + with HTTP.open(Request(url, headers=headers), timeout=5) as response: + return response.status + except HTTPError as error: + return error.code + + +def kid(headers): + jwt_header = headers["Authorization"].split()[1].split(".")[0] + return json.loads(base64.urlsafe_b64decode(jwt_header + "==="))["kid"] + + +def wait_for_stop(bin_dir, env, cwd, run): + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + if dev(bin_dir, env, cwd, "status", run)["status"] == "stopped": + return + time.sleep(0.2) + raise RuntimeError("the run did not finish its TTL cleanup") + + +def verify_deployment_key(root, bin_dir, env, run_root, headers): + """Start the real verifier with a separate trust pin and the same issuer.""" + key = root / "deployment-issuer.key" + checked( + "openssl", "ecparam", "-name", "prime256v1", "-genkey", "-noout", "-out", key + ) + point = checked("openssl", "pkey", "-in", key, "-pubout", "-outform", "DER")[-65:] + assert point[0] == 4 + + def coordinate(value): + return base64.urlsafe_b64encode(value).decode().rstrip("=") + + jwks = run_root / "deployment-jwks" + jwks.touch(mode=0o600) + jwks.write_text( + json.dumps( + { + "keys": [ + { + "kty": "EC", + "alg": "ES256", + "crv": "P-256", + "kid": "deployment-key", + "x": coordinate(point[1:33]), + "y": coordinate(point[33:]), + } + ] + } + ) + ) + config = json.loads((run_root / "breg-runtime.yaml").read_text()) + config["authentication"]["oidc"]["jwksSource"]["documentRef"] = ( + "secret:file/deployment-jwks" + ) + config["audit"]["path"] = str(root / "deployment-audit" / "audit.jsonl") + config["listener"]["bind"] = f"127.0.0.1:{port()}" + runtime_file = root / "deployment-verifier.yaml" + runtime_file.write_text(json.dumps(config)) + process = subprocess.Popen( + [bin_dir / "breg", "--runtime-config", runtime_file], + env={ + key: value for key, value in env.items() if key != "RSTACK_DEV_DATABASE_URL" + }, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + origin = "http://" + config["listener"]["bind"] + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + assert process.poll() is None, "the independently pinned verifier exited" + try: + if get(origin + "/ready", {}) == 200: + break + except OSError: + pass + time.sleep(0.2) + else: + raise RuntimeError("the independently pinned verifier did not become ready") + assert ( + get(origin + "/v1/records/applications?accessProfile=applicant", headers) + == 401 + ) + finally: + process.terminate() + process.wait(timeout=15) + + +def exercise(root, bin_dir, pg, project, ca, url, pg_env): + checkout = root / "licensing" + shutil.copytree(project, checkout) + before = files(checkout) + state = root / "runs" + env = { + **os.environ, + "RSTACK_DEV_STATE_DIR": str(state), + "RSTACK_DEV_DATABASE_URL": url, + "RSTACK_DEV_DATABASE_CA": str(ca), + "RSTACK_BREGCTL_BIN": str(bin_dir / "bregctl"), + "RSTACK_BREG_BIN": str(bin_dir / "breg"), + "RSTACK_CASEWORKCTL_BIN": str(bin_dir / "caseworkctl"), + "RSTACK_CASEWORK_BIN": str(bin_dir / "casework"), + } + try: + exercise_runs(root, bin_dir, pg, checkout, state, env, pg_env) + assert files(checkout) == before + finally: + if state.exists(): + for run_root in state.iterdir(): + dev(bin_dir, env, checkout, "stop", run_root.name) + + +def exercise_runs(root, bin_dir, pg, checkout, state, env, pg_env): + baseline = inventory(pg, pg_env) + with concurrent.futures.ThreadPoolExecutor(2) as pool: + runs = list( + pool.map(lambda _: dev(bin_dir, env, checkout, "start", "."), range(2)) + ) + a, b = runs + assert all(run["journey"]["result"] == "passed" for run in runs) + assert a["run_id"] != b["run_id"] + assert len({url for run in runs for url in run["products"].values()}) == 4 + databases, roles = inventory(pg, pg_env) + assert all(sum(run["run_id"] in item for item in databases) == 3 for run in runs) + assert all(sum(run["run_id"] in item for item in roles) == 6 for run in runs) + for product in ("breg", "casework"): + own_db = next( + name + for name in databases + if a["run_id"] in name and name.endswith(f"_{product}") + ) + other_db = next( + name + for name in databases + if b["run_id"] in name and name.endswith(f"_{product}") + ) + url = urlsplit((state / a["run_id"] / f"{product}-runtime-url").read_text()) + role_env = { + **pg_env, + "PGUSER": unquote(url.username), + "PGPASSWORD": unquote(url.password), + "PGDATABASE": own_db, + } + assert psql(pg, role_env, "SELECT current_database();").strip() == own_db + refused = command( + pg / "psql", + "-XAtq", + "-v", + "ON_ERROR_STOP=1", + env={**role_env, "PGDATABASE": other_db}, + data=b"SELECT 1;", + ) + assert refused.returncode != 0 + for product, alias, path in [ + ("breg", "applicant", "/v1/records/applications?accessProfile=applicant"), + ("casework", "intake", "/v1/review-policies/licensing-approval"), + ]: + first = credential(bin_dir, env, checkout, a["run_id"], alias) + second = credential(bin_dir, env, checkout, b["run_id"], alias) + assert kid(first) != kid(second) + assert get(a["products"][product] + path, first) == 200 + assert get(b["products"][product] + path, second) == 200 + assert get(a["products"][product] + path, second) == 401 + assert get(b["products"][product] + path, first) == 401 + verify_deployment_key( + root, + bin_dir, + env, + state / a["run_id"], + credential(bin_dir, env, checkout, a["run_id"], "applicant"), + ) + print( + "Concurrent licensing journeys passed with isolated product tokens, ports, databases and roles" + ) + + for run in runs: + assert dev(bin_dir, env, checkout, "stop", run["run_id"])["status"] == "stopped" + assert dev(bin_dir, env, checkout, "stop", run["run_id"])["status"] == "stopped" + assert not (state / run["run_id"]).exists() + assert inventory(pg, pg_env) == baseline + + short = dev(bin_dir, env, checkout, "start", ".", "--ttl", "1s") + wait_for_stop(bin_dir, env, checkout, short["run_id"]) + assert inventory(pg, pg_env) == baseline + assert list(state.iterdir()) == [] + + failure_env = {**env, "RSTACK_CASEWORK_BIN": shutil.which("false")} + report = dev(bin_dir, failure_env, checkout, "start", ".", expected=1) + assert "casework exited before readiness" in report["error"], report + assert inventory(pg, pg_env) == baseline, report + assert list(state.iterdir()) == [], report + verify_native_handoff(root, bin_dir, env) + assert inventory(pg, pg_env) == baseline + assert list(state.iterdir()) == [] + print( + "Stop, TTL expiry and failed startup cleaned up; authored product files stayed byte-identical" + ) + + +def verify_native_handoff(root, bin_dir, env): + checkout = root / "native-review" + checkout.mkdir() + for product in ("registry", "casework"): + shutil.copytree(NATIVE_REVIEW / product, checkout / product) + shutil.copyfile(NATIVE_REVIEW / "dev.yaml", checkout / "dev.yaml") + before = files(checkout) + report = dev(bin_dir, env, checkout, "start", ".") + try: + assert report["journey"]["id"] == "native-review-handoff" + assert report["journey"]["result"] == "passed" + assert files(checkout) == before + print( + "Native review handoff, reviewer context and separately authorized BReg apply passed" + ) + finally: + dev(bin_dir, env, checkout, "stop", report["run_id"]) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--bin-dir", type=Path, required=True) + parser.add_argument("--postgres-bin-dir", type=Path, required=True) + parser.add_argument("--scratch-parent", type=Path) + args = parser.parse_args() + with tempfile.TemporaryDirectory( + prefix="rstack-dev-", dir=args.scratch_parent + ) as directory: + root = Path(directory) + pg = args.postgres_bin_dir.resolve() + cluster, ca, url, pg_env = prepare_postgres(root, pg) + checked( + pg / "pg_ctl", "-D", cluster, "-l", root / "postgres.log", "-w", "start" + ) + try: + assert ( + psql( + pg, + pg_env, + "SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid();", + ).strip() + == "t" + ) + exercise( + root, + args.bin_dir.resolve(), + pg, + STACK / "products/platform/examples/licensing", + ca, + url, + pg_env, + ) + finally: + checked(pg / "pg_ctl", "-D", cluster, "-m", "fast", "-w", "stop") + + +if __name__ == "__main__": + main() From 149965e6c848018153a3f0dbf5923448b94b4b1d Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 23:13:25 +0700 Subject: [PATCH 02/34] fix: handle completed tool groups on macOS Signed-off-by: Jeremi Joslin --- .../src/project_dev/products.rs | 37 ++++++++++++++----- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs index 8635cb7377..5a7881f1dd 100644 --- a/crates/registry-bregctl/src/project_dev/products.rs +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -694,23 +694,26 @@ fn child_exited(child: &Child) -> Result { } } -fn signal_group(pid: Pid, signal: Signal) -> Result<()> { - match kill_process_group(pid, signal) { +fn signal_group(child: &Child, signal: Signal) -> Result<()> { + match kill_process_group(child_pid(child)?, signal) { Ok(()) | Err(rustix::io::Errno::SRCH) => Ok(()), + // Darwin excludes zombies from killpg and reports EPERM when there + // are no signalable members. An owned, same-user group with an + // exited leader can already be empty of live processes. + Err(rustix::io::Errno::PERM) if cfg!(target_os = "macos") && child_exited(child)? => Ok(()), Err(error) => Err(error.into()), } } fn stop_child(child: &mut Child) -> Result { - let pid = child_pid(child)?; if !child_exited(child)? { - signal_group(pid, Signal::TERM)?; + signal_group(child, Signal::TERM)?; let deadline = Instant::now() + Duration::from_secs(2); while Instant::now() < deadline && !child_exited(child)? { thread::sleep(POLL); } } - signal_group(pid, Signal::KILL)?; + signal_group(child, Signal::KILL)?; Ok(child.wait()?) } @@ -844,6 +847,24 @@ mod tests { closes_nested_tool_pipe(true); } + #[test] + fn completed_tool_without_descendants_keeps_its_exit_status() { + let mut child = child_command(Path::new("/bin/sh")) + .args(["-c", "exit 7"]) + .spawn() + .unwrap(); + wait_for_leader(&child); + assert_eq!(stop_child(&mut child).unwrap().code(), Some(7)); + } + + fn wait_for_leader(child: &Child) { + let deadline = Instant::now() + Duration::from_secs(2); + while !child_exited(child).unwrap() { + assert!(Instant::now() < deadline, "the tool leader did not exit"); + thread::sleep(POLL); + } + } + fn closes_nested_tool_pipe(leader_exits: bool) { let leader_end = if leader_exits { "exit 0" } else { "wait" }; let mut child = child_command(Path::new("/bin/sh")) @@ -860,12 +881,8 @@ mod tests { let mut ready = String::new(); pipe.read_line(&mut ready).unwrap(); assert_eq!(ready, "ready\n"); - let deadline = Instant::now() + Duration::from_secs(2); if leader_exits { - while !child_exited(&child).unwrap() { - assert!(Instant::now() < deadline, "the tool leader did not exit"); - thread::sleep(POLL); - } + wait_for_leader(&child); } stop_child(&mut child).unwrap(); From f73f53b697e18917e6adc980ea776fad03f090e1 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 00:19:12 +0700 Subject: [PATCH 03/34] fix: align project development with product contracts Signed-off-by: Jeremi Joslin --- .github/workflows/ci.yml | 11 +- Cargo.lock | 1 - crates/registry-bregctl/Cargo.toml | 1 - .../src/project_dev/bindings.rs | 36 +- .../src/project_dev/database.rs | 101 ++-- .../src/project_dev/issuer.rs | 6 +- .../src/project_dev/journey.rs | 28 +- .../registry-bregctl/src/project_dev/mod.rs | 90 +++- .../registry-bregctl/src/project_dev/plan.rs | 464 ++++++++++++++---- .../src/project_dev/products.rs | 166 +++++-- crates/registry-bregctl/tests/project_dev.rs | 107 +++- crates/registry-bregctl/tests/tool_schema.rs | 86 ++++ crates/registry-cli-docs/catalog.json | 90 ++++ crates/registry-rstack/src/cli.rs | 30 ++ crates/registry-rstack/tests/dev.rs | 21 +- .../scripts/generate-breg-configuration.mjs | 8 +- .../generate-breg-configuration.test.mjs | 13 +- editors/configure.py | 1 + editors/tests/test_configure.py | 6 +- .../reviewed-evidence-journey/dev.yaml | 32 +- .../tools/project-dev.v1alpha1.schema.json | 320 +++++++----- .../identifiers/generated/catalog.v1.json | 2 +- products/platform/PROJECT-DEV.md | 20 +- .../config-conventions-exceptions.yaml | 30 ++ products/platform/config-formats.yaml | 59 +++ .../projects/breg-project-dev/dev.yaml | 25 + .../breg-project-dev/registry/registry.yaml | 1 + products/platform/examples/licensing/dev.yaml | 14 +- .../scripts/run-config-conformance.py | 3 + products/platform/scripts/test-rstack-dev.py | 36 +- .../scripts/test_run_config_conformance.py | 8 + 31 files changed, 1476 insertions(+), 340 deletions(-) create mode 100644 products/platform/conformance/projects/breg-project-dev/dev.yaml create mode 100644 products/platform/conformance/projects/breg-project-dev/registry/registry.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea3fe80443..35ddfdf22b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2697,6 +2697,15 @@ jobs: -p registry-breg --features registry-breg/runtime \ -p registry-bregctl -p registry-rstack --bins + - name: Install native PostgreSQL 18 for the project development run + # The Ubuntu runner's PostgreSQL 16 is below BReg's minimum. + shell: bash + run: | + set -euo pipefail + sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + sudo env DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends postgresql-18 + /usr/lib/postgresql/18/bin/postgres --version + - name: Verify the combined licensing development run with native PostgreSQL shell: bash run: | @@ -2704,7 +2713,7 @@ jobs: cargo build --locked -p registry-rstack python3 products/platform/scripts/test-rstack-dev.py \ --bin-dir "${GITHUB_WORKSPACE}/target/debug" \ - --postgres-bin-dir "$(pg_config --bindir)" \ + --postgres-bin-dir /usr/lib/postgresql/18/bin \ --scratch-parent "${RUNNER_TEMP}" - name: Execute the Registry Casework tutorials from a reader directory diff --git a/Cargo.lock b/Cargo.lock index 750b810ea5..b878119670 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6331,7 +6331,6 @@ dependencies = [ "rcgen", "registry-breg", "registry-breg-client", - "registry-casework-core", "registry-cli-api", "registry-cli-reference", "registry-cli-report", diff --git a/crates/registry-bregctl/Cargo.toml b/crates/registry-bregctl/Cargo.toml index 09609c6c4b..4637a037fd 100644 --- a/crates/registry-bregctl/Cargo.toml +++ b/crates/registry-bregctl/Cargo.toml @@ -65,7 +65,6 @@ registry-cli-api.workspace = true registry-cli-reference.workspace = true registry-breg = { workspace = true, features = ["runtime", "tooling"] } registry-breg-client.workspace = true -registry-casework-core.workspace = true registry-evidence-verifier.workspace = true registry-linkml.workspace = true registry-manifest-core.workspace = true diff --git a/crates/registry-bregctl/src/project_dev/bindings.rs b/crates/registry-bregctl/src/project_dev/bindings.rs index b89c7c978f..886f75a0df 100644 --- a/crates/registry-bregctl/src/project_dev/bindings.rs +++ b/crates/registry-bregctl/src/project_dev/bindings.rs @@ -36,7 +36,7 @@ use zeroize::Zeroizing; use super::{ clock, issuer::Issuer, - plan::{self, Person, Plan, Product}, + plan::{self, Plan, Product, TestPerson}, write_private, RunInput, }; @@ -62,17 +62,23 @@ pub(super) struct Bindings { #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(deny_unknown_fields)] pub(super) struct BregReviewAuthority { + #[cfg_attr(feature = "schema", schemars(with = "registry_platform_yaml::LocalId"))] pub(super) id: String, + #[cfg_attr(feature = "schema", schemars(with = "registry_platform_yaml::LocalId"))] pub(super) producer: String, - pub(super) person: String, + #[cfg_attr(feature = "schema", schemars(with = "registry_platform_yaml::LocalId"))] + pub(super) identity: String, } #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(deny_unknown_fields)] pub(super) struct CaseworkContextProvider { + #[cfg_attr(feature = "schema", schemars(with = "registry_platform_yaml::LocalId"))] pub(super) id: String, - pub(super) person: String, + #[cfg_attr(feature = "schema", schemars(with = "registry_platform_yaml::LocalId"))] + pub(super) identity: String, + #[cfg_attr(feature = "schema", schemars(with = "registry_platform_yaml::LocalId"))] pub(super) action: String, } @@ -103,9 +109,9 @@ impl Bindings { ); require_machine_person( plan, - &binding.person, + &binding.identity, Product::Casework, - "/bindings/bregReviewAuthority/person", + "/bindings/bregReviewAuthority/identity", )?; } if let Some(binding) = &self.casework_context_provider { @@ -119,9 +125,9 @@ impl Bindings { ); require_machine_person( plan, - &binding.person, + &binding.identity, Product::Breg, - "/bindings/caseworkContextProvider/person", + "/bindings/caseworkContextProvider/identity", )?; } Ok(()) @@ -139,7 +145,7 @@ fn require_machine_person<'a>( alias: &str, product: Product, field: &str, -) -> Result<&'a Person> { +) -> Result<&'a TestPerson> { ensure!( plan::identifier(alias), "dev.yaml {field} must be a test person alias" @@ -224,7 +230,7 @@ impl RuntimeBindings { prepare_binding( root, &input, - &binding.person, + &binding.identity, Product::Casework, "breg-review-client", &mut clients, @@ -238,7 +244,7 @@ impl RuntimeBindings { prepare_binding( root, &input, - &binding.person, + &binding.identity, Product::Breg, "casework-context-client", &mut clients, @@ -681,7 +687,7 @@ mod tests { people: BTreeMap::from([ ( "producer".to_owned(), - Person { + TestPerson { subject: "review-producer".to_owned(), client: "review-producer-client".to_owned(), grants: BTreeMap::from([( @@ -693,7 +699,7 @@ mod tests { ), ( "context-reader".to_owned(), - Person { + TestPerson { subject: "context-reader".to_owned(), client: "context-reader-client".to_owned(), grants: BTreeMap::from([( @@ -708,11 +714,11 @@ mod tests { breg_review_authority: Some(BregReviewAuthority { id: "casework".to_owned(), producer: "registry".to_owned(), - person: "producer".to_owned(), + identity: "producer".to_owned(), }), casework_context_provider: Some(CaseworkContextProvider { id: "registry".to_owned(), - person: "context-reader".to_owned(), + identity: "context-reader".to_owned(), action: "reviewed-change".to_owned(), }), }, @@ -752,7 +758,7 @@ mod tests { value.people.get_mut("producer").unwrap().grants.clear(); let error = value.bindings.validate(&value).unwrap_err().to_string(); assert!( - error.contains("/bindings/bregReviewAuthority/person"), + error.contains("/bindings/bregReviewAuthority/identity"), "{error}" ); } diff --git a/crates/registry-bregctl/src/project_dev/database.rs b/crates/registry-bregctl/src/project_dev/database.rs index c5c53059f2..0bcc0d482d 100644 --- a/crates/registry-bregctl/src/project_dev/database.rs +++ b/crates/registry-bregctl/src/project_dev/database.rs @@ -46,10 +46,38 @@ pub(super) struct Database { #[derive(Clone)] struct Endpoint { - host: String, + host: UrlHost, port: Option, } +impl Endpoint { + fn product_url(&self, database: &str, user: &str, password: &str) -> Result { + let mut url = + Url::parse("postgresql://localhost/").expect("the fixed PostgreSQL URL base is valid"); + match &self.host { + UrlHost::Domain(host) => url.set_host(Some(host)).map_err(|_| { + anyhow!("the PostgreSQL host cannot be represented in a product URL") + })?, + UrlHost::Ipv4(address) => url.set_ip_host((*address).into()).map_err(|_| { + anyhow!("the PostgreSQL host cannot be represented in a product URL") + })?, + UrlHost::Ipv6(address) => url.set_ip_host((*address).into()).map_err(|_| { + anyhow!("the PostgreSQL host cannot be represented in a product URL") + })?, + } + url.set_port(self.port) + .map_err(|_| anyhow!("the PostgreSQL port cannot be represented in a product URL"))?; + url.set_username(user) + .map_err(|_| anyhow!("the generated PostgreSQL role cannot be represented in a URL"))?; + url.set_password(Some(password)).map_err(|_| { + anyhow!("the generated PostgreSQL password cannot be represented in a URL") + })?; + url.set_path(&format!("/{database}")); + url.query_pairs_mut().append_pair("sslmode", "require"); + Ok(url.into()) + } +} + impl Database { pub(super) fn connect(admin_url: &str, ca_pem: Option<&[u8]>, run_id: &str) -> Result { validate_run_id(run_id)?; @@ -234,20 +262,7 @@ impl Database { } fn product_url(&self, database: &str, user: &str, password: &str) -> Result { - let mut url = - Url::parse("postgresql://localhost/").expect("the fixed PostgreSQL URL base is valid"); - url.set_host(Some(&self.endpoint.host)) - .map_err(|_| anyhow!("the PostgreSQL host cannot be represented in a product URL"))?; - url.set_port(self.endpoint.port) - .map_err(|_| anyhow!("the PostgreSQL port cannot be represented in a product URL"))?; - url.set_username(user) - .map_err(|_| anyhow!("the generated PostgreSQL role cannot be represented in a URL"))?; - url.set_password(Some(password)).map_err(|_| { - anyhow!("the generated PostgreSQL password cannot be represented in a URL") - })?; - url.set_path(&format!("/{database}")); - url.query_pairs_mut().append_pair("sslmode", "require"); - Ok(url.into()) + self.endpoint.product_url(database, user, password) } } @@ -373,11 +388,7 @@ fn admin_config(admin_url: &str) -> Result<(Config, Endpoint)> { bail!("the PostgreSQL admin URL must name one TCP host, user, password, and database"); } - let host = match parsed.host().expect("the host was checked") { - UrlHost::Domain(value) => value.to_owned(), - UrlHost::Ipv4(value) => value.to_string(), - UrlHost::Ipv6(value) => value.to_string(), - }; + let host = parsed.host().expect("the host was checked").to_owned(); Ok(( config, Endpoint { @@ -501,10 +512,12 @@ mod tests { #[test] fn generated_product_urls_contain_only_generated_credentials_and_required_tls() { let endpoint = Endpoint { - host: "db.example".to_owned(), + host: UrlHost::Domain("db.example".to_owned()), port: Some(5543), }; - let url = product_url_for_test(&endpoint, "owned", "runtime", "generated-secret"); + let url = endpoint + .product_url("owned", "runtime", "generated-secret") + .unwrap(); assert_eq!( url, "postgresql://runtime:generated-secret@db.example:5543/owned?sslmode=require" @@ -512,6 +525,34 @@ mod tests { assert!(!url.contains("admin")); } + #[test] + fn generated_product_urls_round_trip_ipv6_hosts_and_credentials() { + for address_text in ["::1", "2001:db8:1234:5678::9"] { + let address = address_text.parse().unwrap(); + let (_, endpoint) = admin_config(&format!( + "postgresql://admin:secret@[{address_text}]:5543/postgres" + )) + .unwrap(); + for (user, database, password) in [ + ("runtime", "owned", "runtime-generated-secret"), + ("migration", "owned_test", "migration-generated-secret"), + ] { + let rendered = endpoint.product_url(database, user, password).unwrap(); + let parsed = Url::parse(&rendered).unwrap(); + assert_eq!(parsed.host(), Some(UrlHost::Ipv6(address)), "{rendered}"); + assert_eq!(parsed.port(), Some(5543), "{rendered}"); + assert_eq!(parsed.username(), user, "{rendered}"); + assert_eq!(parsed.password(), Some(password), "{rendered}"); + assert_eq!(parsed.path(), format!("/{database}"), "{rendered}"); + assert_eq!(parsed.query(), Some("sslmode=require"), "{rendered}"); + assert!( + rendered.contains(&format!("@[{address}]:5543/")), + "{rendered}" + ); + } + } + } + #[test] fn database_access_is_closed_to_public_before_connections_are_enabled() { let names = Names::new(RUN_ID, "casework"); @@ -589,20 +630,4 @@ mod tests { assert!(tls_connector(Some(b"")).is_err()); assert!(tls_connector(Some(b"not a certificate")).is_err()); } - - fn product_url_for_test( - endpoint: &Endpoint, - database: &str, - user: &str, - password: &str, - ) -> String { - let mut url = Url::parse("postgresql://localhost/").unwrap(); - url.set_host(Some(&endpoint.host)).unwrap(); - url.set_port(endpoint.port).unwrap(); - url.set_username(user).unwrap(); - url.set_password(Some(password)).unwrap(); - url.set_path(&format!("/{database}")); - url.query_pairs_mut().append_pair("sslmode", "require"); - url.into() - } } diff --git a/crates/registry-bregctl/src/project_dev/issuer.rs b/crates/registry-bregctl/src/project_dev/issuer.rs index d03c2309b1..03442434ad 100644 --- a/crates/registry-bregctl/src/project_dev/issuer.rs +++ b/crates/registry-bregctl/src/project_dev/issuer.rs @@ -11,7 +11,7 @@ use zeroize::Zeroizing; use super::{ clock, - plan::{self, Person, Product}, + plan::{self, Product, TestPerson}, write_private, RunInput, }; @@ -54,7 +54,7 @@ impl Issuer { pub(super) fn token( &self, input: &RunInput, - person: &Person, + person: &TestPerson, product: Product, expires: u64, ) -> Result> { @@ -65,7 +65,7 @@ impl Issuer { pub(super) async fn sign( &self, input: &RunInput, - person: &Person, + person: &TestPerson, product: Product, expires: u64, ) -> Result> { diff --git a/crates/registry-bregctl/src/project_dev/journey.rs b/crates/registry-bregctl/src/project_dev/journey.rs index fefa083817..ccc8e2f6ad 100644 --- a/crates/registry-bregctl/src/project_dev/journey.rs +++ b/crates/registry-bregctl/src/project_dev/journey.rs @@ -308,12 +308,12 @@ fn capture_value( lookup: impl Fn(&str) -> Result + Copy, ) -> Result { match capture { - Capture::Pointer(pointer) => response + Capture::Pointer { pointer } => response .body .pointer(pointer) .cloned() .context("journey capture pointer was not present"), - Capture::Header(header) => match header.header { + Capture::Header { header } => match header { CapturedHeader::Etag => Ok(Value::String( response .headers @@ -324,13 +324,13 @@ fn capture_value( .to_owned(), )), }, - Capture::Value(capture) => { + Capture::Value { pointer, query } => { let value = response .body - .pointer(&capture.pointer) + .pointer(pointer) .cloned() .context("journey capture pointer was not present")?; - if let Some(query) = &capture.query { + if let Some(query) = query { let text = value .as_str() .context("journey query capture did not select text")?; @@ -345,16 +345,20 @@ fn capture_value( Ok(value) } } - Capture::Find(capture) => { - let expected = expand_json(&capture.r#where.equals, lookup)?; + Capture::Find { + find, + r#where, + pointer, + } => { + let expected = expand_json(&r#where.equals, lookup)?; let items = response .body - .pointer(&capture.find) + .pointer(find) .and_then(Value::as_array) .context("journey find capture did not select an array")?; let mut matches = items .iter() - .filter(|item| item.pointer(&capture.r#where.pointer) == Some(&expected)); + .filter(|item| item.pointer(&r#where.pointer) == Some(&expected)); let item = matches .next() .context("journey find capture found no item")?; @@ -362,7 +366,7 @@ fn capture_value( matches.next().is_none(), "journey find capture matched more than one item" ); - item.pointer(&capture.pointer) + item.pointer(pointer) .cloned() .context("journey find capture pointer was not present") } @@ -546,6 +550,7 @@ mod tests { #[test] fn find_capture_requires_one_matching_item() { let capture: Capture = serde_json::from_value(json!({ + "type": "find", "find": "/items", "where": {"pointer": "/id", "equals": "two"}, "pointer": "/revision" })) @@ -565,9 +570,10 @@ mod tests { #[test] fn expected_status_helpers_keep_the_first_for_reports() { - let status = ExpectedStatus::Many(vec![202, 200]); + let status: ExpectedStatus = serde_json::from_value(json!([202, 200])).unwrap(); assert_eq!(status.first(), 202); assert!(status.contains(200)); + assert_eq!(serde_json::to_value(&status).unwrap(), json!([202, 200])); } #[test] diff --git a/crates/registry-bregctl/src/project_dev/mod.rs b/crates/registry-bregctl/src/project_dev/mod.rs index e149b47e38..a8c6768aca 100644 --- a/crates/registry-bregctl/src/project_dev/mod.rs +++ b/crates/registry-bregctl/src/project_dev/mod.rs @@ -10,6 +10,7 @@ pub(crate) mod plan; mod products; use anyhow::{bail, ensure, Context, Result}; +use registry_platform_yaml::{Diagnostic, Report}; use rustix::fs::{flock, FlockOperation}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; @@ -17,7 +18,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, env, fs::{self, File, OpenOptions}, - io::{BufRead, BufReader, Read, Write}, + io::{self, BufRead, BufReader, Read, Write}, os::unix::{ fs::{DirBuilderExt, MetadataExt, OpenOptionsExt}, process::CommandExt, @@ -57,6 +58,11 @@ pub(crate) struct Args { #[derive(Deserialize)] #[serde(tag = "action", rename_all = "snake_case", deny_unknown_fields)] enum Request { + Check { + project: PathBuf, + deny_warnings: bool, + json: bool, + }, Start { project: PathBuf, ttl: Option, @@ -80,7 +86,8 @@ enum Request { impl Request { fn json(&self) -> bool { match self { - Self::Start { json, .. } + Self::Check { json, .. } + | Self::Start { json, .. } | Self::Status { json, .. } | Self::Token { json, .. } | Self::Stop { json, .. } => *json, @@ -170,6 +177,11 @@ pub(crate) fn run(args: Args, stdout: &mut dyn Write, stderr: &mut dyn Write) -> let machine = request.json(); let starting = matches!(&request, Request::Start { .. }); let outcome = match request { + Request::Check { + project, + deny_warnings, + .. + } => return check(&project, deny_warnings, machine, stdout, stderr), Request::Start { project, ttl, .. } => start(&project, ttl.as_deref()), Request::Status { run_id, .. } => status(&run_id), Request::Token { run_id, person, .. } => token(&run_id, &person), @@ -181,8 +193,9 @@ pub(crate) fn run(args: Args, stdout: &mut dyn Write, stderr: &mut dyn Write) -> (emit(stdout, &value, machine), code) } Err(error) if machine => { - let mut report = - json!({"schema_version": SCHEMA, "status": "error", "error": format!("{error:#}")}); + let mut report = json!({ + "schema_version": SCHEMA, "status": "error", "error": format!("{error:#}") + }); if let Some(pending) = error.downcast_ref::() { report["run_id"] = json!(pending.run_id); report["cleanup_pending"] = json!(true); @@ -248,6 +261,69 @@ fn emit(output: &mut dyn Write, report: &Value, machine: bool) -> std::io::Resul Ok(()) } +fn check( + project: &Path, + deny_warnings: bool, + machine: bool, + stdout: &mut dyn Write, + stderr: &mut dyn Write, +) -> u8 { + let outcome = project + .canonicalize() + .context("cannot open the development project directory") + .and_then(|project| plan::check(&project)); + let (report, exit) = match outcome { + Ok(report) => { + let refused = report.has_errors() || (deny_warnings && report.warning_count() > 0); + (report, u8::from(refused)) + } + Err(error) => match error.downcast::() { + Ok(report) => (report, crate::DOMAIN_REFUSAL_EXIT), + Err(error) => { + let (code, action, exit) = if error.is::() { + ( + "breg.project-dev.check-unavailable", + "Name a project directory containing a readable dev.yaml.", + crate::OPERATIONAL_FAILURE_EXIT, + ) + } else { + ( + "breg.project-dev.invalid-plan", + "Correct the development plan at the reported field.", + crate::DOMAIN_REFUSAL_EXIT, + ) + }; + ( + Report::new(vec![Diagnostic::error(code, "", error.to_string(), action)]), + exit, + ) + } + }, + }; + let result = if machine { + let payload = json!({ + "ok": exit == 0, + "command": "dev-check", + "status": registry_cli_report::Status::from_exit(exit).as_str(), + "diagnostics": report.to_json_value(), + }); + crate::write_ctl_report(stdout, &payload) + .map_err(io::Error::other) + .and_then(|()| writeln!(stdout)) + } else if exit == 0 { + writeln!(stdout, "Development plan is valid") + .and_then(|()| write!(stdout, "{}", report.render_human())) + } else { + write!(stderr, "{}", report.render_human()) + }; + if result.is_ok() { + exit + } else { + let _ = writeln!(stderr, "rstack dev check: output could not be written"); + crate::OPERATIONAL_FAILURE_EXIT + } +} + fn start(project: &Path, ttl: Option<&str>) -> Result { let project = project .canonicalize() @@ -255,8 +331,9 @@ fn start(project: &Path, ttl: Option<&str>) -> Result { let plan = plan::load(&project)?; let schema_people = products::schema_people(&project, &plan)?; let breg_authority = products::breg_authority(&project, &plan)?; + let signals = signals()?; let casework_principals = - products::casework_principals(&project, &plan, breg_authority.as_ref())?; + products::casework_principals(&project, &plan, breg_authority.as_ref(), &signals)?; let ttl_seconds = parse_ttl(ttl.unwrap_or("30m"))?; ensure!( env::var_os(DATABASE_URL).is_some_and(|value| !value.is_empty()), @@ -267,7 +344,6 @@ fn start(project: &Path, ttl: Option<&str>) -> Result { .iter() .map(|entry| Ok((entry.product, binaries(entry.product)?))) .collect::>()?; - let signals = signals()?; let run_id = Uuid::new_v4().simple().to_string(); let base = state_dir(); ensure_private(&base, true)?; @@ -549,7 +625,7 @@ fn token(run_id: &str, person: &str) -> Result { } Ok(json!({ "schema_version": SCHEMA, "run_id": run_id, "status": "token", - "person": person, "credentials": credentials, + "alias": person, "credentials": credentials, })) } diff --git a/crates/registry-bregctl/src/project_dev/plan.rs b/crates/registry-bregctl/src/project_dev/plan.rs index ad20279eb8..6f7320b89f 100644 --- a/crates/registry-bregctl/src/project_dev/plan.rs +++ b/crates/registry-bregctl/src/project_dev/plan.rs @@ -2,7 +2,13 @@ use anyhow::{bail, Context, Result}; use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; -use registry_platform_yaml::{ApiVersion, EnvelopeRule, Expect, FormatSpec, Reader}; +use registry_breg::literal_text::{LiteralText, WRITE_THE_VALUE}; +use registry_platform_yaml::{ + ApiVersion, BoundedU32, Decoded, EnvelopeRule, Expect, FormatSpec, Identified, LocalId, Reader, + Report, Severity, UniqueIdList, UniqueList, +}; +#[cfg(feature = "schema")] +use registry_platform_yaml::{ExternalId, Url as PlatformUrl}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use std::{ @@ -11,14 +17,15 @@ use std::{ io::Read, path::{Component, Path, PathBuf}, }; -use url::Url; +use url::Url as ParsedUrl; + +pub(crate) const API_VERSION: &str = "id.registrystack.org/formats/breg/project-dev/v1alpha1"; +pub(crate) const KIND: &str = "BRegProjectDev"; pub(crate) const FORMAT: FormatSpec<'static> = FormatSpec { - kind: "ProjectDevPlan", + kind: KIND, envelope: EnvelopeRule::ApiVersionKind { - api_versions: &[ApiVersion::current( - "id.registrystack.org/formats/project-dev/v1alpha1", - )], + api_versions: &[ApiVersion::current(API_VERSION)], retired_api_versions: &[], }, removed_keys: &[], @@ -29,6 +36,109 @@ pub(crate) fn schema() -> schemars::Schema { schemars::schema_for!(Plan) } +#[cfg(feature = "schema")] +fn keyed_schema( + generator: &mut schemars::SchemaGenerator, +) -> schemars::Schema { + schemars::json_schema!({ + "type": "object", + "propertyNames": generator.subschema_for::(), + "additionalProperties": generator.subschema_for::(), + }) +} + +#[cfg(feature = "schema")] +struct DerivedId; + +#[cfg(feature = "schema")] +impl schemars::JsonSchema for DerivedId { + fn schema_name() -> std::borrow::Cow<'static, str> { + "DerivedId".into() + } + + fn json_schema(_generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "string", + "maxLength": 64, + "pattern": "^[a-z][a-z0-9_-]{0,63}(\\.[a-z][a-z0-9_-]{0,63})+$", + "description": "A dotted development journey step identifier whose segments are local identifiers.", + }) + } +} + +#[cfg(feature = "schema")] +fn step_id_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "anyOf": [ + generator.subschema_for::(), + generator.subschema_for::(), + ], + }) +} + +#[cfg(feature = "schema")] +fn capture_map_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + let names = schemars::json_schema!({ + "allOf": [ + generator.subschema_for::(), + { + "type": "string", + "maxLength": 64, + "pattern": "^[A-Za-z][A-Za-z0-9-]{0,63}$", + }, + ], + }); + schemars::json_schema!({ + "type": "object", + "propertyNames": names, + "additionalProperties": generator.subschema_for::(), + }) +} + +#[cfg(feature = "schema")] +fn schema_test_people_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + let steps = schemars::json_schema!({ + "type": "object", + "propertyNames": step_id_schema(generator), + "additionalProperties": generator.subschema_for::(), + }); + schemars::json_schema!({ + "type": "object", + "propertyNames": generator.subschema_for::(), + "additionalProperties": steps, + }) +} + +#[cfg(feature = "schema")] +fn claims_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "object", + "propertyNames": generator.subschema_for::(), + "additionalProperties": true, + "x-registry-passthrough": "Claim values are written into the development token as given.", + }) +} + +#[cfg(feature = "schema")] +fn scopes_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "array", + "items": generator.subschema_for::(), + "maxItems": 32, + "uniqueItems": true, + }) +} + +#[cfg(feature = "schema")] +fn expected_json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "object", + "propertyNames": generator.subschema_for::(), + "additionalProperties": true, + "x-registry-passthrough": "Expected values are compared with the product response at the named JSON pointers.", + }) +} + #[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "lowercase")] @@ -50,13 +160,22 @@ impl Product { #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub(super) struct Plan { + #[cfg_attr(feature = "schema", schemars(with = "PlatformUrl"))] pub issuer: String, pub products: Vec, - pub people: BTreeMap, + #[cfg_attr( + feature = "schema", + schemars(schema_with = "keyed_schema::") + )] + pub people: BTreeMap, pub journey: Journey, #[serde(default)] pub bindings: super::bindings::Bindings, #[serde(default)] + #[cfg_attr( + feature = "schema", + schemars(schema_with = "schema_test_people_schema") + )] pub schema_test_people: BTreeMap>, } @@ -66,15 +185,19 @@ pub(super) struct Plan { pub(super) struct ProductPlan { pub product: Product, pub project: PathBuf, + #[cfg_attr(feature = "schema", schemars(with = "ExternalId"))] pub audience: String, } #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(deny_unknown_fields)] -pub(super) struct Person { +pub(super) struct TestPerson { + #[cfg_attr(feature = "schema", schemars(with = "ExternalId"))] pub subject: String, + #[cfg_attr(feature = "schema", schemars(with = "ExternalId"))] pub client: String, + #[serde(deserialize_with = "deserialize_grants")] pub grants: BTreeMap, /// Forward the same authority to both products. Both grants must declare the same scopes and additional claims. #[serde(default, rename = "caseworkContext")] @@ -85,17 +208,60 @@ pub(super) struct Person { #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(deny_unknown_fields)] pub(super) struct Grant { + #[cfg_attr(feature = "schema", schemars(with = "LocalId"))] pub profile: String, + #[serde(deserialize_with = "deserialize_scopes")] + #[cfg_attr(feature = "schema", schemars(schema_with = "scopes_schema"))] pub scopes: Vec, #[serde(default)] + #[cfg_attr(feature = "schema", schemars(schema_with = "claims_schema"))] pub claims: Map, } +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct GrantsDocument { + breg: Option, + casework: Option, +} + +fn deserialize_grants<'de, D>( + deserializer: D, +) -> std::result::Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let grants = GrantsDocument::deserialize(deserializer)?; + Ok([ + grants.breg.map(|grant| (Product::Breg, grant)), + grants.casework.map(|grant| (Product::Casework, grant)), + ] + .into_iter() + .flatten() + .collect()) +} + +fn deserialize_scopes<'de, D>(deserializer: D) -> std::result::Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + UniqueList::::deserialize(deserializer).map(UniqueList::into_vec) +} + +fn deserialize_steps<'de, D>(deserializer: D) -> std::result::Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + UniqueIdList::::deserialize(deserializer).map(UniqueIdList::into_vec) +} + #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(deny_unknown_fields)] pub(super) struct Journey { + #[cfg_attr(feature = "schema", schemars(with = "LocalId"))] pub id: String, + #[serde(deserialize_with = "deserialize_steps")] pub steps: Vec, } @@ -103,22 +269,35 @@ pub(super) struct Journey { #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(deny_unknown_fields)] pub(super) struct Step { + #[cfg_attr(feature = "schema", schemars(schema_with = "step_id_schema"))] pub id: String, pub product: Product, #[serde(rename = "as")] + #[cfg_attr(feature = "schema", schemars(with = "LocalId"))] pub person: String, pub method: Method, pub path: String, pub body: Option, pub raw: Option, #[serde(default)] + #[cfg_attr( + feature = "schema", + schemars(schema_with = "keyed_schema::") + )] pub headers: BTreeMap, pub expect: Expected, #[serde(default)] + #[cfg_attr(feature = "schema", schemars(schema_with = "capture_map_schema"))] pub capture: BTreeMap, pub poll: Option, } +impl Identified for Step { + fn id(&self) -> &str { + &self.id + } +} + #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "UPPERCASE")] @@ -148,39 +327,48 @@ impl From<&Method> for reqwest::Method { pub(super) struct Expected { pub status: ExpectedStatus, #[serde(default)] + #[cfg_attr(feature = "schema", schemars(schema_with = "expected_json_schema"))] pub json: BTreeMap, #[serde(default)] pub checks: Vec, pub code: Option, } -#[derive(Deserialize, Serialize)] +#[derive(Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(untagged)] +#[cfg_attr(feature = "schema", schemars(untagged))] pub(super) enum ExpectedStatus { - One(u16), - Many(Vec), + One(BoundedU32<200, 599>), + Many(Vec>), } +registry_platform_yaml::shape_union!(ExpectedStatus { + scalar => One, + list => Many, +}); + impl ExpectedStatus { pub(super) fn first(&self) -> u16 { match self { - Self::One(status) => *status, - Self::Many(statuses) => statuses[0], + Self::One(status) => status.get() as u16, + Self::Many(statuses) => statuses[0].get() as u16, } } pub(super) fn contains(&self, status: u16) -> bool { match self { - Self::One(expected) => *expected == status, - Self::Many(expected) => expected.contains(&status), + Self::One(expected) => expected.get() == u32::from(status), + Self::Many(expected) => expected + .iter() + .any(|expected| expected.get() == u32::from(status)), } } fn values(&self) -> Vec { match self { - Self::One(status) => vec![*status], - Self::Many(statuses) => statuses.clone(), + Self::One(status) => vec![status.get() as u16], + Self::Many(statuses) => statuses.iter().map(|status| status.get() as u16).collect(), } } } @@ -191,25 +379,30 @@ impl ExpectedStatus { pub(super) struct Check { pub pointer: String, pub equals: Option, + #[cfg_attr(feature = "schema", schemars(range(min = 0, max = 1_048_576)))] pub length: Option, pub contains: Option, } #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(untagged)] +#[serde(tag = "type", rename_all = "kebab-case", deny_unknown_fields)] pub(super) enum Capture { - Pointer(String), - Header(HeaderCapture), - Find(FindCapture), - Value(ValueCapture), -} - -#[derive(Deserialize, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(deny_unknown_fields)] -pub(super) struct HeaderCapture { - pub header: CapturedHeader, + Pointer { + pointer: String, + }, + Header { + header: CapturedHeader, + }, + Find { + find: String, + r#where: FindWhere, + pointer: String, + }, + Value { + pointer: String, + query: Option, + }, } #[derive(Deserialize, Serialize)] @@ -219,23 +412,6 @@ pub(super) enum CapturedHeader { Etag, } -#[derive(Deserialize, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(deny_unknown_fields)] -pub(super) struct ValueCapture { - pub pointer: String, - pub query: Option, -} - -#[derive(Deserialize, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(deny_unknown_fields)] -pub(super) struct FindCapture { - pub find: String, - pub r#where: FindWhere, - pub pointer: String, -} - #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(deny_unknown_fields)] @@ -248,7 +424,11 @@ pub(super) struct FindWhere { #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub(super) struct Poll { + #[serde(rename = "maximumWaitMilliseconds")] + #[cfg_attr(feature = "schema", schemars(range(min = 1, max = 60_000)))] pub timeout_ms: u64, + #[serde(rename = "pollIntervalMilliseconds")] + #[cfg_attr(feature = "schema", schemars(range(min = 50, max = 5_000)))] pub interval_ms: u64, } @@ -261,10 +441,7 @@ pub(super) struct RawBody { } pub(super) fn identifier(value: &str) -> bool { - (1..=64).contains(&value.len()) - && value - .bytes() - .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + LocalId::new(value).is_ok() } fn bounded_atom(value: &str) -> bool { @@ -305,14 +482,49 @@ pub(super) fn read(path: &Path, limit: u64) -> Result> { Ok(bytes) } -pub(super) fn load(root: &Path) -> Result { +fn read_plan(root: &Path) -> Result> { let file = root.join("dev.yaml"); - let bytes = read(&file, 256 * 1024).context("cannot read dev.yaml")?; - let plan = Reader::new("dev.yaml") - .decode::(&bytes, &Expect::one(&FORMAT))? - .value; - plan.validate(root)?; - Ok(plan) + let bytes = read( + &file, + registry_platform_yaml::MAXIMUM_DOCUMENT_BYTES as u64 + 1, + ) + .context("cannot read dev.yaml")?; + Ok(Reader::new(file.display().to_string()) + .with_hook(&mut LiteralText { + remedy: WRITE_THE_VALUE, + }) + .decode::(&bytes, &Expect::one(&FORMAT))?) +} + +fn validate_plan(root: &Path, decoded: &Decoded) -> Result<()> { + if let Err(error) = decoded.value.validate(root) { + let rendered = error.to_string(); + let path = rendered + .strip_prefix("dev.yaml ") + .and_then(|detail| detail.split_once(' ')) + .map_or("", |(path, _)| path); + let diagnostic = decoded.document.diagnostic_at_value( + Severity::Error, + "rstack.dev.invalid-plan", + path, + "The development plan is inconsistent.", + "Correct the named development plan field, then run the check again.", + ); + return Err(decoded.document.report_with([diagnostic]).into()); + } + Ok(()) +} + +pub(crate) fn check(root: &Path) -> Result { + let decoded = read_plan(root)?; + validate_plan(root, &decoded)?; + Ok(decoded.document.report_with(Vec::new())) +} + +pub(super) fn load(root: &Path) -> Result { + let decoded = read_plan(root)?; + validate_plan(root, &decoded)?; + Ok(decoded.value) } impl Plan { @@ -325,7 +537,7 @@ impl Plan { fn validate(&self, root: &Path) -> Result<()> { require( - Url::parse(&self.issuer).is_ok_and(|url| { + ParsedUrl::parse(&self.issuer).is_ok_and(|url| { url.scheme() == "https" && url.host().is_some() && url.username().is_empty() @@ -658,19 +870,21 @@ impl Journey { "must use unique bounded capture names", )?; match capture { - Capture::Pointer(pointer_value) => require( + Capture::Pointer { + pointer: pointer_value, + } => require( pointer(pointer_value), &format!("{field}/capture/{name}"), "must use a JSON pointer", )?, - Capture::Header(_) => {} - Capture::Value(capture) => { + Capture::Header { .. } => {} + Capture::Value { pointer, query } => { require( - pointer(&capture.pointer), + self::pointer(pointer), &format!("{field}/capture/{name}/pointer"), "must be a JSON pointer", )?; - if let Some(query) = &capture.query { + if let Some(query) = query { require( bounded_atom(query), &format!("{field}/capture/{name}/query"), @@ -678,15 +892,17 @@ impl Journey { )?; } } - Capture::Find(capture) => { + Capture::Find { + find, + r#where, + pointer: selected, + } => { require( - pointer(&capture.find) - && pointer(&capture.r#where.pointer) - && pointer(&capture.pointer), + pointer(find) && pointer(&r#where.pointer) && pointer(selected), &format!("{field}/capture/{name}"), "must use JSON pointers", )?; - expand_json(&capture.r#where.equals, |value| { + expand_json(&r#where.equals, |value| { captures .contains(value) .then(|| value.to_owned()) @@ -704,12 +920,7 @@ impl Journey { fn step_identifier(value: &str) -> bool { (1..=64).contains(&value.len()) - && value - .bytes() - .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"-.".contains(&byte)) - && !matches!(value.as_bytes().first(), Some(b'-' | b'.')) - && !value.ends_with('.') - && !value.contains("..") + && (identifier(value) || (value.contains('.') && value.split('.').all(identifier))) } fn capture_identifier(value: &str) -> bool { @@ -806,9 +1017,10 @@ mod tests { ] }, "capture": { - "requestId": {"pointer": "/requestId", "query": "id"}, - "etag": {"header": "etag"}, + "requestId": {"type": "value", "pointer": "/requestId", "query": "id"}, + "etag": {"type": "header", "header": "etag"}, "selected-item": { + "type": "find", "find": "/items", "where": {"pointer": "/id", "equals": "one"}, "pointer": "/value" } @@ -818,7 +1030,7 @@ mod tests { assert!(plan.journey.steps[0].expect.status.contains(201)); assert!(matches!( plan.journey.steps[0].capture["etag"], - Capture::Header(_) + Capture::Header { .. } )); assert!(!plan.people["operator"].casework_context); } @@ -829,19 +1041,103 @@ mod tests { "id": "read.v1", "product": "breg", "as": "operator", "method": "GET", "path": "/v1/records", "expect": {"status": [200, 202]}, - "capture": {"recordId": "/recordIdentifier"}, - "poll": {"timeoutMs": 500, "intervalMs": 50} + "capture": {"draftEtag": {"type": "pointer", "pointer": "/recordIdentifier"}}, + "poll": {"maximumWaitMilliseconds": 500, "pollIntervalMilliseconds": 50} })); + let mut value = serde_json::to_value(plan).unwrap(); + let person = value["people"] + .as_object_mut() + .unwrap() + .remove("operator") + .unwrap(); + value["people"]["date_of_birth"] = person; + value["people"]["date_of_birth"]["grants"]["breg"]["profile"] = json!("date_of_birth"); + value["journey"]["id"] = json!("review_flow"); + value["journey"]["steps"][0]["as"] = json!("date_of_birth"); + value["journey"]["steps"] + .as_array_mut() + .unwrap() + .push(json!({ + "id": "use.v1", "product": "breg", "as": "date_of_birth", + "method": "GET", "path": "/v1/records/{{draftEtag}}", + "expect": {"status": 200} + })); + value["schemaTestPeople"] = json!({ + "review_flow": {"read.v1": "date_of_birth"} + }); + let plan: Plan = serde_json::from_value(value).unwrap(); + plan.validate(&canonical(&root)).unwrap(); } + #[test] + fn identifier_grammars_refuse_values_the_schema_refuses() { + for accepted in ["date-of-birth", "date_of_birth"] { + assert!(identifier(accepted), "local identifier: {accepted}"); + } + for refused in ["-person", "1person", "Person", "person.name"] { + assert!(!identifier(refused), "local identifier: {refused}"); + } + + for accepted in ["read", "read_v1", "read.v1", "read_record.v1_beta"] { + assert!(step_identifier(accepted), "step identifier: {accepted}"); + } + for refused in [ + "-read", "1read", ".read", "read.", "read..v1", "read.-v1", "Read.v1", + ] { + assert!(!step_identifier(refused), "step identifier: {refused}"); + } + + for accepted in ["draftEtag", "DraftEtag", "draft-etag"] { + assert!( + capture_identifier(accepted), + "capture identifier: {accepted}" + ); + } + for refused in ["1draft", "draft_etag", "draft.etag"] { + assert!( + !capture_identifier(refused), + "capture identifier: {refused}" + ); + } + } + + #[test] + fn validation_refuses_a_product_project_outside_the_development_project() { + let (root, mut plan) = plan(json!({ + "id": "read", "product": "breg", "as": "operator", + "method": "GET", "path": "/v1/records", "expect": {"status": 200} + })); + plan.products[0].project = PathBuf::from("../outside"); + + let error = plan.validate(&canonical(&root)).unwrap_err().to_string(); + + assert!(error.contains("/products/0/project"), "{error}"); + } + + #[test] + fn validation_refuses_a_product_project_link_outside_the_development_project() { + let (root, mut plan) = plan(json!({ + "id": "read", "product": "breg", "as": "operator", + "method": "GET", "path": "/v1/records", "expect": {"status": 200} + })); + let outside = tempfile::tempdir().unwrap(); + fs::write(outside.path().join("registry.yaml"), "fixture").unwrap(); + std::os::unix::fs::symlink(outside.path(), root.path().join("registry")).unwrap(); + plan.products[0].project = PathBuf::from("registry"); + + let error = plan.validate(&canonical(&root)).unwrap_err().to_string(); + + assert!(error.contains("/products/0/project"), "{error}"); + } + #[test] fn validation_refuses_polling_a_mutation() { let (root, plan) = plan(json!({ "id": "create", "product": "breg", "as": "operator", "method": "POST", "path": "/v1/records", "body": {}, "expect": {"status": 201}, - "poll": {"timeoutMs": 500, "intervalMs": 50} + "poll": {"maximumWaitMilliseconds": 500, "pollIntervalMilliseconds": 50} })); let error = plan.validate(&canonical(&root)).unwrap_err().to_string(); assert!(error.contains("supported only for GET"), "{error}"); @@ -853,7 +1149,7 @@ mod tests { "id": "read", "product": "breg", "as": "operator", "method": "GET", "path": "/v1/records/{{later}}", "expect": {"status": 200}, - "capture": {"later": "/recordIdentifier"} + "capture": {"later": {"type": "pointer", "pointer": "/recordIdentifier"}} })); let error = format!("{:#}", plan.validate(&canonical(&root)).unwrap_err()); assert!(error.contains("earlier capture"), "{error}"); @@ -865,7 +1161,7 @@ mod tests { "id": "read", "product": "breg", "as": "operator", "method": "GET", "path": "/v1/records", "expect": {"status": 200}, - "poll": {"timeoutMs": 30000, "intervalMs": 50} + "poll": {"maximumWaitMilliseconds": 30000, "pollIntervalMilliseconds": 50} })); let error = plan.validate(&canonical(&root)).unwrap_err().to_string(); assert!(error.contains("at most 250 attempts"), "{error}"); @@ -876,7 +1172,7 @@ mod tests { let (root, plan) = plan(json!({ "id": "read", "product": "breg", "as": "operator", "method": "GET", "path": "/v1/records", "expect": {"status": 200}, - "poll": {"timeoutMs": 60000, "intervalMs": 250} + "poll": {"maximumWaitMilliseconds": 60000, "pollIntervalMilliseconds": 250} })); plan.validate(&canonical(&root)).unwrap(); } @@ -932,7 +1228,7 @@ mod tests { } }, "bindings": {"caseworkContextProvider": { - "id": "registry", "person": "context-reader", "action": "review-record" + "id": "registry", "identity": "context-reader", "action": "review-record" }}, "journey": {"id": "review", "steps": [{ "id": "read", "product": "breg", "as": "reviewer", diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs index 5a7881f1dd..d15599a009 100644 --- a/crates/registry-bregctl/src/project_dev/products.rs +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -14,7 +14,10 @@ use std::{ os::unix::process::CommandExt, path::{Path, PathBuf}, process::{Child, Command, ExitStatus, Stdio}, - sync::{atomic::AtomicBool, Arc}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, thread, time::{Duration, Instant}, }; @@ -418,22 +421,12 @@ impl<'a> ProductRunner<'a> { .stdout(output.try_clone()?) .spawn() .with_context(|| format!("cannot start {} {stage}", product.name()))?; - loop { - if let Err(error) = self.ensure_active() { - stop_child(&mut child)?; - return Err(error); - } - if child_exited(&child)? { - let status = stop_child(&mut child)?; - if status.success() { - return Ok(()); - } - output.rewind()?; - let codes = report_codes(output)?; - bail!("{} {stage} failed{codes}", product.name()); - } - thread::sleep(POLL); + if wait_for_tool(&mut child, || self.ensure_active())?.success() { + return Ok(()); } + output.rewind()?; + let codes = report_codes(output)?; + bail!("{} {stage} failed{codes}", product.name()); } fn ready(&mut self, product: Product, base: &str) -> Result<()> { @@ -610,6 +603,7 @@ pub(super) fn casework_principals( root: &Path, plan: &Plan, breg: Option<&BregAuthority>, + terminate: &AtomicBool, ) -> Result> { let Some(entry) = plan .products @@ -618,23 +612,15 @@ pub(super) fn casework_principals( else { return Ok(BTreeMap::new()); }; - let path = root.join(&entry.project).join("casework.yaml"); - let bytes = plan::read(&path, 1024 * 1024).context("cannot read casework.yaml")?; - let Ok(project) = - registry_casework_core::CaseworkProject::read(&path.display().to_string(), &bytes) - else { - bail!( - "Casework project check failed. Run rstack casework check for the authored diagnostics" - ); - }; + let tool = super::binaries(Product::Casework)?.tool; + let profiles = explain_casework_profiles(&tool, &root.join(&entry.project), terminate)?; let mut principals = BTreeMap::new(); for (name, person) in &plan.people { let Some(grant) = person.grants.get(&Product::Casework) else { continue; }; - let profile = project.value.access_profiles.iter().find(|profile| profile.id == grant.profile) + let claim = profiles.get(&grant.profile).map(String::as_str) .with_context(|| format!("dev.yaml /people/{name}/grants/casework/profile must name a Casework access profile"))?; - let claim = profile.principal_claim.as_str(); ensure!( matches!(claim, "sub" | "registry_principal") || (!plan::reserved_claim(claim) @@ -651,7 +637,82 @@ pub(super) fn casework_principals( "dev.yaml /people/{name}/caseworkContext requires Casework to read sub or the BReg principal claim" ); } - principals.insert(profile.id.clone(), profile.principal_claim.clone()); + principals.insert(grant.profile.clone(), claim.to_owned()); + } + Ok(principals) +} + +fn explain_casework_profiles( + tool: &Path, + project: &Path, + terminate: &AtomicBool, +) -> Result> { + const MAX_REPORT: u64 = 4 * 1024 * 1024; + + let mut child = child_command(tool) + .args(["--format", "json", "explain"]) + .arg(project) + .stdout(Stdio::piped()) + .spawn() + .context("cannot start caseworkctl explain")?; + let stdout = child.stdout.take().expect("the explain output was piped"); + let reader = thread::spawn(move || { + let mut bytes = Vec::new(); + stdout.take(MAX_REPORT + 1).read_to_end(&mut bytes)?; + Ok::<_, std::io::Error>(bytes) + }); + let deadline = Instant::now() + Duration::from_secs(30); + let status = wait_for_tool(&mut child, || { + ensure!( + !terminate.load(Ordering::Relaxed), + "development start was interrupted" + ); + ensure!(Instant::now() < deadline, "caseworkctl explain timed out"); + Ok(()) + })?; + let bytes = reader + .join() + .map_err(|_| anyhow::anyhow!("cannot collect the Casework authoring report"))??; + ensure!( + bytes.len() as u64 <= MAX_REPORT, + "the Casework authoring report exceeds the size limit" + ); + ensure!( + status.success(), + "Casework project check failed. Run rstack casework check for the authored diagnostics" + ); + let report: Value = serde_json::from_slice(&bytes) + .map_err(|_| anyhow::anyhow!("caseworkctl explain did not emit a JSON report"))?; + casework_report_profiles(&report) +} + +fn casework_report_profiles(report: &Value) -> Result> { + ensure!( + report["apiVersion"] == "id.registrystack.org/formats/casework/report/v1alpha2" + && report["kind"] == "CaseworkReport" + && report["command"] == "explain" + && report["status"] == "complete" + && report["ok"] == true, + "caseworkctl explain returned an incompatible report" + ); + // Each explained policy includes the whole project's access profiles. + let profiles = report + .pointer("/reviewPolicies/0/dependencies/accessProfiles") + .and_then(Value::as_array) + .context("caseworkctl explain did not report access profiles")?; + let mut principals = BTreeMap::new(); + for profile in profiles { + let id = profile["id"] + .as_str() + .context("an explained Casework profile has no ID")?; + let claim = profile["principalClaim"] + .as_str() + .filter(|claim| !claim.is_empty()) + .context("an explained Casework profile has no principal claim")?; + ensure!( + principals.insert(id.to_owned(), claim.to_owned()).is_none(), + "caseworkctl explain returned duplicate access profiles" + ); } Ok(principals) } @@ -717,6 +778,19 @@ fn stop_child(child: &mut Child) -> Result { Ok(child.wait()?) } +fn wait_for_tool(child: &mut Child, active: impl Fn() -> Result<()>) -> Result { + loop { + if let Err(error) = active() { + stop_child(child)?; + return Err(error); + } + if child_exited(child)? { + return stop_child(child); + } + thread::sleep(POLL); + } +} + fn report_codes(output: impl Read) -> Result { let mut bytes = Vec::new(); output.take(256 * 1024).read_to_end(&mut bytes)?; @@ -784,7 +858,7 @@ pub(super) fn schema_people(root: &Path, plan: &Plan) -> Result std::process::Output { + let request = serde_json::json!({ + "action": "check", "project": root, "deny_warnings": true, "json": json, + }); + Command::new(env!("CARGO_BIN_EXE_bregctl")) + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request.to_string()) + .env("RSTACK_DEV_STATE_DIR", state) + .env_remove("RSTACK_DEV_DATABASE_URL") + .output() + .unwrap() +} + +#[test] +fn check_validates_the_closed_plan_without_allocating_run_state() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + project(scratch.path(), ""); + + let result = check(scratch.path(), &state, true); + + assert!(result.status.success(), "{result:?}"); + assert!(result.stderr.is_empty(), "{result:?}"); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + assert_eq!(report["kind"], "BRegCtlReport"); + assert_eq!(report["command"], "dev-check"); + assert_eq!(report["status"], "complete"); + assert_eq!(report["ok"], true); + assert_eq!(report["diagnostics"], serde_json::json!([])); + assert!(!state.exists()); +} + +#[test] +fn check_refuses_an_unknown_plan_field_without_allocating_run_state() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + project(scratch.path(), "unexpected: true\n"); + + let result = check(scratch.path(), &state, true); + + assert!(!result.status.success(), "{result:?}"); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + assert_eq!(report["status"], "domain-refusal"); + assert_eq!(report["diagnostics"][0]["code"], "config.unknown-key"); + assert_eq!(report["diagnostics"][0]["path"], "/unexpected"); + assert!(!state.exists()); +} + +#[test] +fn check_renders_both_locations_of_a_duplicate_key() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + project(scratch.path(), "issuer: https://second.example.invalid\n"); + + let result = check(scratch.path(), &state, false); + + assert_eq!(result.status.code(), Some(1)); + assert!(result.stdout.is_empty()); + let human = String::from_utf8(result.stderr).unwrap(); + assert!(human.starts_with("error[yaml.duplicate-key] "), "{human}"); + assert!(human.contains("dev.yaml:17:1 /issuer"), "{human}"); + assert!(human.contains("dev.yaml:4:1 /issuer"), "{human}"); + assert!( + human.contains("next: Keep one definition of the key."), + "{human}" + ); + assert!(!human.contains("second.example.invalid")); + assert!(!state.exists()); +} + +#[test] +fn check_distinguishes_unavailable_input_from_an_invalid_plan() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let result = check(scratch.path(), &state, true); + + assert_eq!(result.status.code(), Some(3)); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + assert_eq!(report["status"], "operational-failure"); + assert_eq!( + report["diagnostics"][0]["code"], + "breg.project-dev.check-unavailable" + ); + assert!(!state.exists()); +} + #[test] fn a_malformed_plan_names_its_field_before_allocating_run_state() { let scratch = tempfile::tempdir().unwrap(); let state = scratch.path().join("state"); fs::write( scratch.path().join("dev.yaml"), - "apiVersion: id.registrystack.org/formats/project-dev/v1alpha1\nkind: ProjectDevPlan\nissuer: https://example.test\nproducts: [{product: coordinator, project: registry, audience: test}]\npeople: {}\njourney: {id: licensing, steps: []}\n", + "apiVersion: id.registrystack.org/formats/breg/project-dev/v1alpha1\nkind: BRegProjectDev\nissuer: https://example.test\nproducts: [{product: coordinator, project: registry, audience: test}]\npeople: {}\njourney: {id: licensing, steps: []}\n", ) .unwrap(); let request = serde_json::json!({ @@ -24,6 +128,7 @@ fn a_malformed_plan_names_its_field_before_allocating_run_state() { assert!(!result.status.success()); let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); assert_eq!(report["schema_version"], "registry.rstack.dev.v1"); + assert!(report.get("diagnostics").is_none()); assert!( report["error"] .as_str() diff --git a/crates/registry-bregctl/tests/tool_schema.rs b/crates/registry-bregctl/tests/tool_schema.rs index 5f30c77ace..88c7637111 100644 --- a/crates/registry-bregctl/tests/tool_schema.rs +++ b/crates/registry-bregctl/tests/tool_schema.rs @@ -87,6 +87,92 @@ fn every_committed_example_satisfies_its_generated_schema() { } } +fn project_dev_with_studio_identifiers() -> Value { + let mut plan = example("products/platform/examples/licensing/dev.yaml"); + let applicant = plan["people"] + .as_object_mut() + .expect("people is a mapping") + .remove("applicant") + .expect("the licensing example has an applicant"); + plan["people"]["date_of_birth"] = applicant; + plan["people"]["date_of_birth"]["grants"]["breg"]["profile"] = json!("date_of_birth"); + plan["journey"]["id"] = json!("review_flow"); + plan["journey"]["steps"][0]["id"] = json!("read.v1"); + plan["journey"]["steps"][0]["as"] = json!("date_of_birth"); + plan["journey"]["steps"][0]["capture"] = + json!({"draftEtag": {"type": "pointer", "pointer": "/etag"}}); + plan["schemaTestPeople"] = json!({ + "review_flow": {"read.v1": "date_of_birth"} + }); + plan +} + +#[test] +fn project_dev_schema_accepts_the_readers_local_derived_and_capture_identifiers() { + let documents = documents().expect("the tool schemas generate"); + let schema = compile(&documents["project-dev.v1alpha1.schema.json"]); + + assert!(schema.is_valid(&project_dev_with_studio_identifiers())); +} + +#[test] +fn project_dev_schema_refuses_identifiers_the_reader_refuses() { + let documents = documents().expect("the tool schemas generate"); + let schema = compile(&documents["project-dev.v1alpha1.schema.json"]); + let valid = project_dev_with_studio_identifiers(); + + for step in ["-read", "1read", ".read", "read.", "read..v1", "read.-v1"] { + let mut changed = valid.clone(); + changed["journey"]["steps"][0]["id"] = json!(step); + assert!(!schema.is_valid(&changed), "step identifier: {step}"); + } + for capture in ["1draft", "draft_etag", "draft.etag"] { + let mut changed = valid.clone(); + let value = changed["journey"]["steps"][0]["capture"] + .as_object_mut() + .unwrap() + .remove("draftEtag") + .unwrap(); + changed["journey"]["steps"][0]["capture"][capture] = value; + assert!(!schema.is_valid(&changed), "capture identifier: {capture}"); + } + for person in ["-person", "1person", "Person", "person.name"] { + let mut changed = valid.clone(); + let value = changed["people"] + .as_object_mut() + .unwrap() + .remove("date_of_birth") + .unwrap(); + changed["people"][person] = value; + assert!(!schema.is_valid(&changed), "person identifier: {person}"); + } + + let mut bad_step_reference = valid.clone(); + bad_step_reference["schemaTestPeople"]["review_flow"] = json!({"read..v1": "date_of_birth"}); + assert!(!schema.is_valid(&bad_step_reference)); + + let mut bad_journey_reference = valid.clone(); + let people = bad_journey_reference["schemaTestPeople"] + .as_object_mut() + .unwrap() + .remove("review_flow") + .unwrap(); + bad_journey_reference["schemaTestPeople"]["review.flow"] = people; + assert!(!schema.is_valid(&bad_journey_reference)); + + let mut bad_person_reference = valid.clone(); + bad_person_reference["schemaTestPeople"]["review_flow"] = json!({"read.v1": "date.of.birth"}); + assert!(!schema.is_valid(&bad_person_reference)); + + let mut bad_person_alias = valid.clone(); + bad_person_alias["journey"]["steps"][0]["as"] = json!("date.of.birth"); + assert!(!schema.is_valid(&bad_person_alias)); + + let mut bad_journey_id = valid; + bad_journey_id["journey"]["id"] = json!("review.flow"); + assert!(!schema.is_valid(&bad_journey_id)); +} + #[test] fn example_scenario_schema_uses_bounded_display_text() { let documents = documents().expect("the tool schemas generate"); diff --git a/crates/registry-cli-docs/catalog.json b/crates/registry-cli-docs/catalog.json index df8852d306..8002f85a12 100644 --- a/crates/registry-cli-docs/catalog.json +++ b/crates/registry-cli-docs/catalog.json @@ -25922,6 +25922,96 @@ ], "constraints": [], "subcommands": [ + { + "name": "check", + "invocation": "rstack dev check", + "about": "Validate a development plan without starting services", + "long_about": null, + "usage": "rstack dev check [OPTIONS] ", + "arguments": [ + { + "display": "", + "description": "Project directory containing dev.yaml", + "always_required": true, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "options": [ + { + "display": "--deny-warnings", + "description": "Refuse an unaccepted warning", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--json", + "description": "Emit the check report as JSON", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--format ", + "description": "Report encoding. Defaults to human even when stdout is piped. Evidence test and fixtures run also accept junit", + "always_required": false, + "repeatable": true, + "default_values": [ + "human" + ], + "possible_values": [ + "human", + "json" + ], + "environment": null + }, + { + "display": "--install-root ", + "description": "Installation generation storage directory", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--bin-dir ", + "description": "Directory for the single public rstack executable link", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--asset-dir ", + "description": "Directory of already downloaded, verified release assets", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "-h, --help", + "description": "Print help", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "constraints": [], + "subcommands": [] + }, { "name": "start", "invocation": "rstack dev start", diff --git a/crates/registry-rstack/src/cli.rs b/crates/registry-rstack/src/cli.rs index fc3c3b31bc..7b7e7e8b8b 100644 --- a/crates/registry-rstack/src/cli.rs +++ b/crates/registry-rstack/src/cli.rs @@ -165,6 +165,18 @@ enum Command { #[derive(Subcommand)] enum DevCommand { + /// Validate a development plan without starting services. + Check { + /// Project directory containing dev.yaml. + #[arg(value_name = "PROJECT")] + project: PathBuf, + /// Refuse an unaccepted warning. + #[arg(long)] + deny_warnings: bool, + /// Emit the check report as JSON. + #[arg(long)] + json: bool, + }, /// Start a development environment for a registry project. Start { /// Project directory containing dev.yaml. @@ -215,6 +227,11 @@ enum DevCommand { #[derive(Serialize)] #[serde(tag = "action", rename_all = "snake_case")] enum DevRequest<'a> { + Check { + project: &'a std::path::Path, + deny_warnings: bool, + json: bool, + }, Start { project: &'a std::path::Path, #[serde(skip_serializing_if = "Option::is_none")] @@ -1323,6 +1340,19 @@ fn first_path_rstack() -> Option { fn dispatch_dev(cli: &Cli, action: &DevCommand, format: ReportFormat) -> Result<(), CliFailure> { let launcher_json = matches!(format, ReportFormat::Json); let (request, database_url, needs_services) = match action { + DevCommand::Check { + project, + deny_warnings, + json, + } => ( + DevRequest::Check { + project, + deny_warnings: *deny_warnings, + json: *json || launcher_json, + }, + None, + false, + ), DevCommand::Start { project, database_url, diff --git a/crates/registry-rstack/tests/dev.rs b/crates/registry-rstack/tests/dev.rs index d4ae903b1b..0d253307d7 100644 --- a/crates/registry-rstack/tests/dev.rs +++ b/crates/registry-rstack/tests/dev.rs @@ -391,6 +391,21 @@ fn source_start_sends_one_json_request_and_binary_set() { fn dev_actions_encode_the_public_contract_and_launcher_json_format() { let tools = SourceTools::new(); for (arguments, expected) in [ + ( + vec![ + "dev", + "check", + tools.project.to_str().unwrap(), + "--deny-warnings", + "--json", + ], + serde_json::json!({ + "action":"check", + "project":tools.project, + "deny_warnings":true, + "json":true + }), + ), ( vec!["--format", "json", "dev", "status", "run-1"], serde_json::json!({"action":"status","run_id":"run-1","json":true}), @@ -449,7 +464,11 @@ fn root_and_dev_help_need_no_installation_or_binary_overrides() { (&["--help"][..], &["dev"][..]), ( &["dev", "--help"][..], - &["start", "status", "token", "stop"][..], + &["check", "start", "status", "token", "stop"][..], + ), + ( + &["dev", "check", "--help"][..], + &["PROJECT", "--deny-warnings", "--json"][..], ), ( &["dev", "start", "--help"][..], diff --git a/docs/site/scripts/generate-breg-configuration.mjs b/docs/site/scripts/generate-breg-configuration.mjs index ab2a744945..ebf775c385 100644 --- a/docs/site/scripts/generate-breg-configuration.mjs +++ b/docs/site/scripts/generate-breg-configuration.mjs @@ -25,8 +25,14 @@ export const CONTRACTS = [ // Files `bregctl` reads beside a project: adopter tooling, published by the // `tool-schema` example under products/breg/generated/tools. Listed in the -// order an adopter meets them, which is the order the reference page shows. +// order an adopter meets them; the generated inventory preserves this order. export const TOOL_CONTRACTS = [ + { + id: 'project-dev', + title: 'dev.yaml', + file: 'tools/project-dev.v1alpha1.schema.json', + reference: 'reference/breg-configuration.mdx', + }, { id: 'dev-clients', title: 'dev-clients.yaml', diff --git a/docs/site/scripts/generate-breg-configuration.test.mjs b/docs/site/scripts/generate-breg-configuration.test.mjs index 1ca4f42cdb..e1d5ceca29 100644 --- a/docs/site/scripts/generate-breg-configuration.test.mjs +++ b/docs/site/scripts/generate-breg-configuration.test.mjs @@ -59,8 +59,8 @@ test('BReg covers every committed tool schema, in the order an adopter meets the const root = new URL('../../../products/breg/generated/tools/', import.meta.url); assert.deepEqual((await readdir(root)).map((name) => `tools/${name}`).sort(), TOOL_CONTRACTS.map((entry) => entry.file.split('/generated/')[1]).sort()); - assert.deepEqual(TOOL_CONTRACTS.map((entry) => entry.id), ['dev-clients', 'example-scenarios', 'journeys', - 'schema-test-credentials', 'migration-descriptor', 'backup-binding', 'model-selection']); + assert.deepEqual(TOOL_CONTRACTS.map((entry) => entry.id), ['project-dev', 'dev-clients', 'example-scenarios', + 'journeys', 'schema-test-credentials', 'migration-descriptor', 'backup-binding', 'model-selection']); }); test('BReg tool and service references read the committed schemas they name', async () => { @@ -71,6 +71,10 @@ test('BReg tool and service references read the committed schemas they name', as for (const entry of [...TOOL_CONTRACTS, ...SERVICE_CONTRACTS]) { const schema = JSON.parse(await readFile(new URL(entry.file.split('/generated/')[1], root), 'utf8')); assert.ok(typeof schema.title === 'string' && schema.title.length > 0, entry.file); + if (entry.id === 'project-dev') { + assert.equal(schema.$id, + 'https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json'); + } } for (const entry of [...document.tools, ...document.services]) { assert.ok(['tooling', 'beta'].includes(entry.status), entry.id); @@ -79,6 +83,11 @@ test('BReg tool and service references read the committed schemas they name', as const fields = (group, id) => new Map(document[group].find((entry) => entry.id === id) .fields.map((field) => [field.key_path, field])); + const projectDev = fields('tools', 'project-dev'); + assert.deepEqual(projectDev.get('kind').values, ['BRegProjectDev']); + assert.ok(projectDev.has('products[].product')); + assert.ok(projectDev.has('journey.steps[].expect.status')); + const descriptor = fields('tools', 'migration-descriptor'); assert.deepEqual(descriptor.get('steps[].type').values, ['chunked-backfill', 'field-encryption-backfill', 'transactional-sql']); diff --git a/editors/configure.py b/editors/configure.py index dd0c63242f..3dd2c7fb26 100644 --- a/editors/configure.py +++ b/editors/configure.py @@ -60,6 +60,7 @@ ), ("products/breg/generated/tools/backup-binding.v1alpha1.schema.json", "*-binding.json"), ("products/breg/generated/tools/dev-clients.v1alpha1.schema.json", "dev-clients.yaml"), + ("products/breg/generated/tools/project-dev.v1alpha1.schema.json", "dev.yaml"), ("products/breg/generated/tools/example-scenarios.v1alpha1.schema.json", "examples/scenarios.json"), ( "products/breg/generated/tools/migration-descriptor.v1alpha1.schema.json", diff --git a/editors/tests/test_configure.py b/editors/tests/test_configure.py index 062ecc4fc8..7944b07897 100644 --- a/editors/tests/test_configure.py +++ b/editors/tests/test_configure.py @@ -97,9 +97,11 @@ def test_breg_schema_snapshots_tasks_and_idempotent_merge(self): settings = json.loads(settings_path.read_text()) self.assertEqual(settings["editor.tabSize"], 4) schemas = settings["yaml.schemas"] - self.assertEqual(len(schemas), 8) + self.assertEqual(len(schemas), 9) dev_clients_schema = (project / ".registry-stack-editor/schemas/dev-clients.v1alpha1.schema.json").as_uri() self.assertEqual(schemas[dev_clients_schema], [str(project / "dev-clients.yaml")]) + project_dev_schema = (project / ".registry-stack-editor/schemas/project-dev.v1alpha1.schema.json").as_uri() + self.assertEqual(schemas[project_dev_schema], [str(project / "dev.yaml")]) project_schema = (project / ".registry-stack-editor/schemas/registry-project.schema.json").as_uri() self.assertEqual(schemas[project_schema], [str(project / "registry.yaml")]) zed = json.loads((self.workspace / ".zed/settings.json").read_text()) @@ -381,7 +383,7 @@ def test_two_products_share_workspace_without_schema_or_task_collision(self): configure.configure("casework", casework, self.workspace, None) configure.configure("breg", breg, self.workspace, None) settings = json.loads(settings_path.read_text()) - self.assertEqual(len(settings["yaml.schemas"]), 15) + self.assertEqual(len(settings["yaml.schemas"]), 16) breg_dev_clients = (breg / ".registry-stack-editor/schemas/dev-clients.v1alpha1.schema.json").as_uri() self.assertEqual(settings["yaml.schemas"][breg_dev_clients], [str(breg / "dev-clients.yaml")]) self.assertEqual(settings["yaml.schemas"][unrelated_uri], [str(self.workspace / "unrelated/registry.yaml")]) diff --git a/products/breg/acceptance/reviewed-evidence-journey/dev.yaml b/products/breg/acceptance/reviewed-evidence-journey/dev.yaml index 48b8477422..520452630c 100644 --- a/products/breg/acceptance/reviewed-evidence-journey/dev.yaml +++ b/products/breg/acceptance/reviewed-evidence-journey/dev.yaml @@ -1,13 +1,13 @@ # yaml-language-server: $schema=https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json -apiVersion: id.registrystack.org/formats/project-dev/v1alpha1 -kind: ProjectDevPlan +apiVersion: id.registrystack.org/formats/breg/project-dev/v1alpha1 +kind: BRegProjectDev issuer: https://issuer.example.test products: - {product: breg, project: registry, audience: "urn:journey:breg"} - {product: casework, project: casework, audience: "urn:journey:casework"} bindings: - bregReviewAuthority: {id: casework, producer: registry, person: producer} - caseworkContextProvider: {id: registry, person: context-status, action: correct-status} + bregReviewAuthority: {id: casework, producer: registry, identity: producer} + caseworkContextProvider: {id: registry, identity: context-status, action: correct-status} people: operator: subject: generic-registry-operator @@ -94,7 +94,7 @@ journey: body: data: {code: DEV-PROOF, registrationNumber: DEV-REG, status: retired, label: Synthetic record} expect: {status: 201} - capture: {record: /data/recordIdentifier} + capture: {record: {type: pointer, pointer: /data/recordIdentifier}} - id: start-proposal product: breg as: operator @@ -103,7 +103,9 @@ journey: headers: {Idempotency-Key: start-proposal} body: {input: {subjectId: "{{record}}", status: active}} expect: {status: 201} - capture: {proposal: /id, draftEtag: {header: etag}} + capture: + proposal: {type: pointer, pointer: /id} + draftEtag: {type: header, header: etag} - id: submit-proposal product: breg as: operator @@ -112,7 +114,9 @@ journey: headers: {If-Match: "{{draftEtag}}", Idempotency-Key: submit-proposal} body: {} expect: {status: 200, json: {/state: submitted}} - capture: {submissionVersion: /submissionVersion, effectDigest: /effectDigest} + capture: + submissionVersion: {type: pointer, pointer: /submissionVersion} + effectDigest: {type: pointer, pointer: /effectDigest} - id: find-task product: casework as: reviewer @@ -120,11 +124,11 @@ journey: path: /v1/review-tasks?queue=reviews headers: {Registry-Source-Profile: reviewer} expect: {status: 200} - poll: {timeoutMs: 60000, intervalMs: 500} + poll: {maximumWaitMilliseconds: 60000, pollIntervalMilliseconds: 500} capture: - task: {find: /items, where: {pointer: /subject/id, equals: "{{proposal}}"}, pointer: /taskId} - request: {find: /items, where: {pointer: /subject/id, equals: "{{proposal}}"}, pointer: /requestId} - taskRevision: {find: /items, where: {pointer: /subject/id, equals: "{{proposal}}"}, pointer: /revision} + task: {type: find, find: /items, where: {pointer: /subject/id, equals: "{{proposal}}"}, pointer: /taskId} + request: {type: find, find: /items, where: {pointer: /subject/id, equals: "{{proposal}}"}, pointer: /requestId} + taskRevision: {type: find, find: /items, where: {pointer: /subject/id, equals: "{{proposal}}"}, pointer: /revision} - id: read-context-from-breg product: casework as: reviewer @@ -143,7 +147,7 @@ journey: path: "/v1/review-tasks/{{task}}/claim" headers: {If-Match: '"{{taskRevision}}"', Idempotency-Key: claim-task, Registry-Source-Profile: reviewer} expect: {status: 200} - capture: {claimRevision: /revision} + capture: {claimRevision: {type: pointer, pointer: /revision}} - id: approve-correction product: casework as: reviewer @@ -164,8 +168,8 @@ journey: method: GET path: "/v1/proposals/{{proposal}}" expect: {status: 200, json: {/decision/status: approved}} - poll: {timeoutMs: 60000, intervalMs: 500} - capture: {approvalEtag: {header: etag}} + poll: {maximumWaitMilliseconds: 60000, pollIntervalMilliseconds: 500} + capture: {approvalEtag: {type: header, header: etag}} - id: apply-correction product: breg as: applier diff --git a/products/breg/generated/tools/project-dev.v1alpha1.schema.json b/products/breg/generated/tools/project-dev.v1alpha1.schema.json index 592b434cd6..efb56218fb 100644 --- a/products/breg/generated/tools/project-dev.v1alpha1.schema.json +++ b/products/breg/generated/tools/project-dev.v1alpha1.schema.json @@ -16,35 +16,102 @@ "additionalProperties": false, "properties": { "id": { - "type": "string" + "$ref": "#/$defs/LocalId" }, - "person": { - "type": "string" + "identity": { + "$ref": "#/$defs/LocalId" }, "producer": { - "type": "string" + "$ref": "#/$defs/LocalId" } }, "required": [ "id", "producer", - "person" + "identity" ], "type": "object" }, "Capture": { - "anyOf": [ + "oneOf": [ { - "type": "string" + "additionalProperties": false, + "properties": { + "pointer": { + "type": "string" + }, + "type": { + "const": "pointer", + "type": "string" + } + }, + "required": [ + "type", + "pointer" + ], + "type": "object" }, { - "$ref": "#/$defs/HeaderCapture" + "additionalProperties": false, + "properties": { + "header": { + "$ref": "#/$defs/CapturedHeader" + }, + "type": { + "const": "header", + "type": "string" + } + }, + "required": [ + "type", + "header" + ], + "type": "object" }, { - "$ref": "#/$defs/FindCapture" + "additionalProperties": false, + "properties": { + "find": { + "type": "string" + }, + "pointer": { + "type": "string" + }, + "type": { + "const": "find", + "type": "string" + }, + "where": { + "$ref": "#/$defs/FindWhere" + } + }, + "required": [ + "type", + "find", + "where", + "pointer" + ], + "type": "object" }, { - "$ref": "#/$defs/ValueCapture" + "additionalProperties": false, + "properties": { + "pointer": { + "type": "string" + }, + "query": { + "type": "string" + }, + "type": { + "const": "value", + "type": "string" + } + }, + "required": [ + "type", + "pointer" + ], + "type": "object" } ] }, @@ -58,18 +125,18 @@ "additionalProperties": false, "properties": { "action": { - "type": "string" + "$ref": "#/$defs/LocalId" }, "id": { - "type": "string" + "$ref": "#/$defs/LocalId" }, - "person": { - "type": "string" + "identity": { + "$ref": "#/$defs/LocalId" } }, "required": [ "id", - "person", + "identity", "action" ], "type": "object" @@ -81,6 +148,7 @@ "equals": true, "length": { "format": "uint", + "maximum": 1048576, "minimum": 0, "type": "integer" }, @@ -93,6 +161,12 @@ ], "type": "object" }, + "DerivedId": { + "description": "A dotted development journey step identifier whose segments are local identifiers.", + "maxLength": 64, + "pattern": "^[a-z][a-z0-9_-]{0,63}(\\.[a-z][a-z0-9_-]{0,63})+$", + "type": "string" + }, "Expected": { "additionalProperties": false, "properties": { @@ -109,7 +183,11 @@ "json": { "additionalProperties": true, "default": {}, - "type": "object" + "propertyNames": { + "$ref": "#/$defs/ExternalId" + }, + "type": "object", + "x-registry-passthrough": "Expected values are compared with the product response at the named JSON pointers." }, "status": { "$ref": "#/$defs/ExpectedStatus" @@ -123,41 +201,28 @@ "ExpectedStatus": { "anyOf": [ { - "format": "uint16", - "maximum": 65535, - "minimum": 0, + "format": "uint32", + "maximum": 599, + "minimum": 200, "type": "integer" }, { "items": { - "format": "uint16", - "maximum": 65535, - "minimum": 0, + "format": "uint32", + "maximum": 599, + "minimum": 200, "type": "integer" }, "type": "array" } ] }, - "FindCapture": { - "additionalProperties": false, - "properties": { - "find": { - "type": "string" - }, - "pointer": { - "type": "string" - }, - "where": { - "$ref": "#/$defs/FindWhere" - } - }, - "required": [ - "find", - "where", - "pointer" - ], - "type": "object" + "ExternalId": { + "description": "An identifier another system issues, written exactly as its issuer gives it.", + "maxLength": 512, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" }, "FindWhere": { "additionalProperties": false, @@ -179,16 +244,22 @@ "claims": { "additionalProperties": true, "default": {}, - "type": "object" + "propertyNames": { + "$ref": "#/$defs/ExternalId" + }, + "type": "object", + "x-registry-passthrough": "Claim values are written into the development token as given." }, "profile": { - "type": "string" + "$ref": "#/$defs/LocalId" }, "scopes": { "items": { - "type": "string" + "$ref": "#/$defs/ExternalId" }, - "type": "array" + "maxItems": 32, + "type": "array", + "uniqueItems": true } }, "required": [ @@ -197,23 +268,11 @@ ], "type": "object" }, - "HeaderCapture": { - "additionalProperties": false, - "properties": { - "header": { - "$ref": "#/$defs/CapturedHeader" - } - }, - "required": [ - "header" - ], - "type": "object" - }, "Journey": { "additionalProperties": false, "properties": { "id": { - "type": "string" + "$ref": "#/$defs/LocalId" }, "steps": { "items": { @@ -228,6 +287,11 @@ ], "type": "object" }, + "LocalId": { + "description": "A local identifier: a lowercase letter, then up to 63 lowercase letters, digits, `_`, or `-`.", + "pattern": "^[a-z][a-z0-9_-]{0,63}$", + "type": "string" + }, "Method": { "enum": [ "GET", @@ -238,57 +302,25 @@ ], "type": "string" }, - "Person": { - "additionalProperties": false, - "properties": { - "caseworkContext": { - "default": false, - "description": "Forward the same authority to both products. Both grants must declare the same scopes and additional claims.", - "type": "boolean" - }, - "client": { - "type": "string" - }, - "grants": { - "additionalProperties": false, - "properties": { - "breg": { - "$ref": "#/$defs/Grant" - }, - "casework": { - "$ref": "#/$defs/Grant" - } - }, - "type": "object" - }, - "subject": { - "type": "string" - } - }, - "required": [ - "subject", - "client", - "grants" - ], - "type": "object" - }, "Poll": { "additionalProperties": false, "properties": { - "intervalMs": { + "maximumWaitMilliseconds": { "format": "uint64", - "minimum": 0, + "maximum": 60000, + "minimum": 1, "type": "integer" }, - "timeoutMs": { + "pollIntervalMilliseconds": { "format": "uint64", - "minimum": 0, + "maximum": 5000, + "minimum": 50, "type": "integer" } }, "required": [ - "timeoutMs", - "intervalMs" + "maximumWaitMilliseconds", + "pollIntervalMilliseconds" ], "type": "object" }, @@ -303,7 +335,7 @@ "additionalProperties": false, "properties": { "audience": { - "type": "string" + "$ref": "#/$defs/ExternalId" }, "product": { "$ref": "#/$defs/Product" @@ -339,7 +371,7 @@ "additionalProperties": false, "properties": { "as": { - "type": "string" + "$ref": "#/$defs/LocalId" }, "body": true, "capture": { @@ -347,6 +379,18 @@ "$ref": "#/$defs/Capture" }, "default": {}, + "propertyNames": { + "allOf": [ + { + "$ref": "#/$defs/ExternalId" + }, + { + "maxLength": 64, + "pattern": "^[A-Za-z][A-Za-z0-9-]{0,63}$", + "type": "string" + } + ] + }, "type": "object" }, "expect": { @@ -357,10 +401,20 @@ "type": "string" }, "default": {}, + "propertyNames": { + "$ref": "#/$defs/ExternalId" + }, "type": "object" }, "id": { - "type": "string" + "anyOf": [ + { + "$ref": "#/$defs/LocalId" + }, + { + "$ref": "#/$defs/DerivedId" + } + ] }, "method": { "$ref": "#/$defs/Method" @@ -388,20 +442,46 @@ ], "type": "object" }, - "ValueCapture": { + "TestPerson": { "additionalProperties": false, "properties": { - "pointer": { - "type": "string" + "caseworkContext": { + "default": false, + "description": "Forward the same authority to both products. Both grants must declare the same scopes and additional claims.", + "type": "boolean" }, - "query": { - "type": "string" + "client": { + "$ref": "#/$defs/ExternalId" + }, + "grants": { + "additionalProperties": false, + "properties": { + "breg": { + "$ref": "#/$defs/Grant" + }, + "casework": { + "$ref": "#/$defs/Grant" + } + }, + "type": "object" + }, + "subject": { + "$ref": "#/$defs/ExternalId" } }, "required": [ - "pointer" + "subject", + "client", + "grants" ], "type": "object" + }, + "Url": { + "description": "An absolute http or https URL with a host preserved as written, no user information, and no backslash, whitespace or control character. DNS case and equivalent IPv6 literals are permitted.", + "format": "uri", + "maxLength": 2048, + "pattern": "^[Hh][Tt][Tt][Pp][Ss]?://[^/?#@\\u005C\\u0000-\\u0020\\u007F-\\u00A0\\u1680\\u2000-\\u200A\\u2028\\u2029\\u202F\\u205F\\u3000]+([/?#][^\\u005C\\u0000-\\u0020\\u007F-\\u00A0\\u1680\\u2000-\\u200A\\u2028\\u2029\\u202F\\u205F\\u3000]*)?$", + "type": "string" } }, "$id": "https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json", @@ -409,7 +489,7 @@ "additionalProperties": false, "properties": { "apiVersion": { - "const": "id.registrystack.org/formats/project-dev/v1alpha1", + "const": "id.registrystack.org/formats/breg/project-dev/v1alpha1", "description": "The format and version this file is written in.", "type": "string" }, @@ -418,19 +498,22 @@ "default": {} }, "issuer": { - "type": "string" + "$ref": "#/$defs/Url" }, "journey": { "$ref": "#/$defs/Journey" }, "kind": { - "const": "ProjectDevPlan", + "const": "BRegProjectDev", "description": "The kind of document this file is.", "type": "string" }, "people": { "additionalProperties": { - "$ref": "#/$defs/Person" + "$ref": "#/$defs/TestPerson" + }, + "propertyNames": { + "$ref": "#/$defs/LocalId" }, "type": "object" }, @@ -443,11 +526,24 @@ "schemaTestPeople": { "additionalProperties": { "additionalProperties": { - "type": "string" + "$ref": "#/$defs/LocalId" + }, + "propertyNames": { + "anyOf": [ + { + "$ref": "#/$defs/LocalId" + }, + { + "$ref": "#/$defs/DerivedId" + } + ] }, "type": "object" }, "default": {}, + "propertyNames": { + "$ref": "#/$defs/LocalId" + }, "type": "object" } }, diff --git a/products/identifiers/generated/catalog.v1.json b/products/identifiers/generated/catalog.v1.json index 1aa55fbee6..c1054bd079 100644 --- a/products/identifiers/generated/catalog.v1.json +++ b/products/identifiers/generated/catalog.v1.json @@ -4996,7 +4996,7 @@ }, "artifact": { "path": "products/breg/generated/tools/project-dev.v1alpha1.schema.json", - "sha256": "cb996ac1008a1e9cac70f9766f4c30e20dc793f91dffb648bf26335c7742e233", + "sha256": "e9877be9b1a191c2ca27af5d998698359cbeaa08aba74749ae0a36cc5f7f7e7f", "mediaType": "application/schema+json" } }, diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index b48d877e1b..c451db1fb5 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -3,18 +3,25 @@ Use `rstack dev` to start a checkout's BReg and Casework projects together. Install matching BReg and Casework components and supply a dedicated development PostgreSQL server over TLS. The database administrator needs to create and -remove roles and databases and install `btree_gist`. Use the platform's trusted -roots or set `RSTACK_DEV_DATABASE_CA` to your server's CA bundle in PEM format. +remove roles and databases and install `btree_gist`. PostgreSQL 17 or newer is +required. Use the platform's trusted roots or set `RSTACK_DEV_DATABASE_CA` to +your server's CA bundle in PEM format. ```sh rstack install breg casework # Set RSTACK_DEV_DATABASE_URL through your shell's secret provider. +rstack dev check ./licensing --json rstack dev start ./licensing --ttl 30m --json rstack dev status RUN_ID rstack dev token RUN_ID --as applicant --json rstack dev stop RUN_ID ``` +Run `rstack dev check PROJECT [--deny-warnings] [--json]` to validate the plan +and its project-local file references offline. With `--json`, the check writes a +`BRegCtlReport` with `command` set to `dev-check`. Use `--deny-warnings` to +refuse any unaccepted warning. + You can also pass `--database-url`. Use the environment variable when the URL includes a password. Start reports the product URLs and the journey result. A failed journey exits 1 and includes the live run ID so you can inspect the @@ -50,13 +57,14 @@ and the authority explicitly shared by both grants. See the [native review example](../breg/acceptance/reviewed-evidence-journey/dev.yaml). Journey steps call relative product API paths and assert the HTTP status and -optional JSON-pointer values. Capture a string or number by JSON pointer and -refer to it as `{{capture-name}}` in a later path, request body, header, or +optional JSON-pointer values. Capture a string or number with a `pointer` +capture and refer to it as `{{capture-name}}` in a later path, request body, header, or expected value. Use URI-safe characters for values in a path. Each step names the test person it runs as. -Use a bounded GET poll for asynchronous delivery or result retrieval. Each -poll can wait up to 60 seconds. A journey can make at most 512 requests and +Use a bounded GET poll for asynchronous delivery or result retrieval. Set +`maximumWaitMilliseconds` up to 60 seconds and choose a +`pollIntervalMilliseconds`. A journey can make at most 512 requests and transfer at most 8 MiB of request and response bodies in total. `token` writes a short-lived bearer header file for each grant. Its JSON report diff --git a/products/platform/config-conventions-exceptions.yaml b/products/platform/config-conventions-exceptions.yaml index df71be0a31..fd83d89688 100644 --- a/products/platform/config-conventions-exceptions.yaml +++ b/products/platform/config-conventions-exceptions.yaml @@ -71,6 +71,36 @@ exceptions: class: protocol-constant reason: RS384 is written as JSON Web Algorithms (RFC 7518) defines it resolution: Permanent. +- rule: CFG-NAME-2 + format: breg/project-dev + location: products/breg/generated/tools/project-dev.v1alpha1.schema.json#/$defs/Method/enum/DELETE + class: protocol-constant + reason: DELETE is written as HTTP methods (RFC 9110) defines it + resolution: Permanent. +- rule: CFG-NAME-2 + format: breg/project-dev + location: products/breg/generated/tools/project-dev.v1alpha1.schema.json#/$defs/Method/enum/GET + class: protocol-constant + reason: GET is written as HTTP methods (RFC 9110) defines it + resolution: Permanent. +- rule: CFG-NAME-2 + format: breg/project-dev + location: products/breg/generated/tools/project-dev.v1alpha1.schema.json#/$defs/Method/enum/PATCH + class: protocol-constant + reason: PATCH is written as HTTP PATCH (RFC 5789) defines it + resolution: Permanent. +- rule: CFG-NAME-2 + format: breg/project-dev + location: products/breg/generated/tools/project-dev.v1alpha1.schema.json#/$defs/Method/enum/POST + class: protocol-constant + reason: POST is written as HTTP methods (RFC 9110) defines it + resolution: Permanent. +- rule: CFG-NAME-2 + format: breg/project-dev + location: products/breg/generated/tools/project-dev.v1alpha1.schema.json#/$defs/Method/enum/PUT + class: protocol-constant + reason: PUT is written as HTTP methods (RFC 9110) defines it + resolution: Permanent. - rule: CFG-NAME-2 format: breg/queries-explanation location: products/breg/contracts/explain/QueriesExplanation.schema.json#/properties/operations/items/properties/filterable/items/properties/wireOperators/items/enum/eq null diff --git a/products/platform/config-formats.yaml b/products/platform/config-formats.yaml index 75a0f0a3d3..f64424772b 100644 --- a/products/platform/config-formats.yaml +++ b/products/platform/config-formats.yaml @@ -719,6 +719,65 @@ formats: boolean: none securityMembers: [] restrictingMembers: [] + - id: breg/project-dev + product: breg + title: Registry Stack project development plan + files: [dev.yaml] + syntax: yaml + audience: authored + stability: experimental + current: + apiVersion: id.registrystack.org/formats/breg/project-dev/v1alpha1 + kind: BRegProjectDev + checkedBy: + file: crates/registry-bregctl/src/project_dev/plan.rs + symbol: API_VERSION + target: + apiVersion: id.registrystack.org/formats/breg/project-dev/v1alpha1 + kind: BRegProjectDev + schema: + path: products/breg/generated/tools/project-dev.v1alpha1.schema.json + id: https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json + origin: generated + generator: cargo run -p registry-bregctl --features schema --example tool-schema -- --output products/breg/generated/tools + driftCheck: products/breg/scripts/check-tool-schemas.sh + reader: + crate: registry-bregctl + file: crates/registry-bregctl/src/project_dev/plan.rs + function: load + type: Plan + check: rstack dev check {project} + example: products/platform/conformance/projects/breg-project-dev/dev.yaml + conformance: + requiredText: /issuer + optionalText: none + integer: /journey/steps/0/expect/status + boolean: none + idList: /journey/steps + set: /people/operator/grants/breg/scopes + reference: /journey/steps/0/as + operand: none + relativePath: none + securityMembers: + - pointer: /issuer + whenOmitted: refused + - pointer: /products/*/audience + whenOmitted: refused + - pointer: /people/*/grants/breg/profile + whenOmitted: refused + - pointer: /people/*/grants/breg/scopes + whenOmitted: refused + - pointer: /people/*/grants/breg/claims + whenOmitted: closed + - pointer: /people/*/grants/casework/profile + whenOmitted: refused + - pointer: /people/*/grants/casework/scopes + whenOmitted: refused + - pointer: /people/*/grants/casework/claims + whenOmitted: closed + - pointer: /people/*/caseworkContext + whenOmitted: closed + restrictingMembers: [/issuer, /products/*/audience, /people/*/grants/breg/profile, /people/*/grants/casework/profile, /people/*/caseworkContext] - id: breg/example-scenarios product: breg title: Example scenarios diff --git a/products/platform/conformance/projects/breg-project-dev/dev.yaml b/products/platform/conformance/projects/breg-project-dev/dev.yaml new file mode 100644 index 0000000000..322bec7874 --- /dev/null +++ b/products/platform/conformance/projects/breg-project-dev/dev.yaml @@ -0,0 +1,25 @@ +# yaml-language-server: $schema=https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json +apiVersion: id.registrystack.org/formats/breg/project-dev/v1alpha1 +kind: BRegProjectDev +issuer: https://issuer.example.invalid +products: + - product: breg + project: registry + audience: urn:example:registry +people: + operator: + subject: operator + client: operator-client + grants: + breg: + profile: operator + scopes: [registry:read] +journey: + id: smoke + steps: + - id: read + product: breg + as: operator + method: GET + path: /v1/records/records + expect: {status: 200} diff --git a/products/platform/conformance/projects/breg-project-dev/registry/registry.yaml b/products/platform/conformance/projects/breg-project-dev/registry/registry.yaml new file mode 100644 index 0000000000..b0e14738b9 --- /dev/null +++ b/products/platform/conformance/projects/breg-project-dev/registry/registry.yaml @@ -0,0 +1 @@ +# This file establishes the referenced product root for development-plan checks. diff --git a/products/platform/examples/licensing/dev.yaml b/products/platform/examples/licensing/dev.yaml index 2869a9cf40..eee70505e6 100644 --- a/products/platform/examples/licensing/dev.yaml +++ b/products/platform/examples/licensing/dev.yaml @@ -1,6 +1,6 @@ # yaml-language-server: $schema=https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json -apiVersion: id.registrystack.org/formats/project-dev/v1alpha1 -kind: ProjectDevPlan +apiVersion: id.registrystack.org/formats/breg/project-dev/v1alpha1 +kind: BRegProjectDev issuer: https://identity.licensing.example.test products: - {product: breg, project: registry, audience: "urn:example:licensing:registry"} @@ -49,7 +49,7 @@ journey: headers: {Idempotency-Key: create-application} body: {data: {reference: application-001, profession: Surveyor}} expect: {status: 201} - capture: {application: /data/recordIdentifier} + capture: {application: {type: pointer, pointer: /data/recordIdentifier}} - id: staff-review-queue product: casework as: administrator @@ -83,14 +83,16 @@ journey: mode: submitted snapshot: {reference: application-001, profession: Surveyor} expect: {status: 201} - capture: {request: /requestId} + capture: {request: {type: pointer, pointer: /requestId}} - id: find-review-task product: casework as: reviewer method: GET path: /v1/review-tasks?queue=applications expect: {status: 200, json: {/items/0/requestId: "{{request}}"}} - capture: {task: /items/0/taskId, task-revision: /items/0/revision} + capture: + task: {type: pointer, pointer: /items/0/taskId} + task-revision: {type: pointer, pointer: /items/0/revision} - id: claim-review product: casework as: reviewer @@ -98,7 +100,7 @@ journey: path: "/v1/review-tasks/{{task}}/claim" headers: {If-Match: '"{{task-revision}}"', Idempotency-Key: claim-review} expect: {status: 200} - capture: {claim-revision: /revision} + capture: {claim-revision: {type: pointer, pointer: /revision}} - id: approve-application product: casework as: reviewer diff --git a/products/platform/scripts/run-config-conformance.py b/products/platform/scripts/run-config-conformance.py index 6ddddd13b4..f927a6c196 100644 --- a/products/platform/scripts/run-config-conformance.py +++ b/products/platform/scripts/run-config-conformance.py @@ -1934,6 +1934,9 @@ def prepare_problems(fmt: Format, staged: "Staged", env: dict[str, str], bin_dir def check_environment(bin_dir: Path) -> dict[str, str]: env = dict(os.environ) env["PATH"] = str(bin_dir) + os.pathsep + env.get("PATH", "") + bregctl = bin_dir / "bregctl" + if bregctl.is_file(): + env["RSTACK_BREGCTL_BIN"] = str(bregctl) env["NO_COLOR"] = "1" env[VARIABLE] = MARKER env.pop(UNSET_VARIABLE, None) diff --git a/products/platform/scripts/test-rstack-dev.py b/products/platform/scripts/test-rstack-dev.py index f9414ed8c9..c6dca72dcf 100644 --- a/products/platform/scripts/test-rstack-dev.py +++ b/products/platform/scripts/test-rstack-dev.py @@ -169,8 +169,17 @@ def files(root): def dev(bin_dir, env, cwd, *args, expected=0): + install_root = Path(env["RSTACK_DEV_STATE_DIR"]).with_name("rstack-install") result = command( - bin_dir / "rstack", "dev", *args, "--json", env=env, cwd=cwd, timeout=390 + bin_dir / "rstack", + "--install-root", + install_root, + "dev", + *args, + "--json", + env=env, + cwd=cwd, + timeout=390, ) try: report = json.loads(result.stdout) @@ -180,7 +189,10 @@ def dev(bin_dir, env, cwd, *args, expected=0): ) from None if result.returncode != expected: raise RuntimeError(f"rstack dev {args[0]} exited {result.returncode}: {report}") - assert report["schema_version"] == "registry.rstack.dev.v1" + if args[0] == "check": + assert report["kind"] == "BRegCtlReport" and report["command"] == "dev-check" + else: + assert report["schema_version"] == "registry.rstack.dev.v1" return report @@ -422,6 +434,22 @@ def main(): prefix="rstack-dev-", dir=args.scratch_parent ) as directory: root = Path(directory) + bin_dir = args.bin_dir.resolve() + project = STACK / "products/platform/examples/licensing" + check_env = { + key: value + for key, value in os.environ.items() + if key not in {"RSTACK_DEV_DATABASE_URL", "RSTACK_DEV_DATABASE_CA"} + } + check_env["RSTACK_BREGCTL_BIN"] = str(bin_dir / "bregctl") + check_env["RSTACK_DEV_STATE_DIR"] = str(root / "offline-check-runs") + report = dev(bin_dir, check_env, project, "check", ".") + assert ( + report["ok"] + and report["status"] == "complete" + and report["diagnostics"] == [] + ) + assert not (root / "offline-check-runs").exists() pg = args.postgres_bin_dir.resolve() cluster, ca, url, pg_env = prepare_postgres(root, pg) checked( @@ -438,9 +466,9 @@ def main(): ) exercise( root, - args.bin_dir.resolve(), + bin_dir, pg, - STACK / "products/platform/examples/licensing", + project, ca, url, pg_env, diff --git a/products/platform/scripts/test_run_config_conformance.py b/products/platform/scripts/test_run_config_conformance.py index 93d015c0fd..7003687e0f 100644 --- a/products/platform/scripts/test_run_config_conformance.py +++ b/products/platform/scripts/test_run_config_conformance.py @@ -1053,6 +1053,14 @@ def test_native_breg_editor_association_pins_kind_version_and_schema(self) -> No class PlanTest(unittest.TestCase): + def test_check_child_selects_the_built_breg_helper_for_source_rstack(self) -> None: + with tempfile.TemporaryDirectory() as directory: + binary = Path(directory) / "bregctl" + binary.write_text("fixture", encoding="utf-8") + with unittest.mock.patch.dict(runner.os.environ, {}, clear=True): + environment = runner.check_environment(Path(directory)) + self.assertEqual(environment["RSTACK_BREGCTL_BIN"], str(binary)) + def test_macos_check_child_restores_the_cargo_runtime_library_path(self) -> None: with ( unittest.mock.patch.object(runner.sys, "platform", "darwin"), From e67d61cdd2e53ecb2baa4fde7e211563274dfd29 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 00:24:33 +0700 Subject: [PATCH 04/34] test: avoid storing development test passwords Signed-off-by: Jeremi Joslin --- .../src/project_dev/database.rs | 97 ++++++++++--------- products/platform/scripts/test-rstack-dev.py | 16 ++- 2 files changed, 56 insertions(+), 57 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/database.rs b/crates/registry-bregctl/src/project_dev/database.rs index 0bcc0d482d..fc3bf7367c 100644 --- a/crates/registry-bregctl/src/project_dev/database.rs +++ b/crates/registry-bregctl/src/project_dev/database.rs @@ -488,24 +488,28 @@ mod tests { #[test] fn admin_urls_require_one_explicit_password_authenticated_tcp_database() { - assert!( - admin_config("postgresql://admin:secret@db.example:5432/postgres?sslmode=require") - .is_ok() - ); - assert!(admin_config("postgres://admin:secret@[::1]:5432/postgres").is_ok()); - for invalid in [ - "http://admin:secret@db.example/postgres", - "postgresql://db.example/postgres", - "postgresql://admin@db.example/postgres", - "postgresql://admin:@db.example/postgres", - "postgresql://admin:secret@db.example", - "postgresql://admin:secret@db.example/one/two", - "postgresql://admin:secret@db.example/postgres?host=other.example", - "postgresql://admin:secret@db.example/postgres?dbname=other", - "postgresql://admin:secret@db.example/postgres?hostaddr=127.0.0.1", - "postgresql:///postgres", - ] { - assert!(admin_config(invalid).is_err(), "accepted {invalid}"); + let password = random_password().unwrap(); + let password = password.as_str(); + let endpoint = format!("postgresql://admin:{password}@db.example"); + assert!(admin_config(&format!("{endpoint}:5432/postgres?sslmode=require")).is_ok()); + assert!(admin_config(&format!("postgres://admin:{password}@[::1]:5432/postgres")).is_ok()); + let invalid = [ + format!("http://admin:{password}@db.example/postgres"), + "postgresql://db.example/postgres".to_owned(), + "postgresql://admin@db.example/postgres".to_owned(), + "postgresql://admin:@db.example/postgres".to_owned(), + endpoint.clone(), + format!("{endpoint}/one/two"), + format!("{endpoint}/postgres?host=other.example"), + format!("{endpoint}/postgres?dbname=other"), + format!("{endpoint}/postgres?hostaddr=127.0.0.1"), + "postgresql:///postgres".to_owned(), + ]; + for (case, url) in invalid.iter().enumerate() { + assert!( + admin_config(url).is_err(), + "accepted invalid admin URL case {case}" + ); } } @@ -515,40 +519,40 @@ mod tests { host: UrlHost::Domain("db.example".to_owned()), port: Some(5543), }; - let url = endpoint - .product_url("owned", "runtime", "generated-secret") - .unwrap(); - assert_eq!( - url, - "postgresql://runtime:generated-secret@db.example:5543/owned?sslmode=require" + let password = random_password().unwrap(); + let password = password.as_str(); + let url = endpoint.product_url("owned", "runtime", password).unwrap(); + assert!( + url == format!("postgresql://runtime:{password}@db.example:5543/owned?sslmode=require"), + "unexpected product connection URL" ); - assert!(!url.contains("admin")); } #[test] fn generated_product_urls_round_trip_ipv6_hosts_and_credentials() { + let admin_password = random_password().unwrap(); + let admin_password = admin_password.as_str(); for address_text in ["::1", "2001:db8:1234:5678::9"] { let address = address_text.parse().unwrap(); let (_, endpoint) = admin_config(&format!( - "postgresql://admin:secret@[{address_text}]:5543/postgres" + "postgresql://admin:{admin_password}@[{address_text}]:5543/postgres" )) .unwrap(); - for (user, database, password) in [ - ("runtime", "owned", "runtime-generated-secret"), - ("migration", "owned_test", "migration-generated-secret"), - ] { + for (user, database) in [("runtime", "owned"), ("migration", "owned_test")] { + let password = random_password().unwrap(); + let password = password.as_str(); let rendered = endpoint.product_url(database, user, password).unwrap(); let parsed = Url::parse(&rendered).unwrap(); - assert_eq!(parsed.host(), Some(UrlHost::Ipv6(address)), "{rendered}"); - assert_eq!(parsed.port(), Some(5543), "{rendered}"); - assert_eq!(parsed.username(), user, "{rendered}"); - assert_eq!(parsed.password(), Some(password), "{rendered}"); - assert_eq!(parsed.path(), format!("/{database}"), "{rendered}"); - assert_eq!(parsed.query(), Some("sslmode=require"), "{rendered}"); + assert_eq!(parsed.host(), Some(UrlHost::Ipv6(address))); + assert_eq!(parsed.port(), Some(5543)); + assert_eq!(parsed.username(), user); assert!( - rendered.contains(&format!("@[{address}]:5543/")), - "{rendered}" + parsed.password() == Some(password), + "unexpected product password" ); + assert_eq!(parsed.path(), format!("/{database}")); + assert_eq!(parsed.query(), Some("sslmode=require")); + assert!(rendered.contains(&format!("@[{address}]:5543/"))); } } } @@ -612,17 +616,14 @@ mod tests { #[test] fn role_statement_carries_a_postgresql_scram_verifier_not_cleartext() { - // postgres-protocol's fixed-salt vector for password `secret`. Its - // production API generates a fresh random salt for every verifier. - let verifier = "SCRAM-SHA-256$4096:AQIDBAUGBwgJCgsMDQ4PEA==$8rrDg00OqaiWXJ7p+sCgHEIaBSHY89ZJl3mfIsf32oY=:05L1f+yZbiN8O0AnO40Og85NNRhvzTS57naKRWCcsIA="; - let sql = create_role_sql("rstack_role", verifier); - assert_eq!( - sql, - format!( - "CREATE ROLE \"rstack_role\" LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOBYPASSRLS PASSWORD '{verifier}'" - ) + let password = random_password().unwrap(); + let verifier = postgres_protocol::password::scram_sha_256(password.as_bytes()); + let sql = create_role_sql("rstack_role", &verifier); + let expected = format!( + "CREATE ROLE \"rstack_role\" LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOBYPASSRLS PASSWORD '{verifier}'" ); - assert!(!sql.contains("PASSWORD 'secret'")); + assert!(verifier.starts_with("SCRAM-SHA-256$")); + assert!(sql == expected && !sql.contains(password.as_str())); } #[test] diff --git a/products/platform/scripts/test-rstack-dev.py b/products/platform/scripts/test-rstack-dev.py index c6dca72dcf..4aaab117e5 100644 --- a/products/platform/scripts/test-rstack-dev.py +++ b/products/platform/scripts/test-rstack-dev.py @@ -52,9 +52,6 @@ def port(): def prepare_postgres(root, pg): password = secrets.token_urlsafe(32) - pwfile = root / "init-password" - pwfile.touch(mode=0o600) - pwfile.write_text(password) cluster = root / "postgres" checked( pg / "initdb", @@ -62,15 +59,14 @@ def prepare_postgres(root, pg): cluster, "-U", "devadmin", - "--pwfile", - pwfile, + "--pwfile=/dev/stdin", "--auth-local=trust", "--auth-host=scram-sha-256", "--no-locale", "-E", "UTF8", + data=f"{password}\n".encode(), ) - pwfile.unlink() ca = root / "ca.pem" checked( "openssl", @@ -278,10 +274,12 @@ def coordinate(value): while time.monotonic() < deadline: assert process.poll() is None, "the independently pinned verifier exited" try: - if get(origin + "/ready", {}) == 200: - break + status = get(origin + "/ready", {}) except OSError: - pass + # The verifier may still be opening its listener. + status = None + if status == 200: + break time.sleep(0.2) else: raise RuntimeError("the independently pinned verifier did not become ready") From 14d8d9d6b7cbba2a59f5d2d6d3999e4a4fd0b890 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 01:08:06 +0700 Subject: [PATCH 05/34] fix: complete development run database setup Signed-off-by: Jeremi Joslin --- .github/workflows/ci.yml | 7 +- .../src/project_dev/bindings.rs | 1 + .../src/project_dev/database.rs | 163 ++++++++++++++++-- .../registry-bregctl/src/project_dev/mod.rs | 30 +++- .../src/project_dev/products.rs | 39 ++++- crates/registry-bregctl/tests/project_dev.rs | 85 ++++++++- products/platform/PROJECT-DEV.md | 2 + products/platform/scripts/test-rstack-dev.py | 134 +++++++++++++- 8 files changed, 431 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35ddfdf22b..02a92deccb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2697,13 +2697,13 @@ jobs: -p registry-breg --features registry-breg/runtime \ -p registry-bregctl -p registry-rstack --bins - - name: Install native PostgreSQL 18 for the project development run + - name: Install native PostgreSQL 18 and PostGIS for project development # The Ubuntu runner's PostgreSQL 16 is below BReg's minimum. shell: bash run: | set -euo pipefail sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y - sudo env DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends postgresql-18 + sudo env DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends postgresql-18 postgresql-18-postgis-3 /usr/lib/postgresql/18/bin/postgres --version - name: Verify the combined licensing development run with native PostgreSQL @@ -2714,7 +2714,8 @@ jobs: python3 products/platform/scripts/test-rstack-dev.py \ --bin-dir "${GITHUB_WORKSPACE}/target/debug" \ --postgres-bin-dir /usr/lib/postgresql/18/bin \ - --scratch-parent "${RUNNER_TEMP}" + --scratch-parent "${RUNNER_TEMP}" \ + --spatial - name: Execute the Registry Casework tutorials from a reader directory shell: bash diff --git a/crates/registry-bregctl/src/project_dev/bindings.rs b/crates/registry-bregctl/src/project_dev/bindings.rs index 886f75a0df..16c52d1b6e 100644 --- a/crates/registry-bregctl/src/project_dev/bindings.rs +++ b/crates/registry-bregctl/src/project_dev/bindings.rs @@ -740,6 +740,7 @@ mod tests { breg_authority: Some(BregAuthority { principal: "sub".to_owned(), purpose: false, + requires_postgis: false, named_clients: BTreeSet::new(), }), casework_principals: BTreeMap::from([("producer".to_owned(), "sub".to_owned())]), diff --git a/crates/registry-bregctl/src/project_dev/database.rs b/crates/registry-bregctl/src/project_dev/database.rs index fc3bf7367c..a55b30e3f2 100644 --- a/crates/registry-bregctl/src/project_dev/database.rs +++ b/crates/registry-bregctl/src/project_dev/database.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -use std::str::FromStr; +use std::{future::Future, str::FromStr, time::Duration}; use anyhow::{anyhow, bail, Context, Result}; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; @@ -21,6 +21,17 @@ const BREG_SCHEMAS: [&str; 5] = [ "registry_derived", "registry_context", ]; +const CONNECTION_TIMEOUT: Duration = Duration::from_secs(10); + +async fn bounded_connection( + connection: impl Future>, + timeout: Duration, +) -> Result { + tokio::time::timeout(timeout, connection) + .await + .map_err(|_| anyhow!("the PostgreSQL connection handshake timed out"))? + .context("the PostgreSQL connection handshake failed") +} pub(super) struct DatabaseUrls { pub runtime: String, @@ -89,7 +100,10 @@ impl Database { .build() .context("cannot initialize the PostgreSQL executor")?; let (client, connection) = runtime - .block_on(admin_config.connect(tls.clone())) + .block_on(bounded_connection( + admin_config.connect(tls.clone()), + CONNECTION_TIMEOUT, + )) .context("cannot connect to the PostgreSQL administrator database over verified TLS")?; let connection_task = runtime.spawn(connection); Ok(Self { @@ -104,8 +118,16 @@ impl Database { }) } - pub(super) fn provision(&mut self, label: &str, breg: bool) -> Result { + pub(super) fn provision( + &mut self, + label: &str, + breg: bool, + requires_postgis: bool, + ) -> Result { validate_label(label)?; + if requires_postgis && !breg { + bail!("PostGIS prerequisites may be provisioned only for a BReg database"); + } let names = Names::new(&self.run_id, label); if self.acquisitions.iter().any(|acquisition| { matches!(acquisition, Acquisition::Database(database) if database == &names.database) @@ -117,8 +139,11 @@ impl Database { let runtime_password = random_password()?; self.create_role(&names.migration_role, &migration_password)?; self.create_role(&names.runtime_role, &runtime_password)?; + if requires_postgis { + self.create_spatial_bbox_role(&names)?; + } self.create_database(&names)?; - self.configure_database(&names, breg)?; + self.configure_database(&names, breg, requires_postgis)?; Ok(DatabaseUrls { runtime: self.product_url(&names.database, &names.runtime_role, &runtime_password)?, @@ -168,7 +193,10 @@ impl Database { fn reconnect_admin(&mut self) -> Result<()> { let (client, connection) = self .runtime - .block_on(self.admin_config.connect(self.tls.clone())) + .block_on(bounded_connection( + self.admin_config.connect(self.tls.clone()), + CONNECTION_TIMEOUT, + )) .context("cannot reconnect to the PostgreSQL administrator database")?; self._connection_task.abort(); self.client = client; @@ -206,14 +234,37 @@ impl Database { password.as_bytes(), )); let statement = Zeroizing::new(create_role_sql(role, &verifier)); - let created = self.runtime.block_on(self.client.batch_execute(&statement)); + self.create_owned_role( + role, + &statement, + "cannot create a least-privilege PostgreSQL role", + ) + } + + fn create_spatial_bbox_role(&mut self, names: &Names) -> Result<()> { + let role = spatial_bbox_role(names)?; + let statement = create_nologin_role_sql(&role); + self.create_owned_role( + &role, + &statement, + "cannot create the BReg spatial bbox role", + ) + } + + fn create_owned_role( + &mut self, + role: &str, + statement: &str, + context: &'static str, + ) -> Result<()> { + let created = self.runtime.block_on(self.client.batch_execute(statement)); if created.is_ok() || created.as_ref().is_err_and(|error| error.is_closed()) { // A closed connection cannot tell us whether PostgreSQL committed // the statement. Retaining the exact run-owned name lets cleanup's // idempotent DROP cover both outcomes without replaying CREATE. self.acquisitions.push(Acquisition::Role(role.to_owned())); } - created.context("cannot create a least-privilege PostgreSQL role") + created.context(context) } fn create_database(&mut self, names: &Names) -> Result<()> { @@ -231,24 +282,48 @@ impl Database { .context("cannot restrict the PostgreSQL database to this run's roles") } - fn configure_database(&self, names: &Names, breg: bool) -> Result<()> { + fn configure_database(&self, names: &Names, breg: bool, requires_postgis: bool) -> Result<()> { let statement = database_setup_sql(names, breg); + let spatial_roles = if requires_postgis { + Some(( + registry_breg::postgres::SqlIdentifier::parse(&names.migration_role) + .map_err(|_| anyhow!("the generated migration role is invalid"))?, + registry_breg::postgres::SqlIdentifier::parse(&names.runtime_role) + .map_err(|_| anyhow!("the generated runtime role is invalid"))?, + )) + } else { + None + }; let mut target = self.admin_config.clone(); target.dbname(&names.database).ssl_mode(SslMode::Require); self.runtime .block_on(async { - let (client, connection) = target - .connect(self.tls.clone()) - .await - .context("cannot connect to the provisioned PostgreSQL database")?; + let (client, connection) = + bounded_connection(target.connect(self.tls.clone()), CONNECTION_TIMEOUT) + .await + .context("cannot connect to the provisioned PostgreSQL database")?; let connection_task = tokio::spawn(connection); - let executed = client.batch_execute(&statement).await; + let executed = async { + client + .batch_execute(&statement) + .await + .context("cannot configure the provisioned PostgreSQL database")?; + if let Some((migration, runtime)) = &spatial_roles { + registry_breg::postgres::provision_postgis_prerequisites( + &client, migration, runtime, + ) + .await + .context("cannot provision BReg PostGIS prerequisites")?; + } + Result::<()>::Ok(()) + } + .await; drop(client); let closed = connection_task .await .context("the PostgreSQL connection task stopped unexpectedly")?; - executed.context("cannot configure the provisioned PostgreSQL database")?; + executed?; closed.context("the PostgreSQL configuration connection failed")?; Result::<()>::Ok(()) }) @@ -283,6 +358,14 @@ impl Names { } } +fn spatial_bbox_role(names: &Names) -> Result { + let runtime = registry_breg::postgres::SqlIdentifier::parse(&names.runtime_role) + .map_err(|_| anyhow!("the generated runtime role is invalid"))?; + Ok(registry_breg::postgres::spatial_bbox_role(&runtime) + .as_str() + .to_owned()) +} + fn validate_run_id(run_id: &str) -> Result<()> { if run_id.len() == 32 && run_id @@ -322,6 +405,13 @@ fn create_role_sql(role: &str, verifier: &str) -> String { ) } +fn create_nologin_role_sql(role: &str) -> String { + format!( + "CREATE ROLE {} NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOBYPASSRLS", + quote_identifier(role) + ) +} + fn database_setup_sql(names: &Names, breg: bool) -> String { if breg { let schemas = BREG_SCHEMAS @@ -446,6 +536,7 @@ fn quote_literal(value: &str) -> String { #[cfg(test)] mod tests { use super::*; + use std::{future::pending, time::Instant}; const RUN_ID: &str = "0123456789abcdef0123456789abcdef"; @@ -486,6 +577,28 @@ mod tests { assert!(names.runtime_role.len() <= 63); } + #[test] + fn administrator_connection_attempt_has_a_deadline() { + let runtime = Builder::new_current_thread().enable_time().build().unwrap(); + let started = Instant::now(); + let result: Result<()> = runtime.block_on(bounded_connection( + pending::>(), + Duration::from_millis(20), + )); + assert!(result.unwrap_err().to_string().contains("timed out")); + assert!(started.elapsed() < Duration::from_secs(1)); + + assert_eq!( + runtime + .block_on(bounded_connection( + async { Ok::<_, tokio_postgres::Error>(7_u8) }, + Duration::from_secs(1), + )) + .unwrap(), + 7 + ); + } + #[test] fn admin_urls_require_one_explicit_password_authenticated_tcp_database() { let password = random_password().unwrap(); @@ -593,6 +706,28 @@ mod tests { } } + #[test] + fn spatial_breg_uses_the_governed_bbox_role_for_its_runtime() { + for label in ["breg_test", "breg"] { + let names = Names::new(RUN_ID, label); + let expected = registry_breg::postgres::spatial_bbox_role( + ®istry_breg::postgres::SqlIdentifier::parse(&names.runtime_role).unwrap(), + ); + assert_eq!(spatial_bbox_role(&names).unwrap(), expected.as_str()); + assert!(expected.as_str().starts_with("breg_bbox_")); + assert!(expected.as_str().len() <= 63); + let statement = create_nologin_role_sql(expected.as_str()); + assert_eq!( + statement, + format!( + "CREATE ROLE {} NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOBYPASSRLS", + quote_identifier(expected.as_str()) + ) + ); + assert!(!statement.contains("PASSWORD")); + } + } + #[test] fn casework_setup_gives_only_migration_role_public_schema_creation() { let names = Names::new(RUN_ID, "casework"); diff --git a/crates/registry-bregctl/src/project_dev/mod.rs b/crates/registry-bregctl/src/project_dev/mod.rs index a8c6768aca..3baf33ae5b 100644 --- a/crates/registry-bregctl/src/project_dev/mod.rs +++ b/crates/registry-bregctl/src/project_dev/mod.rs @@ -48,6 +48,14 @@ const DATABASE_URL: &str = "RSTACK_DEV_DATABASE_URL"; const CA_FILE: &str = "RSTACK_DEV_DATABASE_CA"; const REPORT: &str = "report.json"; const POLL: Duration = Duration::from_millis(100); +// A capture can contain the 1 MiB development plan after JSON escaping, +// selections derived from a second 1 MiB journeys document, facts selected +// from the bounded 4 MiB Casework authoring report, and the remaining paths +// and authority metadata. Keep headroom for those representations while +// retaining one hard bound for both writing and restoring the private input. +const MAXIMUM_CAPTURED_INPUT_BYTES: u64 = + 16 * registry_platform_yaml::MAXIMUM_DOCUMENT_BYTES as u64; +const MAXIMUM_RUN_REPORT_BYTES: u64 = 512 * 1024; #[derive(clap::Args, Debug)] pub(crate) struct Args { @@ -111,6 +119,7 @@ struct RunInput { struct BregAuthority { principal: String, purpose: bool, + requires_postgis: bool, named_clients: BTreeSet, } @@ -360,7 +369,7 @@ fn start(project: &Path, ttl: Option<&str>) -> Result { binaries, }; let preparation = (|| { - write_private(&root.join("input.json"), &serde_json::to_vec(&input)?)?; + write_private(&root.join("input.json"), &capture_input(&input)?)?; Command::new(env::current_exe()?) .arg("project-dev") .arg("--worker") @@ -452,7 +461,7 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { ); ensure_private(root, false)?; let input = Arc::new( - read_json::(&root.join("input.json")) + read_json::(&root.join("input.json"), MAXIMUM_CAPTURED_INPUT_BYTES) .context("cannot restore the captured development plan")?, ); ensure!( @@ -572,7 +581,7 @@ fn status(run_id: &str) -> Result { supervisor_running(&root)?, "run {run_id} lost its supervisor before cleanup" ); - let report: Value = read_json(&root.join(REPORT)) + let report: Value = read_json(&root.join(REPORT), MAXIMUM_RUN_REPORT_BYTES) .context("the development run has not reported readiness")?; if report["expires_at"] .as_u64() @@ -596,7 +605,7 @@ fn token(run_id: &str, person: &str) -> Result { "development run {run_id} is not ready" ); let root = run_dir(run_id)?; - let input: RunInput = read_json(&root.join("input.json"))?; + let input: RunInput = read_json(&root.join("input.json"), MAXIMUM_CAPTURED_INPUT_BYTES)?; let signer = Issuer::load(&root)?; let expires = report["expires_at"] .as_u64() @@ -777,8 +786,17 @@ fn ensure_private(path: &Path, create: bool) -> Result<()> { Ok(()) } -fn read_json(path: &Path) -> Result { - Ok(serde_json::from_slice(&plan::read(path, 512 * 1024)?)?) +fn capture_input(input: &RunInput) -> Result> { + let bytes = serde_json::to_vec(input)?; + ensure!( + bytes.len() as u64 <= MAXIMUM_CAPTURED_INPUT_BYTES, + "captured development input exceeds the internal size limit" + ); + Ok(bytes) +} + +fn read_json(path: &Path, limit: u64) -> Result { + Ok(serde_json::from_slice(&plan::read(path, limit)?)?) } fn write_private(path: &Path, bytes: &[u8]) -> Result<()> { diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs index d15599a009..afd9bf9534 100644 --- a/crates/registry-bregctl/src/project_dev/products.rs +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -159,7 +159,13 @@ impl<'a> ProductRunner<'a> { project: &Path, bind: &str, ) -> Result<(PathBuf, PathBuf)> { - let rehearsal = database.provision("breg_test", true)?; + let requires_postgis = self + .input + .breg_authority + .as_ref() + .expect("BReg authority") + .requires_postgis; + let rehearsal = database.provision("breg_test", true, requires_postgis)?; let empty_package = self.root.join("breg-test-package"); fs::create_dir(&empty_package)?; let config = self.config( @@ -197,7 +203,7 @@ impl<'a> ProductRunner<'a> { ], )?; let package = build.join("package"); - let urls = database.provision("breg", true)?; + let urls = database.provision("breg", true, requires_postgis)?; let config = self.config(Product::Breg, "breg", bind, &package, urls)?; self.tool( Product::Breg, @@ -291,7 +297,7 @@ impl<'a> ProductRunner<'a> { package.as_os_str(), ], )?; - let urls = database.provision("casework", false)?; + let urls = database.provision("casework", false, false)?; let config = self.config(Product::Casework, "casework", bind, &package, urls)?; let arguments = ["--runtime-config".as_ref(), config.as_os_str()]; self.tool(Product::Casework, "apply", &arguments)?; @@ -365,11 +371,14 @@ impl<'a> ProductRunner<'a> { }); config["audit"]["destination"] = json!("file"); config["listener"] = json!({"bind": bind}); + let authority = self.input.breg_authority.as_ref().expect("BReg authority"); + if name == "breg" && authority.requires_postgis { + config["listener"]["publicOrigin"] = json!(format!("http://{bind}")); + } config["identity"] = json!({ "environment": "development", "instanceId": identity, "databaseId": identity, "databaseInitializationEnvironment": "development" }); - let authority = self.input.breg_authority.as_ref().expect("BReg authority"); let mut authority_claims = json!({"principal": authority.principal}); if authority.purpose { authority_claims["purpose"] = json!("registry_purpose"); @@ -596,9 +605,14 @@ pub(super) fn breg_authority(root: &Path, plan: &Plan) -> Result bool { + registry.ddl().requires_postgis +} + pub(super) fn casework_principals( root: &Path, plan: &Plan, @@ -911,6 +925,23 @@ mod tests { sync::mpsc, }; + #[test] + fn postgis_requirement_comes_from_the_compiled_breg_inventory() { + let products = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../products/breg"); + for (project, expected) in [ + ("acceptance/spatial-service-sites", true), + ("examples/minimal", false), + ] { + let compiled = crate::compile( + &products.join(project), + crate::ProfileArg::Production, + "project-dev-test", + ) + .unwrap_or_else(|_| panic!("{project} must compile")); + assert_eq!(breg_requires_postgis(&compiled), expected, "{project}"); + } + } + #[test] fn the_casework_cli_report_owns_profile_principals() { let mut report = json!({ diff --git a/crates/registry-bregctl/tests/project_dev.rs b/crates/registry-bregctl/tests/project_dev.rs index 2bcbd20aca..d8033e58a2 100644 --- a/crates/registry-bregctl/tests/project_dev.rs +++ b/crates/registry-bregctl/tests/project_dev.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 -use std::{fs, process::Command}; +use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; +use std::{fs, os::unix::fs::PermissionsExt, process::Command}; fn project(root: &std::path::Path, extra: &str) { fs::write(root.join("registry.yaml"), "fixture\n").unwrap(); @@ -156,3 +157,85 @@ fn a_run_id_cannot_name_a_path() { assert!(report["error"].as_str().unwrap().contains("run ID")); assert_eq!(fs::read_dir(scratch.path()).unwrap().count(), 0); } + +#[test] +fn worker_restores_a_valid_plan_whose_captured_input_exceeds_512_kib() { + let scratch = tempfile::tempdir().unwrap(); + let project_root = scratch.path().join("project"); + fs::create_dir(&project_root).unwrap(); + fs::write(project_root.join("registry.yaml"), "fixture\n").unwrap(); + let raw = BASE64.encode(vec![b'x'; 400 * 1024]); + let mut plan = serde_json::json!({ + "apiVersion": "id.registrystack.org/formats/breg/project-dev/v1alpha1", + "kind": "BRegProjectDev", + "issuer": "https://issuer.example.invalid", + "products": [{"product": "breg", "project": ".", "audience": "registry"}], + "people": {"operator": { + "subject": "operator", "client": "operator-client", + "grants": {"breg": {"profile": "operator", "scopes": []}} + }}, + "journey": {"id": "smoke", "steps": [{ + "id": "send-raw", "product": "breg", "as": "operator", "method": "POST", + "path": "/v1/records", + "raw": {"contentType": "application/octet-stream", "base64": raw}, + "expect": {"status": 200} + }]} + }); + let document = serde_json::to_vec(&plan).unwrap(); + assert!( + document.len() <= registry_platform_yaml::MAXIMUM_DOCUMENT_BYTES, + "the regression fixture must remain a valid-size development plan" + ); + fs::write(project_root.join("dev.yaml"), &document).unwrap(); + + let checked = check(&project_root, &scratch.path().join("check-state"), true); + assert!( + checked.status.success(), + "the substantial development plan was refused" + ); + + let members = plan.as_object_mut().unwrap(); + members.remove("apiVersion"); + members.remove("kind"); + let run_id = "0123456789abcdef0123456789abcdef"; + let state = scratch.path().join("state"); + let root = state.join(run_id); + fs::create_dir_all(&root).unwrap(); + fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap(); + let root = root.canonicalize().unwrap(); + let false_binary = ["/usr/bin/false", "/bin/false"] + .into_iter() + .map(std::path::Path::new) + .find(|path| path.is_file()) + .unwrap() + .canonicalize() + .unwrap(); + let input = serde_json::to_vec(&serde_json::json!({ + "run_id": run_id, + "project": project_root, + "plan": plan, + "ttl_seconds": 1, + "schema_people": [], + "breg_authority": { + "principal": "sub", "purpose": false, "requires_postgis": false, + "named_clients": [] + }, + "casework_principals": {}, + "binaries": {"breg": {"tool": false_binary, "runtime": false_binary}} + })) + .unwrap(); + assert!(input.len() > 512 * 1024, "captured input: {}", input.len()); + fs::write(root.join("input.json"), input).unwrap(); + + let result = Command::new(env!("CARGO_BIN_EXE_bregctl")) + .args(["project-dev", "--worker"]) + .arg(&root) + .env("RSTACK_DEV_STATE_DIR", &state) + .env_remove("RSTACK_DEV_DATABASE_URL") + .output() + .unwrap(); + let reply: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + let error = reply["error"].as_str().unwrap(); + assert!(error.contains("breg check failed"), "{error}"); + assert!(!error.contains("cannot restore the captured development plan")); +} diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index c451db1fb5..7775bf903b 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -6,6 +6,8 @@ PostgreSQL server over TLS. The database administrator needs to create and remove roles and databases and install `btree_gist`. PostgreSQL 17 or newer is required. Use the platform's trusted roots or set `RSTACK_DEV_DATABASE_CA` to your server's CA bundle in PEM format. +Spatial BReg projects also need PostGIS available on the server and a database +administrator that can install it in the run's databases. ```sh rstack install breg casework diff --git a/products/platform/scripts/test-rstack-dev.py b/products/platform/scripts/test-rstack-dev.py index 4aaab117e5..53dd761330 100644 --- a/products/platform/scripts/test-rstack-dev.py +++ b/products/platform/scripts/test-rstack-dev.py @@ -22,6 +22,7 @@ STACK = Path(__file__).resolve().parents[3] NATIVE_REVIEW = STACK / "products/breg/acceptance/reviewed-evidence-journey" +SPATIAL = STACK / "products/breg/acceptance/spatial-service-sites" HTTP = build_opener(ProxyHandler({})) @@ -152,7 +153,11 @@ def inventory(pg, env): databases = psql( pg, env, "SELECT datname FROM pg_database WHERE datname LIKE 'rstack%';" ) - roles = psql(pg, env, "SELECT rolname FROM pg_roles WHERE rolname LIKE 'rstack%';") + roles = psql( + pg, + env, + "SELECT rolname FROM pg_roles WHERE rolname LIKE 'rstack%' OR rolname LIKE 'breg_bbox_%';", + ) return set(databases.splitlines()), set(roles.splitlines()) @@ -292,7 +297,7 @@ def coordinate(value): process.wait(timeout=15) -def exercise(root, bin_dir, pg, project, ca, url, pg_env): +def exercise(root, bin_dir, pg, project, ca, url, pg_env, spatial): checkout = root / "licensing" shutil.copytree(project, checkout) before = files(checkout) @@ -309,6 +314,8 @@ def exercise(root, bin_dir, pg, project, ca, url, pg_env): } try: exercise_runs(root, bin_dir, pg, checkout, state, env, pg_env) + if spatial: + verify_spatial(root, bin_dir, pg, env, pg_env) assert files(checkout) == before finally: if state.exists(): @@ -422,11 +429,133 @@ def verify_native_handoff(root, bin_dir, env): dev(bin_dir, env, checkout, "stop", report["run_id"]) +def verify_spatial(root, bin_dir, pg, env, pg_env): + checkout = root / "spatial" + registry = checkout / "registry" + shutil.copytree(SPATIAL, registry) + before = files(registry) + fixture = json.loads((registry / "tests/journeys.yaml").read_text()) + people = {} + for journey in fixture["journeys"]: + for step in journey["steps"]: + profile = step["accessProfile"] + claims = step["claims"] + people[profile] = { + "subject": claims["principal"], + "client": "spatial-smoke", + "grants": { + "breg": { + "profile": profile, + "scopes": claims["scopes"], + "claims": { + **claims.get("directClaims", {}), + "registry_actor_kind": claims.get("actorKind", "human"), + "registry_purpose": claims["purpose"], + }, + } + }, + } + with (registry / "fixtures/seed-service-sites.jsonl").open() as records: + site = json.loads(next(records))["data"] + longitude, latitude = site["location"]["coordinates"] + bbox = ( + f"{longitude - 0.01:.6f},{latitude - 0.01:.6f}," + f"{longitude + 0.01:.6f},{latitude + 0.01:.6f}" + ) + plan = { + "apiVersion": "id.registrystack.org/formats/breg/project-dev/v1alpha1", + "kind": "BRegProjectDev", + "issuer": "https://spatial-smoke.example.invalid", + "products": [{"product": "breg", "project": "registry", "audience": "breg"}], + "people": people, + "journey": { + "id": "spatial-smoke", + "steps": [ + { + "id": "create-site", + "product": "breg", + "as": "service-site-admin", + "method": "POST", + "path": "/v1/records/service-sites", + "headers": {"Idempotency-Key": "spatial-smoke"}, + "body": {"data": site}, + "expect": {"status": 201}, + }, + { + "id": "read-bbox", + "product": "breg", + "as": "map-reader", + "method": "GET", + "path": f"/v1/records/service-sites?bbox={bbox}", + "expect": { + "status": 200, + "checks": [{"pointer": "/items", "length": 1}], + }, + }, + ], + }, + } + (checkout / "dev.yaml").write_text(json.dumps(plan)) + baseline = inventory(pg, pg_env) + report = dev(bin_dir, env, checkout, "start", ".") + run = report["run_id"] + try: + assert report["journey"]["result"] == "passed" + origin = report["products"]["breg"] + collection = f"{origin}/v1/gis/collections/service-site.map-reader" + headers = credential(bin_dir, env, checkout, run, "map-reader") + with HTTP.open(Request(collection, headers=headers), timeout=5) as response: + metadata = json.load(response) + items = next( + link["href"] for link in metadata["links"] if link["rel"] == "items" + ) + assert items == f"{collection}/items" + request = Request(f"{items}?bbox={bbox}&f=json", headers=headers) + with HTTP.open(request, timeout=5) as response: + features = json.load(response) + assert features["type"] == "FeatureCollection" + assert len(features["features"]) == 1 + owners = set() + for label in ("breg_test", "breg"): + prefix = f"rstack_{run}_{label}" + db_env = {**pg_env, "PGDATABASE": prefix} + extension = psql( + pg, + db_env, + """SELECT n.nspname + FROM pg_extension e JOIN pg_namespace n ON n.oid = e.extnamespace + WHERE e.extname = 'postgis';""", + ).strip() + assert extension == "registry_spatial_ext" + owner = psql( + pg, + db_env, + f"""SELECT granted.rolname + FROM pg_auth_members membership + JOIN pg_roles grantee ON grantee.oid = membership.member + JOIN pg_roles granted ON granted.oid = membership.roleid + WHERE grantee.rolname = '{prefix}_migration' + AND granted.rolname LIKE 'breg_bbox_%' AND NOT granted.rolcanlogin + AND membership.set_option AND NOT membership.inherit_option + AND NOT membership.admin_option + AND NOT pg_has_role('{prefix}_runtime', granted.oid, 'MEMBER');""", + ).strip() + assert owner.startswith("breg_bbox_") and "\n" not in owner + owners.add(owner) + assert len(owners) == 2 + assert files(registry) == before + finally: + assert dev(bin_dir, env, checkout, "stop", run)["status"] == "stopped" + assert inventory(pg, pg_env) == baseline + print("Spatial BReg bbox journey passed with distinct, cleaned-up spatial roles") + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--bin-dir", type=Path, required=True) parser.add_argument("--postgres-bin-dir", type=Path, required=True) parser.add_argument("--scratch-parent", type=Path) + parser.add_argument("--spatial", action="store_true", help="also verify PostGIS") args = parser.parse_args() with tempfile.TemporaryDirectory( prefix="rstack-dev-", dir=args.scratch_parent @@ -470,6 +599,7 @@ def main(): ca, url, pg_env, + args.spatial, ) finally: checked(pg / "pg_ctl", "-D", cluster, "-m", "fast", "-w", "stop") From c9068be992175113744a17e085424231dfe2aaa7 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 01:41:52 +0700 Subject: [PATCH 06/34] fix: explain plan refusals and bound database operations Signed-off-by: Jeremi Joslin --- .../src/project_dev/bindings.rs | 101 +++-- .../src/project_dev/database.rs | 394 +++++++++++++++--- .../registry-bregctl/src/project_dev/plan.rs | 280 +++++++++---- crates/registry-bregctl/tests/project_dev.rs | 138 ++++++ products/platform/scripts/test-rstack-dev.py | 70 ++++ 5 files changed, 797 insertions(+), 186 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/bindings.rs b/crates/registry-bregctl/src/project_dev/bindings.rs index 16c52d1b6e..3d9a773ee0 100644 --- a/crates/registry-bregctl/src/project_dev/bindings.rs +++ b/crates/registry-bregctl/src/project_dev/bindings.rs @@ -83,30 +83,38 @@ pub(super) struct CaseworkContextProvider { } impl Bindings { - pub(super) fn validate(&self, plan: &Plan) -> Result<()> { + pub(super) fn validate(&self, plan: &Plan) -> plan::SemanticResult<()> { if self.breg_review_authority.is_none() && self.casework_context_provider.is_none() { return Ok(()); } let breg = product_position(plan, Product::Breg); let casework = product_position(plan, Product::Casework); - ensure!( + plan::require( breg.is_some() && casework.is_some(), - "dev.yaml /bindings requires both BReg and Casework products" - ); - ensure!( + "/bindings", + "Run-local bindings require both BReg and Casework products.", + "Add both products to products, or remove bindings.", + )?; + plan::require( breg < casework, - "dev.yaml /products must place BReg before Casework when /bindings is configured" - ); + "/products", + "Products must place BReg before Casework when bindings are configured.", + "Move the BReg product before the Casework product.", + )?; if let Some(binding) = &self.breg_review_authority { - ensure!( + plan::require( plan::identifier(&binding.id), - "dev.yaml /bindings/bregReviewAuthority/id must be a local identifier" - ); - ensure!( + "/bindings/bregReviewAuthority/id", + "The BReg review authority ID must be a local identifier.", + "Set id to a lowercase local identifier of at most 64 characters.", + )?; + plan::require( plan::identifier(&binding.producer), - "dev.yaml /bindings/bregReviewAuthority/producer must be a local identifier" - ); + "/bindings/bregReviewAuthority/producer", + "The BReg review producer must be a local identifier.", + "Set producer to a lowercase local identifier of at most 64 characters.", + )?; require_machine_person( plan, &binding.identity, @@ -115,14 +123,18 @@ impl Bindings { )?; } if let Some(binding) = &self.casework_context_provider { - ensure!( + plan::require( plan::identifier(&binding.id), - "dev.yaml /bindings/caseworkContextProvider/id must be a local identifier" - ); - ensure!( + "/bindings/caseworkContextProvider/id", + "The Casework context provider ID must be a local identifier.", + "Set id to a lowercase local identifier of at most 64 characters.", + )?; + plan::require( plan::identifier(&binding.action), - "dev.yaml /bindings/caseworkContextProvider/action must be a local identifier" - ); + "/bindings/caseworkContextProvider/action", + "The Casework context provider action must be a local identifier.", + "Set action to a lowercase local identifier of at most 64 characters.", + )?; require_machine_person( plan, &binding.identity, @@ -145,26 +157,45 @@ fn require_machine_person<'a>( alias: &str, product: Product, field: &str, -) -> Result<&'a TestPerson> { - ensure!( +) -> plan::SemanticResult<&'a TestPerson> { + plan::require( plan::identifier(alias), - "dev.yaml {field} must be a test person alias" - ); - let person = plan - .people - .get(alias) - .with_context(|| format!("dev.yaml {field} must name a test person"))?; - let grant = person.grants.get(&product).with_context(|| { + field, + "The binding identity must be a test person alias.", + "Set identity to a lowercase local identifier naming a test person.", + )?; + plan::require( + plan.people.contains_key(alias), + field, + "The binding identity does not name a test person.", + "Set identity to the alias of a test person in people.", + )?; + let person = &plan.people[alias]; + plan::require( + person.grants.contains_key(&product), + field, format!( - "dev.yaml {field} must name a person with a {} grant", + "The binding identity must name a person with a {} grant.", product.name() - ) - })?; - ensure!( + ), + format!( + "Add a {} grant to the named test person or choose another identity.", + product.name() + ), + )?; + let grant = &person.grants[&product]; + plan::require( !grant.scopes.is_empty(), - "dev.yaml {field} must name a person whose {} grant has a scope", - product.name() - ); + field, + format!( + "The binding identity must name a person whose {} grant has a scope.", + product.name() + ), + format!( + "Add a scope to the named person's {} grant or choose another identity.", + product.name() + ), + )?; Ok(person) } diff --git a/crates/registry-bregctl/src/project_dev/database.rs b/crates/registry-bregctl/src/project_dev/database.rs index a55b30e3f2..98a5133c17 100644 --- a/crates/registry-bregctl/src/project_dev/database.rs +++ b/crates/registry-bregctl/src/project_dev/database.rs @@ -1,6 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 -use std::{future::Future, str::FromStr, time::Duration}; +use std::{ + future::Future, + str::FromStr, + time::{Duration, Instant}, +}; use anyhow::{anyhow, bail, Context, Result}; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; @@ -22,6 +26,9 @@ const BREG_SCHEMAS: [&str; 5] = [ "registry_context", ]; const CONNECTION_TIMEOUT: Duration = Duration::from_secs(10); +const STATEMENT_TIMEOUT: Duration = Duration::from_secs(10); +const OPERATION_TIMEOUT: Duration = Duration::from_secs(15); +const CLEANUP_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(30); async fn bounded_connection( connection: impl Future>, @@ -33,6 +40,117 @@ async fn bounded_connection( .context("the PostgreSQL connection handshake failed") } +#[derive(Debug, thiserror::Error)] +enum OperationFailure { + #[error("the PostgreSQL {operation} operation exceeded its I/O deadline")] + IoDeadline { operation: &'static str }, + #[error( + "the PostgreSQL {operation} operation was canceled or exceeded its statement deadline" + )] + StatementDeadline { operation: &'static str }, + #[error("the PostgreSQL {operation} operation failed")] + PostgreSql { + operation: &'static str, + #[source] + source: tokio_postgres::Error, + }, +} + +impl OperationFailure { + fn postgres(operation: &'static str, source: tokio_postgres::Error) -> Self { + if source.code() == Some(&tokio_postgres::error::SqlState::QUERY_CANCELED) { + Self::StatementDeadline { operation } + } else { + Self::PostgreSql { operation, source } + } + } + + fn effect_is_unconfirmed(&self) -> bool { + matches!(self, Self::IoDeadline { .. }) + || matches!(self, Self::PostgreSql { source, .. } if source.as_db_error().is_none()) + } +} + +async fn bounded_operation( + operation: &'static str, + future: impl Future>, + timeout: Duration, +) -> std::result::Result { + match tokio::time::timeout(timeout, future).await { + Ok(Ok(value)) => Ok(value), + Ok(Err(error)) => Err(OperationFailure::postgres(operation, error)), + Err(_) => Err(OperationFailure::IoDeadline { operation }), + } +} + +fn retain_create_acquisition( + acquisitions: &mut Vec, + acquisition: Acquisition, + created: &std::result::Result, +) { + if created.is_ok() + || created + .as_ref() + .is_err_and(OperationFailure::effect_is_unconfirmed) + { + acquisitions.push(acquisition); + } +} + +fn apply_statement_deadline(config: &mut Config) { + let deadline = format!("-c statement_timeout={}", STATEMENT_TIMEOUT.as_millis()); + if let Some(existing) = config.get_options().filter(|value| !value.is_empty()) { + config.options(format!("{existing} {deadline}")); + } else { + config.options(deadline); + } +} + +fn remaining_timeout(deadline: Instant, maximum: Duration) -> Result { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + bail!("the PostgreSQL operation deadline expired"); + } + Ok(remaining.min(maximum)) +} + +fn ownership_lock_statement(run_id: &str) -> Result { + let prefix = run_id + .get(..16) + .context("the database run id cannot form an ownership lock")?; + let key = u64::from_str_radix(prefix, 16) + .context("the validated database run id cannot form an ownership lock")? + as i64; + Ok(format!("SELECT pg_advisory_lock({key})")) +} + +fn open_admin_session( + runtime: &Runtime, + config: &Config, + tls: &MakeRustlsConnect, + ownership_lock: &str, + deadline: Instant, +) -> Result<( + Client, + tokio::task::JoinHandle>, +)> { + let (client, connection) = runtime.block_on(bounded_connection( + config.connect(tls.clone()), + remaining_timeout(deadline, CONNECTION_TIMEOUT)?, + ))?; + let connection_task = runtime.spawn(connection); + let locked = runtime.block_on(bounded_operation( + "run ownership lock acquisition", + client.batch_execute(ownership_lock), + remaining_timeout(deadline, OPERATION_TIMEOUT)?, + )); + if let Err(error) = locked { + connection_task.abort(); + return Err(error).context("cannot acquire the PostgreSQL run ownership lock"); + } + Ok((client, connection_task)) +} + pub(super) struct DatabaseUrls { pub runtime: String, pub migration: String, @@ -47,8 +165,10 @@ enum Acquisition { pub(super) struct Database { client: Client, _connection_task: tokio::task::JoinHandle>, + admin_session_failed: bool, tls: MakeRustlsConnect, admin_config: Config, + ownership_lock: String, endpoint: Endpoint, run_id: String, acquisitions: Vec, @@ -94,23 +214,29 @@ impl Database { validate_run_id(run_id)?; let (mut admin_config, endpoint) = admin_config(admin_url)?; admin_config.ssl_mode(SslMode::Require); + apply_statement_deadline(&mut admin_config); let tls = tls_connector(ca_pem)?; let runtime = Builder::new_current_thread() .enable_all() .build() .context("cannot initialize the PostgreSQL executor")?; - let (client, connection) = runtime - .block_on(bounded_connection( - admin_config.connect(tls.clone()), - CONNECTION_TIMEOUT, - )) - .context("cannot connect to the PostgreSQL administrator database over verified TLS")?; - let connection_task = runtime.spawn(connection); + let ownership_lock = ownership_lock_statement(run_id)?; + let session_deadline = Instant::now() + CONNECTION_TIMEOUT + OPERATION_TIMEOUT; + let (client, connection_task) = open_admin_session( + &runtime, + &admin_config, + &tls, + &ownership_lock, + session_deadline, + ) + .context("cannot establish the PostgreSQL administrator session over verified TLS")?; Ok(Self { client, _connection_task: connection_task, + admin_session_failed: false, tls, admin_config, + ownership_lock, endpoint, run_id: run_id.to_owned(), acquisitions: Vec::new(), @@ -156,18 +282,23 @@ impl Database { } pub(super) fn cleanup(&mut self) -> Result<()> { + let deadline = Instant::now() + CLEANUP_ATTEMPT_TIMEOUT; let mut failures = Vec::new(); let mut retained = Vec::new(); let acquisitions = self.acquisitions.clone(); for acquisition in acquisitions.into_iter().rev() { let result = match &acquisition { - Acquisition::Database(name) => self.execute_cleanup(&format!( - "DROP DATABASE IF EXISTS {} WITH (FORCE)", - quote_identifier(name) - )), - Acquisition::Role(name) => { - self.execute_cleanup(&format!("DROP ROLE IF EXISTS {}", quote_identifier(name))) - } + Acquisition::Database(name) => self.execute_cleanup( + &format!( + "DROP DATABASE IF EXISTS {} WITH (FORCE)", + quote_identifier(name) + ), + deadline, + ), + Acquisition::Role(name) => self.execute_cleanup( + &format!("DROP ROLE IF EXISTS {}", quote_identifier(name)), + deadline, + ), }; if let Err(error) = result { failures.push(match &acquisition { @@ -190,40 +321,56 @@ impl Database { } } - fn reconnect_admin(&mut self) -> Result<()> { - let (client, connection) = self - .runtime - .block_on(bounded_connection( - self.admin_config.connect(self.tls.clone()), - CONNECTION_TIMEOUT, - )) - .context("cannot reconnect to the PostgreSQL administrator database")?; - self._connection_task.abort(); + fn reconnect_admin(&mut self, deadline: Instant) -> Result<()> { + // The old server backend can outlive the client-side I/O timeout. Its + // session lock orders this replacement session after an unconfirmed + // CREATE has committed or rolled back, so cleanup cannot run too early. + self.retire_admin_session(); + let (client, connection_task) = open_admin_session( + &self.runtime, + &self.admin_config, + &self.tls, + &self.ownership_lock, + deadline, + ) + .context("cannot reconnect to the PostgreSQL administrator database")?; self.client = client; - self._connection_task = self.runtime.spawn(connection); + self._connection_task = connection_task; + self.admin_session_failed = false; Ok(()) } - fn ensure_admin_connection(&mut self) -> Result<()> { - if self.client.is_closed() { - self.reconnect_admin()?; + fn retire_admin_session(&mut self) { + self._connection_task.abort(); + self.admin_session_failed = true; + } + + fn ensure_admin_connection(&mut self, deadline: Instant) -> Result<()> { + if self.admin_session_failed || self.client.is_closed() { + self.reconnect_admin(deadline)?; } Ok(()) } - fn execute_cleanup(&mut self, statement: &str) -> Result<()> { - self.ensure_admin_connection()?; - match self.runtime.block_on(self.client.batch_execute(statement)) { + fn execute_cleanup(&mut self, statement: &str, deadline: Instant) -> Result<()> { + self.ensure_admin_connection(deadline)?; + let first = self.execute_admin_operation( + statement, + "cleanup", + remaining_timeout(deadline, OPERATION_TIMEOUT)?, + ); + match first { Ok(()) => Ok(()), - Err(first) if self.client.is_closed() => { - self.reconnect_admin()?; - self.runtime - .block_on(self.client.batch_execute(statement)) - .with_context(|| { - format!( - "the PostgreSQL cleanup operation failed after reconnecting ({first})" - ) - }) + Err(first) if first.effect_is_unconfirmed() => { + self.reconnect_admin(deadline)?; + self.execute_admin_operation( + statement, + "cleanup", + remaining_timeout(deadline, OPERATION_TIMEOUT)?, + ) + .with_context(|| { + format!("the PostgreSQL cleanup operation failed after reconnecting ({first})") + }) } Err(error) => Err(error).context("the PostgreSQL cleanup operation failed"), } @@ -257,26 +404,33 @@ impl Database { statement: &str, context: &'static str, ) -> Result<()> { - let created = self.runtime.block_on(self.client.batch_execute(statement)); - if created.is_ok() || created.as_ref().is_err_and(|error| error.is_closed()) { - // A closed connection cannot tell us whether PostgreSQL committed - // the statement. Retaining the exact run-owned name lets cleanup's - // idempotent DROP cover both outcomes without replaying CREATE. - self.acquisitions.push(Acquisition::Role(role.to_owned())); - } - created.context(context) + let created = self.execute_admin_operation(statement, "role creation", OPERATION_TIMEOUT); + // A transport/protocol failure or I/O deadline cannot tell us whether + // PostgreSQL committed the statement. Retaining the exact run-owned name + // lets cleanup's idempotent DROP cover both outcomes without replaying CREATE. + retain_create_acquisition( + &mut self.acquisitions, + Acquisition::Role(role.to_owned()), + &created, + ); + created.map_err(anyhow::Error::new).context(context) } fn create_database(&mut self, names: &Names) -> Result<()> { - let created = self.runtime.block_on(self.client.batch_execute(&format!( + let statement = format!( "CREATE DATABASE {} WITH ALLOW_CONNECTIONS false", quote_identifier(&names.database) - ))); - if created.is_ok() || created.as_ref().is_err_and(|error| error.is_closed()) { - self.acquisitions - .push(Acquisition::Database(names.database.clone())); - } - created.context("cannot create an isolated PostgreSQL database")?; + ); + let created = + self.execute_admin_operation(&statement, "database creation", OPERATION_TIMEOUT); + retain_create_acquisition( + &mut self.acquisitions, + Acquisition::Database(names.database.clone()), + &created, + ); + created + .map_err(anyhow::Error::new) + .context("cannot create an isolated PostgreSQL database")?; self.execute_admin(&database_access_sql(names)) .context("cannot restrict the PostgreSQL database to this run's roles") @@ -304,10 +458,14 @@ impl Database { .await .context("cannot connect to the provisioned PostgreSQL database")?; let connection_task = tokio::spawn(connection); - let executed = async { + let executed = tokio::time::timeout(OPERATION_TIMEOUT, async { client .batch_execute(&statement) .await + .map_err(|error| { + OperationFailure::postgres("database configuration", error) + }) + .map_err(anyhow::Error::new) .context("cannot configure the provisioned PostgreSQL database")?; if let Some((migration, runtime)) = &spatial_roles { registry_breg::postgres::provision_postgis_prerequisites( @@ -317,25 +475,46 @@ impl Database { .context("cannot provision BReg PostGIS prerequisites")?; } Result::<()>::Ok(()) - } - .await; - drop(client); - let closed = connection_task - .await - .context("the PostgreSQL connection task stopped unexpectedly")?; - executed?; - closed.context("the PostgreSQL configuration connection failed")?; - Result::<()>::Ok(()) + }) + .await + .map_err(|_| { + anyhow!( + "the PostgreSQL database configuration operation exceeded its I/O deadline" + ) + }) + .and_then(|result| result); + connection_task.abort(); + executed }) .context("cannot finish PostgreSQL database provisioning") } - fn execute_admin(&self, statement: &str) -> Result<()> { - self.runtime - .block_on(self.client.batch_execute(statement)) + fn execute_admin(&mut self, statement: &str) -> Result<()> { + self.execute_admin_operation(statement, "administrator", OPERATION_TIMEOUT) + .map_err(anyhow::Error::new) .context("the PostgreSQL administrator operation failed") } + fn execute_admin_operation( + &mut self, + statement: &str, + operation: &'static str, + timeout: Duration, + ) -> std::result::Result<(), OperationFailure> { + let executed = self.runtime.block_on(bounded_operation( + operation, + self.client.batch_execute(statement), + timeout, + )); + if executed + .as_ref() + .is_err_and(OperationFailure::effect_is_unconfirmed) + { + self.retire_admin_session(); + } + executed + } + fn product_url(&self, database: &str, user: &str, password: &str) -> Result { self.endpoint.product_url(database, user, password) } @@ -599,6 +778,85 @@ mod tests { ); } + #[test] + fn post_handshake_operation_deadline_marks_the_effect_unconfirmed() { + let runtime = Builder::new_current_thread().enable_time().build().unwrap(); + let started = Instant::now(); + let result: std::result::Result<(), OperationFailure> = + runtime.block_on(bounded_operation( + "role creation", + pending::>(), + Duration::from_millis(20), + )); + let failure = result.unwrap_err(); + assert!(failure.effect_is_unconfirmed()); + assert_eq!( + failure.to_string(), + "the PostgreSQL role creation operation exceeded its I/O deadline" + ); + assert!(started.elapsed() < Duration::from_secs(1)); + } + + #[test] + fn unconfirmed_create_retains_its_exact_acquisition_once() { + let mut acquisitions = Vec::new(); + let created = Err::<(), _>(OperationFailure::IoDeadline { + operation: "role creation", + }); + retain_create_acquisition( + &mut acquisitions, + Acquisition::Role("rstack_exact_role".to_owned()), + &created, + ); + + assert_eq!(acquisitions.len(), 1); + assert!(matches!( + &acquisitions[0], + Acquisition::Role(role) if role == "rstack_exact_role" + )); + } + + #[test] + fn confirmed_statement_cancellation_is_not_tracked_as_a_creation() { + let mut acquisitions = Vec::new(); + let created = Err::<(), _>(OperationFailure::StatementDeadline { + operation: "database creation", + }); + retain_create_acquisition( + &mut acquisitions, + Acquisition::Database("rstack_refused_database".to_owned()), + &created, + ); + assert!(acquisitions.is_empty()); + } + + #[test] + fn operation_deadlines_fit_the_worker_stop_budget() { + let mut config = Config::new(); + config.options("-c application_name=rstack-dev"); + apply_statement_deadline(&mut config); + assert_eq!( + config.get_options(), + Some("-c application_name=rstack-dev -c statement_timeout=10000") + ); + assert!(STATEMENT_TIMEOUT < OPERATION_TIMEOUT); + assert!(OPERATION_TIMEOUT < CLEANUP_ATTEMPT_TIMEOUT); + assert!(CLEANUP_ATTEMPT_TIMEOUT < Duration::from_secs(60)); + } + + #[test] + fn run_ownership_lock_uses_the_first_64_id_bits_as_a_session_key() { + assert_eq!( + ownership_lock_statement(RUN_ID).unwrap(), + "SELECT pg_advisory_lock(81985529216486895)" + ); + assert_eq!( + ownership_lock_statement("fedcba98765432100123456789abcdef").unwrap(), + "SELECT pg_advisory_lock(-81985529216486896)" + ); + assert!(ownership_lock_statement("short").is_err()); + } + #[test] fn admin_urls_require_one_explicit_password_authenticated_tcp_database() { let password = random_password().unwrap(); diff --git a/crates/registry-bregctl/src/project_dev/plan.rs b/crates/registry-bregctl/src/project_dev/plan.rs index 6f7320b89f..9d8ad3ae04 100644 --- a/crates/registry-bregctl/src/project_dev/plan.rs +++ b/crates/registry-bregctl/src/project_dev/plan.rs @@ -1,11 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result}; use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; use registry_breg::literal_text::{LiteralText, WRITE_THE_VALUE}; use registry_platform_yaml::{ - ApiVersion, BoundedU32, Decoded, EnvelopeRule, Expect, FormatSpec, Identified, LocalId, Reader, - Report, Severity, UniqueIdList, UniqueList, + escape_pointer_segment, ApiVersion, BoundedU32, Decoded, EnvelopeRule, Expect, FormatSpec, + Identified, LocalId, Reader, Report, Severity, UniqueIdList, UniqueList, }; #[cfg(feature = "schema")] use registry_platform_yaml::{ExternalId, Url as PlatformUrl}; @@ -466,13 +466,60 @@ pub(super) fn reserved_claim(claim: &str) -> bool { ) || claim.starts_with("registry_grant_") } -fn require(condition: bool, field: &str, message: &str) -> Result<()> { +#[derive(Debug, thiserror::Error)] +#[error("dev.yaml {field} {reason}")] +pub(super) struct SemanticRefusal { + field: String, + reason: String, + remedy: String, +} + +pub(super) type SemanticResult = std::result::Result; + +pub(super) fn require( + condition: bool, + field: &str, + reason: impl Into, + remedy: impl Into, +) -> SemanticResult<()> { if !condition { - bail!("dev.yaml {field} {message}"); + return Err(SemanticRefusal { + field: field.to_owned(), + reason: reason.into(), + remedy: remedy.into(), + }); } Ok(()) } +fn refuse(field: &str, reason: &str, remedy: &str) -> SemanticResult { + Err(SemanticRefusal { + field: field.to_owned(), + reason: reason.to_owned(), + remedy: remedy.to_owned(), + }) +} + +#[derive(Debug, thiserror::Error)] +#[error("capture reference must name an earlier capture")] +struct EarlierCapture; + +fn validate_expansion(result: Result, field: &str) -> SemanticResult { + match result { + Ok(value) => Ok(value), + Err(error) if error.downcast_ref::().is_some() => refuse( + field, + "The capture reference must name an earlier capture.", + "Move the referenced capture to an earlier step or correct the capture name.", + ), + Err(_) => refuse( + field, + "The field has an unclosed capture reference.", + "Close the capture reference with `}}`.", + ), + } +} + pub(super) fn read(path: &Path, limit: u64) -> Result> { let mut bytes = Vec::new(); fs::File::open(path)? @@ -497,18 +544,13 @@ fn read_plan(root: &Path) -> Result> { } fn validate_plan(root: &Path, decoded: &Decoded) -> Result<()> { - if let Err(error) = decoded.value.validate(root) { - let rendered = error.to_string(); - let path = rendered - .strip_prefix("dev.yaml ") - .and_then(|detail| detail.split_once(' ')) - .map_or("", |(path, _)| path); + if let Err(refusal) = decoded.value.validate(root) { let diagnostic = decoded.document.diagnostic_at_value( Severity::Error, "rstack.dev.invalid-plan", - path, - "The development plan is inconsistent.", - "Correct the named development plan field, then run the check again.", + &refusal.field, + &refusal.reason, + &refusal.remedy, ); return Err(decoded.document.report_with([diagnostic]).into()); } @@ -535,7 +577,7 @@ impl Plan { .expect("validated product") } - fn validate(&self, root: &Path) -> Result<()> { + fn validate(&self, root: &Path) -> SemanticResult<()> { require( ParsedUrl::parse(&self.issuer).is_ok_and(|url| { url.scheme() == "https" @@ -547,12 +589,14 @@ impl Plan { && self.issuer.len() <= 512 }), "/issuer", - "must be the project's HTTPS issuer URL", + "The issuer must be the project's bounded HTTPS URL without credentials, query, or fragment.", + "Set issuer to an HTTPS URL with a host and no credentials, query, or fragment.", )?; require( !self.products.is_empty(), "/products", - "must name at least one product", + "The plan must name at least one product.", + "Add a BReg or Casework product to products.", )?; let mut products = BTreeSet::new(); let mut audiences = BTreeSet::new(); @@ -561,12 +605,14 @@ impl Plan { require( products.insert(entry.product), &format!("{field}/product"), - "is repeated", + "The product is repeated.", + "Remove the duplicate product entry.", )?; require( bounded_atom(&entry.audience) && audiences.insert(&entry.audience), &format!("{field}/audience"), - "must be a distinct, bounded audience", + "The audience must be distinct, bounded text without whitespace or control characters.", + "Set audience to a distinct identifier of at most 256 characters without whitespace or control characters.", )?; require( entry.project == Path::new(".") @@ -576,7 +622,8 @@ impl Plan { .components() .all(|part| matches!(part, Component::Normal(_)))), &format!("{field}/project"), - "must be a path inside the project directory", + "The product project must be a relative path inside the development project directory.", + "Set project to `.` or a relative path made only of normal path components.", )?; let manifest = match entry.product { Product::Breg => "registry.yaml", @@ -588,47 +635,55 @@ impl Plan { .canonicalize() .is_ok_and(|path| path.starts_with(root) && path.is_file()), &format!("{field}/project"), - &format!("must contain {manifest} inside the project directory"), + format!("The product project must contain {manifest} inside the development project directory."), + format!("Create {manifest} at this project path inside the development project directory."), )?; } require( (1..=32).contains(&self.people.len()), "/people", - "must define 1 to 32 people", + "The plan must define 1 to 32 test people.", + "Add or remove test people until people contains between 1 and 32 entries.", )?; for (name, person) in &self.people { require( identifier(name), "/people", - "contains an invalid person name", + "A test person name is not a local identifier.", + "Rename the test person with a lowercase local identifier of at most 64 characters.", )?; let field = format!("/people/{name}"); require( bounded_atom(&person.subject), &format!("{field}/subject"), - "must be a bounded subject", + "The test person's subject must be bounded text without whitespace or control characters.", + "Set subject to at most 256 characters without whitespace or control characters.", )?; require( bounded_atom(&person.client), &format!("{field}/client"), - "must be a bounded client ID", + "The test person's client must be a bounded client identifier.", + "Set client to at most 256 characters without whitespace or control characters.", )?; require( !person.grants.is_empty(), &format!("{field}/grants"), - "must name a product grant", + "The test person must have at least one product grant.", + "Add a BReg or Casework grant for this test person.", )?; for (product, grant) in &person.grants { let field = format!("{field}/grants/{}", product.name()); require( products.contains(product), &field, - "must name a started product", + "The grant names a product that is not started.", + "Add the grant's product to products, or remove this grant.", )?; require( identifier(&grant.profile), &format!("{field}/profile"), - "must name a profile", + "The grant profile must be a local identifier.", + "Set profile to a lowercase local identifier of at most 64 characters.", )?; let scopes = grant.scopes.iter().collect::>(); require( @@ -636,13 +691,15 @@ impl Plan { && scopes.len() <= 32 && scopes.iter().all(|scope| bounded_atom(scope)), &format!("{field}/scopes"), - "must contain up to 32 distinct scopes", + "The grant must contain at most 32 distinct bounded scopes.", + "Remove duplicate or excess scopes, and keep each scope within 256 characters without whitespace or control characters.", )?; for claim in grant.claims.keys() { require( bounded_atom(claim) && !reserved_claim(claim), &format!("{field}/claims"), - "contains a reserved or invalid claim", + "The grant contains a reserved or invalid additional claim name.", + "Remove the claim or rename it to bounded text that is not reserved by the token contract.", )?; } } @@ -650,14 +707,16 @@ impl Plan { require( self.bindings.casework_context_provider.is_some(), &format!("{field}/caseworkContext"), - "requires /bindings/caseworkContextProvider", + "Casework context forwarding requires /bindings/caseworkContextProvider.", + "Configure bindings.caseworkContextProvider or remove caseworkContext from this person.", )?; let breg = person.grants.get(&Product::Breg); let casework = person.grants.get(&Product::Casework); require( breg.is_some() && casework.is_some(), &format!("{field}/caseworkContext"), - "requires both BReg and Casework grants", + "Casework context forwarding requires both BReg and Casework grants.", + "Add both product grants to this person or remove caseworkContext.", )?; let (breg, casework) = ( breg.expect("checked grant"), @@ -668,18 +727,25 @@ impl Plan { && breg.scopes.iter().collect::>() == casework.scopes.iter().collect(), &format!("{field}/caseworkContext"), - "requires the same scopes and additional claims in both grants", + "Casework context forwarding requires the same scopes and additional claims in both grants.", + "Make the BReg and Casework grant scopes and additional claims identical.", )?; } } for (journey, steps) in &self.schema_test_people { for (step, person) in steps { + let field = format!( + "/schemaTestPeople/{}/{}", + escape_pointer_segment(journey), + escape_pointer_segment(step) + ); require( self.people .get(person) .is_some_and(|person| person.grants.contains_key(&Product::Breg)), - &format!("/schemaTestPeople/{journey}/{step}"), - "must name a person with a BReg grant", + &field, + "The schema test assignment must name a person with a BReg grant.", + "Set this assignment to a test person that has a BReg grant.", )?; } } @@ -689,7 +755,14 @@ impl Plan { .values() .any(|person| person.grants.contains_key(&product)), "/people", - &format!("must define at least one grant for {}", product.name()), + format!( + "The plan must define at least one grant for {}.", + product.name() + ), + format!( + "Add a {} grant to at least one test person.", + product.name() + ), )?; } self.bindings.validate(self)?; @@ -698,16 +771,18 @@ impl Plan { } impl Journey { - fn validate(&self, plan: &Plan) -> Result<()> { + fn validate(&self, plan: &Plan) -> SemanticResult<()> { require( identifier(&self.id), "/journey/id", - "must be a local identifier", + "The journey ID must be a local identifier.", + "Set id to a lowercase local identifier of at most 64 characters.", )?; require( (1..=32).contains(&self.steps.len()), "/journey/steps", - "must contain 1 to 32 steps", + "The journey must contain 1 to 32 steps.", + "Add or remove steps until the journey contains between 1 and 32 steps.", )?; let mut ids = BTreeSet::new(); let mut captures = BTreeSet::new(); @@ -716,19 +791,22 @@ impl Journey { require( step_identifier(&step.id) && ids.insert(&step.id), &format!("{field}/id"), - "must be a unique local identifier", + "The step ID must be a unique local or dotted derived identifier.", + "Set id to a unique lowercase local identifier or dotted sequence of local identifiers.", )?; require( plan.people .get(&step.person) .is_some_and(|person| person.grants.contains_key(&step.product)), &format!("{field}/as"), - "must name a person with a grant for the step's product", + "The step must name a test person with a grant for its product.", + "Set as to a test person that has a grant for this step's product.", )?; require( valid_path(&step.path), &format!("{field}/path"), - "must be a relative product API path starting with /v1/", + "The request path must be a relative product API path starting with /v1/.", + "Set path to a bounded /v1/ path without whitespace, control characters, backslashes, or fragments.", )?; let statuses = step.expect.status.values(); require( @@ -736,23 +814,27 @@ impl Journey { && statuses.iter().all(|status| (200..=599).contains(status)) && statuses.iter().collect::>().len() == statuses.len(), &format!("{field}/expect/status"), - "must contain 1 to 16 distinct HTTP statuses from 200 to 599", + "Expected status must contain 1 to 16 distinct HTTP statuses from 200 to 599.", + "List between 1 and 16 distinct HTTP status numbers from 200 to 599.", )?; require( step.body.is_none() || step.raw.is_none(), &field, - "must not contain both body and raw", + "A journey step must not contain both body and raw.", + "Remove body or raw so the step declares only one request body form.", )?; require( step.raw.is_none() || !matches!(step.method, Method::Get), &format!("{field}/raw"), - "is supported only for non-GET requests", + "A raw request body is supported only for non-GET requests.", + "Remove raw or change method to a non-GET request method.", )?; if let Some(body) = &step.body { require( serde_json::to_vec(body).is_ok_and(|bytes| bytes.len() <= 1024 * 1024), &format!("{field}/body"), - "must encode to at most 1 MiB", + "The JSON request body must encode to at most 1 MiB.", + "Reduce body until its encoded JSON is at most 1 MiB.", )?; } if let Some(raw) = &step.raw { @@ -764,7 +846,8 @@ impl Journey { .bytes() .all(|byte| byte.is_ascii() && byte != b'\r' && byte != b'\n'), &format!("{field}/raw/contentType"), - "must be a bounded HTTP media type", + "The raw content type must be a bounded ASCII HTTP media type.", + "Set contentType to an ASCII media type with a slash and at most 256 characters.", )?; require( raw.base64.len() <= 1_398_104 @@ -772,22 +855,30 @@ impl Journey { .decode(&raw.base64) .is_ok_and(|bytes| bytes.len() <= 1024 * 1024), &format!("{field}/raw/base64"), - "must be valid base64 encoding at most 1 MiB", + "The raw body must be valid base64 encoding of at most 1 MiB.", + "Encode at most 1 MiB of raw request data as base64.", )?; } if let Some(poll) = &step.poll { require( matches!(step.method, Method::Get), &format!("{field}/poll"), - "is supported only for GET requests", + "Polling is supported only for GET requests.", + "Remove poll or change method to GET.", )?; require( (1..=60_000).contains(&poll.timeout_ms) && (50..=5_000).contains(&poll.interval_ms) - && poll.interval_ms <= poll.timeout_ms && poll.timeout_ms.div_ceil(poll.interval_ms) < 250, &format!("{field}/poll"), - "must use a 1 to 60000 ms timeout, a 50 to 5000 ms interval, and at most 250 attempts", + "Polling must use a 1 to 60000 ms maximum wait, a 50 to 5000 ms interval, and at most 250 attempts.", + "Set maximumWaitMilliseconds and pollIntervalMilliseconds within those bounds and limit the resulting attempts to at most 250.", + )?; + require( + poll.interval_ms <= poll.timeout_ms, + &format!("{field}/poll/pollIntervalMilliseconds"), + "The poll interval must not exceed the maximum wait.", + "Set pollIntervalMilliseconds to no more than maximumWaitMilliseconds.", )?; } for header in step.headers.keys() { @@ -797,44 +888,53 @@ impl Journey { "If-Match" | "Idempotency-Key" | "Registry-Source-Profile" ), &format!("{field}/headers"), - "contains a header outside the supported list", + "The request contains a header outside the supported list.", + "Remove the header or use only If-Match, Idempotency-Key, or Registry-Source-Profile.", )?; } let verify_reference = |value: &str| { captures .contains(value) .then(|| value.to_owned()) - .context("must refer to an earlier capture") + .ok_or_else(|| EarlierCapture.into()) }; - expand(&step.path, verify_reference) - .with_context(|| format!("dev.yaml {field}/path"))?; + validate_expansion( + expand(&step.path, verify_reference), + &format!("{field}/path"), + )?; for value in step.headers.values() { - expand(value, verify_reference) - .with_context(|| format!("dev.yaml {field}/headers"))?; + validate_expansion(expand(value, verify_reference), &format!("{field}/headers"))?; } if let Some(body) = &step.body { - expand_json(body, verify_reference) - .with_context(|| format!("dev.yaml {field}/body"))?; + validate_expansion( + expand_json(body, verify_reference), + &format!("{field}/body"), + )?; } require( step.expect.json.keys().all(|value| pointer(value)), &format!("{field}/expect/json"), - "must use JSON pointers", + "Expected JSON keys must be JSON pointers.", + "Replace each expected JSON key with an empty pointer or one starting with `/`.", )?; for value in step.expect.json.values() { - expand_json(value, verify_reference) - .with_context(|| format!("dev.yaml {field}/expect/json"))?; + validate_expansion( + expand_json(value, verify_reference), + &format!("{field}/expect/json"), + )?; } require( step.expect.checks.len() <= 32, &format!("{field}/expect/checks"), - "must contain at most 32 checks", + "Expected checks must contain at most 32 entries.", + "Remove checks until at most 32 remain.", )?; for check in &step.expect.checks { require( pointer(&check.pointer), &format!("{field}/expect/checks"), - "must use JSON pointers", + "Each expected check must use a JSON pointer.", + "Set each check pointer to an empty pointer or one starting with `/`.", )?; require( usize::from(check.equals.is_some()) @@ -842,32 +942,40 @@ impl Journey { + usize::from(check.contains.is_some()) == 1, &format!("{field}/expect/checks"), - "must contain exactly one of equals, length, or contains", + "Each expected check must contain exactly one of equals, length, or contains.", + "Keep exactly one assertion member in each expected check.", )?; require( check.length.is_none_or(|length| length <= 1024 * 1024), &format!("{field}/expect/checks"), - "contains an excessive length", + "An expected length exceeds 1 MiB.", + "Set length to at most 1048576.", )?; for value in check.equals.iter().chain(&check.contains) { - expand_json(value, verify_reference) - .with_context(|| format!("dev.yaml {field}/expect/checks"))?; + validate_expansion( + expand_json(value, verify_reference), + &format!("{field}/expect/checks"), + )?; } } if let Some(code) = &step.expect.code { require( bounded_atom(code), &format!("{field}/expect/code"), - "must be a bounded problem code", + "The expected problem code must be bounded text without whitespace or control characters.", + "Set code to at most 256 characters without whitespace or control characters.", + )?; + validate_expansion( + expand(code, verify_reference), + &format!("{field}/expect/code"), )?; - expand(code, verify_reference) - .with_context(|| format!("dev.yaml {field}/expect/code"))?; } for (name, capture) in &step.capture { require( capture_identifier(name) && !captures.contains(name.as_str()), &format!("{field}/capture"), - "must use unique bounded capture names", + "Capture names must be unique bounded ASCII identifiers.", + "Rename the capture with a unique identifier of at most 64 ASCII letters, digits, or hyphens, starting with a letter.", )?; match capture { Capture::Pointer { @@ -875,20 +983,23 @@ impl Journey { } => require( pointer(pointer_value), &format!("{field}/capture/{name}"), - "must use a JSON pointer", + "The capture must use a JSON pointer.", + "Set pointer to an empty pointer or one starting with `/`.", )?, Capture::Header { .. } => {} Capture::Value { pointer, query } => { require( self::pointer(pointer), &format!("{field}/capture/{name}/pointer"), - "must be a JSON pointer", + "The value capture pointer must be a JSON pointer.", + "Set pointer to an empty pointer or one starting with `/`.", )?; if let Some(query) = query { require( bounded_atom(query), &format!("{field}/capture/{name}/query"), - "must name a bounded query parameter", + "The value capture query must name a bounded query parameter.", + "Set query to at most 256 characters without whitespace or control characters.", )?; } } @@ -900,15 +1011,18 @@ impl Journey { require( pointer(find) && pointer(&r#where.pointer) && pointer(selected), &format!("{field}/capture/{name}"), - "must use JSON pointers", + "The find capture must use JSON pointers.", + "Set find, where.pointer, and pointer to empty pointers or pointers starting with `/`.", + )?; + validate_expansion( + expand_json(&r#where.equals, |value| { + captures + .contains(value) + .then(|| value.to_owned()) + .ok_or_else(|| EarlierCapture.into()) + }), + &format!("{field}/capture/{name}/find"), )?; - expand_json(&r#where.equals, |value| { - captures - .contains(value) - .then(|| value.to_owned()) - .context("must refer to an earlier capture") - }) - .with_context(|| format!("dev.yaml {field}/capture/{name}/find"))?; } } captures.insert(name.clone()); diff --git a/crates/registry-bregctl/tests/project_dev.rs b/crates/registry-bregctl/tests/project_dev.rs index d8033e58a2..626a9c2a64 100644 --- a/crates/registry-bregctl/tests/project_dev.rs +++ b/crates/registry-bregctl/tests/project_dev.rs @@ -21,6 +21,22 @@ journey:\n id: smoke\n steps:\n - {{id: read, product: breg, as: operator, .unwrap(); } +fn semantic_project(root: &std::path::Path, grant: &str, step_extra: &str) { + fs::write(root.join("registry.yaml"), "fixture\n").unwrap(); + fs::write( + root.join("dev.yaml"), + format!( + "apiVersion: id.registrystack.org/formats/breg/project-dev/v1alpha1\n\ +kind: BRegProjectDev\n\ +issuer: https://issuer.example.invalid\n\ +products:\n - {{product: breg, project: ., audience: private-audience-marker}}\n\ +people:\n operator:\n subject: private-subject-marker\n client: private-client-marker\n grants:\n {grant}: {{profile: private-profile-marker, scopes: []}}\n\ +journey:\n id: smoke\n steps:\n - id: read\n product: breg\n as: operator\n method: GET\n path: /v1/records\n expect: {{status: 200}}\n{step_extra}" + ), + ) + .unwrap(); +} + fn check(root: &std::path::Path, state: &std::path::Path, json: bool) -> std::process::Output { let request = serde_json::json!({ "action": "check", "project": root, "deny_warnings": true, "json": json, @@ -107,6 +123,128 @@ fn check_distinguishes_unavailable_input_from_an_invalid_plan() { assert!(!state.exists()); } +#[test] +fn check_reports_an_actionable_value_free_poll_relationship() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + semantic_project( + scratch.path(), + "breg", + " poll: {maximumWaitMilliseconds: 173, pollIntervalMilliseconds: 211}\n", + ); + + let result = check(scratch.path(), &state, true); + + assert_eq!(result.status.code(), Some(1)); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["code"], "rstack.dev.invalid-plan"); + assert_eq!( + diagnostic["path"], + "/journey/steps/0/poll/pollIntervalMilliseconds" + ); + assert_eq!( + diagnostic["message"], + "The poll interval must not exceed the maximum wait." + ); + assert_eq!( + diagnostic["suggestedAction"], + "Set pollIntervalMilliseconds to no more than maximumWaitMilliseconds." + ); + let output = String::from_utf8(result.stdout).unwrap(); + for supplied in [ + "173", + "211", + "private-audience-marker", + "private-client-marker", + ] { + assert!(!output.contains(supplied), "leaked {supplied}: {output}"); + } + assert!(!state.exists()); +} + +#[test] +fn semantic_diagnostic_escapes_authored_mapping_keys_only_in_its_pointer() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + project( + scratch.path(), + "schemaTestPeople:\n \"private/journey~marker\":\n \"private/step~marker\": private-person-marker\n", + ); + + let result = check(scratch.path(), &state, true); + + assert_eq!(result.status.code(), Some(1)); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + let diagnostic = &report["diagnostics"][0]; + assert_eq!( + diagnostic["path"], + "/schemaTestPeople/private~1journey~0marker/private~1step~0marker" + ); + assert_eq!( + diagnostic["message"], + "The schema test assignment must name a person with a BReg grant." + ); + assert_eq!( + diagnostic["suggestedAction"], + "Set this assignment to a test person that has a BReg grant." + ); + for member in ["message", "suggestedAction"] { + let text = diagnostic[member].as_str().unwrap(); + for supplied in [ + "private/journey~marker", + "private/step~marker", + "private-person-marker", + ] { + assert!(!text.contains(supplied), "leaked {supplied}: {text}"); + } + } + assert!(!state.exists()); +} + +#[test] +fn start_reports_an_actionable_value_free_unstarted_grant() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + semantic_project(scratch.path(), "casework", ""); + let request = serde_json::json!({ + "action": "start", "project": scratch.path(), "json": true, + }); + + let result = Command::new(env!("CARGO_BIN_EXE_bregctl")) + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request.to_string()) + .env("RSTACK_DEV_STATE_DIR", &state) + .env_remove("RSTACK_DEV_DATABASE_URL") + .output() + .unwrap(); + + assert_eq!(result.status.code(), Some(1)); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + let error = report["error"].as_str().unwrap(); + assert!( + error.contains("/people/operator/grants/casework"), + "{error}" + ); + assert!( + error.contains("The grant names a product that is not started."), + "{error}" + ); + assert!( + error.contains("Add the grant's product to products, or remove this grant."), + "{error}" + ); + for supplied in [ + "private-audience-marker", + "private-subject-marker", + "private-client-marker", + "private-profile-marker", + ] { + assert!(!error.contains(supplied), "leaked {supplied}: {error}"); + } + assert!(!state.exists()); +} + #[test] fn a_malformed_plan_names_its_field_before_allocating_run_state() { let scratch = tempfile::tempdir().unwrap(); diff --git a/products/platform/scripts/test-rstack-dev.py b/products/platform/scripts/test-rstack-dev.py index 53dd761330..c93dfa9582 100644 --- a/products/platform/scripts/test-rstack-dev.py +++ b/products/platform/scripts/test-rstack-dev.py @@ -161,6 +161,18 @@ def inventory(pg, env): return set(databases.splitlines()), set(roles.splitlines()) +def administrator_locks(pg, env): + locks = psql( + pg, + env, + """SELECT classid::text || ':' || objid::text || ':' || objsubid::text + FROM pg_locks + WHERE locktype = 'advisory' AND granted + AND database = (SELECT oid FROM pg_database WHERE datname = current_database());""", + ) + return set(locks.splitlines()) + + def files(root): return { path.relative_to(root): hashlib.sha256(path.read_bytes()).digest() @@ -314,6 +326,7 @@ def exercise(root, bin_dir, pg, project, ca, url, pg_env, spatial): } try: exercise_runs(root, bin_dir, pg, checkout, state, env, pg_env) + verify_provisioning_deadline(bin_dir, pg, checkout, state, env, pg_env) if spatial: verify_spatial(root, bin_dir, pg, env, pg_env) assert files(checkout) == before @@ -325,6 +338,7 @@ def exercise(root, bin_dir, pg, project, ca, url, pg_env, spatial): def exercise_runs(root, bin_dir, pg, checkout, state, env, pg_env): baseline = inventory(pg, pg_env) + baseline_locks = administrator_locks(pg, pg_env) with concurrent.futures.ThreadPoolExecutor(2) as pool: runs = list( pool.map(lambda _: dev(bin_dir, env, checkout, "start", "."), range(2)) @@ -332,6 +346,7 @@ def exercise_runs(root, bin_dir, pg, checkout, state, env, pg_env): a, b = runs assert all(run["journey"]["result"] == "passed" for run in runs) assert a["run_id"] != b["run_id"] + assert len(administrator_locks(pg, pg_env) - baseline_locks) == 2 assert len({url for run in runs for url in run["products"].values()}) == 4 databases, roles = inventory(pg, pg_env) assert all(sum(run["run_id"] in item for item in databases) == 3 for run in runs) @@ -404,12 +419,67 @@ def exercise_runs(root, bin_dir, pg, checkout, state, env, pg_env): assert list(state.iterdir()) == [], report verify_native_handoff(root, bin_dir, env) assert inventory(pg, pg_env) == baseline + assert administrator_locks(pg, pg_env) == baseline_locks assert list(state.iterdir()) == [] print( "Stop, TTL expiry and failed startup cleaned up; authored product files stayed byte-identical" ) +def verify_provisioning_deadline(bin_dir, pg, checkout, state, env, pg_env): + baseline = inventory(pg, pg_env) + baseline_locks = administrator_locks(pg, pg_env) + lock = subprocess.Popen( + [pg / "psql", "-XAtq", "-v", "ON_ERROR_STOP=1"], + env={**pg_env, "PGAPPNAME": "rstack-dev-provisioning-lock"}, + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + text=True, + ) + lock_query = """ + SELECT EXISTS ( + SELECT 1 FROM pg_locks locks JOIN pg_stat_activity sessions USING (pid) + WHERE sessions.application_name = 'rstack-dev-provisioning-lock' + AND locks.relation = 'pg_catalog.pg_database'::regclass + AND locks.mode = 'ShareLock' AND locks.granted + ); + """ + try: + # A SHARE lock allows new connections and reads but blocks CREATE DATABASE. + # The idle deadline releases the lock even if the runner stalls. + lock.stdin.write( + "SET lock_timeout = '5s';\n" + "SET idle_in_transaction_session_timeout = '45s';\n" + "BEGIN;\n" + "LOCK TABLE pg_catalog.pg_database IN SHARE MODE;\n" + ) + lock.stdin.flush() + deadline = time.monotonic() + 10 + while psql(pg, pg_env, lock_query).strip() != "t": + assert lock.poll() is None, "the PostgreSQL lock fixture exited" + assert time.monotonic() < deadline, "the PostgreSQL lock was not acquired" + time.sleep(0.1) + + report = dev(bin_dir, env, checkout, "start", ".", expected=1) + error = report["error"] + assert "cannot create an isolated PostgreSQL database" in error, report + assert "statement deadline" in error, report + assert psql(pg, pg_env, lock_query).strip() == "t" + assert inventory(pg, pg_env) == baseline + assert administrator_locks(pg, pg_env) == baseline_locks + assert list(state.iterdir()) == [] + finally: + lock.stdin.close() + try: + lock.wait(timeout=5) + except subprocess.TimeoutExpired: + lock.kill() + lock.wait(timeout=5) + raise RuntimeError("the PostgreSQL lock fixture did not exit") from None + print("A blocked PostgreSQL operation timed out and released earlier resources") + + def verify_native_handoff(root, bin_dir, env): checkout = root / "native-review" checkout.mkdir() From 2fa5d2abc0b1994bcb690a2556081913399bbf12 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 02:17:55 +0700 Subject: [PATCH 07/34] fix: resolve development plan references before allocation Signed-off-by: Jeremi Joslin --- .../registry-bregctl/src/project_dev/mod.rs | 2 +- .../registry-bregctl/src/project_dev/plan.rs | 33 +- .../src/project_dev/products.rs | 380 +++++++++++++++-- crates/registry-bregctl/tests/project_dev.rs | 387 ++++++++++++++++++ products/platform/PROJECT-DEV.md | 16 +- 5 files changed, 761 insertions(+), 57 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/mod.rs b/crates/registry-bregctl/src/project_dev/mod.rs index 3baf33ae5b..5473a23a12 100644 --- a/crates/registry-bregctl/src/project_dev/mod.rs +++ b/crates/registry-bregctl/src/project_dev/mod.rs @@ -338,7 +338,7 @@ fn start(project: &Path, ttl: Option<&str>) -> Result { .canonicalize() .context("cannot open the development project directory")?; let plan = plan::load(&project)?; - let schema_people = products::schema_people(&project, &plan)?; + let schema_people = products::schema_people(&project, &plan, true)?; let breg_authority = products::breg_authority(&project, &plan)?; let signals = signals()?; let casework_principals = diff --git a/crates/registry-bregctl/src/project_dev/plan.rs b/crates/registry-bregctl/src/project_dev/plan.rs index 9d8ad3ae04..3a79cf7d39 100644 --- a/crates/registry-bregctl/src/project_dev/plan.rs +++ b/crates/registry-bregctl/src/project_dev/plan.rs @@ -467,7 +467,7 @@ pub(super) fn reserved_claim(claim: &str) -> bool { } #[derive(Debug, thiserror::Error)] -#[error("dev.yaml {field} {reason}")] +#[error("dev.yaml {field}: {reason} Next: {remedy}")] pub(super) struct SemanticRefusal { field: String, reason: String, @@ -492,7 +492,7 @@ pub(super) fn require( Ok(()) } -fn refuse(field: &str, reason: &str, remedy: &str) -> SemanticResult { +pub(super) fn refuse(field: &str, reason: &str, remedy: &str) -> SemanticResult { Err(SemanticRefusal { field: field.to_owned(), reason: reason.to_owned(), @@ -543,23 +543,30 @@ fn read_plan(root: &Path) -> Result> { .decode::(&bytes, &Expect::one(&FORMAT))?) } +fn refusal_report(decoded: &Decoded, refusal: &SemanticRefusal) -> Report { + let diagnostic = decoded.document.diagnostic_at_value( + Severity::Error, + "rstack.dev.invalid-plan", + &refusal.field, + &refusal.reason, + &refusal.remedy, + ); + decoded.document.report_with([diagnostic]) +} + fn validate_plan(root: &Path, decoded: &Decoded) -> Result<()> { - if let Err(refusal) = decoded.value.validate(root) { - let diagnostic = decoded.document.diagnostic_at_value( - Severity::Error, - "rstack.dev.invalid-plan", - &refusal.field, - &refusal.reason, - &refusal.remedy, - ); - return Err(decoded.document.report_with([diagnostic]).into()); - } - Ok(()) + decoded + .value + .validate(root) + .map_err(|refusal| refusal_report(decoded, &refusal).into()) } pub(crate) fn check(root: &Path) -> Result { let decoded = read_plan(root)?; validate_plan(root, &decoded)?; + if let Err(refusal) = super::products::schema_people(root, &decoded.value, false) { + return Err(refusal_report(&decoded, &refusal).into()); + } Ok(decoded.document.report_with(Vec::new())) } diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs index afd9bf9534..3f234439d0 100644 --- a/crates/registry-bregctl/src/project_dev/products.rs +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -23,7 +23,7 @@ use std::{ }; use super::{ - bindings::RuntimeBindings, + bindings::{Bindings, RuntimeBindings}, clock, database::{Database, DatabaseUrls}, issuer::Issuer, @@ -586,6 +586,36 @@ pub(super) fn breg_authority(root: &Path, plan: &Plan) -> Result = registry + .access_profile_displays() + .iter() + .map(|profile| profile.id.as_str()) + .collect(); + for (name, person) in &plan.people { + if let Some(grant) = person.grants.get(&Product::Breg) { + plan::require( + profiles.contains(grant.profile.as_str()), + &format!( + "/people/{}/grants/breg/profile", + registry_platform_yaml::escape_pointer_segment(name) + ), + "The BReg grant profile does not name an access profile in the BReg project.", + "Set profile to an access profile defined by the BReg project.", + )?; + } + } let inventory = registry_breg::authority::authority_inventory(®istry)?; ensure!( inventory.principal_claims.len() <= 1, @@ -609,6 +639,30 @@ pub(super) fn breg_authority(root: &Path, plan: &Plan) -> Result, + actions: &BTreeSet<&str>, +) -> plan::SemanticResult<()> { + if let Some(binding) = &bindings.breg_review_authority { + plan::require( + review_authorities.contains(binding.id.as_str()), + "/bindings/bregReviewAuthority/id", + "The BReg review authority ID does not name a review authority in the BReg project.", + "Set id to a review authority defined by the BReg project.", + )?; + } + if let Some(binding) = &bindings.casework_context_provider { + plan::require( + actions.contains(binding.action.as_str()), + "/bindings/caseworkContextProvider/action", + "The Casework context provider action does not name an action in the BReg project.", + "Set action to an action defined by the BReg project.", + )?; + } + Ok(()) +} + fn breg_requires_postgis(registry: ®istry_breg::CompiledRegistry) -> bool { registry.ddl().requires_postgis } @@ -627,13 +681,14 @@ pub(super) fn casework_principals( return Ok(BTreeMap::new()); }; let tool = super::binaries(Product::Casework)?.tool; - let profiles = explain_casework_profiles(&tool, &root.join(&entry.project), terminate)?; + let facts = explain_casework_facts(&tool, &root.join(&entry.project), terminate)?; + validate_casework_binding_ids(&plan.bindings, &facts)?; let mut principals = BTreeMap::new(); for (name, person) in &plan.people { let Some(grant) = person.grants.get(&Product::Casework) else { continue; }; - let claim = profiles.get(&grant.profile).map(String::as_str) + let claim = facts.profiles.get(&grant.profile).map(String::as_str) .with_context(|| format!("dev.yaml /people/{name}/grants/casework/profile must name a Casework access profile"))?; ensure!( matches!(claim, "sub" | "registry_principal") @@ -656,11 +711,11 @@ pub(super) fn casework_principals( Ok(principals) } -fn explain_casework_profiles( +fn explain_casework_facts( tool: &Path, project: &Path, terminate: &AtomicBool, -) -> Result> { +) -> Result { const MAX_REPORT: u64 = 4 * 1024 * 1024; let mut child = child_command(tool) @@ -697,10 +752,17 @@ fn explain_casework_profiles( ); let report: Value = serde_json::from_slice(&bytes) .map_err(|_| anyhow::anyhow!("caseworkctl explain did not emit a JSON report"))?; - casework_report_profiles(&report) + casework_report_facts(&report) +} + +#[derive(Debug, Eq, PartialEq)] +struct CaseworkFacts { + profiles: BTreeMap, + trusted_producers: BTreeSet, + context_providers: BTreeSet, } -fn casework_report_profiles(report: &Value) -> Result> { +fn casework_report_facts(report: &Value) -> Result { ensure!( report["apiVersion"] == "id.registrystack.org/formats/casework/report/v1alpha2" && report["kind"] == "CaseworkReport" @@ -728,7 +790,51 @@ fn casework_report_profiles(report: &Value) -> Result> "caseworkctl explain returned duplicate access profiles" ); } - Ok(principals) + Ok(CaseworkFacts { + profiles: principals, + trusted_producers: casework_report_ids(report, "trustedProducers")?, + context_providers: casework_report_ids(report, "contextProviders")?, + }) +} + +fn casework_report_ids(report: &Value, member: &str) -> Result> { + let entries = report[member] + .as_array() + .with_context(|| format!("caseworkctl explain did not report {member}"))?; + let mut ids = BTreeSet::new(); + for entry in entries { + let id = entry["id"] + .as_str() + .with_context(|| format!("a caseworkctl explain {member} entry has no ID"))?; + ensure!( + ids.insert(id.to_owned()), + "caseworkctl explain returned duplicate {member} IDs" + ); + } + Ok(ids) +} + +fn validate_casework_binding_ids( + bindings: &Bindings, + facts: &CaseworkFacts, +) -> plan::SemanticResult<()> { + if let Some(binding) = &bindings.breg_review_authority { + plan::require( + facts.trusted_producers.contains(&binding.producer), + "/bindings/bregReviewAuthority/producer", + "The BReg review producer does not name a trusted producer in the Casework project.", + "Set producer to a trusted producer defined by the Casework project.", + )?; + } + if let Some(binding) = &bindings.casework_context_provider { + plan::require( + facts.context_providers.contains(&binding.id), + "/bindings/caseworkContextProvider/id", + "The Casework context provider ID does not name a context provider in the Casework project.", + "Set id to a context provider defined by the Casework project.", + )?; + } + Ok(()) } impl Drop for ProductRunner<'_> { @@ -853,23 +959,130 @@ pub(super) struct SchemaPerson { person: String, } -pub(super) fn schema_people(root: &Path, plan: &Plan) -> Result> { - let Some(entry) = plan - .products +fn breg_entry(plan: &Plan) -> Option<(usize, &plan::ProductPlan)> { + plan.products .iter() - .find(|entry| entry.product == Product::Breg) - else { - ensure!( - plan.schema_test_people.is_empty(), - "dev.yaml /schemaTestPeople requires a BReg product" - ); + .enumerate() + .find(|(_, entry)| entry.product == Product::Breg) +} + +fn schema_assignment_pointer(journey: &str, step: &str) -> String { + format!( + "/schemaTestPeople/{}/{}", + registry_platform_yaml::escape_pointer_segment(journey), + registry_platform_yaml::escape_pointer_segment(step) + ) +} + +fn fixture_file_refusal(plan: &Plan, field: &str) -> plan::SemanticResult { + if plan.schema_test_people.is_empty() { + plan::refuse( + field, + "The BReg fixture journeys file must be a readable regular file inside the BReg product project.", + "Place tests/journeys.yaml inside the BReg product project without linking it outside that project.", + ) + } else { + plan::refuse( + "/schemaTestPeople", + "The BReg fixture journeys file is missing, unreadable, or outside the BReg product project.", + "Create a readable tests/journeys.yaml inside the BReg product project, or remove schemaTestPeople.", + ) + } +} + +pub(super) fn schema_people( + root: &Path, + plan: &Plan, + required: bool, +) -> plan::SemanticResult> { + let Some((product_index, entry)) = breg_entry(plan) else { + if !plan.schema_test_people.is_empty() { + return plan::refuse( + "/schemaTestPeople", + "Schema test assignments require a BReg product.", + "Add BReg to products, or remove schemaTestPeople.", + ); + } return Ok(Vec::new()); }; - let path = root.join(&entry.project).join("tests/journeys.yaml"); - let bytes = plan::read(&path, 1024 * 1024).context("cannot read BReg tests/journeys.yaml")?; - let steps = - journey_step_profiles(&bytes).context("cannot validate BReg tests/journeys.yaml")?; - let mut used = BTreeSet::new(); + let field = format!("/products/{product_index}/project"); + let product_root = match root.join(&entry.project).canonicalize() { + Ok(path) if path.starts_with(root) && path.is_dir() => path, + Ok(_) | Err(_) => return fixture_file_refusal(plan, &field), + }; + let path = product_root.join("tests/journeys.yaml"); + match fs::symlink_metadata(&path) { + Ok(_) => {} + Err(error) + if error.kind() == std::io::ErrorKind::NotFound + && !required + && plan.schema_test_people.is_empty() => + { + return Ok(Vec::new()); + } + Err(_) => return fixture_file_refusal(plan, &field), + } + let canonical = match path.canonicalize() { + Ok(path) if path.starts_with(&product_root) && path.is_file() => path, + Ok(_) | Err(_) => return fixture_file_refusal(plan, &field), + }; + let bytes = match plan::read( + &canonical, + registry_platform_yaml::MAXIMUM_DOCUMENT_BYTES as u64, + ) { + Ok(bytes) => bytes, + Err(_) => { + return plan::refuse( + if plan.schema_test_people.is_empty() { + &field + } else { + "/schemaTestPeople" + }, + "The BReg fixture journeys file is unreadable or exceeds the 1 MiB limit.", + "Make tests/journeys.yaml readable and no larger than 1 MiB, then run the check again.", + ); + } + }; + let steps = match journey_step_profiles(&bytes) { + Ok(steps) => steps, + Err(_) => { + return plan::refuse( + if plan.schema_test_people.is_empty() { + &field + } else { + "/schemaTestPeople" + }, + "The BReg fixture journeys file is not valid.", + "Run `rstack breg check` for authored diagnostics, then correct tests/journeys.yaml.", + ); + } + }; + let mut fixture_steps = BTreeMap::<&str, BTreeSet<&str>>::new(); + for step in &steps { + fixture_steps + .entry(&step.journey_id) + .or_default() + .insert(&step.step_id); + } + for (journey, assignments) in &plan.schema_test_people { + for step in assignments.keys() { + let field = schema_assignment_pointer(journey, step); + if !fixture_steps.contains_key(journey.as_str()) { + return plan::refuse( + &field, + "The schema test assignment names a BReg fixture journey that does not exist.", + "Set the journey key to a journey defined in tests/journeys.yaml, or remove this assignment.", + ); + } + if !fixture_steps[journey.as_str()].contains(step.as_str()) { + return plan::refuse( + &field, + "The schema test assignment names a BReg fixture step that does not exist in the named journey.", + "Set the step key to a step defined in that fixture journey, or remove this assignment.", + ); + } + } + } let mut bindings = Vec::new(); for step in steps { let eligible = |person: &plan::TestPerson| { @@ -897,29 +1110,26 @@ pub(super) fn schema_people(root: &Path, plan: &Plan) -> Result Bindings { + Bindings { + breg_review_authority: Some(BregReviewAuthority { + id: "review-authority".to_owned(), + producer: "review-producer".to_owned(), + identity: "reviewer".to_owned(), + }), + casework_context_provider: Some(CaseworkContextProvider { + id: "proposal-context".to_owned(), + identity: "context-reader".to_owned(), + action: "review-action".to_owned(), + }), + } + } + + #[test] + fn breg_binding_ids_are_resolved_with_typed_value_free_refusals() { + let bindings = product_bindings(); + let authorities = BTreeSet::from(["review-authority"]); + let actions = BTreeSet::from(["review-action"]); + validate_breg_binding_ids(&bindings, &authorities, &actions).unwrap(); + + for (authorities, actions, field, reason, action) in [ + ( + BTreeSet::new(), + actions.clone(), + "/bindings/bregReviewAuthority/id", + "The BReg review authority ID does not name a review authority in the BReg project.", + "Set id to a review authority defined by the BReg project.", + ), + ( + authorities.clone(), + BTreeSet::new(), + "/bindings/caseworkContextProvider/action", + "The Casework context provider action does not name an action in the BReg project.", + "Set action to an action defined by the BReg project.", + ), + ] { + let error = validate_breg_binding_ids(&bindings, &authorities, &actions) + .unwrap_err() + .to_string(); + assert!(error.contains(field), "{error}"); + assert!(error.contains(reason), "{error}"); + assert!(error.contains(action), "{error}"); + assert!(!error.contains("review-authority"), "{error}"); + assert!(!error.contains("review-action"), "{error}"); + } + } + + #[test] + fn casework_binding_ids_use_the_existing_explain_facts() { + let bindings = product_bindings(); + let mut report = json!({ + "apiVersion": "id.registrystack.org/formats/casework/report/v1alpha2", + "kind": "CaseworkReport", "command": "explain", "status": "complete", "ok": true, + "reviewPolicies": [{"dependencies": {"accessProfiles": [ + {"id": "reviewer", "principalClaim": "sub"} + ]}}], + "trustedProducers": [{"id": "review-producer"}], + "contextProviders": [{"id": "proposal-context"}] + }); + let facts = casework_report_facts(&report).unwrap(); + validate_casework_binding_ids(&bindings, &facts).unwrap(); + + for (member, field, reason, action) in [ + ( + "trustedProducers", + "/bindings/bregReviewAuthority/producer", + "The BReg review producer does not name a trusted producer in the Casework project.", + "Set producer to a trusted producer defined by the Casework project.", + ), + ( + "contextProviders", + "/bindings/caseworkContextProvider/id", + "The Casework context provider ID does not name a context provider in the Casework project.", + "Set id to a context provider defined by the Casework project.", + ), + ] { + report[member] = json!([]); + let facts = casework_report_facts(&report).unwrap(); + let error = validate_casework_binding_ids(&bindings, &facts) + .unwrap_err() + .to_string(); + assert!(error.contains(field), "{error}"); + assert!(error.contains(reason), "{error}"); + assert!(error.contains(action), "{error}"); + assert!(!error.contains("review-producer"), "{error}"); + assert!(!error.contains("proposal-context"), "{error}"); + report[member] = match member { + "trustedProducers" => json!([{"id": "review-producer"}]), + "contextProviders" => json!([{"id": "proposal-context"}]), + _ => unreachable!(), + }; + } + } + #[test] fn cancellation_closes_the_pipe_of_a_nested_tool() { closes_nested_tool_pipe(false); diff --git a/crates/registry-bregctl/tests/project_dev.rs b/crates/registry-bregctl/tests/project_dev.rs index 626a9c2a64..5cab2e0f4c 100644 --- a/crates/registry-bregctl/tests/project_dev.rs +++ b/crates/registry-bregctl/tests/project_dev.rs @@ -37,6 +37,28 @@ journey:\n id: smoke\n steps:\n - id: read\n product: breg\n as: .unwrap(); } +fn fixture_project(root: &std::path::Path, extra: &str) { + project(root, extra); + fs::create_dir(root.join("tests")).unwrap(); + fs::write( + root.join("tests/journeys.yaml"), + concat!( + "apiVersion: id.registrystack.org/formats/breg/journeys/v1\n", + "kind: BRegJourneys\n", + "journeys:\n", + " - id: real-journey\n", + " steps:\n", + " - id: real-step\n", + " recordType: record\n", + " accessProfile: operator\n", + " claims: {principal: fixture-operator}\n", + " request: {type: list}\n", + " expect: {outcome: success, status: 200, count: 0}\n", + ), + ) + .unwrap(); +} + fn check(root: &std::path::Path, state: &std::path::Path, json: bool) -> std::process::Output { let request = serde_json::json!({ "action": "check", "project": root, "deny_warnings": true, "json": json, @@ -50,6 +72,54 @@ fn check(root: &std::path::Path, state: &std::path::Path, json: bool) -> std::pr .unwrap() } +fn start(root: &std::path::Path, state: &std::path::Path) -> std::process::Output { + let request = serde_json::json!({ + "action": "start", "project": root, "json": true, + }); + Command::new(env!("CARGO_BIN_EXE_bregctl")) + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request.to_string()) + .env("RSTACK_DEV_STATE_DIR", state) + .env_remove("RSTACK_DEV_DATABASE_URL") + .output() + .unwrap() +} + +fn assert_dangling_schema_assignment( + extra: &str, + path: &str, + message: &str, + suggested_action: &str, +) { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + fixture_project(scratch.path(), extra); + + let checked = check(scratch.path(), &state, true); + + assert_eq!(checked.status.code(), Some(1), "{checked:?}"); + let report: serde_json::Value = serde_json::from_slice(&checked.stdout).unwrap(); + assert_eq!(report["kind"], "BRegCtlReport"); + assert_eq!(report["command"], "dev-check"); + assert_eq!(report["status"], "domain-refusal"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["code"], "rstack.dev.invalid-plan"); + assert_eq!(diagnostic["path"], path); + assert_eq!(diagnostic["message"], message); + assert_eq!(diagnostic["suggestedAction"], suggested_action); + assert!(!state.exists()); + + let started = start(scratch.path(), &state); + + assert_eq!(started.status.code(), Some(1), "{started:?}"); + let report: serde_json::Value = serde_json::from_slice(&started.stdout).unwrap(); + let error = report["error"].as_str().unwrap(); + assert!(error.contains(path), "{error}"); + assert!(error.contains(message), "{error}"); + assert!(error.contains(suggested_action), "{error}"); + assert!(!state.exists()); +} + #[test] fn check_validates_the_closed_plan_without_allocating_run_state() { let scratch = tempfile::tempdir().unwrap(); @@ -69,6 +139,274 @@ fn check_validates_the_closed_plan_without_allocating_run_state() { assert!(!state.exists()); } +#[test] +fn check_and_start_reject_an_unknown_fixture_journey_without_allocating_state() { + assert_dangling_schema_assignment( + "schemaTestPeople:\n \"private/journey~marker\":\n real-step: operator\n", + "/schemaTestPeople/private~1journey~0marker/real-step", + "The schema test assignment names a BReg fixture journey that does not exist.", + "Set the journey key to a journey defined in tests/journeys.yaml, or remove this assignment.", + ); +} + +#[test] +fn check_and_start_reject_an_unknown_fixture_step_without_allocating_state() { + assert_dangling_schema_assignment( + "schemaTestPeople:\n real-journey:\n \"private/step~marker\": operator\n", + "/schemaTestPeople/real-journey/private~1step~0marker", + "The schema test assignment names a BReg fixture step that does not exist in the named journey.", + "Set the step key to a step defined in that fixture journey, or remove this assignment.", + ); +} + +#[test] +fn check_accepts_a_valid_explicit_fixture_assignment_without_allocating_state() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + fixture_project( + scratch.path(), + "schemaTestPeople:\n real-journey:\n real-step: operator\n", + ); + + let result = check(scratch.path(), &state, true); + + assert!(result.status.success(), "{result:?}"); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + assert_eq!(report["status"], "complete"); + assert_eq!(report["diagnostics"], serde_json::json!([])); + assert!(!state.exists()); +} + +#[test] +fn implicit_fixture_person_refusals_keep_the_intended_assignment_pointer() { + for ambiguous in [false, true] { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + fixture_project(scratch.path(), ""); + let plan = fs::read_to_string(scratch.path().join("dev.yaml")).unwrap(); + let plan = if ambiguous { + plan.replace( + "journey:\n", + " second:\n subject: second\n client: second-client\n grants:\n breg: {profile: operator, scopes: []}\njourney:\n", + ) + } else { + plan.replace("profile: operator", "profile: other") + }; + fs::write(scratch.path().join("dev.yaml"), plan).unwrap(); + + let result = check(scratch.path(), &state, true); + + assert_eq!(result.status.code(), Some(1), "{result:?}"); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + let diagnostic = &report["diagnostics"][0]; + assert_eq!( + diagnostic["path"], + "/schemaTestPeople/real-journey/real-step" + ); + assert_eq!( + diagnostic["message"], + "A BReg fixture step must resolve to exactly one test person with matching profile, scopes, and purpose." + ); + assert_eq!( + diagnostic["suggestedAction"], + "Add or update a matching BReg grant, or select the matching person in schemaTestPeople." + ); + assert!(!state.exists()); + } +} + +#[test] +fn check_and_start_report_an_explicit_assignment_with_no_fixture_file() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + project( + scratch.path(), + "schemaTestPeople:\n real-journey:\n real-step: operator\n", + ); + + let checked = check(scratch.path(), &state, true); + + assert_eq!(checked.status.code(), Some(1), "{checked:?}"); + let report: serde_json::Value = serde_json::from_slice(&checked.stdout).unwrap(); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/schemaTestPeople"); + assert_eq!( + diagnostic["message"], + "The BReg fixture journeys file is missing, unreadable, or outside the BReg product project." + ); + assert_eq!( + diagnostic["suggestedAction"], + "Create a readable tests/journeys.yaml inside the BReg product project, or remove schemaTestPeople." + ); + assert!(!state.exists()); + + let started = start(scratch.path(), &state); + + assert_eq!(started.status.code(), Some(1), "{started:?}"); + let report: serde_json::Value = serde_json::from_slice(&started.stdout).unwrap(); + let error = report["error"].as_str().unwrap(); + assert!(error.contains("/schemaTestPeople"), "{error}"); + assert!( + error.contains("fixture journeys file is missing"), + "{error}" + ); + assert!( + error.contains("Create a readable tests/journeys.yaml"), + "{error}" + ); + assert!(!state.exists()); +} + +#[test] +fn check_and_start_refuse_a_fixture_symlink_outside_the_breg_project() { + let scratch = tempfile::tempdir().unwrap(); + let sibling = tempfile::tempdir_in(scratch.path()).unwrap(); + let state = scratch.path().join("state"); + fixture_project(sibling.path(), ""); + project(scratch.path(), ""); + fs::create_dir(scratch.path().join("breg")).unwrap(); + fs::write(scratch.path().join("breg/registry.yaml"), "fixture\n").unwrap(); + let plan = fs::read_to_string(scratch.path().join("dev.yaml")) + .unwrap() + .replace("product: breg, project: .", "product: breg, project: breg"); + fs::write(scratch.path().join("dev.yaml"), plan).unwrap(); + fs::create_dir(scratch.path().join("breg/tests")).unwrap(); + std::os::unix::fs::symlink( + sibling.path().join("tests/journeys.yaml"), + scratch.path().join("breg/tests/journeys.yaml"), + ) + .unwrap(); + + let checked = check(scratch.path(), &state, true); + + assert_eq!(checked.status.code(), Some(1), "{checked:?}"); + let report: serde_json::Value = serde_json::from_slice(&checked.stdout).unwrap(); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/products/0/project"); + assert_eq!( + diagnostic["message"], + "The BReg fixture journeys file must be a readable regular file inside the BReg product project." + ); + assert_eq!( + diagnostic["suggestedAction"], + "Place tests/journeys.yaml inside the BReg product project without linking it outside that project." + ); + assert!(!state.exists()); + + let started = start(scratch.path(), &state); + + assert_eq!(started.status.code(), Some(1), "{started:?}"); + let report: serde_json::Value = serde_json::from_slice(&started.stdout).unwrap(); + let error = report["error"].as_str().unwrap(); + assert!(error.contains("/products/0/project"), "{error}"); + assert!(error.contains("fixture journeys file must be"), "{error}"); + assert!(error.contains("without linking it outside"), "{error}"); + assert!(!state.exists()); +} + +#[test] +fn check_reports_an_invalid_fixture_as_an_actionable_plan_refusal() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + project( + scratch.path(), + "schemaTestPeople:\n real-journey:\n real-step: operator\n", + ); + fs::create_dir(scratch.path().join("tests")).unwrap(); + fs::write( + scratch.path().join("tests/journeys.yaml"), + "not a fixture\n", + ) + .unwrap(); + + let result = check(scratch.path(), &state, true); + + assert_eq!(result.status.code(), Some(1), "{result:?}"); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/schemaTestPeople"); + assert_eq!( + diagnostic["message"], + "The BReg fixture journeys file is not valid." + ); + assert_eq!( + diagnostic["suggestedAction"], + "Run `rstack breg check` for authored diagnostics, then correct tests/journeys.yaml." + ); + assert!(!state.exists()); +} + +#[test] +fn check_reports_an_oversized_fixture_as_an_actionable_plan_refusal() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + project( + scratch.path(), + "schemaTestPeople:\n real-journey:\n real-step: operator\n", + ); + fs::create_dir(scratch.path().join("tests")).unwrap(); + fs::write( + scratch.path().join("tests/journeys.yaml"), + vec![b' '; 1024 * 1024 + 1], + ) + .unwrap(); + + let result = check(scratch.path(), &state, true); + + assert_eq!(result.status.code(), Some(1), "{result:?}"); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/schemaTestPeople"); + assert_eq!( + diagnostic["message"], + "The BReg fixture journeys file is unreadable or exceeds the 1 MiB limit." + ); + assert_eq!( + diagnostic["suggestedAction"], + "Make tests/journeys.yaml readable and no larger than 1 MiB, then run the check again." + ); + assert!(!state.exists()); +} + +#[test] +fn check_refuses_a_breg_project_directory_that_resolves_outside_the_project() { + let scratch = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + project(scratch.path(), ""); + fs::create_dir(outside.path().join("tests")).unwrap(); + fs::write( + outside.path().join("tests/journeys.yaml"), + include_str!("../../../products/breg/examples/minimal/tests/journeys.yaml"), + ) + .unwrap(); + std::os::unix::fs::symlink( + scratch.path().join("registry.yaml"), + outside.path().join("registry.yaml"), + ) + .unwrap(); + std::os::unix::fs::symlink(outside.path(), scratch.path().join("linked-breg")).unwrap(); + let plan = fs::read_to_string(scratch.path().join("dev.yaml")) + .unwrap() + .replace( + "product: breg, project: .", + "product: breg, project: linked-breg", + ); + fs::write(scratch.path().join("dev.yaml"), plan).unwrap(); + + let result = check(scratch.path(), &state, true); + + assert_eq!(result.status.code(), Some(1), "{result:?}"); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/products/0/project"); + assert_eq!( + diagnostic["message"], + "The BReg fixture journeys file must be a readable regular file inside the BReg product project." + ); + assert!(!state.exists()); +} + #[test] fn check_refuses_an_unknown_plan_field_without_allocating_run_state() { let scratch = tempfile::tempdir().unwrap(); @@ -245,6 +583,55 @@ fn start_reports_an_actionable_value_free_unstarted_grant() { assert!(!state.exists()); } +#[test] +fn start_rejects_a_dangling_breg_grant_profile_before_allocating_state() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + fs::write( + scratch.path().join("registry.yaml"), + include_str!("../../../products/breg/examples/minimal/registry.yaml"), + ) + .unwrap(); + fs::create_dir(scratch.path().join("tests")).unwrap(); + fs::write( + scratch.path().join("tests/journeys.yaml"), + include_str!("../../../products/breg/examples/minimal/tests/journeys.yaml"), + ) + .unwrap(); + fs::write( + scratch.path().join("dev.yaml"), + "apiVersion: id.registrystack.org/formats/breg/project-dev/v1alpha1\n\ +kind: BRegProjectDev\n\ +issuer: https://issuer.example.invalid\n\ +products:\n - {product: breg, project: ., audience: registry}\n\ +people:\n editor:\n subject: editor\n client: editor-client\n grants:\n breg:\n profile: directory-editor\n scopes: [registry:directory:write]\n claims: {registry_purpose: directory-administration}\n unused:\n subject: unused\n client: unused-client\n grants:\n breg: {profile: private-profile-marker, scopes: []}\n\ +journey:\n id: smoke\n steps:\n - {id: read, product: breg, as: editor, method: GET, path: /v1/records, expect: {status: 200}}\n", + ) + .unwrap(); + + let result = start(scratch.path(), &state); + + assert_eq!(result.status.code(), Some(1), "{result:?}"); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + let error = report["error"].as_str().unwrap(); + assert!( + error.contains("/people/unused/grants/breg/profile"), + "{error}" + ); + assert!( + error.contains( + "The BReg grant profile does not name an access profile in the BReg project." + ), + "{error}" + ); + assert!( + error.contains("Set profile to an access profile defined by the BReg project."), + "{error}" + ); + assert!(!error.contains("private-profile-marker"), "{error}"); + assert!(!state.exists()); +} + #[test] fn a_malformed_plan_names_its_field_before_allocating_run_state() { let scratch = tempfile::tempdir().unwrap(); diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index 7775bf903b..475f7be24d 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -19,10 +19,11 @@ rstack dev token RUN_ID --as applicant --json rstack dev stop RUN_ID ``` -Run `rstack dev check PROJECT [--deny-warnings] [--json]` to validate the plan -and its project-local file references offline. With `--json`, the check writes a -`BRegCtlReport` with `command` set to `dev-check`. Use `--deny-warnings` to -refuse any unaccepted warning. +Run `rstack dev check PROJECT [--deny-warnings] [--json]` to validate the plan, +the product directories it names, and BReg schema-test assignments offline. +With `--json`, the check writes a `BRegCtlReport` with `command` set to +`dev-check`. Use `--deny-warnings` to refuse any unaccepted warning. +`start` also validates the authored products with their own tooling. You can also pass `--database-url`. Use the environment variable when the URL includes a password. Start reports the product URLs and the journey result. @@ -44,9 +45,10 @@ List the project's issuer, product directories and audiences in startup order, and test people with synthetic subjects and exact product grants. A grant selects the access profile, scopes, and additional claims. Each selected profile gets the test person's subject in its principal claim. The products enforce their -ordinary permissions. If several people match a BReg schema-test step, select -its alias under `schemaTestPeople..`. The run obtains the -normal schema-test receipt before packaging. +ordinary permissions. If several people match a step in the BReg project's +`tests/journeys.yaml`, select its alias under +`schemaTestPeople..`. The run obtains the normal +schema-test receipt before packaging. For a native review handoff, `bindings.bregReviewAuthority` identifies the Casework producer, and `bindings.caseworkContextProvider` identifies the BReg From 6b531c07a3e2832f8f7016b19e86f0fd091657ac Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 03:02:15 +0700 Subject: [PATCH 08/34] fix: validate product references offline and enforce poll deadlines Signed-off-by: Jeremi Joslin --- .../src/project_dev/journey.rs | 253 ++++++++++- .../registry-bregctl/src/project_dev/mod.rs | 69 ++- .../registry-bregctl/src/project_dev/plan.rs | 33 +- .../src/project_dev/products.rs | 398 +++++++++++++++--- crates/registry-bregctl/tests/project_dev.rs | 303 ++++++++++++- products/platform/PROJECT-DEV.md | 10 +- .../breg-project-dev/registry/registry.yaml | 67 ++- products/platform/scripts/test-rstack-dev.py | 54 ++- 8 files changed, 1074 insertions(+), 113 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/journey.rs b/crates/registry-bregctl/src/project_dev/journey.rs index ccc8e2f6ad..742a38d617 100644 --- a/crates/registry-bregctl/src/project_dev/journey.rs +++ b/crates/registry-bregctl/src/project_dev/journey.rs @@ -156,9 +156,13 @@ fn execute_step( .poll .as_ref() .map(|poll| Instant::now() + Duration::from_millis(poll.timeout_ms)); + let mut last_status = 0; loop { active()?; + if poll_deadline.is_some_and(|deadline| Instant::now() >= deadline) { + return Ok((last_status, false, BTreeMap::new())); + } add_attempt(attempts)?; add_transfer(transferred, request_bytes)?; let token = issuer.token(input, person, step.product, clock()? + 300)?; @@ -180,8 +184,12 @@ fn execute_step( .header(reqwest::header::CONTENT_TYPE, &spec.content_type) .body(body.clone()); } - let (response, response_bytes) = runtime.block_on(read_response(request))?; - add_transfer(transferred, response_bytes)?; + let Some(response) = + read_response_before_deadline(runtime, request, poll_deadline, transferred)? + else { + return Ok((last_status, false, BTreeMap::new())); + }; + last_status = response.status; let captures = matches_expectation(step, &response, lookup)? .then(|| capture_updates(step, &response, lookup)) .and_then(Result::ok); @@ -443,7 +451,31 @@ fn add_attempt(attempts: &mut usize) -> Result<()> { Ok(()) } -async fn read_response(request: reqwest::RequestBuilder) -> Result<(ResponseData, usize)> { +fn read_response_before_deadline( + runtime: &tokio::runtime::Runtime, + request: reqwest::RequestBuilder, + deadline: Option, + transferred: &mut usize, +) -> Result> { + let response = read_response(request, transferred); + let Some(deadline) = deadline else { + return runtime.block_on(response).map(Some); + }; + if Instant::now() >= deadline { + return Ok(None); + } + let timeout = tokio::time::Instant::from_std(deadline); + match runtime.block_on(async { tokio::time::timeout_at(timeout, response).await }) { + Ok(response) if Instant::now() < deadline => response.map(Some), + Ok(_) => Ok(None), + Err(_) => Ok(None), + } +} + +async fn read_response( + request: reqwest::RequestBuilder, + transferred: &mut usize, +) -> Result { let mut response = request.send().await?; let status = response.status().as_u16(); let headers = response.headers().clone(); @@ -454,17 +486,14 @@ async fn read_response(request: reqwest::RequestBuilder) -> Result<(ResponseData "journey response is too large" ); bytes.extend_from_slice(&chunk); + add_transfer(transferred, chunk.len())?; } - let length = bytes.len(); let body = serde_json::from_slice(&bytes).unwrap_or(Value::Null); - Ok(( - ResponseData { - status, - headers, - body, - }, - length, - )) + Ok(ResponseData { + status, + headers, + body, + }) } #[cfg(test)] @@ -472,11 +501,211 @@ mod tests { use super::*; use crate::project_dev::plan::ExpectedStatus; use serde_json::json; + use std::{ + io::{Read, Write}, + net::{Ipv4Addr, TcpListener, TcpStream}, + path::PathBuf, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, Mutex, + }, + }; + + static POLL_TEST_LOCK: Mutex<()> = Mutex::new(()); fn no_capture(_: &str) -> Result { anyhow::bail!("unexpected capture") } + struct SyntheticServer { + url: String, + requests: Arc, + thread: thread::JoinHandle<()>, + } + + impl SyntheticServer { + fn finish(self) -> usize { + self.thread.join().unwrap(); + self.requests.load(Ordering::SeqCst) + } + } + + fn synthetic_server(responses: Vec<(Duration, u16, &'static str)>) -> SyntheticServer { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + listener.set_nonblocking(true).unwrap(); + let url = format!("http://{}/", listener.local_addr().unwrap()); + let requests = Arc::new(AtomicUsize::new(0)); + let observed = requests.clone(); + let thread = thread::spawn(move || { + let end = Instant::now() + Duration::from_millis(500); + let mut response = 0; + while Instant::now() < end { + match listener.accept() { + Ok((mut stream, _)) => { + observed.fetch_add(1, Ordering::SeqCst); + stream.set_nonblocking(false).unwrap(); + read_request(&mut stream); + let &(delay, status, body) = responses + .get(response) + .unwrap_or_else(|| responses.last().unwrap()); + response += 1; + let reason = if status == 200 { "OK" } else { "Accepted" }; + let headers = format!( + "HTTP/1.1 {status} {reason}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + if write_to_client(&mut stream, headers.as_bytes()) { + thread::sleep(delay); + write_to_client(&mut stream, body.as_bytes()); + } + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(1)); + } + Err(error) => panic!("synthetic server accept failed: {error}"), + } + } + }); + SyntheticServer { + url, + requests, + thread, + } + } + + fn read_request(stream: &mut TcpStream) { + stream + .set_read_timeout(Some(Duration::from_secs(1))) + .unwrap(); + let mut request = Vec::new(); + let mut chunk = [0u8; 2048]; + while request.len() < 16 * 1024 && !request.windows(4).any(|part| part == b"\r\n\r\n") { + match stream.read(&mut chunk) { + Ok(0) | Err(_) => break, + Ok(length) => request.extend_from_slice(&chunk[..length]), + } + } + assert!(request.windows(4).any(|part| part == b"\r\n\r\n")); + } + + fn write_to_client(stream: &mut TcpStream, bytes: &[u8]) -> bool { + match stream.write_all(bytes) { + Ok(()) => true, + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::NotConnected + ) => + { + false + } + Err(error) => panic!("synthetic server write failed: {error}"), + } + } + + fn polling_input(maximum_wait: u64, interval: u64) -> RunInput { + let plan = serde_json::from_value(json!({ + "issuer": "https://issuer.example.invalid", + "products": [{"product": "breg", "project": ".", "audience": "registry"}], + "people": {"reader": { + "subject": "reader", "client": "reader-client", + "grants": {"breg": {"profile": "reader", "scopes": ["registry:read"]}} + }}, + "journey": {"id": "poll", "steps": [{ + "id": "wait", "product": "breg", "as": "reader", "method": "GET", + "path": "/v1/result", "expect": {"status": 200, "json": {"/state": "ready"}}, + "poll": { + "maximumWaitMilliseconds": maximum_wait, + "pollIntervalMilliseconds": interval + } + }]} + })) + .unwrap(); + RunInput { + run_id: "0123456789abcdef0123456789abcdef".to_owned(), + project: PathBuf::from("/tmp/project"), + plan, + ttl_seconds: 300, + schema_people: Vec::new(), + breg_authority: Some(super::super::BregAuthority { + principal: "sub".to_owned(), + purpose: false, + requires_postgis: false, + named_clients: Default::default(), + }), + casework_principals: BTreeMap::new(), + binaries: BTreeMap::new(), + } + } + + #[test] + fn poll_refuses_a_matching_response_that_finishes_after_its_deadline() { + let _lock = POLL_TEST_LOCK + .lock() + .unwrap_or_else(|error| error.into_inner()); + let server = synthetic_server(vec![ + (Duration::ZERO, 202, r#"{"state":"waiting"}"#), + (Duration::from_millis(500), 200, r#"{"state":"ready"}"#), + ]); + let root = tempfile::tempdir().unwrap(); + let issuer = Issuer::create(root.path()).unwrap(); + let input = polling_input(300, 50); + let urls = BTreeMap::from([(Product::Breg, server.url.clone())]); + + let report = run(&input, &issuer, &urls, || Ok(())).unwrap(); + + assert_eq!(report.result, "failed"); + assert_eq!(report.steps[0].status, 202); + assert_eq!(server.finish(), 2); + } + + #[test] + fn poll_starts_no_retry_after_its_deadline() { + let _lock = POLL_TEST_LOCK + .lock() + .unwrap_or_else(|error| error.into_inner()); + let server = synthetic_server(vec![(Duration::ZERO, 202, r#"{"state":"waiting"}"#)]); + let root = tempfile::tempdir().unwrap(); + let issuer = Issuer::create(root.path()).unwrap(); + let input = polling_input(300, 300); + let urls = BTreeMap::from([(Product::Breg, server.url.clone())]); + + let report = run(&input, &issuer, &urls, || Ok(())).unwrap(); + + assert_eq!(report.result, "failed"); + assert_eq!(report.steps[0].status, 202); + assert_eq!(server.finish(), 1); + } + + #[test] + fn poll_sends_nothing_with_zero_remaining_budget() { + let _lock = POLL_TEST_LOCK + .lock() + .unwrap_or_else(|error| error.into_inner()); + let server = synthetic_server(vec![(Duration::ZERO, 200, r#"{"state":"ready"}"#)]); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let request = reqwest::Client::new().get(format!("{}v1/result", server.url)); + let mut transferred = 0; + + let response = read_response_before_deadline( + &runtime, + request, + Some(Instant::now()), + &mut transferred, + ) + .unwrap(); + + assert!(response.is_none()); + assert_eq!(transferred, 0); + assert_eq!(server.finish(), 0); + } + #[test] fn exact_capture_reference_preserves_scalar_type() { let captures = BTreeMap::from([("revision", json!(7))]); diff --git a/crates/registry-bregctl/src/project_dev/mod.rs b/crates/registry-bregctl/src/project_dev/mod.rs index 5473a23a12..cd609fe02e 100644 --- a/crates/registry-bregctl/src/project_dev/mod.rs +++ b/crates/registry-bregctl/src/project_dev/mod.rs @@ -289,6 +289,21 @@ fn check( Err(error) => match error.downcast::() { Ok(report) => (report, crate::DOMAIN_REFUSAL_EXIT), Err(error) => { + if let Some(unavailable) = error.downcast_ref::() { + let report = Report::new(vec![Diagnostic::error( + "breg.project-dev.check-unavailable", + unavailable.field(), + unavailable.reason(), + unavailable.remedy(), + )]); + return write_check_report( + report, + crate::OPERATIONAL_FAILURE_EXIT, + machine, + stdout, + stderr, + ); + } let (code, action, exit) = if error.is::() { ( "breg.project-dev.check-unavailable", @@ -309,6 +324,16 @@ fn check( } }, }; + write_check_report(report, exit, machine, stdout, stderr) +} + +fn write_check_report( + report: Report, + exit: u8, + machine: bool, + stdout: &mut dyn Write, + stderr: &mut dyn Write, +) -> u8 { let result = if machine { let payload = json!({ "ok": exit == 0, @@ -339,10 +364,8 @@ fn start(project: &Path, ttl: Option<&str>) -> Result { .context("cannot open the development project directory")?; let plan = plan::load(&project)?; let schema_people = products::schema_people(&project, &plan, true)?; - let breg_authority = products::breg_authority(&project, &plan)?; let signals = signals()?; - let casework_principals = - products::casework_principals(&project, &plan, breg_authority.as_ref(), &signals)?; + let inspection = products::inspect(&project, &plan, &signals)?; let ttl_seconds = parse_ttl(ttl.unwrap_or("30m"))?; ensure!( env::var_os(DATABASE_URL).is_some_and(|value| !value.is_empty()), @@ -364,8 +387,8 @@ fn start(project: &Path, ttl: Option<&str>) -> Result { plan, ttl_seconds, schema_people, - breg_authority, - casework_principals, + breg_authority: inspection.breg_authority, + casework_principals: inspection.casework_principals, binaries, }; let preparation = (|| { @@ -701,25 +724,33 @@ fn stopping(root: &Path, signals: &AtomicBool) -> bool { signals.load(Ordering::Relaxed) || root.join("stop").exists() } +fn binary(name: &str) -> Result { + let path = env::var_os(name) + .map(PathBuf::from) + .with_context(|| format!("{name} is required for the development run"))?; + let path = path + .canonicalize() + .with_context(|| format!("cannot resolve {name}"))?; + ensure!(path.is_file(), "{name} must be a binary file"); + Ok(path) +} + +fn product_tool(product: Product) -> Result { + binary(match product { + Product::Breg => "RSTACK_BREGCTL_BIN", + Product::Casework => "RSTACK_CASEWORKCTL_BIN", + }) +} + fn binaries(product: Product) -> Result { - let get = |name: &str| -> Result { - let path = env::var_os(name) - .map(PathBuf::from) - .with_context(|| format!("{name} is required for the development run"))?; - let path = path - .canonicalize() - .with_context(|| format!("cannot resolve {name}"))?; - ensure!(path.is_file(), "{name} must be a binary file"); - Ok(path) - }; Ok(match product { Product::Breg => Binaries { - tool: get("RSTACK_BREGCTL_BIN")?, - runtime: get("RSTACK_BREG_BIN")?, + tool: product_tool(product)?, + runtime: binary("RSTACK_BREG_BIN")?, }, Product::Casework => Binaries { - tool: get("RSTACK_CASEWORKCTL_BIN")?, - runtime: get("RSTACK_CASEWORK_BIN")?, + tool: product_tool(product)?, + runtime: binary("RSTACK_CASEWORK_BIN")?, }, }) } diff --git a/crates/registry-bregctl/src/project_dev/plan.rs b/crates/registry-bregctl/src/project_dev/plan.rs index 3a79cf7d39..1c6a737e15 100644 --- a/crates/registry-bregctl/src/project_dev/plan.rs +++ b/crates/registry-bregctl/src/project_dev/plan.rs @@ -474,6 +474,20 @@ pub(super) struct SemanticRefusal { remedy: String, } +impl SemanticRefusal { + pub(super) fn new( + field: impl Into, + reason: impl Into, + remedy: impl Into, + ) -> Self { + Self { + field: field.into(), + reason: reason.into(), + remedy: remedy.into(), + } + } +} + pub(super) type SemanticResult = std::result::Result; pub(super) fn require( @@ -483,21 +497,13 @@ pub(super) fn require( remedy: impl Into, ) -> SemanticResult<()> { if !condition { - return Err(SemanticRefusal { - field: field.to_owned(), - reason: reason.into(), - remedy: remedy.into(), - }); + return Err(SemanticRefusal::new(field, reason, remedy)); } Ok(()) } pub(super) fn refuse(field: &str, reason: &str, remedy: &str) -> SemanticResult { - Err(SemanticRefusal { - field: field.to_owned(), - reason: reason.to_owned(), - remedy: remedy.to_owned(), - }) + Err(SemanticRefusal::new(field, reason, remedy)) } #[derive(Debug, thiserror::Error)] @@ -567,6 +573,13 @@ pub(crate) fn check(root: &Path) -> Result { if let Err(refusal) = super::products::schema_people(root, &decoded.value, false) { return Err(refusal_report(&decoded, &refusal).into()); } + let terminate = super::signals()?; + if let Err(error) = super::products::inspect(root, &decoded.value, &terminate) { + return match error.downcast::() { + Ok(refusal) => Err(refusal_report(&decoded, &refusal).into()), + Err(error) => Err(error), + }; + } Ok(decoded.document.report_with(Vec::new())) } diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs index 3f234439d0..688f32c788 100644 --- a/crates/registry-bregctl/src/project_dev/products.rs +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -568,24 +568,121 @@ fn copy_project(source: &Path, target: &Path) -> Result<()> { copy_dir(source, target, 0, &mut 0, &mut 0) } -pub(super) fn breg_authority(root: &Path, plan: &Plan) -> Result> { - let Some(entry) = plan +pub(super) struct ProductInspection { + pub(super) breg_authority: Option, + pub(super) casework_principals: BTreeMap, +} + +#[derive(Debug, thiserror::Error)] +#[error("{reason}")] +pub(super) struct InspectionUnavailable { + field: String, + reason: String, + remedy: String, +} + +impl InspectionUnavailable { + fn new(field: &str, reason: &str, remedy: &str) -> Self { + Self { + field: field.to_owned(), + reason: reason.to_owned(), + remedy: remedy.to_owned(), + } + } + + pub(super) fn field(&self) -> &str { + &self.field + } + + pub(super) fn reason(&self) -> &str { + &self.reason + } + + pub(super) fn remedy(&self) -> &str { + &self.remedy + } +} + +struct SelectedProduct { + field: String, + root: PathBuf, +} + +pub(super) fn inspect( + root: &Path, + plan: &Plan, + terminate: &AtomicBool, +) -> Result { + let breg_authority = breg_authority(root, plan)?; + let casework_principals = casework_principals(root, plan, breg_authority.as_ref(), terminate)?; + Ok(ProductInspection { + breg_authority, + casework_principals, + }) +} + +fn product_root( + root: &Path, + plan: &Plan, + product: Product, +) -> plan::SemanticResult> { + let Some((index, entry)) = plan .products .iter() - .find(|entry| entry.product == Product::Breg) + .enumerate() + .find(|(_, entry)| entry.product == product) else { return Ok(None); }; - let registry = crate::compile( - &root.join(&entry.project), - crate::ProfileArg::Production, - "project-dev", - ) - .map_err(|_| { - anyhow::anyhow!( - "BReg project check failed; run rstack breg check for the authored diagnostics" + let field = format!("/products/{index}/project"); + let project = root.join(&entry.project).canonicalize().map_err(|_| { + plan::SemanticRefusal::new( + &field, + "The selected product project directory cannot be resolved.", + "Restore the product project directory inside the development project.", + ) + })?; + plan::require( + project.starts_with(root) && project.is_dir(), + &field, + "The selected product project directory is outside the development project.", + "Select a product project directory inside the development project.", + )?; + let manifest_name = match product { + Product::Breg => "registry.yaml", + Product::Casework => "casework.yaml", + }; + let manifest = project.join(manifest_name).canonicalize().map_err(|_| { + plan::SemanticRefusal::new( + &field, + "The selected product manifest cannot be resolved.", + format!("Restore {manifest_name} inside the selected product project."), ) })?; + plan::require( + manifest.starts_with(&project) && manifest.is_file(), + &field, + "The selected product manifest is outside the selected product project.", + format!("Place {manifest_name} inside the selected product project."), + )?; + Ok(Some(SelectedProduct { + field, + root: project, + })) +} + +fn breg_authority(root: &Path, plan: &Plan) -> Result> { + let Some(project) = product_root(root, plan, Product::Breg)? else { + return Ok(None); + }; + let registry = crate::compile(&project.root, crate::ProfileArg::Production, "project-dev") + .map_err(|_| { + plan::SemanticRefusal::new( + &project.field, + "The BReg project did not pass its owning validation.", + "Run `rstack breg check` for authored diagnostics, then correct the BReg project.", + ) + })?; let review_authorities = registry .review_authorities() .iter() @@ -617,20 +714,7 @@ pub(super) fn breg_authority(root: &Path, plan: &Plan) -> Result Result, +) -> plan::SemanticResult { + plan::require( + principal_claims.len() <= 1, + product_field, + "The BReg access profiles select more than one principal claim.", + "Use one principal claim across the BReg project's access profiles.", + )?; + let principal = principal_claims + .iter() + .next() + .cloned() + .unwrap_or_else(|| "sub".to_owned()); + for (name, person) in &plan.people { + if let Some(grant) = person.grants.get(&Product::Breg) { + plan::require( + !grant.claims.contains_key(&principal), + &format!( + "/people/{}/grants/breg/claims", + registry_platform_yaml::escape_pointer_segment(name) + ), + "The BReg grant claims cannot replace the principal claim selected by the BReg project.", + "Remove the selected principal claim from this grant's additional claims.", + )?; + } + } + Ok(principal) +} + fn validate_breg_binding_ids( bindings: &Bindings, review_authorities: &BTreeSet<&str>, @@ -667,44 +783,82 @@ fn breg_requires_postgis(registry: ®istry_breg::CompiledRegistry) -> bool { registry.ddl().requires_postgis } -pub(super) fn casework_principals( +fn casework_principals( root: &Path, plan: &Plan, breg: Option<&BregAuthority>, terminate: &AtomicBool, ) -> Result> { - let Some(entry) = plan - .products - .iter() - .find(|entry| entry.product == Product::Casework) - else { + let Some(project) = product_root(root, plan, Product::Casework)? else { return Ok(BTreeMap::new()); }; - let tool = super::binaries(Product::Casework)?.tool; - let facts = explain_casework_facts(&tool, &root.join(&entry.project), terminate)?; + let tool = super::product_tool(Product::Casework).map_err(|_| { + InspectionUnavailable::new( + &project.field, + "The Casework project cannot be inspected because its owning tool is unavailable.", + "Install the matching Casework tools or provide RSTACK_CASEWORKCTL_BIN, then run the check again.", + ) + })?; + let facts = match explain_casework_facts(&tool, &project.root, terminate) { + Ok(facts) => facts, + Err(error) if error.is::() => { + return Err(plan::SemanticRefusal::new( + &project.field, + "The Casework project did not pass validation by its owning tool.", + "Run `rstack casework check` for authored diagnostics, then correct the Casework project.", + ).into()); + } + Err(_) => { + return Err(InspectionUnavailable::new( + &project.field, + "The selected `caseworkctl` helper could not inspect the Casework project.", + "Install the matching Casework tools or provide RSTACK_CASEWORKCTL_BIN, then run the check again.", + ).into()); + } + }; validate_casework_binding_ids(&plan.bindings, &facts)?; + Ok(casework_principals_from_facts(plan, breg, &facts)?) +} + +fn casework_principals_from_facts( + plan: &Plan, + breg: Option<&BregAuthority>, + facts: &CaseworkFacts, +) -> plan::SemanticResult> { let mut principals = BTreeMap::new(); for (name, person) in &plan.people { let Some(grant) = person.grants.get(&Product::Casework) else { continue; }; - let claim = facts.profiles.get(&grant.profile).map(String::as_str) - .with_context(|| format!("dev.yaml /people/{name}/grants/casework/profile must name a Casework access profile"))?; - ensure!( + let escaped = registry_platform_yaml::escape_pointer_segment(name); + let claim = facts.profiles.get(&grant.profile).map(String::as_str).ok_or_else(|| { + plan::SemanticRefusal::new( + format!("/people/{escaped}/grants/casework/profile"), + "The Casework grant profile does not name an access profile in the Casework project.", + "Set profile to an access profile defined by the Casework project.", + ) + })?; + plan::require( matches!(claim, "sub" | "registry_principal") || (!plan::reserved_claim(claim) && !matches!(claim, "registry_actor_kind" | "registry_purpose")), - "dev.yaml /people/{name}/grants/casework/profile names an incompatible principal claim" - ); - ensure!( + &format!("/people/{escaped}/grants/casework/profile"), + "The Casework access profile selects an incompatible principal claim.", + "Choose a Casework access profile whose principal claim is `sub`, `registry_principal`, or a non-reserved custom claim.", + )?; + plan::require( !grant.claims.contains_key(claim), - "dev.yaml /people/{name}/grants/casework/claims cannot replace the Casework principal claim" - ); + &format!("/people/{escaped}/grants/casework/claims"), + "The Casework grant claims cannot replace the principal claim selected by its access profile.", + "Remove the selected principal claim from this grant's additional claims.", + )?; if person.casework_context { - ensure!( + plan::require( claim == "sub" || breg.is_some_and(|authority| authority.principal == claim), - "dev.yaml /people/{name}/caseworkContext requires Casework to read sub or the BReg principal claim" - ); + &format!("/people/{escaped}/caseworkContext"), + "Casework context forwarding requires Casework to read `sub` or the BReg principal claim.", + "Choose a compatible Casework access profile or remove caseworkContext.", + )?; } principals.insert(grant.profile.clone(), claim.to_owned()); } @@ -734,7 +888,7 @@ fn explain_casework_facts( let status = wait_for_tool(&mut child, || { ensure!( !terminate.load(Ordering::Relaxed), - "development start was interrupted" + "development product inspection was interrupted" ); ensure!(Instant::now() < deadline, "caseworkctl explain timed out"); Ok(()) @@ -746,15 +900,44 @@ fn explain_casework_facts( bytes.len() as u64 <= MAX_REPORT, "the Casework authoring report exceeds the size limit" ); - ensure!( - status.success(), - "Casework project check failed. Run rstack casework check for the authored diagnostics" - ); let report: Value = serde_json::from_slice(&bytes) .map_err(|_| anyhow::anyhow!("caseworkctl explain did not emit a JSON report"))?; - casework_report_facts(&report) + if status.success() { + return casework_report_facts(&report); + } + if status.code() == Some(1) && casework_domain_refusal(&report) { + return Err(InvalidCaseworkProject.into()); + } + anyhow::bail!("caseworkctl explain did not complete successfully") +} + +fn casework_domain_refusal(report: &Value) -> bool { + report["apiVersion"] == "id.registrystack.org/formats/casework/report/v1alpha2" + && report["kind"] == "CaseworkReport" + && report["command"] == "explain" + && report["status"] == "domain-refusal" + && report["ok"] == false + && report["diagnostics"].as_array().is_some_and(|diagnostics| { + diagnostics.iter().any(|diagnostic| { + diagnostic["severity"] == "error" + && diagnostic["code"] + .as_str() + .is_some_and(|value| !value.is_empty()) + && diagnostic["path"].is_string() + && diagnostic["message"] + .as_str() + .is_some_and(|value| !value.is_empty()) + && diagnostic["suggestedAction"] + .as_str() + .is_some_and(|value| !value.is_empty()) + }) + }) } +#[derive(Debug, thiserror::Error)] +#[error("the Casework project did not pass its owning validation")] +struct InvalidCaseworkProject; + #[derive(Debug, Eq, PartialEq)] struct CaseworkFacts { profiles: BTreeMap, @@ -1188,6 +1371,121 @@ mod tests { .contains("incompatible report")); } + fn authority_plan(breg_claims: Value, casework_claims: Value, context: bool) -> Plan { + serde_json::from_value(json!({ + "issuer": "https://issuer.example.invalid", + "products": [ + {"product": "breg", "project": "breg", "audience": "registry"}, + {"product": "casework", "project": "casework", "audience": "review"} + ], + "people": {"operator": { + "subject": "operator", "client": "operator-client", + "grants": { + "breg": {"profile": "operator", "scopes": [], "claims": breg_claims}, + "casework": {"profile": "reviewer", "scopes": [], "claims": casework_claims} + }, + "caseworkContext": context + }}, + "journey": {"id": "smoke", "steps": []} + })) + .unwrap() + } + + #[test] + fn breg_authority_restrictions_are_typed_and_located() { + let plan = authority_plan(json!({}), json!({}), false); + let error = breg_principal( + &plan, + "/products/0/project", + &BTreeSet::from(["registry_principal".to_owned(), "sub".to_owned()]), + ) + .unwrap_err() + .to_string(); + assert!(error.contains("/products/0/project"), "{error}"); + assert!( + error.contains("The BReg access profiles select more than one principal claim."), + "{error}" + ); + assert!( + error.contains("Use one principal claim across the BReg project's access profiles."), + "{error}" + ); + + let plan = authority_plan( + json!({"registry_principal": "private-claim-marker"}), + json!({}), + false, + ); + let error = breg_principal( + &plan, + "/products/0/project", + &BTreeSet::from(["registry_principal".to_owned()]), + ) + .unwrap_err() + .to_string(); + assert!( + error.contains("/people/operator/grants/breg/claims"), + "{error}" + ); + assert!( + error.contains("cannot replace the principal claim"), + "{error}" + ); + assert!(!error.contains("private-claim-marker"), "{error}"); + } + + #[test] + fn casework_authority_restrictions_are_typed_and_located() { + let facts = |principal: &str| CaseworkFacts { + profiles: BTreeMap::from([("reviewer".to_owned(), principal.to_owned())]), + trusted_producers: BTreeSet::new(), + context_providers: BTreeSet::new(), + }; + + let plan = authority_plan(json!({}), json!({}), false); + let error = casework_principals_from_facts(&plan, None, &facts("registry_actor_kind")) + .unwrap_err() + .to_string(); + assert!( + error.contains("/people/operator/grants/casework/profile"), + "{error}" + ); + assert!(error.contains("incompatible principal claim"), "{error}"); + + let plan = authority_plan(json!({}), json!({"sub": "private-claim-marker"}), false); + let error = casework_principals_from_facts(&plan, None, &facts("sub")) + .unwrap_err() + .to_string(); + assert!( + error.contains("/people/operator/grants/casework/claims"), + "{error}" + ); + assert!( + error.contains("cannot replace the principal claim"), + "{error}" + ); + assert!(!error.contains("private-claim-marker"), "{error}"); + + let plan = authority_plan(json!({}), json!({}), true); + let breg = BregAuthority { + principal: "registry_principal".to_owned(), + purpose: false, + requires_postgis: false, + named_clients: BTreeSet::new(), + }; + let error = casework_principals_from_facts(&plan, Some(&breg), &facts("custom_principal")) + .unwrap_err() + .to_string(); + assert!( + error.contains("/people/operator/caseworkContext"), + "{error}" + ); + assert!( + error.contains("requires Casework to read `sub` or the BReg principal claim"), + "{error}" + ); + } + fn product_bindings() -> Bindings { Bindings { breg_review_authority: Some(BregReviewAuthority { diff --git a/crates/registry-bregctl/tests/project_dev.rs b/crates/registry-bregctl/tests/project_dev.rs index 5cab2e0f4c..97d1e88e2c 100644 --- a/crates/registry-bregctl/tests/project_dev.rs +++ b/crates/registry-bregctl/tests/project_dev.rs @@ -4,7 +4,13 @@ use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; use std::{fs, os::unix::fs::PermissionsExt, process::Command}; fn project(root: &std::path::Path, extra: &str) { - fs::write(root.join("registry.yaml"), "fixture\n").unwrap(); + fs::write( + root.join("registry.yaml"), + include_str!( + "../../../products/platform/conformance/projects/breg-project-dev/registry/registry.yaml" + ), + ) + .unwrap(); fs::write( root.join("dev.yaml"), format!( @@ -85,6 +91,118 @@ fn start(root: &std::path::Path, state: &std::path::Path) -> std::process::Outpu .unwrap() } +fn check_with_casework_tool( + root: &std::path::Path, + state: &std::path::Path, + tool: &std::path::Path, +) -> std::process::Output { + let request = serde_json::json!({ + "action": "check", "project": root, "deny_warnings": true, "json": true, + }); + Command::new(env!("CARGO_BIN_EXE_bregctl")) + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request.to_string()) + .env("RSTACK_DEV_STATE_DIR", state) + .env("RSTACK_CASEWORKCTL_BIN", tool) + .env_remove("RSTACK_CASEWORK_BIN") + .env_remove("RSTACK_DEV_DATABASE_URL") + .output() + .unwrap() +} + +fn start_with_casework_tool( + root: &std::path::Path, + state: &std::path::Path, + tool: &std::path::Path, +) -> std::process::Output { + let request = serde_json::json!({ + "action": "start", "project": root, "json": true, + }); + Command::new(env!("CARGO_BIN_EXE_bregctl")) + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request.to_string()) + .env("RSTACK_DEV_STATE_DIR", state) + .env("RSTACK_CASEWORKCTL_BIN", tool) + .env_remove("RSTACK_CASEWORK_BIN") + .env_remove("RSTACK_DEV_DATABASE_URL") + .output() + .unwrap() +} + +fn fake_casework_tool(root: &std::path::Path) -> std::path::PathBuf { + let tool = root.join("caseworkctl"); + fs::write( + &tool, + concat!( + "#!/usr/bin/env bash\n", + "set -euo pipefail\n", + "[[ $# -eq 4 ]]\n", + "[[ $1 == --format && $2 == json && $3 == explain ]]\n", + "[[ -f $4/casework.yaml ]]\n", + "printf '%s\\n' '", + "{\"apiVersion\":\"id.registrystack.org/formats/casework/report/v1alpha2\",", + "\"kind\":\"CaseworkReport\",\"command\":\"explain\",\"status\":\"complete\",\"ok\":true,", + "\"reviewPolicies\":[{\"dependencies\":{\"accessProfiles\":[{\"id\":\"reviewer\",\"principalClaim\":\"sub\"}]}}],", + "\"trustedProducers\":[{\"id\":\"review-producer\"}],", + "\"contextProviders\":[{\"id\":\"proposal-context\"}]}", + "'\n", + ), + ) + .unwrap(); + let mut permissions = fs::metadata(&tool).unwrap().permissions(); + permissions.set_mode(0o700); + fs::set_permissions(&tool, permissions).unwrap(); + tool +} + +fn casework_reference_project( + root: &std::path::Path, + casework_profile: &str, + bindings: &str, +) -> std::path::PathBuf { + fs::write( + root.join("registry.yaml"), + include_str!( + "../../../products/platform/conformance/projects/breg-project-dev/registry/registry.yaml" + ), + ) + .unwrap(); + fs::create_dir(root.join("casework")).unwrap(); + fs::write(root.join("casework/casework.yaml"), "fixture\n").unwrap(); + fs::create_dir(root.join("tests")).unwrap(); + fs::write( + root.join("tests/journeys.yaml"), + concat!( + "apiVersion: id.registrystack.org/formats/breg/journeys/v1\n", + "kind: BRegJourneys\n", + "journeys:\n", + " - id: real-journey\n", + " steps:\n", + " - id: real-step\n", + " recordType: record\n", + " accessProfile: operator\n", + " claims: {principal: fixture-operator}\n", + " request: {type: list}\n", + " expect: {outcome: success, status: 200, count: 0}\n", + ), + ) + .unwrap(); + fs::write( + root.join("dev.yaml"), + format!( + "apiVersion: id.registrystack.org/formats/breg/project-dev/v1alpha1\n\ +kind: BRegProjectDev\n\ +issuer: https://issuer.example.invalid\n\ +products:\n - {{product: breg, project: ., audience: registry}}\n - {{product: casework, project: casework, audience: review}}\n\ +people:\n operator:\n subject: operator\n client: operator-client\n grants:\n breg: {{profile: operator, scopes: [registry:read]}}\n reviewer:\n subject: reviewer\n client: reviewer-client\n grants:\n casework: {{profile: {casework_profile}, scopes: [casework:review]}}\n\ +journey:\n id: smoke\n steps:\n - {{id: read, product: breg, as: operator, method: GET, path: /v1/records, expect: {{status: 200}}}}\n\ +{bindings}" + ), + ) + .unwrap(); + fake_casework_tool(root) +} + fn assert_dangling_schema_assignment( extra: &str, path: &str, @@ -609,6 +727,27 @@ journey:\n id: smoke\n steps:\n - {id: read, product: breg, as: editor, met ) .unwrap(); + let checked = check(scratch.path(), &state, true); + + assert_eq!(checked.status.code(), Some(1), "{checked:?}"); + let report: serde_json::Value = serde_json::from_slice(&checked.stdout).unwrap(); + assert_eq!(report["status"], "domain-refusal"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/people/unused/grants/breg/profile"); + assert_eq!( + diagnostic["message"], + "The BReg grant profile does not name an access profile in the BReg project." + ); + assert_eq!( + diagnostic["suggestedAction"], + "Set profile to an access profile defined by the BReg project." + ); + assert!(!checked + .stdout + .windows(22) + .any(|value| value == b"private-profile-marker")); + assert!(!state.exists()); + let result = start(scratch.path(), &state); assert_eq!(result.status.code(), Some(1), "{result:?}"); @@ -632,6 +771,160 @@ journey:\n id: smoke\n steps:\n - {id: read, product: breg, as: editor, met assert!(!state.exists()); } +#[test] +fn check_and_start_resolve_casework_profiles_and_bindings_with_the_owning_report() { + for (profile, bindings, path, message, action, supplied) in [ + ( + "private-profile-marker", + "", + "/people/reviewer/grants/casework/profile", + "The Casework grant profile does not name an access profile in the Casework project.", + "Set profile to an access profile defined by the Casework project.", + "private-profile-marker", + ), + ( + "reviewer", + "bindings:\n bregReviewAuthority: {id: local-review, producer: private-producer-marker, identity: reviewer}\n", + "/bindings/bregReviewAuthority/producer", + "The BReg review producer does not name a trusted producer in the Casework project.", + "Set producer to a trusted producer defined by the Casework project.", + "private-producer-marker", + ), + ( + "reviewer", + "bindings:\n caseworkContextProvider: {id: private-provider-marker, identity: operator, action: review-action}\n", + "/bindings/caseworkContextProvider/id", + "The Casework context provider ID does not name a context provider in the Casework project.", + "Set id to a context provider defined by the Casework project.", + "private-provider-marker", + ), + ] { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let tool = casework_reference_project(scratch.path(), profile, bindings); + + let checked = check_with_casework_tool(scratch.path(), &state, &tool); + + assert_eq!(checked.status.code(), Some(1), "{checked:?}"); + let report: serde_json::Value = serde_json::from_slice(&checked.stdout).unwrap(); + assert_eq!(report["status"], "domain-refusal"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], path); + assert_eq!(diagnostic["message"], message); + assert_eq!(diagnostic["suggestedAction"], action); + assert!(!diagnostic.to_string().contains(supplied)); + assert!(!state.exists()); + + let started = start_with_casework_tool(scratch.path(), &state, &tool); + + assert_eq!(started.status.code(), Some(1), "{started:?}"); + let report: serde_json::Value = serde_json::from_slice(&started.stdout).unwrap(); + let error = report["error"].as_str().unwrap(); + assert!(error.contains(path), "{error}"); + assert!(error.contains(message), "{error}"); + assert!(error.contains(action), "{error}"); + assert!(!error.contains(supplied), "{error}"); + assert!(!state.exists()); + } +} + +#[test] +fn check_distinguishes_an_unavailable_casework_helper_from_an_invalid_project() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let tool = casework_reference_project(scratch.path(), "reviewer", ""); + let request = serde_json::json!({ + "action": "check", "project": scratch.path(), "deny_warnings": true, "json": true, + }); + + let unavailable = Command::new(env!("CARGO_BIN_EXE_bregctl")) + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request.to_string()) + .env("RSTACK_DEV_STATE_DIR", &state) + .env_remove("RSTACK_CASEWORKCTL_BIN") + .env_remove("RSTACK_CASEWORK_BIN") + .env_remove("RSTACK_DEV_DATABASE_URL") + .output() + .unwrap(); + + assert_eq!(unavailable.status.code(), Some(3), "{unavailable:?}"); + let report: serde_json::Value = serde_json::from_slice(&unavailable.stdout).unwrap(); + assert_eq!(report["status"], "operational-failure"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/products/1/project"); + assert_eq!( + diagnostic["message"], + "The Casework project cannot be inspected because its owning tool is unavailable." + ); + assert_eq!( + diagnostic["suggestedAction"], + "Install the matching Casework tools or provide RSTACK_CASEWORKCTL_BIN, then run the check again." + ); + assert!(!state.exists()); + + fs::write( + &tool, + concat!( + "#!/usr/bin/env bash\n", + "set -euo pipefail\n", + "printf '%s\\n' '", + "{\"ok\":false,\"command\":\"explain\",\"status\":\"operational-failure\",", + "\"apiVersion\":\"id.registrystack.org/formats/casework/report/v1alpha2\",", + "\"kind\":\"CaseworkReport\",\"version\":\"0.46.1-dev\",", + "\"diagnostics\":[{\"severity\":\"error\",\"code\":\"caseworkctl.operational-failure\",", + "\"path\":\"runtime\",\"message\":\"A dependency was unavailable.\",", + "\"suggestedAction\":\"Restore the dependency, then retry.\"}]}'\n", + "exit 3\n", + ), + ) + .unwrap(); + let helper_failure = check_with_casework_tool(scratch.path(), &state, &tool); + + assert_eq!(helper_failure.status.code(), Some(3), "{helper_failure:?}"); + let report: serde_json::Value = serde_json::from_slice(&helper_failure.stdout).unwrap(); + assert_eq!(report["status"], "operational-failure"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/products/1/project"); + assert_eq!( + diagnostic["message"], + "The selected `caseworkctl` helper could not inspect the Casework project." + ); + assert!(!state.exists()); + + fs::write( + &tool, + concat!( + "#!/usr/bin/env bash\n", + "set -euo pipefail\n", + "printf '%s\\n' '", + "{\"ok\":false,\"command\":\"explain\",\"status\":\"domain-refusal\",", + "\"apiVersion\":\"id.registrystack.org/formats/casework/report/v1alpha2\",", + "\"kind\":\"CaseworkReport\",\"version\":\"0.46.1-dev\",", + "\"diagnostics\":[{\"severity\":\"error\",\"code\":\"casework.project.invalid\",", + "\"path\":\"/reviewPolicies\",\"message\":\"The authored project was refused.\",", + "\"suggestedAction\":\"Correct the Casework project.\"}]}'\n", + "exit 1\n", + ), + ) + .unwrap(); + let invalid = check_with_casework_tool(scratch.path(), &state, &tool); + + assert_eq!(invalid.status.code(), Some(1), "{invalid:?}"); + let report: serde_json::Value = serde_json::from_slice(&invalid.stdout).unwrap(); + assert_eq!(report["status"], "domain-refusal"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/products/1/project"); + assert_eq!( + diagnostic["message"], + "The Casework project did not pass validation by its owning tool." + ); + assert_eq!( + diagnostic["suggestedAction"], + "Run `rstack casework check` for authored diagnostics, then correct the Casework project." + ); + assert!(!state.exists()); +} + #[test] fn a_malformed_plan_names_its_field_before_allocating_run_state() { let scratch = tempfile::tempdir().unwrap(); @@ -688,7 +981,13 @@ fn worker_restores_a_valid_plan_whose_captured_input_exceeds_512_kib() { let scratch = tempfile::tempdir().unwrap(); let project_root = scratch.path().join("project"); fs::create_dir(&project_root).unwrap(); - fs::write(project_root.join("registry.yaml"), "fixture\n").unwrap(); + fs::write( + project_root.join("registry.yaml"), + include_str!( + "../../../products/platform/conformance/projects/breg-project-dev/registry/registry.yaml" + ), + ) + .unwrap(); let raw = BASE64.encode(vec![b'x'; 400 * 1024]); let mut plan = serde_json::json!({ "apiVersion": "id.registrystack.org/formats/breg/project-dev/v1alpha1", diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index 475f7be24d..f001cfe746 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -19,11 +19,11 @@ rstack dev token RUN_ID --as applicant --json rstack dev stop RUN_ID ``` -Run `rstack dev check PROJECT [--deny-warnings] [--json]` to validate the plan, -the product directories it names, and BReg schema-test assignments offline. -With `--json`, the check writes a `BRegCtlReport` with `command` set to -`dev-check`. Use `--deny-warnings` to refuse any unaccepted warning. -`start` also validates the authored products with their own tooling. +Run `rstack dev check PROJECT [--deny-warnings] [--json]` to validate the plan +and resolve its fixture, grant, and binding references with the installed +product authoring tools. The check works with the database offline. With +`--json`, it writes a `BRegCtlReport` with `command` set to `dev-check`. +Use `--deny-warnings` to refuse any unaccepted warning. You can also pass `--database-url`. Use the environment variable when the URL includes a password. Start reports the product URLs and the journey result. diff --git a/products/platform/conformance/projects/breg-project-dev/registry/registry.yaml b/products/platform/conformance/projects/breg-project-dev/registry/registry.yaml index b0e14738b9..2175f1a432 100644 --- a/products/platform/conformance/projects/breg-project-dev/registry/registry.yaml +++ b/products/platform/conformance/projects/breg-project-dev/registry/registry.yaml @@ -1 +1,66 @@ -# This file establishes the referenced product root for development-plan checks. +# This deliberately small product is compiled by `rstack dev check`; it has no +# journeys fixture because the conformance plan has no explicit assignments. +apiVersion: id.registrystack.org/formats/breg/project/v1alpha1 +kind: BRegProject +project: + id: conformance-registry + canonicalBaseIri: https://registry.example.invalid + version: 0.1.0 + defaultLanguage: en + display: + label: Conformance registry + description: Minimal product for the development-plan conformance corpus. +package: + sourceRevision: conformance-registry-0.1.0 +datasets: + - id: records + display: {label: Records, description: Conformance records.} +reviewAuthorities: + - id: local-review + protocol: id.registrystack.org/protocols/review/v1 + display: {label: Local review, description: Review authority used by development-plan checks.} +recordTypes: + - id: record + route: records + primaryDataset: records + mutationMode: mutable + display: {label: Record, description: A conformance record.} + fields: + - {id: name, type: string, required: true, minimumLength: 1, maximumLength: 100, classification: internal} + - {id: owner, type: string, required: true, minimumLength: 1, maximumLength: 100, classification: internal} +actions: + - id: review-action + display: {label: Review action, description: Create one conformance record.} + execution: {mode: immediate} + inputs: + - {id: name, type: string, required: true, minimumLength: 1, maximumLength: 100, classification: internal} + - {id: owner, type: string, required: true, minimumLength: 1, maximumLength: 100, classification: internal} + plan: + type: fixed + effects: + - id: record + target: {recordType: record} + operation: create + set: + name: {fromInput: name} + owner: {fromInput: owner} +accessProfiles: + - id: operator + default: true + principalClaim: registry_principal + requiredScopes: unrestricted + permissions: + records: + - recordType: record + operations: [get, list] + readableFields: [name, owner] + rowBoundaries: + - {field: owner, claim: registry_principal, operator: equals} + actions: + - action: review-action + operations: [invoke] + targets: + - recordType: record + rowBoundaries: + - {field: owner, claim: registry_principal, operator: equals} + results: [record] diff --git a/products/platform/scripts/test-rstack-dev.py b/products/platform/scripts/test-rstack-dev.py index c93dfa9582..e2728e0aae 100644 --- a/products/platform/scripts/test-rstack-dev.py +++ b/products/platform/scripts/test-rstack-dev.py @@ -620,6 +620,45 @@ def verify_spatial(root, bin_dir, pg, env, pg_env): print("Spatial BReg bbox journey passed with distinct, cleaned-up spatial roles") +def verify_offline_plan(root, bin_dir, project): + state = root / "offline-check-runs" + check_env = { + key: value + for key, value in os.environ.items() + if key + not in { + "RSTACK_DEV_DATABASE_URL", + "RSTACK_DEV_DATABASE_CA", + "RSTACK_BREG_BIN", + "RSTACK_CASEWORK_BIN", + } + } + check_env["RSTACK_BREGCTL_BIN"] = str(bin_dir / "bregctl") + check_env["RSTACK_CASEWORKCTL_BIN"] = str(bin_dir / "caseworkctl") + check_env["RSTACK_DEV_STATE_DIR"] = str(state) + report = dev(bin_dir, check_env, project, "check", ".") + assert report["ok"] and report["status"] == "complete" + assert report["diagnostics"] == [] + assert not state.exists() + + invalid = root / "offline-invalid" + shutil.copytree(project, invalid) + plan = invalid / "dev.yaml" + authored = plan.read_text() + profile = "profile: intake" + unknown = "missing-check-profile" + assert authored.count(profile) == 1 + plan.write_text(authored.replace(profile, f"profile: {unknown}")) + refused = dev(bin_dir, check_env, invalid, "check", ".", expected=1) + [diagnostic] = refused["diagnostics"] + assert refused["status"] == "domain-refusal" + assert diagnostic["path"] == "/people/intake/grants/casework/profile" + assert diagnostic["suggestedAction"] + assert unknown not in diagnostic["message"] + diagnostic["suggestedAction"] + assert not state.exists() + print("Offline checks resolve product profiles with only the authoring tools") + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--bin-dir", type=Path, required=True) @@ -633,20 +672,7 @@ def main(): root = Path(directory) bin_dir = args.bin_dir.resolve() project = STACK / "products/platform/examples/licensing" - check_env = { - key: value - for key, value in os.environ.items() - if key not in {"RSTACK_DEV_DATABASE_URL", "RSTACK_DEV_DATABASE_CA"} - } - check_env["RSTACK_BREGCTL_BIN"] = str(bin_dir / "bregctl") - check_env["RSTACK_DEV_STATE_DIR"] = str(root / "offline-check-runs") - report = dev(bin_dir, check_env, project, "check", ".") - assert ( - report["ok"] - and report["status"] == "complete" - and report["diagnostics"] == [] - ) - assert not (root / "offline-check-runs").exists() + verify_offline_plan(root, bin_dir, project) pg = args.postgres_bin_dir.resolve() cluster, ca, url, pg_env = prepare_postgres(root, pg) checked( From 6dd2fa4f6e7875f84c53ca5a63bf951bb08a9060 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 00:47:29 +0700 Subject: [PATCH 09/34] feat(casework): list access profiles in caseworkctl explain Signed-off-by: Jeremi Joslin --- crates/registry-caseworkctl/src/project.rs | 11 +++++++- crates/registry-caseworkctl/src/report.rs | 6 ++++ .../tests/report_examples.rs | 15 ++++++++++ products/casework/CHANGELOG.md | 2 ++ .../casework/contracts/cli/report.schema.json | 28 +++++++++++++++++++ 5 files changed, 61 insertions(+), 1 deletion(-) diff --git a/crates/registry-caseworkctl/src/project.rs b/crates/registry-caseworkctl/src/project.rs index 9ab977f186..e5076aea17 100644 --- a/crates/registry-caseworkctl/src/project.rs +++ b/crates/registry-caseworkctl/src/project.rs @@ -778,8 +778,17 @@ pub(super) fn explain(project: &Path) -> Result { "refusals": ["untrusted producer", "policy binding mismatch", "concealed source", "changed subject or contract", "stale task revision", "ineligible reviewer"], })) }).collect::,registry_casework_core::ReviewPolicyError>>()?; + let mut access_profiles = policy + .access_profiles + .iter() + .map(|profile| crate::report::ExplainAccessProfile { + id: profile.id.clone(), + principal_claim: profile.principal_claim.clone(), + }) + .collect::>(); + access_profiles.sort_by(|a, b| a.id.cmp(&b.id)); Ok( - json!({"ok":true,"command":"explain","project":policy.project,"reviewPolicies":reviews,"contextProviders":policy.context_providers,"clocks":policy.clocks,"calendars":policy.calendars,"trustedProducers":policy.trusted_producers,"staffing":{"next":"due ordered eligible visible tasks","holdings":"claimed or assigned tasks under current visibility","caseload":"atomic preview and revision-fenced apply","routing":"once per task or stage from compiled minimized fields","clockRecompute":"pinned policy closure"},"authority":"producer, human reviewer and downstream executor authenticate independently"}), + json!({"ok":true,"command":"explain","project":policy.project,"accessProfiles":access_profiles,"reviewPolicies":reviews,"contextProviders":policy.context_providers,"clocks":policy.clocks,"calendars":policy.calendars,"trustedProducers":policy.trusted_producers,"staffing":{"next":"due ordered eligible visible tasks","holdings":"claimed or assigned tasks under current visibility","caseload":"atomic preview and revision-fenced apply","routing":"once per task or stage from compiled minimized fields","clockRecompute":"pinned policy closure"},"authority":"producer, human reviewer and downstream executor authenticate independently"}), ) } pub(super) fn test(project: &Path, deny_warnings: bool) -> Result { diff --git a/crates/registry-caseworkctl/src/report.rs b/crates/registry-caseworkctl/src/report.rs index 4ed3641f9c..601805f6d2 100644 --- a/crates/registry-caseworkctl/src/report.rs +++ b/crates/registry-caseworkctl/src/report.rs @@ -345,8 +345,14 @@ payload!(ExplainStaffing { routing: String, clock_recompute: String }); +payload!(ExplainAccessProfile { + #[cfg_attr(feature = "schema", schemars(with = "registry_platform_yaml::LocalId"))] + id: String, + principal_claim: String +}); payload!(ExplainData { project: ProjectIdentity, + access_profiles: Vec, review_policies: Vec, context_providers: Vec, clocks: Vec, diff --git a/crates/registry-caseworkctl/tests/report_examples.rs b/crates/registry-caseworkctl/tests/report_examples.rs index c79bc7a381..4550d71f56 100644 --- a/crates/registry-caseworkctl/tests/report_examples.rs +++ b/crates/registry-caseworkctl/tests/report_examples.rs @@ -205,3 +205,18 @@ fn stage_outcomes_example_passes_offline_authoring_commands() { assert_eq!(report["status"], "complete"); } } + +#[test] +fn explain_lists_every_access_profile_at_the_top_level() { + let root = repo_root(); + let (report, _) = output(&root, &["explain", STANDALONE], 0); + assert_eq!( + report["accessProfiles"], + serde_json::json!([ + {"id": "administrator", "principalClaim": "sub"}, + {"id": "requester", "principalClaim": "sub"}, + {"id": "staff", "principalClaim": "sub"}, + {"id": "supervisor", "principalClaim": "sub"} + ]) + ); +} diff --git a/products/casework/CHANGELOG.md b/products/casework/CHANGELOG.md index d01a406c6b..76a207dd51 100644 --- a/products/casework/CHANGELOG.md +++ b/products/casework/CHANGELOG.md @@ -7,6 +7,8 @@ `caseworkctl explain` lists them. A stage that omits the list offers every policy outcome. +- `caseworkctl explain` adds a top-level `accessProfiles` list, sorted by `id`, with each access profile's `id` and `principalClaim`, so a caller no longer reads them from a review policy's `dependencies`. The member is additive. + ## 0.46.0 - 2026-11-08 - Each access profile may carry an optional `label` and `description`. Neither is matched against a caller, a profile that writes neither reads as before, and rewording them leaves a policy's behavioral binding unchanged. Casework keeps both in the review policy description, which any caller who may read that description, requesters included, can see. diff --git a/products/casework/contracts/cli/report.schema.json b/products/casework/contracts/cli/report.schema.json index ab9637319a..e063563756 100644 --- a/products/casework/contracts/cli/report.schema.json +++ b/products/casework/contracts/cli/report.schema.json @@ -803,6 +803,22 @@ ], "type": "object" }, + "ExplainAccessProfile": { + "additionalProperties": false, + "properties": { + "id": { + "$ref": "#/$defs/LocalId" + }, + "principalClaim": { + "type": "string" + } + }, + "required": [ + "id", + "principalClaim" + ], + "type": "object" + }, "ExplainPolicy": { "additionalProperties": false, "properties": { @@ -2737,6 +2753,11 @@ "ok" ] }, + { + "required": [ + "accessProfiles" + ] + }, { "required": [ "authority" @@ -2782,6 +2803,7 @@ "then": { "required": [ "project", + "accessProfiles", "reviewPolicies", "contextProviders", "clocks", @@ -2794,6 +2816,12 @@ } ], "properties": { + "accessProfiles": { + "items": { + "$ref": "#/$defs/ExplainAccessProfile" + }, + "type": "array" + }, "apiVersion": { "const": "id.registrystack.org/formats/casework/report/v1alpha2", "type": "string" From b9f416a2ad7fe6cd0e3ea926b88f6914ad0e01c9 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 01:35:46 +0700 Subject: [PATCH 10/34] fix(bregctl): log dev product output, record process groups, check grants Send product tool and runtime output to owner-only logs under the run directory and append the log tail to startup failures. Record child process groups in processes.json. Refuse unknown BReg grant profiles at their pointer and encrypted fields before provisioning. Read Casework access profiles from the top-level accessProfiles of caseworkctl explain. Signed-off-by: Jeremi Joslin --- .../src/project_dev/products.rs | 434 +++++++++++++++--- crates/registry-bregctl/tests/project_dev.rs | 1 + 2 files changed, 374 insertions(+), 61 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs index 688f32c788..92bc4f25db 100644 --- a/crates/registry-bregctl/src/project_dev/products.rs +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -9,9 +9,9 @@ use serde_json::{json, Value}; use std::{ collections::{BTreeMap, BTreeSet}, fs, - io::{Read, Seek}, + io::{Read, Seek, SeekFrom}, net::{Ipv4Addr, TcpListener}, - os::unix::process::CommandExt, + os::unix::{fs::OpenOptionsExt, process::CommandExt}, path::{Path, PathBuf}, process::{Child, Command, ExitStatus, Stdio}, sync::{ @@ -28,7 +28,7 @@ use super::{ database::{Database, DatabaseUrls}, issuer::Issuer, plan::{self, Plan, Product}, - stopping, write_private, BregAuthority, RunInput, POLL, + private_directory, stopping, write_private, BregAuthority, RunInput, POLL, }; pub(super) struct ProductRunner<'a> { @@ -139,14 +139,18 @@ impl<'a> ProductRunner<'a> { if product == Product::Casework { command.arg("serve"); } - // Released runtimes own their listeners. If a competing process - // takes this port in the handoff, startup fails and is cleaned up. + let log = open_log(self.root, &format!("{}-runtime", product.name()))?; + command.stdout(log.try_clone()?).stderr(log); + // Released runtimes own their listeners, so the reservation is + // released before the spawn. A competing process can take the port + // in that gap; the runtime then fails to bind and exits, which the + // readiness wait reports along with the end of its log. drop(reservation); let child = command - .stdout(Stdio::null()) .spawn() .with_context(|| format!("cannot start {}", product.name()))?; self.children.push((product, child)); + self.record_processes()?; self.ready(product, url)?; } Ok(urls) @@ -422,20 +426,27 @@ impl<'a> ProductRunner<'a> { fn tool(&self, product: Product, stage: &str, args: &[&std::ffi::OsStr]) -> Result<()> { self.ensure_active()?; - let mut output = tempfile::tempfile_in(self.root)?; - let mut child = child_command(&self.input.binaries[&product].tool) - .arg(stage) - .args(args) - .args(["--format", "json"]) - .stdout(output.try_clone()?) - .spawn() - .with_context(|| format!("cannot start {} {stage}", product.name()))?; - if wait_for_tool(&mut child, || self.ensure_active())?.success() { - return Ok(()); - } - output.rewind()?; - let codes = report_codes(output)?; - bail!("{} {stage} failed{codes}", product.name()); + let mut arguments = vec![std::ffi::OsStr::new(stage)]; + arguments.extend_from_slice(args); + arguments.extend(["--format", "json"].map(std::ffi::OsStr::new)); + run_logged_tool( + &self.input.binaries[&product].tool, + &arguments, + self.root, + (product.name(), stage), + || self.ensure_active(), + ) + } + + /// Record every live child's process group so a later `stop` can reach + /// processes whose supervisor died. + fn record_processes(&self) -> Result<()> { + let entries: Vec<_> = self + .children + .iter() + .map(|(product, child)| (*product, child.id())) + .collect(); + write_processes(self.root, &entries) } fn ready(&mut self, product: Product, base: &str) -> Result<()> { @@ -443,32 +454,47 @@ impl<'a> ProductRunner<'a> { let url = format!("{base}/ready"); loop { self.ensure_active()?; - ensure!( - self.services_alive()?, - "{} exited before readiness", - product.name() - ); + self.ensure_alive(product)?; if self .runtime .block_on(async { self.http.get(&url).send().await }) .is_ok_and(|response| response.status().is_success()) { - ensure!( - self.services_alive()?, - "{} exited before readiness", - product.name() - ); + self.ensure_alive(product)?; return Ok(()); } ensure!( Instant::now() < end, - "{} did not become ready", - product.name() + "{} did not become ready{}", + product.name(), + self.runtime_log_tail(product) ); thread::sleep(POLL); } } + /// Fail while waiting on `waiting`, naming whichever child exited. + fn ensure_alive(&self, waiting: Product) -> Result<()> { + for (product, child) in &self.children { + if child_exited(child)? { + let detail = self.runtime_log_tail(*product); + if *product == waiting { + bail!("{} exited before readiness{detail}", product.name()); + } + bail!( + "{} exited while {} was starting{detail}", + product.name(), + waiting.name() + ); + } + } + Ok(()) + } + + fn runtime_log_tail(&self, product: Product) -> String { + failure_detail(&log_path(self.root, &format!("{}-runtime", product.name()))) + } + pub(super) fn services_alive(&self) -> Result { for (_, child) in &self.children { if child_exited(child)? { @@ -500,6 +526,9 @@ impl<'a> ProductRunner<'a> { false } }); + // Drop reaped groups from the record so a later recovery cannot signal + // a reused process group ID. + self.record_processes()?; ensure!(failures.is_empty(), "cannot stop {}", failures.join(", ")); drop(self.bindings.take()); Ok(()) @@ -514,6 +543,100 @@ impl<'a> ProductRunner<'a> { } } +fn log_path(root: &Path, name: &str) -> PathBuf { + root.join("logs").join(format!("{name}.log")) +} + +/// Open the owner-only log for one product stage, appending to earlier output. +fn open_log(root: &Path, name: &str) -> Result { + let directory = root.join("logs"); + match private_directory(&directory) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error).context("cannot create the development log directory"), + } + fs::OpenOptions::new() + .create(true) + .append(true) + .mode(0o600) + .open(log_path(root, name)) + .with_context(|| format!("cannot open the {name} log")) +} + +const LOG_TAIL_LINES: usize = 20; +const LOG_TAIL_BYTES: u64 = 4096; + +/// The last lines of a log, at most 4 KiB, decoded lossily. An unreadable +/// log yields an empty tail; the caller reports the path either way. +fn log_tail(path: &Path) -> String { + let Ok(mut file) = fs::File::open(path) else { + return String::new(); + }; + let mut bytes = Vec::new(); + let read = file + .metadata() + .map(|meta| meta.len()) + .and_then(|length| file.seek(SeekFrom::Start(length.saturating_sub(LOG_TAIL_BYTES)))) + .and_then(|_| file.take(LOG_TAIL_BYTES).read_to_end(&mut bytes)); + if read.is_err() { + return String::new(); + } + let text = String::from_utf8_lossy(&bytes); + let lines: Vec<_> = text.lines().collect(); + lines[lines.len().saturating_sub(LOG_TAIL_LINES)..] + .join("\n") + .trim() + .to_owned() +} + +/// The suffix appended to a stage failure: the log path and its last lines. +fn failure_detail(path: &Path) -> String { + let tail = log_tail(path); + if tail.is_empty() { + format!(" (log: {})", path.display()) + } else { + format!(" (log: {}); last lines:\n{tail}", path.display()) + } +} + +/// Run a product tool with its stderr in `/logs/.log`. Stdout is +/// the JSON report, which supplies the diagnostic codes of a failure. +fn run_logged_tool( + binary: &Path, + args: &[&std::ffi::OsStr], + root: &Path, + (product, stage): (&str, &str), + active: impl Fn() -> Result<()>, +) -> Result<()> { + let name = format!("{product}-{stage}"); + let mut output = tempfile::tempfile_in(root)?; + let mut child = child_command(binary) + .args(args) + .stdout(output.try_clone()?) + .stderr(open_log(root, &name)?) + .spawn() + .with_context(|| format!("cannot start {product} {stage}"))?; + if wait_for_tool(&mut child, active)?.success() { + return Ok(()); + } + output.rewind()?; + let codes = report_codes(output)?; + bail!( + "{product} {stage} failed{codes}{}", + failure_detail(&log_path(root, &name)) + ); +} + +/// Record each child's process group. Children start as group leaders, so the +/// group ID is the PID. +fn write_processes(root: &Path, children: &[(Product, u32)]) -> Result<()> { + let entries: Vec<_> = children + .iter() + .map(|(product, pid)| json!({"product": product.name(), "pid": pid, "pgid": pid})) + .collect(); + write_private(&root.join("processes.json"), &serde_json::to_vec(&entries)?) +} + fn random_secret() -> Result> { use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; let mut bytes = zeroize::Zeroizing::new([0u8; 32]); @@ -683,6 +806,7 @@ fn breg_authority(root: &Path, plan: &Plan) -> Result> { "Run `rstack breg check` for authored diagnostics, then correct the BReg project.", ) })?; + refuse_encrypted_fields(®istry)?; let review_authorities = registry .review_authorities() .iter() @@ -695,24 +819,7 @@ fn breg_authority(root: &Path, plan: &Plan) -> Result> { .map(|action| action.id.as_str()) .collect(); validate_breg_binding_ids(&plan.bindings, &review_authorities, &actions)?; - let profiles: BTreeSet<_> = registry - .access_profile_displays() - .iter() - .map(|profile| profile.id.as_str()) - .collect(); - for (name, person) in &plan.people { - if let Some(grant) = person.grants.get(&Product::Breg) { - plan::require( - profiles.contains(grant.profile.as_str()), - &format!( - "/people/{}/grants/breg/profile", - registry_platform_yaml::escape_pointer_segment(name) - ), - "The BReg grant profile does not name an access profile in the BReg project.", - "Set profile to an access profile defined by the BReg project.", - )?; - } - } + check_breg_grants(®istry, &plan.people)?; let inventory = registry_breg::authority::authority_inventory(®istry)?; let principal = breg_principal(plan, &project.field, &inventory.principal_claims)?; Ok(Some(BregAuthority { @@ -779,6 +886,55 @@ fn validate_breg_binding_ids( Ok(()) } +/// The generated runtime configuration has no field-encryption provider, so a +/// project with encrypted fields would only fail late and obscurely. +fn refuse_encrypted_fields(registry: ®istry_breg::CompiledRegistry) -> Result<()> { + for entity in registry.entities().values() { + for field in entity.fields.values() { + ensure!( + field.encryption.is_none(), + "rstack dev does not configure field encryption yet; {}.{} is encrypted", + entity.id, + field.id + ); + } + } + Ok(()) +} + +/// Every BReg grant must name an access profile the project declares, +/// including profiles that carry only proposal permissions and so belong to no +/// entity. Scopes are not checked here: a profile's required scopes are a +/// per-entity rule the runtime applies, and a mismatch surfaces when the +/// journey runs. +fn check_breg_grants( + registry: ®istry_breg::CompiledRegistry, + people: &BTreeMap, +) -> plan::SemanticResult<()> { + let profiles: BTreeSet<_> = registry + .access_profile_displays() + .iter() + .map(|profile| profile.id.as_str()) + .collect(); + for (name, person) in people { + if let Some(grant) = person.grants.get(&Product::Breg) { + plan::require( + profiles.contains(grant.profile.as_str()), + &format!( + "/people/{}/grants/breg/profile", + registry_platform_yaml::escape_pointer_segment(name) + ), + "The BReg grant profile does not name an access profile in the BReg project.", + format!( + "Set profile to an access profile defined by the BReg project. It declares: {}.", + profiles.iter().copied().collect::>().join(", ") + ), + )?; + } + } + Ok(()) +} + fn breg_requires_postgis(registry: ®istry_breg::CompiledRegistry) -> bool { registry.ddl().requires_postgis } @@ -954,9 +1110,8 @@ fn casework_report_facts(report: &Value) -> Result { && report["ok"] == true, "caseworkctl explain returned an incompatible report" ); - // Each explained policy includes the whole project's access profiles. let profiles = report - .pointer("/reviewPolicies/0/dependencies/accessProfiles") + .pointer("/accessProfiles") .and_then(Value::as_array) .context("caseworkctl explain did not report access profiles")?; let mut principals = BTreeMap::new(); @@ -1315,6 +1470,7 @@ mod tests { use crate::project_dev::bindings::{Bindings, BregReviewAuthority, CaseworkContextProvider}; use std::{ io::{BufRead, BufReader}, + os::unix::fs::PermissionsExt, sync::mpsc, }; @@ -1340,10 +1496,11 @@ mod tests { let mut report = json!({ "apiVersion": "id.registrystack.org/formats/casework/report/v1alpha2", "kind": "CaseworkReport", "command": "explain", "status": "complete", "ok": true, - "reviewPolicies": [{"dependencies": {"accessProfiles": [ - {"id": "reviewer", "principalClaim": "sub", "requiredScopes": ["casework:staff"]}, - {"id": "producer", "principalClaim": "producer_identity"} - ]}}], + "reviewPolicies": [], + "accessProfiles": [ + {"id": "producer", "principalClaim": "producer_identity"}, + {"id": "reviewer", "principalClaim": "sub"} + ], "trustedProducers": [], "contextProviders": [] }); @@ -1354,12 +1511,10 @@ mod tests { ("reviewer".to_owned(), "sub".to_owned()), ]) ); - let profiles = report - .pointer_mut("/reviewPolicies/0/dependencies/accessProfiles") - .unwrap() + report["accessProfiles"] .as_array_mut() - .unwrap(); - profiles.push(json!({"id": "reviewer", "principalClaim": "different"})); + .unwrap() + .push(json!({"id": "reviewer", "principalClaim": "different"})); assert!(casework_report_facts(&report) .unwrap_err() .to_string() @@ -1582,6 +1737,163 @@ mod tests { } } + #[test] + fn a_report_without_the_access_profile_list_is_refused() { + let report = json!({ + "apiVersion": "id.registrystack.org/formats/casework/report/v1alpha2", + "kind": "CaseworkReport", "command": "explain", "status": "complete", "ok": true, + "reviewPolicies": [{"dependencies": {"accessProfiles": [ + {"id": "reviewer", "principalClaim": "sub"} + ]}}] + }); + assert!(casework_report_facts(&report) + .unwrap_err() + .to_string() + .contains("did not report access profiles")); + } + + #[test] + fn a_failed_tool_leaves_its_stderr_in_an_owner_only_log_and_the_error() { + let scratch = tempfile::tempdir().unwrap(); + let root = scratch.path(); + let error = run_logged_tool( + Path::new("/bin/sh"), + &["-c".as_ref(), "echo boom >&2; exit 3".as_ref()], + root, + ("breg", "test"), + || Ok(()), + ) + .unwrap_err() + .to_string(); + assert!(error.contains("breg test failed"), "{error}"); + assert!(error.contains("boom"), "{error}"); + let logs = root.join("logs"); + let log = logs.join("breg-test.log"); + assert_eq!(fs::read_to_string(&log).unwrap(), "boom\n"); + assert!(error.contains(&log.display().to_string()), "{error}"); + assert_eq!( + fs::metadata(&logs).unwrap().permissions().mode() & 0o777, + 0o700 + ); + assert_eq!( + fs::metadata(&log).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + + #[test] + fn a_successful_tool_leaves_no_failure_text() { + let scratch = tempfile::tempdir().unwrap(); + run_logged_tool( + Path::new("/bin/sh"), + &["-c".as_ref(), "echo fine >&2".as_ref()], + scratch.path(), + ("breg", "test"), + || Ok(()), + ) + .unwrap(); + } + + #[test] + fn the_log_tail_keeps_the_last_twenty_lines_within_four_kib() { + let scratch = tempfile::tempdir().unwrap(); + let path = scratch.path().join("log"); + let lines: Vec<_> = (1..=30).map(|number| format!("line {number}")).collect(); + fs::write(&path, lines.join("\n")).unwrap(); + let tail = log_tail(&path); + assert_eq!(tail.lines().count(), 20); + assert!(tail.starts_with("line 11") && tail.ends_with("line 30")); + + fs::write(&path, format!("{}\nlast", "x".repeat(10_000))).unwrap(); + let tail = log_tail(&path); + assert!(tail.len() <= 4096 && tail.ends_with("last")); + + fs::write(&path, [b'a', 0xff, b'\n', b'b']).unwrap(); + assert_eq!(log_tail(&path), "a\u{fffd}\nb"); + } + + #[test] + fn spawned_processes_are_recorded_as_product_pid_and_group() { + let scratch = tempfile::tempdir().unwrap(); + write_processes( + scratch.path(), + &[(Product::Breg, 4242), (Product::Casework, 4343)], + ) + .unwrap(); + let path = scratch.path().join("processes.json"); + assert_eq!( + serde_json::from_slice::(&fs::read(&path).unwrap()).unwrap(), + json!([ + {"product": "breg", "pid": 4242, "pgid": 4242}, + {"product": "casework", "pid": 4343, "pgid": 4343} + ]) + ); + assert_eq!( + fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + + fn minimal_registry(edit: impl FnOnce(&str) -> String) -> registry_breg::CompiledRegistry { + let source = + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../products/breg/examples/minimal"); + let scratch = tempfile::tempdir().unwrap(); + let project = scratch.path().join("project"); + copy_project(&source, &project).unwrap(); + let registry = project.join("registry.yaml"); + fs::write(®istry, edit(&fs::read_to_string(®istry).unwrap())).unwrap(); + crate::compile(&project, crate::ProfileArg::Production, "project-dev-test") + .unwrap_or_else(|_| panic!("the edited project must compile")) + } + + fn breg_person(profile: &str) -> plan::TestPerson { + serde_json::from_value(json!({ + "subject": "someone", "client": "client", + "grants": {"breg": {"profile": profile, "scopes": []}} + })) + .unwrap() + } + + #[test] + fn a_breg_grant_naming_an_unknown_profile_is_refused_at_its_pointer() { + let registry = minimal_registry(str::to_owned); + let people = BTreeMap::from([ + ("editor".to_owned(), breg_person("directory-editor")), + ("typo".to_owned(), breg_person("directory-edtor")), + ]); + let error = check_breg_grants(®istry, &people) + .unwrap_err() + .to_string(); + assert!( + error.contains("/people/typo/grants/breg/profile"), + "{error}" + ); + assert!(error.contains("directory-editor"), "{error}"); + let valid = BTreeMap::from([("editor".to_owned(), breg_person("directory-editor"))]); + check_breg_grants(®istry, &valid).unwrap(); + } + + #[test] + fn encrypted_fields_are_refused_before_provisioning() { + let registry = minimal_registry(|source| { + let source = source.replacen( + "mutationMode: mutable\n display: {label: Contact", + "mutationMode: mutable\n classification: restricted\n display: {label: Contact", + 1, + ); + source.replacen( + "{id: email, type: string, required: true, minimumLength: 1, maximumLength: 254, classification: internal}\n - {id: jurisdiction", + "{id: email, type: string, required: true, minimumLength: 1, maximumLength: 254, classification: restricted, encrypted: true}\n - {id: jurisdiction", + 1, + ) + }); + assert_eq!( + refuse_encrypted_fields(®istry).unwrap_err().to_string(), + "rstack dev does not configure field encryption yet; contact.email is encrypted" + ); + refuse_encrypted_fields(&minimal_registry(str::to_owned)).unwrap(); + } + #[test] fn cancellation_closes_the_pipe_of_a_nested_tool() { closes_nested_tool_pipe(false); diff --git a/crates/registry-bregctl/tests/project_dev.rs b/crates/registry-bregctl/tests/project_dev.rs index 97d1e88e2c..e9854e3da1 100644 --- a/crates/registry-bregctl/tests/project_dev.rs +++ b/crates/registry-bregctl/tests/project_dev.rs @@ -142,6 +142,7 @@ fn fake_casework_tool(root: &std::path::Path) -> std::path::PathBuf { "printf '%s\\n' '", "{\"apiVersion\":\"id.registrystack.org/formats/casework/report/v1alpha2\",", "\"kind\":\"CaseworkReport\",\"command\":\"explain\",\"status\":\"complete\",\"ok\":true,", + "\"accessProfiles\":[{\"id\":\"reviewer\",\"principalClaim\":\"sub\"}],", "\"reviewPolicies\":[{\"dependencies\":{\"accessProfiles\":[{\"id\":\"reviewer\",\"principalClaim\":\"sub\"}]}}],", "\"trustedProducers\":[{\"id\":\"review-producer\"}],", "\"contextProviders\":[{\"id\":\"proposal-context\"}]}", From 0b327f89e1d326a786804b09d4a6dcb6126680be Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 01:45:09 +0700 Subject: [PATCH 11/34] refactor(bregctl): name every database object a dev run can own Signed-off-by: Jeremi Joslin --- .../src/project_dev/database.rs | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/crates/registry-bregctl/src/project_dev/database.rs b/crates/registry-bregctl/src/project_dev/database.rs index 98a5133c17..dc11fa2adb 100644 --- a/crates/registry-bregctl/src/project_dev/database.rs +++ b/crates/registry-bregctl/src/project_dev/database.rs @@ -518,6 +518,33 @@ impl Database { fn product_url(&self, database: &str, user: &str, password: &str) -> Result { self.endpoint.product_url(database, user, password) } + + /// Names of the databases and roles this run still owns, oldest first. + #[allow(dead_code)] + pub(super) fn pending_names(&self) -> Vec { + self.acquisitions + .iter() + .map(|acquisition| match acquisition { + Acquisition::Database(name) | Acquisition::Role(name) => name.clone(), + }) + .collect() + } +} + +/// Every database and role name `provision` can create for one label of a +/// run, in creation order. Recovery after a lost supervisor uses this list, +/// since the spatial bbox role does not carry the run id. +#[allow(dead_code)] +pub(super) fn run_object_names(run_id: &str, label: &str, spatial: bool) -> Result> { + validate_run_id(run_id)?; + validate_label(label)?; + let names = Names::new(run_id, label); + let mut owned = vec![names.migration_role.clone(), names.runtime_role.clone()]; + if spatial { + owned.push(spatial_bbox_role(&names)?); + } + owned.push(names.database); + Ok(owned) } struct Names { @@ -986,6 +1013,23 @@ mod tests { } } + #[test] + fn run_object_names_list_every_name_provision_creates() { + for (label, spatial) in [("breg", true), ("breg_test", false), ("casework", false)] { + let names = Names::new(RUN_ID, label); + let mut expected = vec![names.migration_role.clone(), names.runtime_role.clone()]; + if spatial { + let bbox = spatial_bbox_role(&names).unwrap(); + assert!(!bbox.contains(RUN_ID), "the bbox role is not run-scoped"); + expected.push(bbox); + } + expected.push(names.database.clone()); + assert_eq!(run_object_names(RUN_ID, label, spatial).unwrap(), expected); + } + assert!(run_object_names("not-a-run", "breg", false).is_err()); + assert!(run_object_names(RUN_ID, "other", false).is_err()); + } + #[test] fn casework_setup_gives_only_migration_role_public_schema_creation() { let names = Names::new(RUN_ID, "casework"); From 4b776d6d7c73b07e4e31cf3cd6c37648a20b4269 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 01:51:52 +0700 Subject: [PATCH 12/34] fix(bregctl): explain failed dev journey steps and report interruptions distinctly Signed-off-by: Jeremi Joslin --- .../src/project_dev/journey.rs | 202 +++++++++++++++--- 1 file changed, 176 insertions(+), 26 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/journey.rs b/crates/registry-bregctl/src/project_dev/journey.rs index 742a38d617..3c11b455f7 100644 --- a/crates/registry-bregctl/src/project_dev/journey.rs +++ b/crates/registry-bregctl/src/project_dev/journey.rs @@ -36,6 +36,21 @@ pub(super) struct StepReport { status: u16, expected_status: u16, result: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + reason: Option, +} + +const REASON_LIMIT: usize = 512; + +fn bounded_reason(text: &str) -> String { + text.chars().take(REASON_LIMIT).collect() +} + +/// What one step observed. `reason` is set when the step failed. +struct StepOutcome { + status: u16, + reason: Option, + updates: BTreeMap, } struct ResponseData { @@ -82,15 +97,29 @@ pub(super) fn run( &mut transferred, &mut attempts, ); - let (status, passed, updates) = outcome.unwrap_or_else(|_| (0, false, BTreeMap::new())); + let outcome = match outcome { + Ok(outcome) => outcome, + Err(error) => { + // A stop signal or the startup deadline interrupts the + // journey; it is not a failure of this step. + active()?; + StepOutcome { + status: 0, + reason: Some(bounded_reason(&format!("{error:#}"))), + updates: BTreeMap::new(), + } + } + }; + let passed = outcome.reason.is_none(); if passed { - captures.extend(updates); + captures.extend(outcome.updates); } report.steps.push(StepReport { id: step.id.clone(), - status, + status: outcome.status, expected_status: step.expect.status.first(), result: if passed { "passed" } else { "failed" }, + reason: outcome.reason, }); if !passed { report.result = "failed"; @@ -112,7 +141,7 @@ fn execute_step( active: &impl Fn() -> Result<()>, transferred: &mut usize, attempts: &mut usize, -) -> Result<(u16, bool, BTreeMap)> { +) -> Result { let lookup = |name: &str| { captures .get(name) @@ -157,11 +186,12 @@ fn execute_step( .as_ref() .map(|poll| Instant::now() + Duration::from_millis(poll.timeout_ms)); let mut last_status = 0; + let mut last_reason = None; loop { active()?; if poll_deadline.is_some_and(|deadline| Instant::now() >= deadline) { - return Ok((last_status, false, BTreeMap::new())); + return Ok(poll_timed_out(last_status, last_reason)); } add_attempt(attempts)?; add_transfer(transferred, request_bytes)?; @@ -187,26 +217,53 @@ fn execute_step( let Some(response) = read_response_before_deadline(runtime, request, poll_deadline, transferred)? else { - return Ok((last_status, false, BTreeMap::new())); + return Ok(poll_timed_out(last_status, last_reason)); }; last_status = response.status; - let captures = matches_expectation(step, &response, lookup)? - .then(|| capture_updates(step, &response, lookup)) - .and_then(Result::ok); - if let Some(updates) = captures { - return Ok((response.status, true, updates)); - } + let reason = match first_mismatch(step, &response, lookup)? { + Some(reason) => reason, + None => match capture_updates(step, &response, lookup) { + Ok(updates) => { + return Ok(StepOutcome { + status: response.status, + reason: None, + updates, + }) + } + Err(error) => format!("capture failed: {error:#}"), + }, + }; + let failed = |reason: &str| StepOutcome { + status: response.status, + reason: Some(bounded_reason(reason)), + updates: BTreeMap::new(), + }; let Some((poll, deadline)) = step.poll.as_ref().zip(poll_deadline) else { - return Ok((response.status, false, BTreeMap::new())); + return Ok(failed(&reason)); }; let now = Instant::now(); if now >= deadline { - return Ok((response.status, false, BTreeMap::new())); + return Ok(failed(&reason)); } + last_reason = Some(reason); thread::sleep(Duration::from_millis(poll.interval_ms).min(deadline - now)); } } +/// A poll whose deadline passed keeps the last completed status and says why +/// that response did not match. +fn poll_timed_out(status: u16, last_reason: Option) -> StepOutcome { + let reason = match last_reason { + Some(reason) => format!("poll deadline passed; last response: {reason}"), + None => "poll deadline passed before a response arrived".to_owned(), + }; + StepOutcome { + status, + reason: Some(bounded_reason(&reason)), + updates: BTreeMap::new(), + } +} + fn request_url(base: &str, path: &str) -> Result { ensure!(plan::valid_path(path), "invalid journey path"); let base = url::Url::parse(base)?; @@ -234,32 +291,68 @@ fn request_url(base: &str, path: &str) -> Result { Ok(url) } -fn matches_expectation( +/// Describe the first expectation the response misses, as +/// ` expected observed `. +fn first_mismatch( step: &Step, response: &ResponseData, lookup: impl Fn(&str) -> Result + Copy, -) -> Result { +) -> Result> { + let observed_at = |pointer: &str| { + response + .body + .pointer(pointer) + .map_or_else(|| "nothing".to_owned(), Value::to_string) + }; if !step.expect.status.contains(response.status) { - return Ok(false); + return Ok(Some(format!( + "/status expected {} observed {}", + serde_json::to_string(&step.expect.status)?, + response.status + ))); } for (pointer, expected) in &step.expect.json { - if response.body.pointer(pointer) != Some(&expand_json(expected, lookup)?) { - return Ok(false); + let expected = expand_json(expected, lookup)?; + if response.body.pointer(pointer) != Some(&expected) { + return Ok(Some(format!( + "{pointer} expected {expected} observed {}", + observed_at(pointer) + ))); } } if let Some(code) = &step.expect.code { - if response.body.pointer("/code").and_then(Value::as_str) - != Some(expand_text(code, lookup)?.as_str()) - { - return Ok(false); + let expected = expand_text(code, lookup)?; + if response.body.pointer("/code").and_then(Value::as_str) != Some(expected.as_str()) { + return Ok(Some(format!( + "/code expected {} observed {}", + Value::String(expected), + observed_at("/code") + ))); } } for check in &step.expect.checks { if !matches_check(check, &response.body, lookup)? { - return Ok(false); + return Ok(Some(format!( + "{} expected {} observed {}", + check.pointer, + describe_check(check, lookup)?, + observed_at(&check.pointer) + ))); } } - Ok(true) + Ok(None) +} + +fn describe_check(check: &Check, lookup: impl Fn(&str) -> Result + Copy) -> Result { + Ok(if let Some(expected) = &check.equals { + expand_json(expected, lookup)?.to_string() + } else if let Some(length) = check.length { + format!("length {length}") + } else if let Some(expected) = &check.contains { + format!("containing {}", expand_json(expected, lookup)?) + } else { + "a check".to_owned() + }) } fn matches_check( @@ -659,6 +752,10 @@ mod tests { assert_eq!(report.result, "failed"); assert_eq!(report.steps[0].status, 202); + assert!(report.steps[0] + .reason + .as_deref() + .is_some_and(|reason| reason.starts_with("poll deadline passed; last response: "))); assert_eq!(server.finish(), 2); } @@ -677,6 +774,10 @@ mod tests { assert_eq!(report.result, "failed"); assert_eq!(report.steps[0].status, 202); + assert!(report.steps[0] + .reason + .as_deref() + .is_some_and(|reason| reason.starts_with("poll deadline passed; last response: "))); assert_eq!(server.finish(), 1); } @@ -772,10 +873,59 @@ mod tests { headers: HeaderMap::new(), body: json!({"code": "missing"}), }; - assert!(matches_expectation(&step, &response, no_capture).unwrap()); + assert!(first_mismatch(&step, &response, no_capture) + .unwrap() + .is_none()); assert_eq!(step.expect.status.first(), 404); } + #[test] + fn mismatch_names_the_pointer_expected_and_observed_values() { + let step: Step = serde_json::from_value(json!({ + "id": "read", "product": "breg", "as": "person", "method": "GET", + "path": "/v1/records", "expect": {"status": 200, "json": {"/state": "open"}} + })) + .unwrap(); + let mut response = ResponseData { + status: 500, + headers: HeaderMap::new(), + body: json!({"state": "closed"}), + }; + assert_eq!( + first_mismatch(&step, &response, no_capture) + .unwrap() + .unwrap(), + "/status expected 200 observed 500" + ); + response.status = 200; + assert_eq!( + first_mismatch(&step, &response, no_capture) + .unwrap() + .unwrap(), + "/state expected \"open\" observed \"closed\"" + ); + response.body = json!({"state": "open"}); + assert!(first_mismatch(&step, &response, no_capture) + .unwrap() + .is_none()); + } + + #[test] + fn reasons_are_bounded_and_omitted_when_the_step_passes() { + assert_eq!(bounded_reason(&"é".repeat(2000)).chars().count(), 512); + let report = StepReport { + id: "a".into(), + status: 200, + expected_status: 200, + result: "passed", + reason: None, + }; + assert!(serde_json::to_value(&report) + .unwrap() + .get("reason") + .is_none()); + } + #[test] fn find_capture_requires_one_matching_item() { let capture: Capture = serde_json::from_value(json!({ From 449c43fa9cc06159a62db1435b51df4de81ca421 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 01:51:53 +0700 Subject: [PATCH 13/34] fix(bregctl): keep failed dev run logs, bound cleanup and report a lost supervisor with recovery facts Signed-off-by: Jeremi Joslin --- .../src/project_dev/database.rs | 2 - .../registry-bregctl/src/project_dev/mod.rs | 548 ++++++++++++++++-- 2 files changed, 514 insertions(+), 36 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/database.rs b/crates/registry-bregctl/src/project_dev/database.rs index dc11fa2adb..0da1236a39 100644 --- a/crates/registry-bregctl/src/project_dev/database.rs +++ b/crates/registry-bregctl/src/project_dev/database.rs @@ -520,7 +520,6 @@ impl Database { } /// Names of the databases and roles this run still owns, oldest first. - #[allow(dead_code)] pub(super) fn pending_names(&self) -> Vec { self.acquisitions .iter() @@ -534,7 +533,6 @@ impl Database { /// Every database and role name `provision` can create for one label of a /// run, in creation order. Recovery after a lost supervisor uses this list, /// since the spatial bbox role does not carry the run id. -#[allow(dead_code)] pub(super) fn run_object_names(run_id: &str, label: &str, spatial: bool) -> Result> { validate_run_id(run_id)?; validate_label(label)?; diff --git a/crates/registry-bregctl/src/project_dev/mod.rs b/crates/registry-bregctl/src/project_dev/mod.rs index cd609fe02e..b0d675a35b 100644 --- a/crates/registry-bregctl/src/project_dev/mod.rs +++ b/crates/registry-bregctl/src/project_dev/mod.rs @@ -47,6 +47,8 @@ const REQUEST: &str = "RSTACK_PROJECT_DEV_REQUEST"; const DATABASE_URL: &str = "RSTACK_DEV_DATABASE_URL"; const CA_FILE: &str = "RSTACK_DEV_DATABASE_CA"; const REPORT: &str = "report.json"; +const PENDING: &str = "cleanup-pending.json"; +const CLEANUP_CAP: Duration = Duration::from_secs(30 * 60); const POLL: Duration = Duration::from_millis(100); // A capture can contain the 1 MiB development plan after JSON escaping, // selections derived from a second 1 MiB journeys document, facts selected @@ -143,6 +145,140 @@ struct CleanupPending { detail: String, } +/// A run that cannot finish by itself: its supervisor is gone, or cleanup +/// gave up. `facts` carries what an operator needs to release it by hand. +#[derive(Debug, thiserror::Error)] +#[error("{}", recovery_message(run_id, facts))] +struct RecoveryNeeded { + run_id: String, + facts: Value, +} + +fn recovery_message(run_id: &str, facts: &Value) -> String { + let mut text = match facts["kind"].as_str() { + Some("cleanup-pending") => { + format!("run {run_id} gave up cleanup; release these resources by hand") + } + _ => format!( + "run {run_id} lost its supervisor before cleanup; release these resources by hand" + ), + }; + let mut line = |entry: String| { + text.push_str("\n "); + text.push_str(&entry); + }; + if let Some(directory) = facts["run_directory"].as_str() { + line(format!("run directory {directory}")); + } + for process in facts["processes"].as_array().into_iter().flatten() { + line(format!( + "process {} pid {} pgid {}", + process["product"], process["pid"], process["pgid"] + )); + } + for object in facts["objects"].as_array().into_iter().flatten() { + let roles: Vec<_> = object["roles"] + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .collect(); + line(format!( + "{} database {} roles {}", + object["label"].as_str().unwrap_or("?"), + object["database"].as_str().unwrap_or("?"), + roles.join(", ") + )); + } + for name in facts["databases_and_roles"] + .as_array() + .into_iter() + .flatten() + { + line(format!("database or role {}", name.as_str().unwrap_or("?"))); + } + if let Some(reason) = facts["unavailable"].as_str() { + line(format!("some facts are unavailable: {reason}")); + } + text +} + +/// Everything a lost supervisor leaves behind: the processes it recorded and +/// every database and role its plan provisions. +fn lost_supervisor_facts(root: &Path, run_id: &str) -> Value { + let mut facts = json!({ + "kind": "lost-supervisor", + "run_directory": root, + "processes": recorded_processes(root), + "objects": [], + }); + match provisioned_objects(root, run_id) { + Ok(objects) => facts["objects"] = json!(objects), + Err(error) => facts["unavailable"] = json!(format!("{error:#}")), + } + facts +} + +fn recorded_processes(root: &Path) -> Value { + read_json::(&root.join("processes.json"), MAXIMUM_RUN_REPORT_BYTES) + .ok() + .filter(Value::is_array) + .unwrap_or_else(|| json!([])) +} + +fn provisioned_objects(root: &Path, run_id: &str) -> Result> { + let input: RunInput = read_json(&root.join("input.json"), MAXIMUM_CAPTURED_INPUT_BYTES) + .context("cannot read the captured development plan")?; + let spatial = input + .breg_authority + .as_ref() + .is_some_and(|authority| authority.requires_postgis); + let mut labels = Vec::new(); + for entry in &input.plan.products { + match entry.product { + Product::Breg => { + labels.push(("breg_test", spatial)); + labels.push(("breg", spatial)); + } + Product::Casework => labels.push(("casework", false)), + } + } + labels + .into_iter() + .map(|(label, spatial)| { + let mut names = database::run_object_names(run_id, label, spatial)?; + let database = names.pop().context("run has no database name")?; + Ok(json!({"label": label, "database": database, "roles": names})) + }) + .collect() +} + +fn pending_report(run_id: &str, names: Vec, processes: Value) -> Value { + json!({"run_id": run_id, "databases_and_roles": names, "processes": processes}) +} + +/// The delays between cleanup attempts: 2 s doubling to 30 s, until their +/// sum reaches the cap. +fn cleanup_schedule(cap: Duration) -> impl Iterator { + let mut next = Duration::from_secs(2); + let mut total = Duration::ZERO; + std::iter::from_fn(move || { + if total >= cap { + return None; + } + let delay = next; + total += delay; + next = (next * 2).min(Duration::from_secs(30)); + Some(delay) + }) +} + +/// The wall clock decides expiry, because the monotonic clock stops while a +/// laptop sleeps. +fn expired(now_unix: u64, expires_at: u64) -> bool { + now_unix >= expires_at +} + fn startup_error(root: &Path, run_id: &str, error: anyhow::Error) -> anyhow::Error { if root.exists() { CleanupPending { @@ -209,6 +345,10 @@ pub(crate) fn run(args: Args, stdout: &mut dyn Write, stderr: &mut dyn Write) -> report["run_id"] = json!(pending.run_id); report["cleanup_pending"] = json!(true); } + if let Some(recovery) = error.downcast_ref::() { + report["run_id"] = json!(recovery.run_id); + report["recovery"] = recovery.facts.clone(); + } (writeln!(stdout, "{report}"), 1) } Err(error) => (writeln!(stderr, "rstack dev: {error:#}"), 1), @@ -227,6 +367,12 @@ fn emit(output: &mut dyn Write, report: &Value, machine: bool) -> std::io::Resul let run = report["run_id"].as_str().expect("run report"); let status = report["status"].as_str().expect("run status"); writeln!(output, "Run {run} {status}")?; + if status == "cleanup-pending" { + let message = recovery_message(run, &report["recovery"]); + for line in message.lines().skip(1) { + writeln!(output, "{line}")?; + } + } if let Some(products) = report["products"].as_object() { for (product, url) in products { writeln!(output, " {product} {}", url.as_str().expect("product URL"))?; @@ -245,9 +391,15 @@ fn emit(output: &mut dyn Write, report: &Value, machine: bool) -> std::io::Resul " {} expected HTTP {}, observed {}", step["id"], step["expected_status"], step["status"] )?; + if let Some(reason) = step["reason"].as_str() { + writeln!(output, " {reason}")?; + } } } } + if let Some(logs) = report["logs"].as_str() { + writeln!(output, " Logs {logs}")?; + } if let Some(people) = report["people"].as_array() { for name in people { writeln!( @@ -393,6 +545,7 @@ fn start(project: &Path, ttl: Option<&str>) -> Result { }; let preparation = (|| { write_private(&root.join("input.json"), &capture_input(&input)?)?; + let supervisor_log = open_supervisor_log(&root)?; Command::new(env::current_exe()?) .arg("project-dev") .arg("--worker") @@ -401,15 +554,16 @@ fn start(project: &Path, ttl: Option<&str>) -> Result { .process_group(0) .stdin(Stdio::null()) .stdout(Stdio::piped()) - .stderr(Stdio::null()) + .stderr(Stdio::from(supervisor_log)) .spawn() .context("cannot start the development run supervisor") })(); let mut child = match preparation { Ok(child) => child, Err(error) => { - fs::remove_dir_all(&root).context("cannot remove unstarted run directory")?; - return Err(error); + let kept = remove_failed_run(&root, &input.run_id) + .context("cannot remove unstarted run directory")?; + return Err(note_kept_logs(error, kept)); } }; let stream = child.stdout.take().expect("startup channel"); @@ -417,7 +571,7 @@ fn start(project: &Path, ttl: Option<&str>) -> Result { thread::spawn(move || { let mut line = String::new(); let result = BufReader::new(stream) - .take(512 * 1024) + .take(MAXIMUM_RUN_REPORT_BYTES) .read_line(&mut line) .context("cannot read startup report") .and_then(|_| { @@ -432,6 +586,7 @@ fn start(project: &Path, ttl: Option<&str>) -> Result { Ok(Ok(WorkerReply::Ready { report })) => return Ok(report), Ok(Ok(WorkerReply::Failed { error })) => { let error = anyhow::anyhow!(error); + let kept_logs = failed_logs_dir(&base, &input.run_id); if root.join("worker.lock").exists() { if let Err(cleanup) = stop(&input.run_id) { return Err(startup_error( @@ -443,16 +598,21 @@ fn start(project: &Path, ttl: Option<&str>) -> Result { )); } } else if root.exists() { - fs::remove_dir_all(&root).context("cannot remove unstarted run directory")?; + remove_failed_run(&root, &input.run_id) + .context("cannot remove unstarted run directory")?; } - return Err(error); + return Err(note_kept_logs( + error, + kept_logs.exists().then_some(kept_logs), + )); } Ok(Err(error)) => { if child.try_wait()?.is_some() && root.exists() && !root.join("worker.lock").exists() { - fs::remove_dir_all(&root).context("cannot remove unstarted run directory")?; + remove_failed_run(&root, &input.run_id) + .context("cannot remove unstarted run directory")?; } return Err(startup_error(&root, &input.run_id, error)); } @@ -500,6 +660,7 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { .open(root.join("worker.lock"))?; flock(&lock, FlockOperation::LockExclusive)?; let mut database = None; + let mut expiry = None; let prepared = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { products.check()?; let ca = env::var_os(CA_FILE) @@ -523,12 +684,13 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { "a product exited during the journey" ); let expires_at = clock()? + input.ttl_seconds; + expiry = Some(expires_at); let report = json!({ "schema_version": SCHEMA, "run_id": input.run_id, "status": "ready", "products": urls, "people": input.plan.people.keys().collect::>(), - "journey": journey, "expires_at": expires_at, + "journey": journey, "expires_at": expires_at, "logs": root.join("logs"), }); - write_private(&root.join(REPORT), &serde_json::to_vec(&report)?)?; + write_private(&root.join(REPORT), &capture_report(&report)?)?; Ok::<_, anyhow::Error>(report) })) .unwrap_or_else(|_| Err(anyhow::anyhow!("development run startup failed internally"))); @@ -540,7 +702,10 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { failure = Some(format!("cannot report development run readiness: {error}")); } else { let deadline = Instant::now() + Duration::from_secs(input.ttl_seconds); - while Instant::now() < deadline && !stopping(root, &signals) { + while Instant::now() < deadline + && !stopping(root, &signals) + && !expiry.is_some_and(|at| clock().map_or(true, |now| expired(now, at))) + { if !products.services_alive().unwrap_or(false) { break; } @@ -553,6 +718,7 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { let stopped = simple_report(&input.run_id, "stopping"); let mark_stopping = write_private(&root.join(REPORT), &serde_json::to_vec(&stopped)?); let mut told_parent = false; + let mut delays = cleanup_schedule(CLEANUP_CAP); loop { // Keep the exact ownership list until both the supervised processes // and their database resources have been released. @@ -571,9 +737,28 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { ); told_parent = true; } - thread::sleep(Duration::from_secs(2)); + let Some(delay) = delays.next() else { + let names = database + .as_ref() + .map_or_else(Vec::new, Database::pending_names); + let pending = pending_report(&input.run_id, names, recorded_processes(root)); + write_private(&root.join(PENDING), &serde_json::to_vec(&pending)?)?; + bail!( + "run {} gave up cleanup after {} minutes; run rstack dev status {} for the resources to release", + input.run_id, + CLEANUP_CAP.as_secs() / 60, + input.run_id + ); + }; + thread::sleep(delay); + } + // A failed run keeps its logs outside the run directory. + let removed = if failure.is_some() { + keep_logs(root, &input.run_id).map(drop) + } else { + Ok(()) } - let removed = fs::remove_dir_all(root).context("cannot remove run scratch files"); + .and_then(|()| fs::remove_dir_all(root).context("cannot remove run scratch files")); if let Some(error) = failure.filter(|_| !told_parent) { let error = match removed.as_ref() { Ok(()) => error, @@ -600,10 +785,16 @@ fn status(run_id: &str) -> Result { return Ok(simple_report(run_id, "stopped")); } let observed = (|| { - ensure!( - supervisor_running(&root)?, - "run {run_id} lost its supervisor before cleanup" - ); + if let Ok(pending) = read_json::(&root.join(PENDING), MAXIMUM_RUN_REPORT_BYTES) { + return Ok(pending_status(&root, run_id, pending)); + } + if supervisor_gone(&root)? { + return Err(RecoveryNeeded { + run_id: run_id.to_owned(), + facts: lost_supervisor_facts(&root, run_id), + } + .into()); + } let report: Value = read_json(&root.join(REPORT), MAXIMUM_RUN_REPORT_BYTES) .context("the development run has not reported readiness")?; if report["expires_at"] @@ -620,6 +811,17 @@ fn status(run_id: &str) -> Result { observed } +fn pending_status(root: &Path, run_id: &str, pending: Value) -> Value { + let mut report = simple_report(run_id, "cleanup-pending"); + report["recovery"] = json!({ + "kind": "cleanup-pending", + "run_directory": root, + "databases_and_roles": pending["databases_and_roles"], + "processes": pending["processes"], + }); + report +} + fn token(run_id: &str, person: &str) -> Result { ensure!(plan::identifier(person), "invalid test person name"); let report = status(run_id)?; @@ -657,7 +859,7 @@ fn token(run_id: &str, person: &str) -> Result { } Ok(json!({ "schema_version": SCHEMA, "run_id": run_id, "status": "token", - "alias": person, "credentials": credentials, + "alias": person, "credentials": credentials, "expires_at": expires, })) } @@ -666,7 +868,13 @@ fn stop(run_id: &str) -> Result { if !root.exists() { return Ok(simple_report(run_id, "stopped")); } - ensure_private(&root, false)?; + if let Err(error) = ensure_private(&root, false) { + // The supervisor can finish and remove the run between the checks. + if !root.exists() { + return Ok(simple_report(run_id, "stopped")); + } + return Err(error); + } match OpenOptions::new() .write(true) .create_new(true) @@ -684,14 +892,26 @@ fn stop(run_id: &str) -> Result { while root.exists() { ensure!( Instant::now() < deadline, - "run {run_id} has not completed cleanup; check the PostgreSQL server and retry stop" + "run {run_id} is still cleaning up or its cleanup has stalled; run rstack dev status {run_id} to see its state" ); thread::sleep(POLL); - if root.exists() && root.join(REPORT).exists() { - let alive = supervisor_running(&root); - if root.exists() { - ensure!(alive?, "run {run_id} lost its supervisor before cleanup"); + if !root.exists() { + break; + } + if let Ok(pending) = read_json::(&root.join(PENDING), MAXIMUM_RUN_REPORT_BYTES) { + return Err(RecoveryNeeded { + run_id: run_id.to_owned(), + facts: pending_status(&root, run_id, pending)["recovery"].clone(), + } + .into()); + } + let gone = supervisor_gone(&root); + if root.exists() && gone? { + return Err(RecoveryNeeded { + run_id: run_id.to_owned(), + facts: lost_supervisor_facts(&root, run_id), } + .into()); } } Ok(simple_report(run_id, "stopped")) @@ -701,6 +921,21 @@ fn simple_report(run_id: &str, status: &str) -> Value { json!({"schema_version": SCHEMA, "run_id": run_id, "status": status}) } +/// Whether a supervisor that took its lock has since vanished. The lock file +/// appears just before the lock is taken, so a first sighting is confirmed +/// after one poll before it counts. +fn supervisor_gone(root: &Path) -> Result { + for attempt in 0..2 { + if !root.join("worker.lock").exists() || supervisor_running(root)? { + return Ok(false); + } + if attempt == 0 { + thread::sleep(POLL); + } + } + Ok(true) +} + fn supervisor_running(root: &Path) -> Result { ensure_private(root, false)?; let lock = @@ -724,10 +959,13 @@ fn stopping(root: &Path, signals: &AtomicBool) -> bool { signals.load(Ordering::Relaxed) || root.join("stop").exists() } -fn binary(name: &str) -> Result { - let path = env::var_os(name) - .map(PathBuf::from) - .with_context(|| format!("{name} is required for the development run"))?; +fn binary(product: Product, name: &str) -> Result { + let path = env::var_os(name).map(PathBuf::from).with_context(|| { + format!( + "{name} is not set; run rstack install {} or set {name}", + product.name() + ) + })?; let path = path .canonicalize() .with_context(|| format!("cannot resolve {name}"))?; @@ -736,21 +974,24 @@ fn binary(name: &str) -> Result { } fn product_tool(product: Product) -> Result { - binary(match product { - Product::Breg => "RSTACK_BREGCTL_BIN", - Product::Casework => "RSTACK_CASEWORKCTL_BIN", - }) + binary( + product, + match product { + Product::Breg => "RSTACK_BREGCTL_BIN", + Product::Casework => "RSTACK_CASEWORKCTL_BIN", + }, + ) } fn binaries(product: Product) -> Result { Ok(match product { Product::Breg => Binaries { tool: product_tool(product)?, - runtime: binary("RSTACK_BREG_BIN")?, + runtime: binary(product, "RSTACK_BREG_BIN")?, }, Product::Casework => Binaries { tool: product_tool(product)?, - runtime: binary("RSTACK_CASEWORK_BIN")?, + runtime: binary(product, "RSTACK_CASEWORK_BIN")?, }, }) } @@ -826,6 +1067,82 @@ fn capture_input(input: &RunInput) -> Result> { Ok(bytes) } +fn capture_report(report: &Value) -> Result> { + let bytes = serde_json::to_vec(report)?; + ensure!( + bytes.len() as u64 <= MAXIMUM_RUN_REPORT_BYTES, + "development run report exceeds the internal size limit" + ); + Ok(bytes) +} + +/// Create `/logs` and the owner-only supervisor log inside it. +fn open_supervisor_log(root: &Path) -> Result { + match private_directory(&root.join("logs")) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error).context("cannot create the development log directory"), + } + OpenOptions::new() + .create(true) + .append(true) + .mode(0o600) + .open(root.join("logs").join("supervisor.log")) + .context("cannot open the supervisor log") +} + +fn failed_logs_dir(base: &Path, run_id: &str) -> PathBuf { + base.join("failed").join(run_id).join("logs") +} + +fn has_log_content(logs: &Path) -> Result { + let entries = match fs::read_dir(logs) { + Ok(entries) => entries, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error).context("cannot read the development log directory"), + }; + for entry in entries { + if entry?.metadata()?.len() > 0 { + return Ok(true); + } + } + Ok(false) +} + +/// Move a failed run's non-empty logs to `/failed//logs`. +fn keep_logs(root: &Path, run_id: &str) -> Result> { + let logs = root.join("logs"); + if !has_log_content(&logs)? { + return Ok(None); + } + let base = root.parent().context("run directory has no parent")?; + let target = failed_logs_dir(base, run_id); + let run = target.parent().expect("failed run directory"); + let failed = run.parent().expect("failed directory"); + ensure_private(failed, true)?; + match private_directory(run) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error).context("cannot create the failed run log directory"), + } + ensure_private(run, false)?; + fs::rename(&logs, &target).context("cannot keep the failed run logs")?; + Ok(Some(target)) +} + +fn remove_failed_run(root: &Path, run_id: &str) -> Result> { + let kept = keep_logs(root, run_id)?; + fs::remove_dir_all(root)?; + Ok(kept) +} + +fn note_kept_logs(error: anyhow::Error, kept: Option) -> anyhow::Error { + match kept { + Some(path) => anyhow::anyhow!("{error:#}; logs kept at {}", path.display()), + None => error, + } +} + fn read_json(path: &Path, limit: u64) -> Result { Ok(serde_json::from_slice(&plan::read(path, limit)?)?) } @@ -844,7 +1161,36 @@ fn clock() -> Result { #[cfg(test)] mod tests { - use super::parse_ttl; + use super::*; + + const RUN_ID: &str = "0123456789abcdef0123456789abcdef"; + + fn fake_run(root: &Path, products: Value, postgis: bool) { + let input = json!({ + "run_id": RUN_ID, "project": "/project", "ttl_seconds": 60, + "plan": { + "issuer": "https://issuer.example.invalid", + "products": products, + "people": {"operator": { + "subject": "operator", "client": "operator-client", + "grants": {"breg": {"profile": "operator", "scopes": []}} + }}, + "journey": {"id": "smoke", "steps": [{ + "id": "read", "product": "breg", "as": "operator", + "method": "GET", "path": "/v1/records", + "expect": {"status": 200} + }]} + }, + "schema_people": [], + "breg_authority": { + "principal": "p", "purpose": false, + "requires_postgis": postgis, "named_clients": [] + }, + "casework_principals": {}, + "binaries": {} + }); + fs::write(root.join("input.json"), input.to_string()).unwrap(); + } #[test] fn ttl_is_bounded_and_requires_a_unit() { @@ -855,4 +1201,138 @@ mod tests { assert!(parse_ttl(invalid).is_err(), "{invalid}"); } } + + #[test] + fn expiry_follows_the_wall_clock() { + assert!(!expired(99, 100)); + assert!(expired(100, 100)); + assert!(expired(101, 100)); + } + + #[test] + fn cleanup_backs_off_from_two_to_thirty_seconds_and_stops_at_the_cap() { + let delays: Vec<_> = cleanup_schedule(CLEANUP_CAP).map(|d| d.as_secs()).collect(); + assert_eq!(delays[..6], [2, 4, 8, 16, 30, 30]); + let total: u64 = delays.iter().sum(); + assert!(total >= CLEANUP_CAP.as_secs()); + assert!(total - delays.last().unwrap() < CLEANUP_CAP.as_secs()); + assert_eq!(cleanup_schedule(Duration::ZERO).count(), 0); + } + + #[test] + fn lost_supervisor_facts_name_processes_databases_and_roles() { + let scratch = tempfile::tempdir().unwrap(); + let root = scratch.path(); + fake_run( + root, + json!([ + {"product": "breg", "project": ".", "audience": "registry"}, + {"product": "casework", "project": ".", "audience": "casework"} + ]), + true, + ); + fs::write( + root.join("processes.json"), + json!([{"product": "breg", "pid": 4242, "pgid": 4242}]).to_string(), + ) + .unwrap(); + let facts = lost_supervisor_facts(root, RUN_ID); + assert_eq!(facts["kind"], "lost-supervisor"); + assert_eq!(facts["run_directory"], json!(root)); + assert_eq!(facts["processes"][0]["pgid"], 4242); + let labels: Vec<_> = facts["objects"] + .as_array() + .unwrap() + .iter() + .map(|object| object["label"].as_str().unwrap()) + .collect(); + assert_eq!(labels, ["breg_test", "breg", "casework"]); + let breg = &facts["objects"][1]; + assert_eq!(breg["database"], format!("rstack_{RUN_ID}_breg")); + // Migration, runtime and the spatial bbox role. + assert_eq!(breg["roles"].as_array().unwrap().len(), 3); + assert_eq!(facts["objects"][2]["roles"].as_array().unwrap().len(), 2); + let text = recovery_message(RUN_ID, &facts); + assert!( + text.contains("process \"breg\" pid 4242 pgid 4242"), + "{text}" + ); + assert!( + text.contains(&format!("breg database rstack_{RUN_ID}_breg")), + "{text}" + ); + } + + #[test] + fn lost_supervisor_facts_survive_missing_files() { + let scratch = tempfile::tempdir().unwrap(); + let facts = lost_supervisor_facts(scratch.path(), RUN_ID); + assert_eq!(facts["processes"], json!([])); + assert!(facts["unavailable"].is_string()); + } + + #[test] + fn a_dead_supervisor_is_detected_only_once_its_lock_file_exists() { + let scratch = tempfile::tempdir().unwrap(); + let root = scratch.path().canonicalize().unwrap(); + fs::set_permissions(&root, std::os::unix::fs::PermissionsExt::from_mode(0o700)).unwrap(); + assert!(!supervisor_gone(&root).unwrap()); + let lock = File::create(root.join("worker.lock")).unwrap(); + assert!(supervisor_gone(&root).unwrap()); + flock(&lock, FlockOperation::NonBlockingLockExclusive).unwrap(); + assert!(!supervisor_gone(&root).unwrap()); + } + + #[test] + fn pending_cleanup_reports_resources_and_processes() { + let report = pending_report(RUN_ID, vec!["db".into(), "role".into()], json!([])); + assert_eq!(report["databases_and_roles"], json!(["db", "role"])); + let status = pending_status(Path::new("/run"), RUN_ID, report); + assert_eq!(status["status"], "cleanup-pending"); + assert_eq!(status["recovery"]["databases_and_roles"][1], "role"); + let text = recovery_message(RUN_ID, &status["recovery"]); + assert!(text.contains("database or role role"), "{text}"); + } + + #[test] + fn failed_run_logs_move_out_of_the_run_directory() { + let scratch = tempfile::tempdir().unwrap(); + let base = scratch.path().canonicalize().unwrap(); + let root = base.join(RUN_ID); + private_directory(&root).unwrap(); + open_supervisor_log(&root).unwrap(); + assert_eq!(keep_logs(&root, RUN_ID).unwrap(), None, "empty logs stay"); + fs::write(root.join("logs").join("breg-check.log"), "boom").unwrap(); + let kept = remove_failed_run(&root, RUN_ID).unwrap().unwrap(); + assert_eq!(kept, failed_logs_dir(&base, RUN_ID)); + assert!(!root.exists()); + assert_eq!( + fs::read_to_string(kept.join("breg-check.log")).unwrap(), + "boom" + ); + assert_eq!( + fs::metadata(base.join("failed")).unwrap().mode() & 0o777, + 0o700 + ); + assert_eq!( + fs::metadata(kept.join("supervisor.log")).unwrap().mode() & 0o777, + 0o600 + ); + // `failed` is not a run. + assert!(run_dir("failed").is_err()); + } + + #[test] + fn run_reports_are_bounded() { + assert!(capture_report(&json!({"a": 1})).is_ok()); + let big = json!({"a": "x".repeat(MAXIMUM_RUN_REPORT_BYTES as usize)}); + assert!(capture_report(&big).is_err()); + } + + #[test] + fn a_missing_binary_names_the_install_command() { + let error = binaries(Product::Casework).err().unwrap().to_string(); + assert!(error.contains("rstack install casework"), "{error}"); + assert!(error.contains("RSTACK_CASEWORK_BIN"), "{error}"); + } } From 5ef5c1477a8153158d88926bd995200df288c575 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 01:52:24 +0700 Subject: [PATCH 14/34] fix(bregctl): let a least-privilege admin run dev databases and name PostGIS and URL refusals Signed-off-by: Jeremi Joslin --- .../src/project_dev/database.rs | 229 +++++++++++++++++- 1 file changed, 217 insertions(+), 12 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/database.rs b/crates/registry-bregctl/src/project_dev/database.rs index 0da1236a39..4fb99a8924 100644 --- a/crates/registry-bregctl/src/project_dev/database.rs +++ b/crates/registry-bregctl/src/project_dev/database.rs @@ -29,6 +29,16 @@ const CONNECTION_TIMEOUT: Duration = Duration::from_secs(10); const STATEMENT_TIMEOUT: Duration = Duration::from_secs(10); const OPERATION_TIMEOUT: Duration = Duration::from_secs(15); const CLEANUP_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(30); +const KEEPALIVE_IDLE: Duration = Duration::from_secs(30); +/// Lets a role holding CREATEROLE use the roles it creates. PostgreSQL 16 and +/// later grant such a creator only ADMIN on them, which is not enough for +/// `CREATE SCHEMA ... AUTHORIZATION` or `CREATE DATABASE ... OWNER`. +const CREATEROLE_SELF_GRANT: &str = "set,inherit"; +/// Mirrors the BReg kernel's supported PostGIS floor +/// (`MIN_POSTGIS_MAJOR` and `MIN_POSTGIS_MINOR` in +/// `registry-breg/src/postgres/roles.rs`), so the runner can name it before +/// the kernel refuses with a generic connection error. +const MIN_POSTGIS: (u32, u32) = (3, 5); async fn bounded_connection( connection: impl Future>, @@ -97,12 +107,48 @@ fn retain_create_acquisition( } } -fn apply_statement_deadline(config: &mut Config) { - let deadline = format!("-c statement_timeout={}", STATEMENT_TIMEOUT.as_millis()); +/// Startup options for every administrator session: the statement deadline and +/// the CREATEROLE self-grant. The comma in the self-grant value is not a +/// separator for the server's startup option parser, which splits on spaces. +fn apply_session_options(config: &mut Config) { + let added = format!( + "-c statement_timeout={} -c createrole_self_grant={CREATEROLE_SELF_GRANT}", + STATEMENT_TIMEOUT.as_millis() + ); if let Some(existing) = config.get_options().filter(|value| !value.is_empty()) { - config.options(format!("{existing} {deadline}")); + config.options(format!("{existing} {added}")); } else { - config.options(deadline); + config.options(added); + } +} + +fn postgis_version_supported(version: &str) -> bool { + let mut parts = version.split('.'); + let mut next = || parts.next().and_then(|value| value.parse::().ok()); + match (next(), next()) { + (Some(major), Some(minor)) => (major, minor) >= MIN_POSTGIS, + _ => false, + } +} + +/// Refuses a spatial BReg run when the server does not offer a supported +/// PostGIS. `available` is the `default_version` of the `postgis` row in +/// `pg_available_extensions`, or `None` when there is no such row. +fn require_postgis(available: Option<&str>) -> Result<()> { + match available { + None => bail!( + "spatial BReg needs PostGIS {}.{} or later installed on the PostgreSQL server; \ + the server does not offer the postgis extension", + MIN_POSTGIS.0, + MIN_POSTGIS.1 + ), + Some(version) if !postgis_version_supported(version) => bail!( + "spatial BReg needs PostGIS {}.{} or later installed on the PostgreSQL server; \ + the server offers {version}", + MIN_POSTGIS.0, + MIN_POSTGIS.1 + ), + Some(_) => Ok(()), } } @@ -214,7 +260,7 @@ impl Database { validate_run_id(run_id)?; let (mut admin_config, endpoint) = admin_config(admin_url)?; admin_config.ssl_mode(SslMode::Require); - apply_statement_deadline(&mut admin_config); + apply_session_options(&mut admin_config); let tls = tls_connector(ca_pem)?; let runtime = Builder::new_current_thread() .enable_all() @@ -261,6 +307,10 @@ impl Database { bail!("the {label} database was already provisioned for this run"); } + if requires_postgis { + self.check_postgis_available()?; + } + let migration_password = random_password()?; let runtime_password = random_password()?; self.create_role(&names.migration_role, &migration_password)?; @@ -472,7 +522,11 @@ impl Database { &client, migration, runtime, ) .await - .context("cannot provision BReg PostGIS prerequisites")?; + .context( + "cannot provision BReg PostGIS prerequisites; the PostgreSQL \ + administrator must be allowed to CREATE EXTENSION postgis, \ + which normally needs a superuser", + )?; } Result::<()>::Ok(()) }) @@ -489,6 +543,34 @@ impl Database { .context("cannot finish PostgreSQL database provisioning") } + /// Reads the PostGIS version the server offers, before this run acquires + /// any role or database. + fn check_postgis_available(&mut self) -> Result<()> { + let queried = self.runtime.block_on(bounded_operation( + "PostGIS availability check", + self.client.query_opt( + "SELECT default_version FROM pg_available_extensions WHERE name = 'postgis'", + &[], + ), + OPERATION_TIMEOUT, + )); + if queried + .as_ref() + .is_err_and(OperationFailure::effect_is_unconfirmed) + { + self.retire_admin_session(); + } + let row = queried + .map_err(anyhow::Error::new) + .context("cannot check that the PostgreSQL server offers PostGIS")?; + let version = row + .map(|row| row.try_get::<_, Option>(0)) + .transpose() + .context("cannot read the PostGIS version the PostgreSQL server offers")? + .flatten(); + require_postgis(version.as_deref()) + } + fn execute_admin(&mut self, statement: &str) -> Result<()> { self.execute_admin_operation(statement, "administrator", OPERATION_TIMEOUT) .map_err(anyhow::Error::new) @@ -641,8 +723,8 @@ fn database_setup_sql(names: &Names, breg: bool) -> String { } fn admin_config(admin_url: &str) -> Result<(Config, Endpoint)> { - let parsed = - Url::parse(admin_url).map_err(|_| anyhow!("the PostgreSQL admin URL is invalid"))?; + let parsed = Url::parse(admin_url) + .map_err(|_| anyhow!("the PostgreSQL admin URL is not a parseable URL"))?; if !matches!(parsed.scheme(), "postgres" | "postgresql") || parsed.cannot_be_a_base() || parsed.fragment().is_some() @@ -669,9 +751,41 @@ fn admin_config(admin_url: &str) -> Result<(Config, Endpoint)> { "the PostgreSQL admin URL must not override its endpoint or credentials in the query" ); } + for (key, value) in parsed.query_pairs() { + match key.as_ref() { + "sslmode" => { + if !matches!(value.as_ref(), "require" | "verify-ca" | "verify-full") { + bail!( + "the PostgreSQL admin URL sslmode must be require, verify-ca, or \ + verify-full; the runner always uses verified TLS" + ); + } + } + "sslrootcert" => bail!( + "the PostgreSQL admin URL must not set sslrootcert; set RSTACK_DEV_DATABASE_CA \ + to a CA PEM file instead" + ), + "connect_timeout" | "application_name" => {} + other => bail!("the PostgreSQL admin URL query parameter {other} is not supported"), + } + } - let config = - Config::from_str(admin_url).map_err(|_| anyhow!("the PostgreSQL admin URL is invalid"))?; + // tokio-postgres does not parse verify-ca or verify-full, and the runner + // sets its own TLS mode, so the checked sslmode is not passed on. + let mut sanitized = parsed.clone(); + let kept: Vec<(String, String)> = parsed + .query_pairs() + .filter(|(key, _)| key != "sslmode") + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect(); + if kept.is_empty() { + sanitized.set_query(None); + } else { + sanitized.query_pairs_mut().clear().extend_pairs(kept); + } + let mut config = Config::from_str(sanitized.as_str()) + .map_err(|_| anyhow!("the PostgreSQL admin URL has an invalid query parameter value"))?; + config.keepalives_idle(KEEPALIVE_IDLE); if config.get_hosts().len() != 1 || !matches!(config.get_hosts().first(), Some(Host::Tcp(_))) || !config.get_hostaddrs().is_empty() @@ -859,10 +973,19 @@ mod tests { fn operation_deadlines_fit_the_worker_stop_budget() { let mut config = Config::new(); config.options("-c application_name=rstack-dev"); - apply_statement_deadline(&mut config); + apply_session_options(&mut config); assert_eq!( config.get_options(), - Some("-c application_name=rstack-dev -c statement_timeout=10000") + Some( + "-c application_name=rstack-dev -c statement_timeout=10000 \ + -c createrole_self_grant=set,inherit" + ) + ); + let mut bare = Config::new(); + apply_session_options(&mut bare); + assert_eq!( + bare.get_options(), + Some("-c statement_timeout=10000 -c createrole_self_grant=set,inherit") ); assert!(STATEMENT_TIMEOUT < OPERATION_TIMEOUT); assert!(OPERATION_TIMEOUT < CLEANUP_ATTEMPT_TIMEOUT); @@ -909,6 +1032,88 @@ mod tests { } } + #[test] + fn admin_url_query_keys_and_ssl_modes_are_checked_by_name() { + let password = random_password().unwrap(); + let base = format!( + "postgresql://admin:{}@db.example/postgres", + password.as_str() + ); + for accepted in [ + "sslmode=require", + "sslmode=verify-ca", + "sslmode=verify-full", + "connect_timeout=5", + "application_name=rstack", + ] { + assert!( + admin_config(&format!("{base}?{accepted}")).is_ok(), + "{accepted}" + ); + } + for (query, expected) in [ + ("sslmode=disable", "always uses verified TLS"), + ("sslmode=allow", "always uses verified TLS"), + ("sslmode=prefer", "always uses verified TLS"), + ("sslrootcert=/tmp/ca.pem", "RSTACK_DEV_DATABASE_CA"), + ( + "options=-csearch_path%3Dx", + "query parameter options is not supported", + ), + ( + "channel_binding=require", + "query parameter channel_binding is not supported", + ), + ] { + let message = admin_config(&format!("{base}?{query}")) + .err() + .unwrap_or_else(|| panic!("accepted {query}")) + .to_string(); + assert!(message.contains(expected), "{query}: {message}"); + assert!( + !message.contains(password.as_str()), + "{query} leaked the URL" + ); + } + let unparseable = admin_config("not a url").err().unwrap().to_string(); + assert!(unparseable.contains("not a parseable URL"), "{unparseable}"); + let bad_value = admin_config(&format!("{base}?connect_timeout=soon")) + .err() + .unwrap() + .to_string(); + assert!( + bad_value.contains("invalid query parameter value"), + "{bad_value}" + ); + assert!(!bad_value.contains(password.as_str())); + } + + #[test] + fn admin_config_keeps_idle_connections_alive() { + let (config, _) = admin_config("postgresql://admin:secret@db.example/postgres").unwrap(); + assert_eq!(config.get_keepalives_idle(), Duration::from_secs(30)); + } + + #[test] + fn spatial_runs_are_refused_without_a_supported_postgis() { + assert!(require_postgis(Some("3.5.2")).is_ok()); + assert!(require_postgis(Some("3.5")).is_ok()); + assert!(require_postgis(Some("3.10.0")).is_ok()); + assert!(require_postgis(Some("4.0.0")).is_ok()); + let missing = require_postgis(None).unwrap_err().to_string(); + assert!( + missing.starts_with( + "spatial BReg needs PostGIS 3.5 or later installed on the PostgreSQL server" + ), + "{missing}" + ); + for old in ["3.4.9", "2.5.5", "garbage", ""] { + let message = require_postgis(Some(old)).unwrap_err().to_string(); + assert!(message.contains("PostGIS 3.5 or later"), "{old}: {message}"); + assert!(message.contains(&format!("offers {old}")), "{message}"); + } + } + #[test] fn generated_product_urls_contain_only_generated_credentials_and_required_tls() { let endpoint = Endpoint { From 4bac0ae29e46b96823be4ba285ec7b8c31f60d00 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 01:59:06 +0700 Subject: [PATCH 15/34] docs(bregctl): describe every dev plan field in the generated schema Also reports plan refusals under breg.project-dev.invalid-plan, the code bregctl already uses for the same class. Signed-off-by: Jeremi Joslin --- .../src/project_dev/bindings.rs | 11 ++ .../registry-bregctl/src/project_dev/plan.rs | 74 ++++++++- crates/registry-bregctl/tests/project_dev.rs | 4 +- .../tools/project-dev.v1alpha1.schema.json | 154 ++++++++++++++---- .../identifiers/generated/catalog.v1.json | 4 +- 5 files changed, 207 insertions(+), 40 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/bindings.rs b/crates/registry-bregctl/src/project_dev/bindings.rs index 3d9a773ee0..1f12ac5b00 100644 --- a/crates/registry-bregctl/src/project_dev/bindings.rs +++ b/crates/registry-bregctl/src/project_dev/bindings.rs @@ -48,36 +48,47 @@ const TOKEN_LIFETIME_SECONDS: u64 = 300; const CLOCK_LEEWAY_SECONDS: u64 = 5; const MAXIMUM_REPLAY_ENTRIES: usize = 256; +/// Run-local connections between BReg and Casework. Configuring either one requires both products, with BReg listed before Casework. #[derive(Default, Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub(super) struct Bindings { + /// Connects BReg to Casework as a review authority. BReg authenticates to Casework as a machine identity. #[serde(default, skip_serializing_if = "Option::is_none")] pub(super) breg_review_authority: Option, + /// Connects Casework to BReg as a context provider. Casework authenticates to BReg as a machine identity. #[serde(default, skip_serializing_if = "Option::is_none")] pub(super) casework_context_provider: Option, } +/// A BReg review authority that the run connects to Casework. #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(deny_unknown_fields)] pub(super) struct BregReviewAuthority { + /// Local identifier of the review authority. #[cfg_attr(feature = "schema", schemars(with = "registry_platform_yaml::LocalId"))] pub(super) id: String, + /// Local identifier of the review producer. #[cfg_attr(feature = "schema", schemars(with = "registry_platform_yaml::LocalId"))] pub(super) producer: String, + /// Name of the test person, in `people`, whose Casework grant BReg uses to authenticate. That grant needs at least one scope. #[cfg_attr(feature = "schema", schemars(with = "registry_platform_yaml::LocalId"))] pub(super) identity: String, } +/// A Casework context provider that the run connects to BReg. #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(deny_unknown_fields)] pub(super) struct CaseworkContextProvider { + /// Local identifier of the context provider. #[cfg_attr(feature = "schema", schemars(with = "registry_platform_yaml::LocalId"))] pub(super) id: String, + /// Name of the test person, in `people`, whose BReg grant Casework uses to authenticate. That grant needs at least one scope. #[cfg_attr(feature = "schema", schemars(with = "registry_platform_yaml::LocalId"))] pub(super) identity: String, + /// Local identifier of the BReg action that the provider pairs with. #[cfg_attr(feature = "schema", schemars(with = "registry_platform_yaml::LocalId"))] pub(super) action: String, } diff --git a/crates/registry-bregctl/src/project_dev/plan.rs b/crates/registry-bregctl/src/project_dev/plan.rs index 1c6a737e15..ff58823104 100644 --- a/crates/registry-bregctl/src/project_dev/plan.rs +++ b/crates/registry-bregctl/src/project_dev/plan.rs @@ -139,6 +139,7 @@ fn expected_json_schema(generator: &mut schemars::SchemaGenerator) -> schemars:: }) } +/// A product the development run can start. #[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "lowercase")] @@ -156,21 +157,28 @@ impl Product { } } +/// The development plan: the products to start, the test people who call them, and the journey to run. #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub(super) struct Plan { + /// HTTPS URL, at most 512 characters and without credentials, query or fragment, that the run's tokens name as their issuer. #[cfg_attr(feature = "schema", schemars(with = "PlatformUrl"))] pub issuer: String, + /// The products to start, each with its project directory and token audience. List each product at most once. pub products: Vec, + /// The test people, keyed by a lowercase local identifier. Define 1 to 32, and give at least one grant to every started product. #[cfg_attr( feature = "schema", schemars(schema_with = "keyed_schema::") )] pub people: BTreeMap, + /// The ordered sequence of HTTP requests the run sends to the started products. pub journey: Journey, + /// Run-local connections between BReg and Casework. Leave empty when the products do not call each other. #[serde(default)] pub bindings: super::bindings::Bindings, + /// Chooses the test person for a BReg fixture step in tests/journeys.yaml, keyed by journey identifier and then step identifier. Needed only when several people could run the step; every named person needs a BReg grant. #[serde(default)] #[cfg_attr( feature = "schema", @@ -179,24 +187,32 @@ pub(super) struct Plan { pub schema_test_people: BTreeMap>, } +/// One product to start and where its project lives. #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(deny_unknown_fields)] pub(super) struct ProductPlan { + /// Which product to start. pub product: Product, + /// Relative path from the plan's directory to the product project, or `.` for the plan's own directory. It must contain registry.yaml for BReg or casework.yaml for Casework. pub project: PathBuf, + /// Token audience this product accepts: 1 to 256 characters without whitespace or control characters, distinct from the other products' audiences. #[cfg_attr(feature = "schema", schemars(with = "ExternalId"))] pub audience: String, } +/// A named test identity that signs in to the started products. #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(deny_unknown_fields)] pub(super) struct TestPerson { + /// Subject claim of this person's tokens: 1 to 256 characters without whitespace or control characters. #[cfg_attr(feature = "schema", schemars(with = "ExternalId"))] pub subject: String, + /// Client identifier of this person's tokens: 1 to 256 characters without whitespace or control characters. #[cfg_attr(feature = "schema", schemars(with = "ExternalId"))] pub client: String, + /// The authority this person holds in each product, keyed by `breg` or `casework`. Give at least one grant, only for products listed in `products`. #[serde(deserialize_with = "deserialize_grants")] pub grants: BTreeMap, /// Forward the same authority to both products. Both grants must declare the same scopes and additional claims. @@ -204,15 +220,19 @@ pub(super) struct TestPerson { pub casework_context: bool, } +/// The authority a test person holds in one product. #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(deny_unknown_fields)] pub(super) struct Grant { + /// Local identifier of the product profile this grant uses. #[cfg_attr(feature = "schema", schemars(with = "LocalId"))] pub profile: String, + /// Distinct scopes written into the token, at most 32, each 1 to 256 characters without whitespace or control characters. #[serde(deserialize_with = "deserialize_scopes")] #[cfg_attr(feature = "schema", schemars(schema_with = "scopes_schema"))] pub scopes: Vec, + /// Additional token claims, keyed by claim name. Names must not be reserved token claims such as `iss`, `aud`, `sub`, `scope` or any `registry_grant_` claim. #[serde(default)] #[cfg_attr(feature = "schema", schemars(schema_with = "claims_schema"))] pub claims: Map, @@ -255,40 +275,55 @@ where UniqueIdList::::deserialize(deserializer).map(UniqueIdList::into_vec) } +/// An ordered list of requests the run sends and checks. #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(deny_unknown_fields)] pub(super) struct Journey { + /// Local identifier naming this journey. #[cfg_attr(feature = "schema", schemars(with = "LocalId"))] pub id: String, + /// The steps, run in order. Include 1 to 32 with distinct identifiers. A step may refer to captures from earlier steps only. #[serde(deserialize_with = "deserialize_steps")] pub steps: Vec, } +/// One HTTP request, what to expect back, and what to remember from the response. #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(deny_unknown_fields)] pub(super) struct Step { + /// Unique step identifier: a local identifier, or a dotted sequence of local identifiers, at most 64 characters. #[cfg_attr(feature = "schema", schemars(schema_with = "step_id_schema"))] pub id: String, + /// The product that receives the request. pub product: Product, + /// The test person who sends the request. The person needs a grant for this step's product. #[serde(rename = "as")] #[cfg_attr(feature = "schema", schemars(with = "LocalId"))] pub person: String, + /// The HTTP method. pub method: Method, + /// Request path, starting with `/v1/`, at most 2048 characters, without whitespace, control characters, backslashes or fragments. It may contain `{{name}}` references to earlier captures. pub path: String, + /// JSON request body, at most 1 MiB once encoded. String values may contain `{{name}}` references to earlier captures. Do not combine with `raw`. pub body: Option, + /// Non-JSON request body. Allowed only for methods other than GET. Do not combine with `body`. pub raw: Option, + /// Extra request headers, keyed by header name. Only `If-Match`, `Idempotency-Key` and `Registry-Source-Profile` are supported. Values may contain `{{name}}` references to earlier captures. #[serde(default)] #[cfg_attr( feature = "schema", schemars(schema_with = "keyed_schema::") )] pub headers: BTreeMap, + /// What the response must look like for the step to pass. pub expect: Expected, + /// Values to remember from the response, keyed by a capture name. Later steps refer to a capture as `{{name}}` in paths, headers, bodies and expectations. Names are unique across the journey. #[serde(default)] #[cfg_attr(feature = "schema", schemars(schema_with = "capture_map_schema"))] pub capture: BTreeMap, + /// Repeat a GET request until it passes or the wait runs out. Allowed only for GET. pub poll: Option, } @@ -298,6 +333,7 @@ impl Identified for Step { } } +/// An HTTP request method. #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "UPPERCASE")] @@ -321,25 +357,33 @@ impl From<&Method> for reqwest::Method { } } +/// The conditions a response must meet. #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(deny_unknown_fields)] pub(super) struct Expected { + /// Accepted HTTP status: one status, or a list of 1 to 16 distinct statuses, each from 200 to 599. pub status: ExpectedStatus, + /// Exact values the response body must hold, keyed by JSON pointer (empty for the whole body, otherwise starting with `/`). String values may contain `{{name}}` references to earlier captures. #[serde(default)] #[cfg_attr(feature = "schema", schemars(schema_with = "expected_json_schema"))] pub json: BTreeMap, + /// Further assertions about the response body, at most 32. Each one tests a single JSON pointer. #[serde(default)] pub checks: Vec, + /// Problem code the error response must carry, 1 to 256 characters without whitespace or control characters. It may contain `{{name}}` references to earlier captures. pub code: Option, } +/// One accepted HTTP status or a list of them. #[derive(Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(untagged)] #[cfg_attr(feature = "schema", schemars(untagged))] pub(super) enum ExpectedStatus { + /// A single accepted status from 200 to 599. One(BoundedU32<200, 599>), + /// Several accepted statuses, each from 200 to 599; the response may carry any one of them. Many(Vec>), } @@ -373,70 +417,98 @@ impl ExpectedStatus { } } +/// An assertion about one value in the response body. Set exactly one of `equals`, `length` or `contains`. #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(deny_unknown_fields)] pub(super) struct Check { + /// JSON pointer to the value to test: empty for the whole body, otherwise starting with `/`. pub pointer: String, + /// The value at the pointer must equal this. String values may contain `{{name}}` references to earlier captures. pub equals: Option, + /// The array, object or string at the pointer must have this many items, entries or characters, at most 1048576. #[cfg_attr(feature = "schema", schemars(range(min = 0, max = 1_048_576)))] pub length: Option, + /// The array at the pointer must hold this item, the string must contain this text, or the object must have this key. String values may contain `{{name}}` references to earlier captures. pub contains: Option, } +/// A value remembered from a response, selected by `type`. #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(tag = "type", rename_all = "kebab-case", deny_unknown_fields)] pub(super) enum Capture { + /// Remember the value at a JSON pointer in the response body. Pointer { + /// JSON pointer into the response body: empty for the whole body, otherwise starting with `/`. pointer: String, }, + /// Remember a response header. Header { + /// The response header to remember. header: CapturedHeader, }, + /// Remember a value from the one array item that matches a condition. The step fails if no item or more than one item matches. Find { + /// JSON pointer to the array in the response body to search. find: String, + /// The condition an item must meet. r#where: FindWhere, + /// JSON pointer, relative to the matching item, of the value to remember. pointer: String, }, + /// Remember the value at a JSON pointer, optionally reading one query parameter from a URL found there. Value { + /// JSON pointer into the response body: empty for the whole body, otherwise starting with `/`. pointer: String, + /// Name of a query parameter. When set, the value at the pointer must be a URL or relative reference, and the parameter's value is remembered instead. At most 256 characters without whitespace or control characters. query: Option, }, } +/// A response header a capture can remember. #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "lowercase")] pub(super) enum CapturedHeader { + /// The entity tag, usable as an `If-Match` value in a later step. Etag, } +/// The condition a `find` capture uses to pick an array item. #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(deny_unknown_fields)] pub(super) struct FindWhere { + /// JSON pointer, relative to each array item, of the value to compare. pub pointer: String, + /// The value an item must hold at the pointer to match. String values may contain `{{name}}` references to earlier captures. pub equals: Value, } +/// How long and how often to repeat a GET request. The wait divided by the interval, rounded up, must be below 250 attempts. #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub(super) struct Poll { + /// Longest total wait in milliseconds, from 1 to 60000. #[serde(rename = "maximumWaitMilliseconds")] #[cfg_attr(feature = "schema", schemars(range(min = 1, max = 60_000)))] pub timeout_ms: u64, + /// Pause between attempts in milliseconds, from 50 to 5000 and not above the maximum wait. #[serde(rename = "pollIntervalMilliseconds")] #[cfg_attr(feature = "schema", schemars(range(min = 50, max = 5_000)))] pub interval_ms: u64, } +/// A request body sent as given, without JSON encoding. #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub(super) struct RawBody { + /// HTTP media type for the Content-Type header, containing a slash, 1 to 256 ASCII characters. pub content_type: String, + /// The body as standard base64, at most 1 MiB once decoded. pub base64: String, } @@ -552,7 +624,7 @@ fn read_plan(root: &Path) -> Result> { fn refusal_report(decoded: &Decoded, refusal: &SemanticRefusal) -> Report { let diagnostic = decoded.document.diagnostic_at_value( Severity::Error, - "rstack.dev.invalid-plan", + "breg.project-dev.invalid-plan", &refusal.field, &refusal.reason, &refusal.remedy, diff --git a/crates/registry-bregctl/tests/project_dev.rs b/crates/registry-bregctl/tests/project_dev.rs index e9854e3da1..c116a1572c 100644 --- a/crates/registry-bregctl/tests/project_dev.rs +++ b/crates/registry-bregctl/tests/project_dev.rs @@ -222,7 +222,7 @@ fn assert_dangling_schema_assignment( assert_eq!(report["command"], "dev-check"); assert_eq!(report["status"], "domain-refusal"); let diagnostic = &report["diagnostics"][0]; - assert_eq!(diagnostic["code"], "rstack.dev.invalid-plan"); + assert_eq!(diagnostic["code"], "breg.project-dev.invalid-plan"); assert_eq!(diagnostic["path"], path); assert_eq!(diagnostic["message"], message); assert_eq!(diagnostic["suggestedAction"], suggested_action); @@ -595,7 +595,7 @@ fn check_reports_an_actionable_value_free_poll_relationship() { assert_eq!(result.status.code(), Some(1)); let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); let diagnostic = &report["diagnostics"][0]; - assert_eq!(diagnostic["code"], "rstack.dev.invalid-plan"); + assert_eq!(diagnostic["code"], "breg.project-dev.invalid-plan"); assert_eq!( diagnostic["path"], "/journey/steps/0/poll/pollIntervalMilliseconds" diff --git a/products/breg/generated/tools/project-dev.v1alpha1.schema.json b/products/breg/generated/tools/project-dev.v1alpha1.schema.json index efb56218fb..028109f595 100644 --- a/products/breg/generated/tools/project-dev.v1alpha1.schema.json +++ b/products/breg/generated/tools/project-dev.v1alpha1.schema.json @@ -2,27 +2,34 @@ "$defs": { "Bindings": { "additionalProperties": false, + "description": "Run-local connections between BReg and Casework. Configuring either one requires both products, with BReg listed before Casework.", "properties": { "bregReviewAuthority": { - "$ref": "#/$defs/BregReviewAuthority" + "$ref": "#/$defs/BregReviewAuthority", + "description": "Connects BReg to Casework as a review authority. BReg authenticates to Casework as a machine identity." }, "caseworkContextProvider": { - "$ref": "#/$defs/CaseworkContextProvider" + "$ref": "#/$defs/CaseworkContextProvider", + "description": "Connects Casework to BReg as a context provider. Casework authenticates to BReg as a machine identity." } }, "type": "object" }, "BregReviewAuthority": { "additionalProperties": false, + "description": "A BReg review authority that the run connects to Casework.", "properties": { "id": { - "$ref": "#/$defs/LocalId" + "$ref": "#/$defs/LocalId", + "description": "Local identifier of the review authority." }, "identity": { - "$ref": "#/$defs/LocalId" + "$ref": "#/$defs/LocalId", + "description": "Name of the test person, in `people`, whose Casework grant BReg uses to authenticate. That grant needs at least one scope." }, "producer": { - "$ref": "#/$defs/LocalId" + "$ref": "#/$defs/LocalId", + "description": "Local identifier of the review producer." } }, "required": [ @@ -33,11 +40,14 @@ "type": "object" }, "Capture": { + "description": "A value remembered from a response, selected by `type`.", "oneOf": [ { "additionalProperties": false, + "description": "Remember the value at a JSON pointer in the response body.", "properties": { "pointer": { + "description": "JSON pointer into the response body: empty for the whole body, otherwise starting with `/`.", "type": "string" }, "type": { @@ -53,9 +63,11 @@ }, { "additionalProperties": false, + "description": "Remember a response header.", "properties": { "header": { - "$ref": "#/$defs/CapturedHeader" + "$ref": "#/$defs/CapturedHeader", + "description": "The response header to remember." }, "type": { "const": "header", @@ -70,11 +82,14 @@ }, { "additionalProperties": false, + "description": "Remember a value from the one array item that matches a condition. The step fails if no item or more than one item matches.", "properties": { "find": { + "description": "JSON pointer to the array in the response body to search.", "type": "string" }, "pointer": { + "description": "JSON pointer, relative to the matching item, of the value to remember.", "type": "string" }, "type": { @@ -82,7 +97,8 @@ "type": "string" }, "where": { - "$ref": "#/$defs/FindWhere" + "$ref": "#/$defs/FindWhere", + "description": "The condition an item must meet." } }, "required": [ @@ -95,11 +111,14 @@ }, { "additionalProperties": false, + "description": "Remember the value at a JSON pointer, optionally reading one query parameter from a URL found there.", "properties": { "pointer": { + "description": "JSON pointer into the response body: empty for the whole body, otherwise starting with `/`.", "type": "string" }, "query": { + "description": "Name of a query parameter. When set, the value at the pointer must be a URL or relative reference, and the parameter's value is remembered instead. At most 256 characters without whitespace or control characters.", "type": "string" }, "type": { @@ -116,22 +135,25 @@ ] }, "CapturedHeader": { - "enum": [ - "etag" - ], + "const": "etag", + "description": "The entity tag, usable as an `If-Match` value in a later step.", "type": "string" }, "CaseworkContextProvider": { "additionalProperties": false, + "description": "A Casework context provider that the run connects to BReg.", "properties": { "action": { - "$ref": "#/$defs/LocalId" + "$ref": "#/$defs/LocalId", + "description": "Local identifier of the BReg action that the provider pairs with." }, "id": { - "$ref": "#/$defs/LocalId" + "$ref": "#/$defs/LocalId", + "description": "Local identifier of the context provider." }, "identity": { - "$ref": "#/$defs/LocalId" + "$ref": "#/$defs/LocalId", + "description": "Name of the test person, in `people`, whose BReg grant Casework uses to authenticate. That grant needs at least one scope." } }, "required": [ @@ -143,16 +165,23 @@ }, "Check": { "additionalProperties": false, + "description": "An assertion about one value in the response body. Set exactly one of `equals`, `length` or `contains`.", "properties": { - "contains": true, - "equals": true, + "contains": { + "description": "The array at the pointer must hold this item, the string must contain this text, or the object must have this key. String values may contain `{{name}}` references to earlier captures." + }, + "equals": { + "description": "The value at the pointer must equal this. String values may contain `{{name}}` references to earlier captures." + }, "length": { + "description": "The array, object or string at the pointer must have this many items, entries or characters, at most 1048576.", "format": "uint", "maximum": 1048576, "minimum": 0, "type": "integer" }, "pointer": { + "description": "JSON pointer to the value to test: empty for the whole body, otherwise starting with `/`.", "type": "string" } }, @@ -169,20 +198,24 @@ }, "Expected": { "additionalProperties": false, + "description": "The conditions a response must meet.", "properties": { "checks": { "default": [], + "description": "Further assertions about the response body, at most 32. Each one tests a single JSON pointer.", "items": { "$ref": "#/$defs/Check" }, "type": "array" }, "code": { + "description": "Problem code the error response must carry, 1 to 256 characters without whitespace or control characters. It may contain `{{name}}` references to earlier captures.", "type": "string" }, "json": { "additionalProperties": true, "default": {}, + "description": "Exact values the response body must hold, keyed by JSON pointer (empty for the whole body, otherwise starting with `/`). String values may contain `{{name}}` references to earlier captures.", "propertyNames": { "$ref": "#/$defs/ExternalId" }, @@ -190,7 +223,8 @@ "x-registry-passthrough": "Expected values are compared with the product response at the named JSON pointers." }, "status": { - "$ref": "#/$defs/ExpectedStatus" + "$ref": "#/$defs/ExpectedStatus", + "description": "Accepted HTTP status: one status, or a list of 1 to 16 distinct statuses, each from 200 to 599." } }, "required": [ @@ -201,12 +235,14 @@ "ExpectedStatus": { "anyOf": [ { + "description": "A single accepted status from 200 to 599.", "format": "uint32", "maximum": 599, "minimum": 200, "type": "integer" }, { + "description": "Several accepted statuses, each from 200 to 599; the response may carry any one of them.", "items": { "format": "uint32", "maximum": 599, @@ -215,7 +251,8 @@ }, "type": "array" } - ] + ], + "description": "One accepted HTTP status or a list of them." }, "ExternalId": { "description": "An identifier another system issues, written exactly as its issuer gives it.", @@ -226,9 +263,13 @@ }, "FindWhere": { "additionalProperties": false, + "description": "The condition a `find` capture uses to pick an array item.", "properties": { - "equals": true, + "equals": { + "description": "The value an item must hold at the pointer to match. String values may contain `{{name}}` references to earlier captures." + }, "pointer": { + "description": "JSON pointer, relative to each array item, of the value to compare.", "type": "string" } }, @@ -240,10 +281,12 @@ }, "Grant": { "additionalProperties": false, + "description": "The authority a test person holds in one product.", "properties": { "claims": { "additionalProperties": true, "default": {}, + "description": "Additional token claims, keyed by claim name. Names must not be reserved token claims such as `iss`, `aud`, `sub`, `scope` or any `registry_grant_` claim.", "propertyNames": { "$ref": "#/$defs/ExternalId" }, @@ -251,9 +294,11 @@ "x-registry-passthrough": "Claim values are written into the development token as given." }, "profile": { - "$ref": "#/$defs/LocalId" + "$ref": "#/$defs/LocalId", + "description": "Local identifier of the product profile this grant uses." }, "scopes": { + "description": "Distinct scopes written into the token, at most 32, each 1 to 256 characters without whitespace or control characters.", "items": { "$ref": "#/$defs/ExternalId" }, @@ -270,11 +315,14 @@ }, "Journey": { "additionalProperties": false, + "description": "An ordered list of requests the run sends and checks.", "properties": { "id": { - "$ref": "#/$defs/LocalId" + "$ref": "#/$defs/LocalId", + "description": "Local identifier naming this journey." }, "steps": { + "description": "The steps, run in order. Include 1 to 32 with distinct identifiers. A step may refer to captures from earlier steps only.", "items": { "$ref": "#/$defs/Step" }, @@ -293,6 +341,7 @@ "type": "string" }, "Method": { + "description": "An HTTP request method.", "enum": [ "GET", "POST", @@ -304,14 +353,17 @@ }, "Poll": { "additionalProperties": false, + "description": "How long and how often to repeat a GET request. The wait divided by the interval, rounded up, must be below 250 attempts.", "properties": { "maximumWaitMilliseconds": { + "description": "Longest total wait in milliseconds, from 1 to 60000.", "format": "uint64", "maximum": 60000, "minimum": 1, "type": "integer" }, "pollIntervalMilliseconds": { + "description": "Pause between attempts in milliseconds, from 50 to 5000 and not above the maximum wait.", "format": "uint64", "maximum": 5000, "minimum": 50, @@ -325,6 +377,7 @@ "type": "object" }, "Product": { + "description": "A product the development run can start.", "enum": [ "breg", "casework" @@ -333,14 +386,18 @@ }, "ProductPlan": { "additionalProperties": false, + "description": "One product to start and where its project lives.", "properties": { "audience": { - "$ref": "#/$defs/ExternalId" + "$ref": "#/$defs/ExternalId", + "description": "Token audience this product accepts: 1 to 256 characters without whitespace or control characters, distinct from the other products' audiences." }, "product": { - "$ref": "#/$defs/Product" + "$ref": "#/$defs/Product", + "description": "Which product to start." }, "project": { + "description": "Relative path from the plan's directory to the product project, or `.` for the plan's own directory. It must contain registry.yaml for BReg or casework.yaml for Casework.", "type": "string" } }, @@ -353,11 +410,14 @@ }, "RawBody": { "additionalProperties": false, + "description": "A request body sent as given, without JSON encoding.", "properties": { "base64": { + "description": "The body as standard base64, at most 1 MiB once decoded.", "type": "string" }, "contentType": { + "description": "HTTP media type for the Content-Type header, containing a slash, 1 to 256 ASCII characters.", "type": "string" } }, @@ -369,16 +429,21 @@ }, "Step": { "additionalProperties": false, + "description": "One HTTP request, what to expect back, and what to remember from the response.", "properties": { "as": { - "$ref": "#/$defs/LocalId" + "$ref": "#/$defs/LocalId", + "description": "The test person who sends the request. The person needs a grant for this step's product." + }, + "body": { + "description": "JSON request body, at most 1 MiB once encoded. String values may contain `{{name}}` references to earlier captures. Do not combine with `raw`." }, - "body": true, "capture": { "additionalProperties": { "$ref": "#/$defs/Capture" }, "default": {}, + "description": "Values to remember from the response, keyed by a capture name. Later steps refer to a capture as `{{name}}` in paths, headers, bodies and expectations. Names are unique across the journey.", "propertyNames": { "allOf": [ { @@ -394,13 +459,15 @@ "type": "object" }, "expect": { - "$ref": "#/$defs/Expected" + "$ref": "#/$defs/Expected", + "description": "What the response must look like for the step to pass." }, "headers": { "additionalProperties": { "type": "string" }, "default": {}, + "description": "Extra request headers, keyed by header name. Only `If-Match`, `Idempotency-Key` and `Registry-Source-Profile` are supported. Values may contain `{{name}}` references to earlier captures.", "propertyNames": { "$ref": "#/$defs/ExternalId" }, @@ -414,22 +481,28 @@ { "$ref": "#/$defs/DerivedId" } - ] + ], + "description": "Unique step identifier: a local identifier, or a dotted sequence of local identifiers, at most 64 characters." }, "method": { - "$ref": "#/$defs/Method" + "$ref": "#/$defs/Method", + "description": "The HTTP method." }, "path": { + "description": "Request path, starting with `/v1/`, at most 2048 characters, without whitespace, control characters, backslashes or fragments. It may contain `{{name}}` references to earlier captures.", "type": "string" }, "poll": { - "$ref": "#/$defs/Poll" + "$ref": "#/$defs/Poll", + "description": "Repeat a GET request until it passes or the wait runs out. Allowed only for GET." }, "product": { - "$ref": "#/$defs/Product" + "$ref": "#/$defs/Product", + "description": "The product that receives the request." }, "raw": { - "$ref": "#/$defs/RawBody" + "$ref": "#/$defs/RawBody", + "description": "Non-JSON request body. Allowed only for methods other than GET. Do not combine with `body`." } }, "required": [ @@ -444,6 +517,7 @@ }, "TestPerson": { "additionalProperties": false, + "description": "A named test identity that signs in to the started products.", "properties": { "caseworkContext": { "default": false, @@ -451,10 +525,12 @@ "type": "boolean" }, "client": { - "$ref": "#/$defs/ExternalId" + "$ref": "#/$defs/ExternalId", + "description": "Client identifier of this person's tokens: 1 to 256 characters without whitespace or control characters." }, "grants": { "additionalProperties": false, + "description": "The authority this person holds in each product, keyed by `breg` or `casework`. Give at least one grant, only for products listed in `products`.", "properties": { "breg": { "$ref": "#/$defs/Grant" @@ -466,7 +542,8 @@ "type": "object" }, "subject": { - "$ref": "#/$defs/ExternalId" + "$ref": "#/$defs/ExternalId", + "description": "Subject claim of this person's tokens: 1 to 256 characters without whitespace or control characters." } }, "required": [ @@ -487,6 +564,7 @@ "$id": "https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, + "description": "The development plan: the products to start, the test people who call them, and the journey to run.", "properties": { "apiVersion": { "const": "id.registrystack.org/formats/breg/project-dev/v1alpha1", @@ -495,13 +573,16 @@ }, "bindings": { "$ref": "#/$defs/Bindings", - "default": {} + "default": {}, + "description": "Run-local connections between BReg and Casework. Leave empty when the products do not call each other." }, "issuer": { - "$ref": "#/$defs/Url" + "$ref": "#/$defs/Url", + "description": "HTTPS URL, at most 512 characters and without credentials, query or fragment, that the run's tokens name as their issuer." }, "journey": { - "$ref": "#/$defs/Journey" + "$ref": "#/$defs/Journey", + "description": "The ordered sequence of HTTP requests the run sends to the started products." }, "kind": { "const": "BRegProjectDev", @@ -512,12 +593,14 @@ "additionalProperties": { "$ref": "#/$defs/TestPerson" }, + "description": "The test people, keyed by a lowercase local identifier. Define 1 to 32, and give at least one grant to every started product.", "propertyNames": { "$ref": "#/$defs/LocalId" }, "type": "object" }, "products": { + "description": "The products to start, each with its project directory and token audience. List each product at most once.", "items": { "$ref": "#/$defs/ProductPlan" }, @@ -541,6 +624,7 @@ "type": "object" }, "default": {}, + "description": "Chooses the test person for a BReg fixture step in tests/journeys.yaml, keyed by journey identifier and then step identifier. Needed only when several people could run the step; every named person needs a BReg grant.", "propertyNames": { "$ref": "#/$defs/LocalId" }, diff --git a/products/identifiers/generated/catalog.v1.json b/products/identifiers/generated/catalog.v1.json index c1054bd079..0a8544ba4c 100644 --- a/products/identifiers/generated/catalog.v1.json +++ b/products/identifiers/generated/catalog.v1.json @@ -4996,7 +4996,7 @@ }, "artifact": { "path": "products/breg/generated/tools/project-dev.v1alpha1.schema.json", - "sha256": "e9877be9b1a191c2ca27af5d998698359cbeaa08aba74749ae0a36cc5f7f7e7f", + "sha256": "30a0fcc71d24875309ae9cd96f41305add32ad69e89dcc9b4cbd1622478eeb43", "mediaType": "application/schema+json" } }, @@ -5212,7 +5212,7 @@ }, "artifact": { "path": "products/casework/contracts/cli/report.schema.json", - "sha256": "442c29fe6db3bd93c2a35a5303e341d8aaa70c95d58dcd589fee9fff529fdff4", + "sha256": "9d701cf095137be5e1ce0203943626bcf7005597f3e5748527b510fabe180cb4", "mediaType": "application/schema+json" } }, From 03264b8af72dc15953497ca9c765e987f4830d72 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 01:28:34 +0700 Subject: [PATCH 16/34] fix: honour dev --json for launcher failures and require only the plan's components Signed-off-by: Jeremi Joslin --- crates/registry-rstack/src/cli.rs | 118 +++++++++++++++++-------- crates/registry-rstack/tests/dev.rs | 128 ++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+), 35 deletions(-) diff --git a/crates/registry-rstack/src/cli.rs b/crates/registry-rstack/src/cli.rs index 7b7e7e8b8b..4c9eed4676 100644 --- a/crates/registry-rstack/src/cli.rs +++ b/crates/registry-rstack/src/cli.rs @@ -384,6 +384,7 @@ pub fn run() -> i32 { let arguments: Vec = std::env::args_os().collect(); let (duplicate_format, json_requested, format_supplied, requested_format) = report_arguments(&arguments); + let json_requested = json_requested || dev_json_requested(&arguments); if duplicate_format { return emit_failure(2, "--format may be supplied only once", json_requested); } @@ -453,6 +454,16 @@ fn report_arguments(arguments: &[OsString]) -> (bool, bool, bool, Option 1, json, occurrences > 0, requested) } +/// Whether a `dev` subcommand carries its own `--json` before any `--`. +fn dev_json_requested(arguments: &[OsString]) -> bool { + command_tokens(arguments).first() == Some(&"dev") + && arguments + .iter() + .skip(1) + .take_while(|argument| *argument != "--") + .any(|argument| argument == "--json") +} + fn requested_report_format(value: &OsString) -> RequestedReportFormat { match value.to_str() { Some("human") => RequestedReportFormat::Human, @@ -1337,9 +1348,56 @@ fn first_path_rstack() -> Option { None } +/// What a development action needs from the installed components. Only +/// `start` runs services; Casework is needed only when the plan lists it. +#[derive(Default)] +struct Needs { + services: bool, + casework: bool, +} + +impl Needs { + fn components(&self) -> Vec<&'static str> { + if self.casework { + vec!["breg", "casework"] + } else { + vec!["breg"] + } + } + + fn variables(&self) -> Vec<&'static str> { + let mut variables = vec!["RSTACK_BREGCTL_BIN"]; + if self.services { + variables.push("RSTACK_BREG_BIN"); + } + if self.services && self.casework { + variables.push("RSTACK_CASEWORKCTL_BIN"); + variables.push("RSTACK_CASEWORK_BIN"); + } + variables + } +} + +/// Whether the project's dev.yaml lists a casework product. An unreadable or +/// oversized plan counts as using Casework, so the launcher never skips a +/// binary the helper might need; the helper reports the real plan error. +fn plan_uses_casework(project: &std::path::Path) -> bool { + use std::io::Read; + const LIMIT: u64 = 1024 * 1024; + let mut text = String::new(); + let read = std::fs::File::open(project.join("dev.yaml")) + .and_then(|file| file.take(LIMIT + 1).read_to_string(&mut text)); + if read.is_err() || text.len() as u64 > LIMIT { + return true; + } + text.lines() + .filter_map(|line| line.split_once("product:")) + .any(|(_, value)| value.trim_start().starts_with("casework")) +} + fn dispatch_dev(cli: &Cli, action: &DevCommand, format: ReportFormat) -> Result<(), CliFailure> { let launcher_json = matches!(format, ReportFormat::Json); - let (request, database_url, needs_services) = match action { + let (request, database_url, needs) = match action { DevCommand::Check { project, deny_warnings, @@ -1351,7 +1409,7 @@ fn dispatch_dev(cli: &Cli, action: &DevCommand, format: ReportFormat) -> Result< json: *json || launcher_json, }, None, - false, + Needs::default(), ), DevCommand::Start { project, @@ -1365,7 +1423,10 @@ fn dispatch_dev(cli: &Cli, action: &DevCommand, format: ReportFormat) -> Result< json: *json || launcher_json, }, database_url.as_deref(), - true, + Needs { + services: true, + casework: plan_uses_casework(project), + }, ), DevCommand::Status { run_id, json } => ( DevRequest::Status { @@ -1373,7 +1434,7 @@ fn dispatch_dev(cli: &Cli, action: &DevCommand, format: ReportFormat) -> Result< json: *json || launcher_json, }, None, - false, + Needs::default(), ), DevCommand::Token { run_id, @@ -1386,7 +1447,7 @@ fn dispatch_dev(cli: &Cli, action: &DevCommand, format: ReportFormat) -> Result< json: *json || launcher_json, }, None, - false, + Needs::default(), ), DevCommand::Stop { run_id, json } => ( DevRequest::Stop { @@ -1394,7 +1455,7 @@ fn dispatch_dev(cli: &Cli, action: &DevCommand, format: ReportFormat) -> Result< json: *json || launcher_json, }, None, - false, + Needs::default(), ), }; let request = serde_json::to_string(&request).map_err(|error| { @@ -1403,12 +1464,7 @@ fn dispatch_dev(cli: &Cli, action: &DevCommand, format: ReportFormat) -> Result< let selected = selected_state(cli)?; let (mut command, lease) = if let Some((lease, selection, manifest)) = selected { - let required_components = if needs_services { - &["breg", "casework"][..] - } else { - &["breg"][..] - }; - for component in required_components { + for component in needs.components() { if !selection.components.iter().any(|name| name == component) { return Err(CliFailure::domain(format!( "component {component} is missing; run rstack install {component}" @@ -1423,16 +1479,7 @@ fn dispatch_dev(cli: &Cli, action: &DevCommand, format: ReportFormat) -> Result< } let mut command = ProcessCommand::new(executable); set_selected_generation_environment(&mut command, lease.path(), &selection, &manifest)?; - let required_variables = if needs_services { - &[ - "RSTACK_BREGCTL_BIN", - "RSTACK_BREG_BIN", - "RSTACK_CASEWORKCTL_BIN", - "RSTACK_CASEWORK_BIN", - ][..] - } else { - &["RSTACK_BREGCTL_BIN"][..] - }; + let required_variables = needs.variables(); let missing: Vec<_> = required_variables .iter() .filter(|variable| { @@ -1450,16 +1497,7 @@ fn dispatch_dev(cli: &Cli, action: &DevCommand, format: ReportFormat) -> Result< } (command, Some(lease)) } else { - let variables = if needs_services { - &[ - "RSTACK_BREGCTL_BIN", - "RSTACK_BREG_BIN", - "RSTACK_CASEWORKCTL_BIN", - "RSTACK_CASEWORK_BIN", - ][..] - } else { - &["RSTACK_BREGCTL_BIN"][..] - }; + let variables = needs.variables(); let mut paths = Vec::with_capacity(variables.len()); let mut missing = Vec::new(); for variable in variables { @@ -1467,13 +1505,23 @@ fn dispatch_dev(cli: &Cli, action: &DevCommand, format: ReportFormat) -> Result< .filter(|value| !value.is_empty()) .map(PathBuf::from); match path { - Some(path) if path.is_file() => paths.push((*variable, path)), - _ => missing.push(*variable), + Some(path) if path.is_file() => paths.push((variable, path)), + _ => missing.push(variable), } } if !missing.is_empty() { return Err(CliFailure::domain(format!( - "without a managed installation, rstack dev requires existing source binary overrides: {}", + "without a managed installation, run rstack install {}, or set existing source binary overrides: {}", + needs + .components() + .into_iter() + .filter(|component| { + missing + .iter() + .any(|variable| variable.starts_with(&format!("RSTACK_{}", component.to_ascii_uppercase()))) + }) + .collect::>() + .join(" "), missing.join(", ") ))); } diff --git a/crates/registry-rstack/tests/dev.rs b/crates/registry-rstack/tests/dev.rs index 0d253307d7..5d6cd35e0a 100644 --- a/crates/registry-rstack/tests/dev.rs +++ b/crates/registry-rstack/tests/dev.rs @@ -484,3 +484,131 @@ fn root_and_dev_help_need_no_installation_or_binary_overrides() { } } } + +fn write_plan(project: &Path, products: &[&str]) { + let entries: Vec = products + .iter() + .map(|product| { + format!(" - {{product: {product}, project: {product}, audience: \"urn:example:{product}\"}}\n") + }) + .collect(); + fs::write( + project.join("dev.yaml"), + format!( + "issuer: https://issuer.example\nproducts:\n{}", + entries.concat() + ), + ) + .unwrap(); +} + +#[test] +fn dev_subcommand_json_makes_launcher_failures_json() { + let tools = SourceTools::new(); + let output = tools + .empty_command() + .args(["dev", "start", "/missing", "--json"]) + .output() + .unwrap(); + + assert_eq!(output.status.code(), Some(1), "{output:?}"); + assert!(output.stderr.is_empty(), "{output:?}"); + let report: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(report["schema_version"], "registry.rstack.error.v1"); + assert_eq!(report["status"], "refused"); + assert_eq!(report["error"]["class"], "domain"); + + let usage = tools + .empty_command() + .args(["dev", "token", "run-1", "--json"]) + .output() + .unwrap(); + assert_eq!(usage.status.code(), Some(2), "{usage:?}"); + let report: Value = serde_json::from_slice(&usage.stdout).unwrap(); + assert_eq!(report["schema_version"], "registry.rstack.error.v1"); + assert_eq!(report["error"]["class"], "usage"); + + let human = tools + .empty_command() + .args(["dev", "start", "/missing"]) + .output() + .unwrap(); + assert_eq!(human.status.code(), Some(1), "{human:?}"); + assert!(human.stdout.is_empty(), "{human:?}"); + assert!(String::from_utf8(human.stderr) + .unwrap() + .starts_with("rstack: ")); +} + +#[test] +fn breg_only_plan_does_not_require_casework_binaries() { + let tools = SourceTools::new(); + write_plan(&tools.project, &["breg"]); + let output = tools + .empty_command() + .args(["dev", "start"]) + .arg(&tools.project) + .env("RSTACK_BREGCTL_BIN", &tools.bregctl) + .env("RSTACK_BREG_BIN", &tools.breg) + .output() + .unwrap(); + + assert!(output.status.success(), "{output:?}"); + let stdout = String::from_utf8(output.stdout.clone()).unwrap(); + assert_eq!(output_field(&stdout, "caseworkctl"), ""); + assert_eq!(output_field(&stdout, "casework"), ""); +} + +#[test] +fn casework_plan_passes_casework_binaries_and_names_missing_installs() { + let tools = SourceTools::new(); + write_plan(&tools.project, &["breg", "casework"]); + let output = tools + .command() + .args(["dev", "start"]) + .arg(&tools.project) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + let stdout = String::from_utf8(output.stdout).unwrap(); + assert_eq!( + output_field(&stdout, "casework"), + tools.casework.to_str().unwrap() + ); + + let missing = tools + .empty_command() + .args(["dev", "start"]) + .arg(&tools.project) + .env("RSTACK_BREGCTL_BIN", &tools.bregctl) + .env("RSTACK_BREG_BIN", &tools.breg) + .output() + .unwrap(); + assert_eq!(missing.status.code(), Some(1), "{missing:?}"); + let stderr = String::from_utf8(missing.stderr).unwrap(); + assert!(stderr.contains("rstack install casework"), "{stderr}"); + assert!(!stderr.contains("rstack install breg"), "{stderr}"); + assert!(stderr.contains("RSTACK_CASEWORKCTL_BIN"), "{stderr}"); +} + +#[test] +fn missing_breg_names_the_install_command_before_the_overrides() { + let tools = SourceTools::new(); + write_plan(&tools.project, &["breg", "casework"]); + let output = tools + .empty_command() + .args(["dev", "start"]) + .arg(&tools.project) + .output() + .unwrap(); + + assert_eq!(output.status.code(), Some(1), "{output:?}"); + let stderr = String::from_utf8(output.stderr).unwrap(); + let install = stderr + .find("rstack install breg casework") + .unwrap_or_else(|| panic!("{stderr}")); + let overrides = stderr + .find("RSTACK_BREGCTL_BIN") + .unwrap_or_else(|| panic!("{stderr}")); + assert!(install < overrides, "{stderr}"); +} From 06d3a2f1c5b5cbf760d438cc61b9327ddb0fc4cb Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 01:30:10 +0700 Subject: [PATCH 17/34] refactor: read the dev plan's products with the YAML reader Signed-off-by: Jeremi Joslin --- Cargo.lock | 1 + crates/registry-rstack/Cargo.toml | 1 + crates/registry-rstack/src/cli.rs | 36 +++++++++++++++++++--------- crates/registry-rstack/tests/dev.rs | 37 +++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b878119670..fc659c6a13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8174,6 +8174,7 @@ dependencies = [ "registry-platform-buildinfo", "registry-platform-canonical-json", "registry-platform-config", + "registry-platform-yaml", "registry-thunderid-tooling", "rustix 1.1.5", "serde", diff --git a/crates/registry-rstack/Cargo.toml b/crates/registry-rstack/Cargo.toml index 2c427270fb..6ee9fdebfa 100644 --- a/crates/registry-rstack/Cargo.toml +++ b/crates/registry-rstack/Cargo.toml @@ -22,6 +22,7 @@ hex.workspace = true registry-platform-buildinfo.workspace = true registry-platform-canonical-json.workspace = true registry-platform-config.workspace = true +registry-platform-yaml.workspace = true registry-thunderid-tooling.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/crates/registry-rstack/src/cli.rs b/crates/registry-rstack/src/cli.rs index 4c9eed4676..cda7a57406 100644 --- a/crates/registry-rstack/src/cli.rs +++ b/crates/registry-rstack/src/cli.rs @@ -1378,21 +1378,35 @@ impl Needs { } } -/// Whether the project's dev.yaml lists a casework product. An unreadable or -/// oversized plan counts as using Casework, so the launcher never skips a -/// binary the helper might need; the helper reports the real plan error. +/// Whether the project's dev.yaml lists a casework product under +/// `/products/*/product`. A missing, unreadable, oversized or unparsable plan, +/// or one without a products list, counts as using Casework, so the launcher +/// never skips a binary the helper might need; the helper reports the real +/// plan error. fn plan_uses_casework(project: &std::path::Path) -> bool { + use registry_platform_yaml::{NodeValue, Reader, MAXIMUM_DOCUMENT_BYTES}; use std::io::Read; - const LIMIT: u64 = 1024 * 1024; - let mut text = String::new(); - let read = std::fs::File::open(project.join("dev.yaml")) - .and_then(|file| file.take(LIMIT + 1).read_to_string(&mut text)); - if read.is_err() || text.len() as u64 > LIMIT { + let limit = MAXIMUM_DOCUMENT_BYTES as u64; + let file = project.join("dev.yaml"); + let mut bytes = Vec::new(); + let read = std::fs::File::open(&file) + .and_then(|handle| handle.take(limit + 1).read_to_end(&mut bytes)); + if read.is_err() || bytes.len() as u64 > limit { return true; } - text.lines() - .filter_map(|line| line.split_once("product:")) - .any(|(_, value)| value.trim_start().starts_with("casework")) + let Ok(Some(root)) = Reader::new(file.display().to_string()).scan(&bytes) else { + return true; + }; + let Some(NodeValue::Sequence(products)) = root.pointer("/products").map(|node| &node.value) + else { + return true; + }; + products.iter().any(|entry| { + !matches!( + entry.pointer("/product").map(|node| &node.value), + Some(NodeValue::String(text)) if text.text != "casework" + ) + }) } fn dispatch_dev(cli: &Cli, action: &DevCommand, format: ReportFormat) -> Result<(), CliFailure> { diff --git a/crates/registry-rstack/tests/dev.rs b/crates/registry-rstack/tests/dev.rs index 5d6cd35e0a..360a1b3a8f 100644 --- a/crates/registry-rstack/tests/dev.rs +++ b/crates/registry-rstack/tests/dev.rs @@ -612,3 +612,40 @@ fn missing_breg_names_the_install_command_before_the_overrides() { .unwrap_or_else(|| panic!("{stderr}")); assert!(install < overrides, "{stderr}"); } + +#[test] +fn casework_text_outside_the_products_list_does_not_require_casework() { + let tools = SourceTools::new(); + fs::write( + tools.project.join("dev.yaml"), + "# product: casework\nissuer: https://issuer.example\nproducts:\n - {product: breg, project: breg, audience: \"urn:example:breg\"}\njourney:\n steps:\n - name: \"product: casework\"\n", + ) + .unwrap(); + let output = tools + .empty_command() + .args(["dev", "start"]) + .arg(&tools.project) + .env("RSTACK_BREGCTL_BIN", &tools.bregctl) + .env("RSTACK_BREG_BIN", &tools.breg) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); +} + +#[test] +fn unparsable_plan_still_requires_casework() { + let tools = SourceTools::new(); + fs::write(tools.project.join("dev.yaml"), "products: [\n").unwrap(); + let output = tools + .empty_command() + .args(["dev", "start"]) + .arg(&tools.project) + .env("RSTACK_BREGCTL_BIN", &tools.bregctl) + .env("RSTACK_BREG_BIN", &tools.breg) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(1), "{output:?}"); + assert!(String::from_utf8(output.stderr) + .unwrap() + .contains("rstack install casework")); +} From 9f840fb68d7ac36e6630d339fe79439bd3ac453a Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 02:10:51 +0700 Subject: [PATCH 18/34] test(rstack): prove least-privilege admin, kept logs and lost-supervisor recovery in the dev acceptance run Signed-off-by: Jeremi Joslin --- products/platform/scripts/test-rstack-dev.py | 217 +++++++++++++++++-- 1 file changed, 204 insertions(+), 13 deletions(-) diff --git a/products/platform/scripts/test-rstack-dev.py b/products/platform/scripts/test-rstack-dev.py index e2728e0aae..a302206a95 100644 --- a/products/platform/scripts/test-rstack-dev.py +++ b/products/platform/scripts/test-rstack-dev.py @@ -9,7 +9,9 @@ import hashlib import json import os +import re import secrets +import signal import shutil import socket import subprocess @@ -239,6 +241,53 @@ def wait_for_stop(bin_dir, env, cwd, run): raise RuntimeError("the run did not finish its TTL cleanup") +def live_runs(state): + """Runs that may still be alive; `failed/` only holds kept logs.""" + if not state.exists(): + return [] + return [path for path in state.iterdir() if path.name != "failed"] + + +def kept_logs(state, error): + """The log directory a failed run's error names, after checking it.""" + marker = "logs kept at " + assert marker in error, error + logs = Path(error.rsplit(marker, 1)[1].strip()) + assert logs.name == "logs" and logs.parent.parent == state / "failed", logs + assert logs.is_dir() and logs.stat().st_mode & 0o777 == 0o700, logs + assert (logs / "supervisor.log").is_file(), logs + return logs + + +def eventually(check, message, timeout=10): + deadline = time.monotonic() + timeout + while not check(): + assert time.monotonic() < deadline, message + time.sleep(0.1) + + +def accepts_connections(url): + parts = urlsplit(url) + with socket.socket() as probe: + probe.settimeout(2) + return probe.connect_ex((parts.hostname, parts.port)) == 0 + + +def product_pids(bin_dir): + """Processes running this build's BReg or Casework (not their ctl tools).""" + pattern = f"{re.escape(str(bin_dir))}/(breg|casework)( |$)" + result = command("pgrep", "-f", pattern) + assert result.returncode in (0, 1), "pgrep failed" + return result.stdout.split() + + +def assert_nothing_listening(bin_dir, urls=()): + """No product URL accepts a connection and no product process remains.""" + for url in urls: + eventually(lambda: not accepts_connections(url), f"{url} is still listening") + eventually(lambda: not product_pids(bin_dir), "a product process is still alive") + + def verify_deployment_key(root, bin_dir, env, run_root, headers): """Start the real verifier with a separate trust pin and the same issuer.""" key = root / "deployment-issuer.key" @@ -309,15 +358,31 @@ def coordinate(value): process.wait(timeout=15) +def create_owner(pg, pg_env, url): + """A least-privilege administrator: it can create roles and databases only.""" + password = secrets.token_urlsafe(32) + psql( + pg, + pg_env, + f"CREATE ROLE devowner LOGIN CREATEROLE CREATEDB PASSWORD '{password}';", + ) + parts = urlsplit(url) + owner_url = ( + f"postgresql://devowner:{password}@{parts.hostname}:{parts.port}/postgres" + ) + return owner_url, {**pg_env, "PGUSER": "devowner", "PGPASSWORD": password} + + def exercise(root, bin_dir, pg, project, ca, url, pg_env, spatial): checkout = root / "licensing" shutil.copytree(project, checkout) before = files(checkout) state = root / "runs" + owner_url, owner_env = create_owner(pg, pg_env, url) env = { **os.environ, "RSTACK_DEV_STATE_DIR": str(state), - "RSTACK_DEV_DATABASE_URL": url, + "RSTACK_DEV_DATABASE_URL": owner_url, "RSTACK_DEV_DATABASE_CA": str(ca), "RSTACK_BREGCTL_BIN": str(bin_dir / "bregctl"), "RSTACK_BREG_BIN": str(bin_dir / "breg"), @@ -326,14 +391,17 @@ def exercise(root, bin_dir, pg, project, ca, url, pg_env, spatial): } try: exercise_runs(root, bin_dir, pg, checkout, state, env, pg_env) + verify_lost_supervisor(bin_dir, pg, checkout, state, env, pg_env, owner_env) verify_provisioning_deadline(bin_dir, pg, checkout, state, env, pg_env) + verify_spatial_refusal(root, bin_dir, pg, state, env, pg_env) if spatial: - verify_spatial(root, bin_dir, pg, env, pg_env) + verify_spatial( + root, bin_dir, pg, {**env, "RSTACK_DEV_DATABASE_URL": url}, pg_env + ) assert files(checkout) == before finally: - if state.exists(): - for run_root in state.iterdir(): - dev(bin_dir, env, checkout, "stop", run_root.name) + for run_root in live_runs(state): + dev(bin_dir, env, checkout, "stop", run_root.name) def exercise_runs(root, bin_dir, pg, checkout, state, env, pg_env): @@ -406,21 +474,32 @@ def exercise_runs(root, bin_dir, pg, checkout, state, env, pg_env): assert dev(bin_dir, env, checkout, "stop", run["run_id"])["status"] == "stopped" assert not (state / run["run_id"]).exists() assert inventory(pg, pg_env) == baseline + assert_nothing_listening( + bin_dir, [url for run in runs for url in run["products"].values()] + ) short = dev(bin_dir, env, checkout, "start", ".", "--ttl", "1s") wait_for_stop(bin_dir, env, checkout, short["run_id"]) assert inventory(pg, pg_env) == baseline - assert list(state.iterdir()) == [] + assert live_runs(state) == [] + assert_nothing_listening(bin_dir, short["products"].values()) - failure_env = {**env, "RSTACK_CASEWORK_BIN": shutil.which("false")} + failing = root / "failing-casework" + failing.write_text("#!/bin/sh\nprintf 'boom\\n' >&2\nexit 1\n") + failing.chmod(0o700) + failure_env = {**env, "RSTACK_CASEWORK_BIN": str(failing)} report = dev(bin_dir, failure_env, checkout, "start", ".", expected=1) assert "casework exited before readiness" in report["error"], report + assert "boom" in report["error"], report + logs = kept_logs(state, report["error"]) + assert any("boom" in log.read_text() for log in logs.glob("casework-*.log")), report assert inventory(pg, pg_env) == baseline, report - assert list(state.iterdir()) == [], report + assert live_runs(state) == [], report + assert_nothing_listening(bin_dir) verify_native_handoff(root, bin_dir, env) assert inventory(pg, pg_env) == baseline assert administrator_locks(pg, pg_env) == baseline_locks - assert list(state.iterdir()) == [] + assert live_runs(state) == [] print( "Stop, TTL expiry and failed startup cleaned up; authored product files stayed byte-identical" ) @@ -468,7 +547,9 @@ def verify_provisioning_deadline(bin_dir, pg, checkout, state, env, pg_env): assert psql(pg, pg_env, lock_query).strip() == "t" assert inventory(pg, pg_env) == baseline assert administrator_locks(pg, pg_env) == baseline_locks - assert list(state.iterdir()) == [] + assert live_runs(state) == [] + if "logs kept at " in error: + kept_logs(state, error) finally: lock.stdin.close() try: @@ -480,6 +561,90 @@ def verify_provisioning_deadline(bin_dir, pg, checkout, state, env, pg_env): print("A blocked PostgreSQL operation timed out and released earlier resources") +def listener_pids(url): + result = command( + "lsof", "-nP", "-Fp", f"-iTCP:{urlsplit(url).port}", "-sTCP:LISTEN" + ) + return {line[1:] for line in result.stdout.decode().split() if line[0] == "p"} + + +def recover_lost_supervisor(pg, owner_env, recovery): + """Release a lost run by hand, from the facts `stop` reports. + + Order: stop each process group (TERM, then KILL), drop every database + (WITH (FORCE) ends leftover sessions), drop every role, then remove the + run directory. + """ + for process in recovery["processes"]: + group = process["pgid"] + os.killpg(group, signal.SIGTERM) + + def gone(group=group): + try: + os.killpg(group, 0) + except ProcessLookupError: + return True + return False + + try: + eventually(gone, "a process group ignored SIGTERM", timeout=10) + except AssertionError: + os.killpg(group, signal.SIGKILL) + eventually(gone, "a process group survived SIGKILL", timeout=10) + objects = recovery["objects"] + statements = [ + f'DROP DATABASE IF EXISTS "{item["database"]}" WITH (FORCE);' + for item in objects + ] + [f'DROP ROLE IF EXISTS "{role}";' for item in objects for role in item["roles"]] + psql(pg, owner_env, "\n".join(statements) + "\n") + shutil.rmtree(recovery["run_directory"]) + + +def verify_lost_supervisor(bin_dir, pg, checkout, state, env, pg_env, owner_env): + baseline = inventory(pg, pg_env) + baseline_locks = administrator_locks(pg, pg_env) + report = dev(bin_dir, env, checkout, "start", ".") + run = report["run_id"] + worker = command( + "pgrep", "-f", f"project-dev --worker {re.escape(str(state / run))}" + ) + [supervisor] = worker.stdout.split() + os.kill(int(supervisor), signal.SIGKILL) + status = dev(bin_dir, env, checkout, "status", run, expected=1) + stopped = dev(bin_dir, env, checkout, "stop", run, expected=1) + for lost in (status, stopped): + assert lost["recovery"]["kind"] == "lost-supervisor", lost + assert lost["run_id"] == run, lost + recovery = stopped["recovery"] + assert recovery["run_directory"] == str(state / run), recovery + processes = recovery["processes"] + assert sorted(p["product"] for p in processes) == ["breg", "casework"], recovery + for process in processes: + os.kill(process["pid"], 0) + assert os.getpgid(process["pid"]) == process["pgid"], recovery + listeners = listener_pids(report["products"][process["product"]]) + assert str(process["pid"]) in listeners, recovery + # The pattern behind assert_nothing_listening sees the live products. + assert len(product_pids(bin_dir)) == 2, recovery + databases, roles = inventory(pg, pg_env) + assert {item["database"] for item in recovery["objects"]} == ( + databases - baseline[0] + ), recovery + assert {role for item in recovery["objects"] for role in item["roles"]} == ( + roles - baseline[1] + ), recovery + recover_lost_supervisor(pg, owner_env, recovery) + assert_nothing_listening(bin_dir, report["products"].values()) + assert inventory(pg, pg_env) == baseline + eventually( + lambda: administrator_locks(pg, pg_env) == baseline_locks, + "the lost supervisor's administrator lock was not released", + ) + assert live_runs(state) == [] + assert dev(bin_dir, env, checkout, "status", run)["status"] == "stopped" + print("A lost supervisor was recovered by hand from the facts stop reported") + + def verify_native_handoff(root, bin_dir, env): checkout = root / "native-review" checkout.mkdir() @@ -499,11 +664,11 @@ def verify_native_handoff(root, bin_dir, env): dev(bin_dir, env, checkout, "stop", report["run_id"]) -def verify_spatial(root, bin_dir, pg, env, pg_env): - checkout = root / "spatial" +def spatial_project(root, name): + """Copy the spatial registry and write a plan that creates and reads a site.""" + checkout = root / name registry = checkout / "registry" shutil.copytree(SPATIAL, registry) - before = files(registry) fixture = json.loads((registry / "tests/journeys.yaml").read_text()) people = {} for journey in fixture["journeys"]: @@ -566,6 +731,32 @@ def verify_spatial(root, bin_dir, pg, env, pg_env): }, } (checkout / "dev.yaml").write_text(json.dumps(plan)) + return checkout, registry, bbox + + +def verify_spatial_refusal(root, bin_dir, pg, state, env, pg_env): + """A spatial start is refused cleanly when the administrator cannot offer PostGIS.""" + checkout, _, _ = spatial_project(root, "spatial-refusal") + baseline = inventory(pg, pg_env) + offered = psql( + pg, + pg_env, + "SELECT default_version FROM pg_available_extensions WHERE name = 'postgis';", + ).strip() + supported = bool(offered) and tuple(map(int, offered.split(".")[:2])) >= (3, 5) + report = dev(bin_dir, env, checkout, "start", ".", expected=1) + if supported: + assert "must be allowed to CREATE EXTENSION postgis" in report["error"], report + else: + assert "PostGIS 3.5 or later installed" in report["error"], report + assert inventory(pg, pg_env) == baseline, report + assert live_runs(state) == [], report + print("A spatial start without PostGIS rights was refused and left nothing behind") + + +def verify_spatial(root, bin_dir, pg, env, pg_env): + checkout, registry, bbox = spatial_project(root, "spatial") + before = files(registry) baseline = inventory(pg, pg_env) report = dev(bin_dir, env, checkout, "start", ".") run = report["run_id"] From 85272d7c9779dc865d5ddf5c297452569c1f2ab5 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 02:16:17 +0700 Subject: [PATCH 19/34] fix(bregctl): accept dev grants that name proposal-only BReg profiles Signed-off-by: Jeremi Joslin --- crates/registry-bregctl/src/project_dev/products.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs index 92bc4f25db..ee64c33f7e 100644 --- a/crates/registry-bregctl/src/project_dev/products.rs +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -1873,6 +1873,19 @@ mod tests { check_breg_grants(®istry, &valid).unwrap(); } + #[test] + fn a_breg_grant_may_name_a_profile_with_only_proposal_permissions() { + let source = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../products/breg/acceptance/reviewed-evidence-journey/registry"); + let scratch = tempfile::tempdir().unwrap(); + let project = scratch.path().join("project"); + copy_project(&source, &project).unwrap(); + let registry = crate::compile(&project, crate::ProfileArg::Production, "project-dev-test") + .unwrap_or_else(|_| panic!("the reviewed evidence project must compile")); + let people = BTreeMap::from([("applier".to_owned(), breg_person("applier"))]); + check_breg_grants(®istry, &people).unwrap(); + } + #[test] fn encrypted_fields_are_refused_before_provisioning() { let registry = minimal_registry(|source| { From 36de460b242ffda2ddab6fe304bd93968f384e8c Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 02:16:18 +0700 Subject: [PATCH 20/34] fix(bregctl): remove a dev run directory in one step so status never sees it half-deleted Signed-off-by: Jeremi Joslin --- .../registry-bregctl/src/project_dev/mod.rs | 37 ++++++++++++++++++- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/mod.rs b/crates/registry-bregctl/src/project_dev/mod.rs index b0d675a35b..461602a197 100644 --- a/crates/registry-bregctl/src/project_dev/mod.rs +++ b/crates/registry-bregctl/src/project_dev/mod.rs @@ -758,7 +758,7 @@ fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { } else { Ok(()) } - .and_then(|()| fs::remove_dir_all(root).context("cannot remove run scratch files")); + .and_then(|()| remove_run_dir(root)); if let Some(error) = failure.filter(|_| !told_parent) { let error = match removed.as_ref() { Ok(()) => error, @@ -1132,10 +1132,26 @@ fn keep_logs(root: &Path, run_id: &str) -> Result> { fn remove_failed_run(root: &Path, run_id: &str) -> Result> { let kept = keep_logs(root, run_id)?; - fs::remove_dir_all(root)?; + remove_run_dir(root)?; Ok(kept) } +/// Where a run directory goes while it is deleted. The name is not a run ID, +/// so `status` and `stop` never read it as a run. +fn removing_dir(root: &Path) -> Result { + let name = root.file_name().context("run directory has no name")?; + Ok(root.with_file_name(format!(".{}.removing", name.to_string_lossy()))) +} + +/// Renames the run directory away before deleting it, so `status` sees either +/// the whole run or no run, never a run whose report is already deleted. +fn remove_run_dir(root: &Path) -> Result<()> { + let doomed = removing_dir(root)?; + fs::rename(root, &doomed).context("cannot remove run scratch files")?; + fs::remove_dir_all(&doomed) + .with_context(|| format!("cannot remove run scratch files at {}", doomed.display())) +} + fn note_kept_logs(error: anyhow::Error, kept: Option) -> anyhow::Error { match kept { Some(path) => anyhow::anyhow!("{error:#}; logs kept at {}", path.display()), @@ -1294,6 +1310,23 @@ mod tests { assert!(text.contains("database or role role"), "{text}"); } + #[test] + fn a_run_directory_disappears_in_one_step_and_leaves_nothing() { + let scratch = tempfile::tempdir().unwrap(); + let base = scratch.path().canonicalize().unwrap(); + let root = base.join(RUN_ID); + private_directory(&root).unwrap(); + fs::write(root.join(REPORT), "{}").unwrap(); + private_directory(&root.join("logs")).unwrap(); + fs::write(root.join("logs").join("breg-check.log"), "boom").unwrap(); + let doomed = removing_dir(&root).unwrap(); + assert_eq!(doomed.parent(), Some(base.as_path())); + assert!(run_dir(&doomed.file_name().unwrap().to_string_lossy()).is_err()); + remove_run_dir(&root).unwrap(); + assert!(!root.exists()); + assert_eq!(fs::read_dir(&base).unwrap().count(), 0); + } + #[test] fn failed_run_logs_move_out_of_the_run_directory() { let scratch = tempfile::tempdir().unwrap(); From ecda759de4ea94f5c1c94d0dd2b0f45098ca4898 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 02:17:50 +0700 Subject: [PATCH 21/34] test(rstack): ignore dev runs being deleted and check none are left behind Signed-off-by: Jeremi Joslin --- products/platform/scripts/test-rstack-dev.py | 26 +++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/products/platform/scripts/test-rstack-dev.py b/products/platform/scripts/test-rstack-dev.py index a302206a95..b2032d7c46 100644 --- a/products/platform/scripts/test-rstack-dev.py +++ b/products/platform/scripts/test-rstack-dev.py @@ -11,8 +11,8 @@ import os import re import secrets -import signal import shutil +import signal import socket import subprocess import tempfile @@ -242,10 +242,25 @@ def wait_for_stop(bin_dir, env, cwd, run): def live_runs(state): - """Runs that may still be alive; `failed/` only holds kept logs.""" + """Runs that may still be alive; `failed/` only holds kept logs and a + `.RUN_ID.removing` directory is a stopped run whose files are being deleted.""" if not state.exists(): return [] - return [path for path in state.iterdir() if path.name != "failed"] + return [ + path + for path in state.iterdir() + if path.name != "failed" and not path.name.startswith(".") + ] + + +def nothing_left_behind(state): + """Every stopped run's files are eventually deleted.""" + eventually( + lambda: ( + not state.exists() or all(path.name == "failed" for path in state.iterdir()) + ), + "a stopped run's files were not deleted", + ) def kept_logs(state, error): @@ -284,7 +299,9 @@ def product_pids(bin_dir): def assert_nothing_listening(bin_dir, urls=()): """No product URL accepts a connection and no product process remains.""" for url in urls: - eventually(lambda: not accepts_connections(url), f"{url} is still listening") + eventually( + lambda url=url: not accepts_connections(url), f"{url} is still listening" + ) eventually(lambda: not product_pids(bin_dir), "a product process is still alive") @@ -399,6 +416,7 @@ def exercise(root, bin_dir, pg, project, ca, url, pg_env, spatial): root, bin_dir, pg, {**env, "RSTACK_DEV_DATABASE_URL": url}, pg_env ) assert files(checkout) == before + nothing_left_behind(state) finally: for run_root in live_runs(state): dev(bin_dir, env, checkout, "stop", run_root.name) From 5200fb9bcc56cb86a27fa9bfe03d618852b36574 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 02:06:23 +0700 Subject: [PATCH 22/34] docs(platform): document dev run admin rules, recovery and logs Signed-off-by: Jeremi Joslin --- products/breg/CHANGELOG.md | 4 + products/casework/CHANGELOG.md | 4 +- products/platform/CHANGELOG.md | 5 + products/platform/PROJECT-DEV.md | 192 +++++++++++++++++++++++++++---- 4 files changed, 180 insertions(+), 25 deletions(-) diff --git a/products/breg/CHANGELOG.md b/products/breg/CHANGELOG.md index 0efb10562f..a6b6783269 100644 --- a/products/breg/CHANGELOG.md +++ b/products/breg/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +- `bregctl project-dev` runs BReg projects, including spatial projects, from a + `breg/project-dev` v1alpha1 plan. It is an alpha format and may change. + `rstack dev` is its entry point. + ## 0.46.0 - 2026-11-08 - A `record_timer.worker.iteration_failed` warning carries a closed `error` diff --git a/products/casework/CHANGELOG.md b/products/casework/CHANGELOG.md index 76a207dd51..925bbb49a4 100644 --- a/products/casework/CHANGELOG.md +++ b/products/casework/CHANGELOG.md @@ -7,7 +7,9 @@ `caseworkctl explain` lists them. A stage that omits the list offers every policy outcome. -- `caseworkctl explain` adds a top-level `accessProfiles` list, sorted by `id`, with each access profile's `id` and `principalClaim`, so a caller no longer reads them from a review policy's `dependencies`. The member is additive. +- `caseworkctl explain` adds a top-level `accessProfiles` list, sorted by `id`, + with each access profile's `id` and `principalClaim`, so a caller no longer + reads them from a review policy's `dependencies`. The member is additive. ## 0.46.0 - 2026-11-08 diff --git a/products/platform/CHANGELOG.md b/products/platform/CHANGELOG.md index 45fddf7cac..ac4a9e0fc6 100644 --- a/products/platform/CHANGELOG.md +++ b/products/platform/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +- `rstack dev check|start|status|token|stop` runs a checkout's BReg and Casework + projects together against a PostgreSQL server you supply. See + [PROJECT-DEV.md](PROJECT-DEV.md). It is an alpha developer tool and may + change. + ## 0.46.0 - 2026-11-08 - `registry-platform-text`: `TextStep::StripNonspacingMarksIn` replaces diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index f001cfe746..1268ceb6a0 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -2,12 +2,9 @@ Use `rstack dev` to start a checkout's BReg and Casework projects together. Install matching BReg and Casework components and supply a dedicated development -PostgreSQL server over TLS. The database administrator needs to create and -remove roles and databases and install `btree_gist`. PostgreSQL 17 or newer is -required. Use the platform's trusted roots or set `RSTACK_DEV_DATABASE_CA` to -your server's CA bundle in PEM format. -Spatial BReg projects also need PostGIS available on the server and a database -administrator that can install it in the run's databases. +PostgreSQL server over TLS. `rstack dev` does not start PostgreSQL and does not +use Docker: you supply the server. The products' own development modes are +unchanged by `rstack dev`. ```sh rstack install breg casework @@ -25,8 +22,7 @@ product authoring tools. The check works with the database offline. With `--json`, it writes a `BRegCtlReport` with `command` set to `dev-check`. Use `--deny-warnings` to refuse any unaccepted warning. -You can also pass `--database-url`. Use the environment variable when the URL -includes a password. Start reports the product URLs and the journey result. +Start reports the product URLs and the journey result. A failed journey exits 1 and includes the live run ID so you can inspect the result and stop the run. Startup failures clean up or report the run ID if cleanup is pending. The TTL starts at readiness and defaults to 30 minutes. @@ -38,9 +34,80 @@ reviewer's decision. Point `start` at the directory containing `dev.yaml`. Product files remain byte-for-byte unchanged. Generated runtime settings, keys, and packages live in the run's owner-only scratch directory. +## Supply the PostgreSQL server + +The server must be PostgreSQL 17 or newer and must serve TLS. The runner always +verifies the server certificate and host name. It trusts the platform's roots, +or the CA bundle in PEM format named by `RSTACK_DEV_DATABASE_CA`. + +The administrator role named in the URL needs `LOGIN CREATEROLE CREATEDB`. +Superuser is not needed for a non-spatial run. The runner sets +`createrole_self_grant` on its own sessions, so you do not configure it. The +administrator also installs `btree_gist`, which PostgreSQL treats as a trusted +extension. + +The administrator URL always carries a password. Set it through +`RSTACK_DEV_DATABASE_URL`, because `--database-url` is visible to other local +users in the process list. The URL must be a `postgres://` or `postgresql://` +URL with one TCP host, a user, a password and one database name. Percent-encode +special characters in the password. The URL accepts these query parameters and +refuses all others: + +- `sslmode`: `require`, `verify-ca` or `verify-full`. The runner verifies the + certificate and host name whichever you choose. +- `connect_timeout` and `application_name`. +- `sslrootcert` is refused. Set `RSTACK_DEV_DATABASE_CA` instead. + +### Spatial projects + +A spatial BReg project needs PostGIS 3.5 or later installed on the server. The +administrator must be allowed to `CREATE EXTENSION postgis`, which normally +means a superuser or the managed service's equivalent. The run checks that the +server offers a supported PostGIS before it creates anything. + +### A local server + +This recipe starts a PostgreSQL 17 or newer server on `127.0.0.1` with a +private certificate authority, and creates a least-privilege administrator. It +uses the PostgreSQL and OpenSSL command-line tools. Keep `DEV` outside any +repository. + +```sh +DEV="$HOME/rstack-dev-postgres" +mkdir -p "$DEV" +initdb -D "$DEV/data" -U postgres --auth=scram-sha-256 --pwprompt -E UTF8 + +openssl req -new -x509 -newkey rsa:2048 -nodes -days 30 -subj /CN=rstack-dev-ca \ + -addext basicConstraints=critical,CA:TRUE -keyout "$DEV/ca.key" -out "$DEV/ca.pem" +openssl req -new -newkey rsa:2048 -nodes -subj /CN=localhost \ + -keyout "$DEV/data/server.key" -out "$DEV/server.csr" +chmod 600 "$DEV/data/server.key" +printf 'subjectAltName=DNS:localhost,IP:127.0.0.1\nextendedKeyUsage=serverAuth\n' \ + > "$DEV/server.ext" +openssl x509 -req -in "$DEV/server.csr" -CA "$DEV/ca.pem" -CAkey "$DEV/ca.key" \ + -CAcreateserial -days 30 -extfile "$DEV/server.ext" -out "$DEV/data/server.crt" + +printf "listen_addresses = '127.0.0.1'\nport = 5432\nssl = on\n" \ + >> "$DEV/data/postgresql.conf" +pg_ctl -D "$DEV/data" -l "$DEV/postgres.log" -w start + +CONNECTION="host=127.0.0.1 port=5432 user=postgres dbname=postgres sslmode=verify-full sslrootcert=$DEV/ca.pem" +psql "$CONNECTION" -c "CREATE ROLE devadmin LOGIN CREATEROLE CREATEDB" +psql "$CONNECTION" -c '\password devadmin' + +export RSTACK_DEV_DATABASE_CA="$DEV/ca.pem" +# Set RSTACK_DEV_DATABASE_URL to +# postgresql://devadmin:PASSWORD@127.0.0.1:5432/postgres?sslmode=verify-full +``` + +For a spatial project, install PostGIS for this server and create the +administrator as a superuser instead. + ## Write the development plan Use the [versioned plan schema](../breg/generated/tools/project-dev.v1alpha1.schema.json). +The plan file starts with the schema modeline that editors read, as the +[licensing example](examples/licensing/dev.yaml) does. List the project's issuer, product directories and audiences in startup order, and test people with synthetic subjects and exact product grants. A grant selects the access profile, scopes, and additional claims. Each selected profile gets @@ -50,6 +117,15 @@ ordinary permissions. If several people match a step in the BReg project's `schemaTestPeople..`. The run obtains the normal schema-test receipt before packaging. +Every BReg grant must name an access profile that one of the BReg project's +entities declares. The run refuses a grant that does not, before it provisions +anything. The error names the plan pointer, for example +`/people/NAME/grants/breg/profile`, and lists the profiles the project +declares. A Casework grant must name an access profile of the Casework project, +reported at `/people/NAME/grants/casework/profile`. The run does not compare +scopes with the profile: a profile's required scopes are a per-entity rule, and +a mismatch appears as a failed journey step. + For a native review handoff, `bindings.bregReviewAuthority` identifies the Casework producer, and `bindings.caseworkContextProvider` identifies the BReg context provider and action. The run pairs the provider on a private copy @@ -60,21 +136,50 @@ token subject or the BReg principal claim. The token carries both audiences and the authority explicitly shared by both grants. See the [native review example](../breg/acceptance/reviewed-evidence-journey/dev.yaml). -Journey steps call relative product API paths and assert the HTTP status and -optional JSON-pointer values. Capture a string or number with a `pointer` -capture and refer to it as `{{capture-name}}` in a later path, request body, header, or -expected value. Use URI-safe characters for values in a path. Each step names -the test person it runs as. +### Journey steps -Use a bounded GET poll for asynchronous delivery or result retrieval. Set +A journey runs 1 to 32 steps in order. Each step names the test person it runs +as (`as`), the product, the method, and a relative path that starts with `/v1/`. +A step sends one of two request bodies: + +- `body` is a JSON body. +- `raw` is a non-JSON body for methods other than GET: `contentType` and + `base64`. + +A step can also set the `If-Match`, `Idempotency-Key` and +`Registry-Source-Profile` headers. Use URI-safe characters for values in a path. + +Each step asserts the response with `expect`: + +- `status` is one accepted HTTP status from 200 to 599, or a list of up to 16. +- `json` maps JSON pointers to exact expected values. +- `checks` lists up to 32 assertions. Each tests one JSON pointer with exactly + one of `equals`, `length` or `contains`. +- `code` is the problem code an error response must carry. + +A step remembers values from its response under `capture`. Refer to a capture +as `{{capture-name}}` in a later path, request body, header, or expected value. +Capture names are unique across the journey. A capture has one `type`: + +- `pointer` remembers the string or number at a JSON pointer in the body. +- `header` remembers a response header. `etag` is the supported header. +- `find` searches the array at a JSON pointer for the one item whose `where` + pointer equals a value, and remembers the item's `pointer` value. The step + fails if no item or more than one item matches. +- `value` remembers the value at a pointer. With `query`, the value must be a + URL, and the capture is that query parameter's value. + +Use a bounded GET `poll` for asynchronous delivery or result retrieval. Set `maximumWaitMilliseconds` up to 60 seconds and choose a -`pollIntervalMilliseconds`. A journey can make at most 512 requests and -transfer at most 8 MiB of request and response bodies in total. +`pollIntervalMilliseconds` from 50 to 5000. A journey can make at most 512 +requests and transfer at most 8 MiB of request and response bodies in total. `token` writes a short-lived bearer header file for each grant. Its JSON report -gives the file path and the selected profile. Pass the file to curl with -`--header @HEADER_FILE`. With BReg, include the selected profile as the -`accessProfile` query parameter. With Casework, the header file carries the profile. +gives the person's `alias`, an `expires_at` time, and a `credentials` list whose +entries carry the `product`, the selected `profile` and the `header_file` path. +Pass the file to curl with `--header @HEADER_FILE`. With BReg, include the +selected profile as the `accessProfile` query parameter. With Casework, the +header file carries the profile. ## Run isolation @@ -85,8 +190,47 @@ Use synthetic test people and keep the issuer's private directory under your user account. Stop and TTL expiry terminate the services and remove the databases, roles, -keys, and run files. If PostgreSQL temporarily becomes unavailable, the -supervisor retries cleanup and `stop` reports pending work. Keep the supervisor -alive to finish cleanup when PostgreSQL returns. After an uncatchable -supervisor exit or machine failure, identify the exact databases and roles by -run ID before removing them. +keys, and run files. A spatial BReg run also creates a `NOLOGIN` role whose name +starts with `breg_bbox_`; cleanup removes it with the others. + +The state directory holds one directory per run. It defaults to +`rstack-dev-UID` in the system temporary directory. Set `RSTACK_DEV_STATE_DIR` +to use another location. + +### Logs + +During a run, `/logs/supervisor.log` holds the supervisor's log and +`/logs/-.log` holds each product stage's log, for example +`breg-check.log`. `status` names the logs directory. When a start fails, the run +directory is removed but its logs are kept at `/failed//logs`, +and the error names that path. Delete kept logs yourself. + +### Recovery + +If PostgreSQL becomes unavailable during cleanup, the supervisor retries with +backoff for up to 30 minutes. It then writes `cleanup-pending.json` in the run +directory, and `status` reports `cleanup-pending` with the remaining databases, +roles and processes. If the supervisor is lost, `stop` and `status` report the +run directory, the process groups, and the exact database and role names. + +Release the listed resources by hand, in this order: + +1. Stop each listed process group. +2. As the administrator, run `DROP DATABASE IF EXISTS "NAME" WITH (FORCE)` for + each listed database. +3. As the administrator, run `DROP ROLE IF EXISTS "NAME"` for each listed role, + including a `breg_bbox_*` role from a spatial run. +4. Remove the run directory. + +### Errors and limitations + +With `--json`, a failed `rstack dev` command prints one JSON error object with +`schema_version` (`registry.rstack.dev.v1`), `status` set to `error`, and +`error`. It adds `run_id` when the run needs cleanup and `recovery` with the +resources listed above. Failures that occur before the helper starts, such as a +missing component, print an `registry.rstack.error.v1` object with `status` set +to `refused` and an `error` object holding `class` and `message`. The reports of +`rstack dev check` use `BRegCtlReport` instead. + +A BReg project with encrypted fields is refused before anything is provisioned: +`rstack dev does not configure field encryption yet; ENTITY.FIELD is encrypted`. From 5ff937abf6e62accc9b0f0b94cea4000de688ea6 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 02:07:14 +0700 Subject: [PATCH 23/34] docs(platform): group dev run failure handling under its own heading Signed-off-by: Jeremi Joslin --- products/platform/PROJECT-DEV.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index 1268ceb6a0..b0a572d194 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -197,6 +197,8 @@ The state directory holds one directory per run. It defaults to `rstack-dev-UID` in the system temporary directory. Set `RSTACK_DEV_STATE_DIR` to use another location. +## When a run fails + ### Logs During a run, `/logs/supervisor.log` holds the supervisor's log and @@ -215,7 +217,7 @@ run directory, the process groups, and the exact database and role names. Release the listed resources by hand, in this order: -1. Stop each listed process group. +1. Stop each listed process group, for example `kill -TERM -PGID`. 2. As the administrator, run `DROP DATABASE IF EXISTS "NAME" WITH (FORCE)` for each listed database. 3. As the administrator, run `DROP ROLE IF EXISTS "NAME"` for each listed role, @@ -228,7 +230,7 @@ With `--json`, a failed `rstack dev` command prints one JSON error object with `schema_version` (`registry.rstack.dev.v1`), `status` set to `error`, and `error`. It adds `run_id` when the run needs cleanup and `recovery` with the resources listed above. Failures that occur before the helper starts, such as a -missing component, print an `registry.rstack.error.v1` object with `status` set +missing component, print a `registry.rstack.error.v1` object with `status` set to `refused` and an `error` object holding `class` and `message`. The reports of `rstack dev check` use `BRegCtlReport` instead. From 585b1fdefef4da2def3d470740685cea2bbfaf4e Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 02:18:19 +0700 Subject: [PATCH 24/34] docs(platform): name proposal-only BReg profiles and interrupted run deletion in the dev guide Signed-off-by: Jeremi Joslin --- products/platform/PROJECT-DEV.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index b0a572d194..451f54fa22 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -117,8 +117,9 @@ ordinary permissions. If several people match a step in the BReg project's `schemaTestPeople..`. The run obtains the normal schema-test receipt before packaging. -Every BReg grant must name an access profile that one of the BReg project's -entities declares. The run refuses a grant that does not, before it provisions +Every BReg grant must name an access profile the BReg project declares, +including a profile that carries only proposal permissions and so belongs to no +entity. The run refuses a grant that does not, before it provisions anything. The error names the plan pointer, for example `/people/NAME/grants/breg/profile`, and lists the profiles the project declares. A Casework grant must name an access profile of the Casework project, @@ -195,7 +196,9 @@ starts with `breg_bbox_`; cleanup removes it with the others. The state directory holds one directory per run. It defaults to `rstack-dev-UID` in the system temporary directory. Set `RSTACK_DEV_STATE_DIR` -to use another location. +to use another location. A run directory is renamed to `..removing` +before it is deleted, and `status` and `stop` ignore that name. If the deletion +is interrupted, delete that directory yourself. ## When a run fails From 76a8c4e4f1709f99198f58d17b1813e1057cd58b Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 02:23:34 +0700 Subject: [PATCH 25/34] test(bregctl): give the binding-ID test the access profile list caseworkctl explain reports Signed-off-by: Jeremi Joslin --- crates/registry-bregctl/src/project_dev/products.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs index ee64c33f7e..e291e02fc8 100644 --- a/crates/registry-bregctl/src/project_dev/products.rs +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -1696,9 +1696,8 @@ mod tests { let mut report = json!({ "apiVersion": "id.registrystack.org/formats/casework/report/v1alpha2", "kind": "CaseworkReport", "command": "explain", "status": "complete", "ok": true, - "reviewPolicies": [{"dependencies": {"accessProfiles": [ - {"id": "reviewer", "principalClaim": "sub"} - ]}}], + "reviewPolicies": [], + "accessProfiles": [{"id": "reviewer", "principalClaim": "sub"}], "trustedProducers": [{"id": "review-producer"}], "contextProviders": [{"id": "proposal-context"}] }); From bf045f717bb4710af6104501194b6750e0a6afe5 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 02:23:41 +0700 Subject: [PATCH 26/34] docs(platform): say dev start refuses binding IDs the projects do not declare Signed-off-by: Jeremi Joslin --- products/platform/PROJECT-DEV.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index 451f54fa22..0d97d06cdc 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -129,7 +129,10 @@ a mismatch appears as a failed journey step. For a native review handoff, `bindings.bregReviewAuthority` identifies the Casework producer, and `bindings.caseworkContextProvider` identifies the BReg -context provider and action. The run pairs the provider on a private copy +context provider and action. Before it provisions anything, `start` refuses a +binding that names a review authority or action the BReg project does not +declare, or a producer or context provider the Casework project does not +declare. The run pairs the provider on a private copy before packaging. A reviewer whose Casework context call forwards the bearer to BReg declares both grants, one shared client, and `caseworkContext: true`. Both grants declare the same scopes and additional claims. Casework reads the From ed1b22c38d18569a9c2ca931e80303698ef9c155 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 03:14:00 +0700 Subject: [PATCH 27/34] fix(bregctl): locate dev encrypted-field refusals and match the BReg grant remedy to Casework dev check now inspects the products, so an encrypted BReg field is refused at the product's plan pointer instead of as an unlocated plan error. The BReg grant remedy uses the same wording as the Casework one, and the guide says check refuses grants, bindings and encrypted fields as start does. Signed-off-by: Jeremi Joslin --- .../src/project_dev/products.rs | 44 ++++++++++++------- products/platform/PROJECT-DEV.md | 23 +++++----- 2 files changed, 41 insertions(+), 26 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs index e291e02fc8..79ce78d7c1 100644 --- a/crates/registry-bregctl/src/project_dev/products.rs +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -806,7 +806,7 @@ fn breg_authority(root: &Path, plan: &Plan) -> Result> { "Run `rstack breg check` for authored diagnostics, then correct the BReg project.", ) })?; - refuse_encrypted_fields(®istry)?; + refuse_encrypted_fields(®istry, &project.field)?; let review_authorities = registry .review_authorities() .iter() @@ -888,15 +888,23 @@ fn validate_breg_binding_ids( /// The generated runtime configuration has no field-encryption provider, so a /// project with encrypted fields would only fail late and obscurely. -fn refuse_encrypted_fields(registry: ®istry_breg::CompiledRegistry) -> Result<()> { +fn refuse_encrypted_fields( + registry: ®istry_breg::CompiledRegistry, + product_field: &str, +) -> plan::SemanticResult<()> { for entity in registry.entities().values() { for field in entity.fields.values() { - ensure!( + plan::require( field.encryption.is_none(), - "rstack dev does not configure field encryption yet; {}.{} is encrypted", - entity.id, - field.id - ); + product_field, + format!( + "The BReg project encrypts {}.{}, and rstack dev does not configure field \ + encryption yet.", + entity.id, field.id + ), + "Run this BReg project with `rstack breg` and a field-encryption provider, or \ + select a project without encrypted fields.", + )?; } } Ok(()) @@ -925,10 +933,7 @@ fn check_breg_grants( registry_platform_yaml::escape_pointer_segment(name) ), "The BReg grant profile does not name an access profile in the BReg project.", - format!( - "Set profile to an access profile defined by the BReg project. It declares: {}.", - profiles.iter().copied().collect::>().join(", ") - ), + "Set profile to an access profile defined by the BReg project.", )?; } } @@ -1867,7 +1872,11 @@ mod tests { error.contains("/people/typo/grants/breg/profile"), "{error}" ); - assert!(error.contains("directory-editor"), "{error}"); + assert!( + error.ends_with("Next: Set profile to an access profile defined by the BReg project."), + "{error}" + ); + assert!(!error.contains("directory-edtor"), "{error}"); let valid = BTreeMap::from([("editor".to_owned(), breg_person("directory-editor"))]); check_breg_grants(®istry, &valid).unwrap(); } @@ -1900,10 +1909,15 @@ mod tests { ) }); assert_eq!( - refuse_encrypted_fields(®istry).unwrap_err().to_string(), - "rstack dev does not configure field encryption yet; contact.email is encrypted" + refuse_encrypted_fields(®istry, "/products/0/project") + .unwrap_err() + .to_string(), + "dev.yaml /products/0/project: The BReg project encrypts contact.email, and rstack \ + dev does not configure field encryption yet. Next: Run this BReg project with \ + `rstack breg` and a field-encryption provider, or select a project without \ + encrypted fields." ); - refuse_encrypted_fields(&minimal_registry(str::to_owned)).unwrap(); + refuse_encrypted_fields(&minimal_registry(str::to_owned), "/products/0/project").unwrap(); } #[test] diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index 0d97d06cdc..f5016ef457 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -119,20 +119,19 @@ schema-test receipt before packaging. Every BReg grant must name an access profile the BReg project declares, including a profile that carries only proposal permissions and so belongs to no -entity. The run refuses a grant that does not, before it provisions -anything. The error names the plan pointer, for example -`/people/NAME/grants/breg/profile`, and lists the profiles the project -declares. A Casework grant must name an access profile of the Casework project, -reported at `/people/NAME/grants/casework/profile`. The run does not compare +entity. `rstack dev check` refuses a grant that does not, and so does `start`, +before it provisions anything. The error names the plan pointer, for example +`/people/NAME/grants/breg/profile`. A Casework grant must name an access profile +of the Casework project, reported at `/people/NAME/grants/casework/profile`. The run does not compare scopes with the profile: a profile's required scopes are a per-entity rule, and a mismatch appears as a failed journey step. For a native review handoff, `bindings.bregReviewAuthority` identifies the Casework producer, and `bindings.caseworkContextProvider` identifies the BReg -context provider and action. Before it provisions anything, `start` refuses a -binding that names a review authority or action the BReg project does not -declare, or a producer or context provider the Casework project does not -declare. The run pairs the provider on a private copy +context provider and action. `rstack dev check` refuses a binding that names a +review authority or action the BReg project does not declare, or a producer or +context provider the Casework project does not declare; `start` refuses it +before it provisions anything. The run pairs the provider on a private copy before packaging. A reviewer whose Casework context call forwards the bearer to BReg declares both grants, one shared client, and `caseworkContext: true`. Both grants declare the same scopes and additional claims. Casework reads the @@ -240,5 +239,7 @@ missing component, print a `registry.rstack.error.v1` object with `status` set to `refused` and an `error` object holding `class` and `message`. The reports of `rstack dev check` use `BRegCtlReport` instead. -A BReg project with encrypted fields is refused before anything is provisioned: -`rstack dev does not configure field encryption yet; ENTITY.FIELD is encrypted`. +`rstack dev` does not configure field encryption yet. `rstack dev check` and +`start` refuse a BReg project with encrypted fields at the product's +`/products/N/project` pointer, and the error names the encrypted +`ENTITY.FIELD`; `start` refuses it before it provisions anything. From 13a5ce5fe117bbc3c1d01460c212f336f7ade449 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 03:14:01 +0700 Subject: [PATCH 28/34] test(bregctl): check the dev install hint without depending on which variables are set Signed-off-by: Jeremi Joslin --- crates/registry-bregctl/src/project_dev/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/mod.rs b/crates/registry-bregctl/src/project_dev/mod.rs index 461602a197..c141c561ad 100644 --- a/crates/registry-bregctl/src/project_dev/mod.rs +++ b/crates/registry-bregctl/src/project_dev/mod.rs @@ -1364,8 +1364,10 @@ mod tests { #[test] fn a_missing_binary_names_the_install_command() { - let error = binaries(Product::Casework).err().unwrap().to_string(); + let error = binary(Product::Casework, "RSTACK_DEV_TEST_UNSET_BIN") + .unwrap_err() + .to_string(); assert!(error.contains("rstack install casework"), "{error}"); - assert!(error.contains("RSTACK_CASEWORK_BIN"), "{error}"); + assert!(error.contains("RSTACK_DEV_TEST_UNSET_BIN"), "{error}"); } } From 2ec143269195f2a201ce2b308a63aaec55d4aa41 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 06:27:23 +0700 Subject: [PATCH 29/34] fix(bregctl): report the expiry dev tokens carry The local issuer caps each token at five minutes, but `rstack dev token` reported the run's expiry as the tokens' own, so a client that refreshed on that time kept sending expired tokens. The report and the issuer now share one expiry rule. Signed-off-by: Jeremi Joslin --- .../src/project_dev/issuer.rs | 24 +++++++++++++++++-- .../registry-bregctl/src/project_dev/mod.rs | 4 +++- products/platform/PROJECT-DEV.md | 6 +++-- products/platform/scripts/test-rstack-dev.py | 3 +++ 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/issuer.rs b/crates/registry-bregctl/src/project_dev/issuer.rs index 03442434ad..4c093247ae 100644 --- a/crates/registry-bregctl/src/project_dev/issuer.rs +++ b/crates/registry-bregctl/src/project_dev/issuer.rs @@ -6,7 +6,7 @@ use registry_platform_crypto::{ generate_private_jwk, jws::sign_compact_jws, GeneratedKeyAlgorithm, LocalJwkSigner, PrivateJwk, }; use serde_json::{json, Value}; -use std::{path::Path, time::Duration}; +use std::path::Path; use zeroize::Zeroizing; use super::{ @@ -15,6 +15,15 @@ use super::{ write_private, RunInput, }; +/// How long a token the local issuer signs stays valid. +pub(super) const TOKEN_LIFETIME_SECONDS: u64 = 300; + +/// When a token issued at `now` expires: one lifetime later, or with the run +/// if that comes first. +pub(super) fn token_expiry(run_expires: u64, now: u64) -> u64 { + run_expires.min(now.saturating_add(TOKEN_LIFETIME_SECONDS)) +} + pub(super) struct Issuer { signer: LocalJwkSigner, runtime: tokio::runtime::Runtime, @@ -70,7 +79,7 @@ impl Issuer { expires: u64, ) -> Result> { let now = clock()?; - let expires = expires.min(now + Duration::from_secs(300).as_secs()); + let expires = token_expiry(expires, now); ensure!(expires > now, "the development run has expired"); let plan = &input.plan; let grant = &person.grants[&product]; @@ -123,3 +132,14 @@ impl Issuer { Ok(Zeroizing::new(token)) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_token_expires_five_minutes_after_issue_or_with_the_run() { + assert_eq!(token_expiry(10_000, 1_000), 1_000 + TOKEN_LIFETIME_SECONDS); + assert_eq!(token_expiry(1_100, 1_000), 1_100); + } +} diff --git a/crates/registry-bregctl/src/project_dev/mod.rs b/crates/registry-bregctl/src/project_dev/mod.rs index c141c561ad..31b4ac4a24 100644 --- a/crates/registry-bregctl/src/project_dev/mod.rs +++ b/crates/registry-bregctl/src/project_dev/mod.rs @@ -832,9 +832,11 @@ fn token(run_id: &str, person: &str) -> Result { let root = run_dir(run_id)?; let input: RunInput = read_json(&root.join("input.json"), MAXIMUM_CAPTURED_INPUT_BYTES)?; let signer = Issuer::load(&root)?; - let expires = report["expires_at"] + let run_expires = report["expires_at"] .as_u64() .context("run expiry is missing")?; + // Report the expiry the tokens carry, which is earlier than the run's. + let expires = issuer::token_expiry(run_expires, clock()?); let subject = input .plan .people diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index f5016ef457..a065d84451 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -178,8 +178,10 @@ Use a bounded GET `poll` for asynchronous delivery or result retrieval. Set requests and transfer at most 8 MiB of request and response bodies in total. `token` writes a short-lived bearer header file for each grant. Its JSON report -gives the person's `alias`, an `expires_at` time, and a `credentials` list whose -entries carry the `product`, the selected `profile` and the `header_file` path. +gives the person's `alias`, the `expires_at` time when the tokens expire +(five minutes after issue, or with the run if that is sooner), and a +`credentials` list whose entries carry the `product`, the selected `profile` +and the `header_file` path. Pass the file to curl with `--header @HEADER_FILE`. With BReg, include the selected profile as the `accessProfile` query parameter. With Casework, the header file carries the profile. diff --git a/products/platform/scripts/test-rstack-dev.py b/products/platform/scripts/test-rstack-dev.py index b2032d7c46..97dba72c73 100644 --- a/products/platform/scripts/test-rstack-dev.py +++ b/products/platform/scripts/test-rstack-dev.py @@ -212,7 +212,10 @@ def dev(bin_dir, env, cwd, *args, expected=0): def credential(bin_dir, env, cwd, run, alias): + before = time.time() report = dev(bin_dir, env, cwd, "token", run, "--as", alias) + # The report gives the expiry the tokens carry, at most five minutes away. + assert before < report["expires_at"] <= time.time() + 300, report [entry] = report["credentials"] header_file = Path(entry["header_file"]) assert header_file.stat().st_mode & 0o077 == 0 From 44f4abd635ec1a04dfb9423a516b1f8d687f49b7 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 06:27:27 +0700 Subject: [PATCH 30/34] fix(bregctl): refuse dev captures that refer to their own step A step's captures are all read from its own response, so a `find` capture whose `where` value used a sibling capture passed validation and then failed at run time. Validation now makes a step's captures available only to later steps. Signed-off-by: Jeremi Joslin --- .../registry-bregctl/src/project_dev/plan.rs | 24 ++++++++++++++++++- products/platform/PROJECT-DEV.md | 3 ++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/plan.rs b/crates/registry-bregctl/src/project_dev/plan.rs index ff58823104..3f9797f6cd 100644 --- a/crates/registry-bregctl/src/project_dev/plan.rs +++ b/crates/registry-bregctl/src/project_dev/plan.rs @@ -1117,8 +1117,10 @@ impl Journey { )?; } } - captures.insert(name.clone()); } + // A step's captures are all read from its own response, so they + // become available only to later steps. + captures.extend(step.capture.keys().cloned()); } Ok(()) } @@ -1361,6 +1363,26 @@ mod tests { assert!(error.contains("earlier capture"), "{error}"); } + #[test] + fn validation_refuses_a_find_capture_that_refers_to_its_own_step() { + let (root, plan) = plan(json!({ + "id": "list", "product": "breg", "as": "operator", + "method": "GET", "path": "/v1/records", + "expect": {"status": 200}, + "capture": { + "first": {"type": "pointer", "pointer": "/items/0/recordIdentifier"}, + "second": { + "type": "find", "find": "/items", + "where": {"pointer": "/recordIdentifier", "equals": "{{first}}"}, + "pointer": "/status" + } + } + })); + let error = format!("{:#}", plan.validate(&canonical(&root)).unwrap_err()); + assert!(error.contains("earlier capture"), "{error}"); + assert!(error.contains("/capture/second/find"), "{error}"); + } + #[test] fn validation_refuses_unbounded_poll_attempts() { let (root, plan) = plan(json!({ diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index a065d84451..fa90e53357 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -161,7 +161,8 @@ Each step asserts the response with `expect`: - `code` is the problem code an error response must carry. A step remembers values from its response under `capture`. Refer to a capture -as `{{capture-name}}` in a later path, request body, header, or expected value. +as `{{capture-name}}` in a later step's path, request body, header, expected +value, or `find` value. A step cannot use its own captures. Capture names are unique across the journey. A capture has one `type`: - `pointer` remembers the string or number at a JSON pointer in the body. From 40a855bd3b14ee9b60662ce19b54bba89bef1c88 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 06:27:56 +0700 Subject: [PATCH 31/34] fix(bregctl): require URI audiences for products a dev binding calls A binding requests tokens for the other product's audience as an OAuth resource, and the products accept that resource only as an absolute URI. A bare audience such as `review` passed `rstack dev check` and failed when the product loaded its configuration. Validation now refuses it at the audience's plan pointer. Signed-off-by: Jeremi Joslin --- .../src/project_dev/bindings.rs | 43 +++++++++++++++++++ .../registry-bregctl/src/project_dev/plan.rs | 4 +- crates/registry-bregctl/tests/project_dev.rs | 2 +- products/platform/PROJECT-DEV.md | 4 +- 4 files changed, 49 insertions(+), 4 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/bindings.rs b/crates/registry-bregctl/src/project_dev/bindings.rs index 1f12ac5b00..094f30d247 100644 --- a/crates/registry-bregctl/src/project_dev/bindings.rs +++ b/crates/registry-bregctl/src/project_dev/bindings.rs @@ -112,6 +112,14 @@ impl Bindings { "Products must place BReg before Casework when bindings are configured.", "Move the BReg product before the Casework product.", )?; + // Each binding requests a token for the other product's audience as + // its OAuth resource, which the products accept only as an absolute URI. + if self.breg_review_authority.is_some() { + require_resource_audience(plan, casework, "bregReviewAuthority")?; + } + if self.casework_context_provider.is_some() { + require_resource_audience(plan, breg, "caseworkContextProvider")?; + } if let Some(binding) = &self.breg_review_authority { plan::require( @@ -163,6 +171,22 @@ fn product_position(plan: &Plan, product: Product) -> Option { .position(|entry| entry.product == product) } +fn require_resource_audience( + plan: &Plan, + position: Option, + binding: &str, +) -> plan::SemanticResult<()> { + let Some(index) = position else { + return Ok(()); + }; + plan::require( + registry_platform_httputil::valid_resource_uri(&plan.products[index].audience), + &format!("/products/{index}/audience"), + format!("The audience must be an absolute URI because /bindings/{binding} requests tokens for it as an OAuth resource."), + "Set audience to an absolute URI without a fragment or credentials, such as `urn:example:review`.", + ) +} + fn require_machine_person<'a>( plan: &'a Plan, alias: &str, @@ -806,6 +830,25 @@ mod tests { ); } + #[test] + fn validation_refuses_a_bound_audience_that_is_not_a_resource_uri() { + let mut value = plan(); + value.products[1].audience = "review".to_owned(); + let error = value.bindings.validate(&value).unwrap_err().to_string(); + assert!(error.contains("/products/1/audience"), "{error}"); + assert!(error.contains("absolute URI"), "{error}"); + + let mut value = plan(); + value.products[0].audience = "registry".to_owned(); + let error = value.bindings.validate(&value).unwrap_err().to_string(); + assert!(error.contains("/products/0/audience"), "{error}"); + + let mut value = plan(); + value.bindings.casework_context_provider = None; + value.products[0].audience = "registry".to_owned(); + value.bindings.validate(&value).unwrap(); + } + async fn sign_assertion( key: &PrivateJwk, client: &str, diff --git a/crates/registry-bregctl/src/project_dev/plan.rs b/crates/registry-bregctl/src/project_dev/plan.rs index 3f9797f6cd..44154e410e 100644 --- a/crates/registry-bregctl/src/project_dev/plan.rs +++ b/crates/registry-bregctl/src/project_dev/plan.rs @@ -1430,8 +1430,8 @@ mod tests { json!({ "issuer": "https://issuer.example.invalid", "products": [ - {"product": "breg", "project": ".", "audience": "registry"}, - {"product": "casework", "project": ".", "audience": "review"} + {"product": "breg", "project": ".", "audience": "urn:example:registry"}, + {"product": "casework", "project": ".", "audience": "urn:example:review"} ], "people": { "context-reader": { diff --git a/crates/registry-bregctl/tests/project_dev.rs b/crates/registry-bregctl/tests/project_dev.rs index c116a1572c..f895516d1a 100644 --- a/crates/registry-bregctl/tests/project_dev.rs +++ b/crates/registry-bregctl/tests/project_dev.rs @@ -194,7 +194,7 @@ fn casework_reference_project( "apiVersion: id.registrystack.org/formats/breg/project-dev/v1alpha1\n\ kind: BRegProjectDev\n\ issuer: https://issuer.example.invalid\n\ -products:\n - {{product: breg, project: ., audience: registry}}\n - {{product: casework, project: casework, audience: review}}\n\ +products:\n - {{product: breg, project: ., audience: \"urn:example:registry\"}}\n - {{product: casework, project: casework, audience: \"urn:example:review\"}}\n\ people:\n operator:\n subject: operator\n client: operator-client\n grants:\n breg: {{profile: operator, scopes: [registry:read]}}\n reviewer:\n subject: reviewer\n client: reviewer-client\n grants:\n casework: {{profile: {casework_profile}, scopes: [casework:review]}}\n\ journey:\n id: smoke\n steps:\n - {{id: read, product: breg, as: operator, method: GET, path: /v1/records, expect: {{status: 200}}}}\n\ {bindings}" diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index fa90e53357..9a7e1b86b9 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -131,7 +131,9 @@ Casework producer, and `bindings.caseworkContextProvider` identifies the BReg context provider and action. `rstack dev check` refuses a binding that names a review authority or action the BReg project does not declare, or a producer or context provider the Casework project does not declare; `start` refuses it -before it provisions anything. The run pairs the provider on a private copy +before it provisions anything. A binding requests tokens for the other +product's audience as an OAuth resource, so that audience must be an absolute +URI, such as `urn:example:review`. The run pairs the provider on a private copy before packaging. A reviewer whose Casework context call forwards the bearer to BReg declares both grants, one shared client, and `caseworkContext: true`. Both grants declare the same scopes and additional claims. Casework reads the From 749c31a6071c782a1c4da480acd077819b69ec39 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 06:27:57 +0700 Subject: [PATCH 32/34] fix(bregctl): bound how long stop waits for the dev token server Stop waited for every open token request to finish, so a client that sent headers and never finished its body held `rstack dev stop` forever. The server now gets two seconds to drain, then open requests are dropped. Signed-off-by: Jeremi Joslin --- .../src/project_dev/bindings.rs | 65 +++++++++++++++++-- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/bindings.rs b/crates/registry-bregctl/src/project_dev/bindings.rs index 094f30d247..f53cd91bb5 100644 --- a/crates/registry-bregctl/src/project_dev/bindings.rs +++ b/crates/registry-bregctl/src/project_dev/bindings.rs @@ -45,6 +45,8 @@ const FORM_LIMIT: usize = 16 * 1024; const ASSERTION_LIMIT: usize = 12 * 1024; const ASSERTION_LIFETIME_SECONDS: u64 = 300; const TOKEN_LIFETIME_SECONDS: u64 = 300; +/// How long stop lets the token endpoint finish requests already in progress. +const TOKEN_SERVER_SHUTDOWN_GRACE: Duration = Duration::from_secs(2); const CLOCK_LEEWAY_SECONDS: u64 = 5; const MAXIMUM_REPLAY_ENTRIES: usize = 256; @@ -522,12 +524,25 @@ fn start_token_server( let Ok(listener) = tokio::net::TcpListener::from_std(listener) else { return; }; - let _ = axum::serve(listener, app) - .with_graceful_shutdown(async move { - let _ = shutdown_rx.await; - }) - .await; + let (drain_tx, drain_rx) = tokio::sync::oneshot::channel::<()>(); + let server = async move { + let _ = axum::serve(listener, app) + .with_graceful_shutdown(async move { + let _ = drain_rx.await; + }) + .await; + }; + tokio::pin!(server); + tokio::select! { + () = &mut server => return, + _ = shutdown_rx => {} + } + let _ = drain_tx.send(()); + // A client that never finishes its request must not hold up + // stop; whatever is still open after the grace is dropped. + let _ = tokio::time::timeout(TOKEN_SERVER_SHUTDOWN_GRACE, server).await; }); + runtime.shutdown_timeout(TOKEN_SERVER_SHUTDOWN_GRACE); })?; Ok(TokenServer { endpoint, @@ -849,6 +864,46 @@ mod tests { value.bindings.validate(&value).unwrap(); } + #[test] + fn stopping_the_token_server_does_not_wait_for_an_unfinished_request() { + use std::io::Write; + + let root = tempfile::tempdir().unwrap(); + let issuer = Arc::new(Issuer::create(root.path()).unwrap()); + let origins = BTreeMap::from([ + (Product::Breg, "http://127.0.0.1:31001".to_owned()), + (Product::Casework, "http://127.0.0.1:31002".to_owned()), + ]); + let runtime = + RuntimeBindings::prepare(root.path(), Arc::new(input(plan())), issuer, &origins) + .unwrap(); + let endpoint = runtime.server.as_ref().unwrap().endpoint.clone(); + let address = endpoint + .trim_start_matches("http://") + .trim_end_matches("/oauth2/token") + .to_owned(); + let mut client = std::net::TcpStream::connect(address).unwrap(); + client + .write_all( + b"POST /oauth2/token HTTP/1.1\r\nHost: localhost\r\n\ + Content-Type: application/x-www-form-urlencoded\r\n\ + Content-Length: 100\r\n\r\ngrant_type=", + ) + .unwrap(); + thread::sleep(Duration::from_millis(200)); + + let (stopped, wait) = std::sync::mpsc::channel(); + thread::spawn(move || { + drop(runtime); + let _ = stopped.send(()); + }); + assert!( + wait.recv_timeout(Duration::from_secs(10)).is_ok(), + "the token server waited for a client that never finished its request" + ); + drop(client); + } + async fn sign_assertion( key: &PrivateJwk, client: &str, From 019563b493d1e0f969b79e1d3450e43c18761ccf Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 06:48:21 +0700 Subject: [PATCH 33/34] test(bregctl): give the dev poll tests room on a busy machine A journey run spends about 200 ms building its client before the first request, and the poll tests allowed 300 ms per poll inside a 500 ms server window, so an occasional stall failed them. The budgets are now one second inside a two-second window; the late body still arrives after the deadline. Signed-off-by: Jeremi Joslin --- crates/registry-bregctl/src/project_dev/journey.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/registry-bregctl/src/project_dev/journey.rs b/crates/registry-bregctl/src/project_dev/journey.rs index 3c11b455f7..0cd7596abf 100644 --- a/crates/registry-bregctl/src/project_dev/journey.rs +++ b/crates/registry-bregctl/src/project_dev/journey.rs @@ -623,6 +623,11 @@ mod tests { } } + /// How long a synthetic server accepts requests. A journey run spends a few + /// hundred milliseconds before its first request, so the poll budgets below + /// leave room for a busy machine. + const SERVER_WINDOW: Duration = Duration::from_secs(2); + fn synthetic_server(responses: Vec<(Duration, u16, &'static str)>) -> SyntheticServer { let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); listener.set_nonblocking(true).unwrap(); @@ -630,7 +635,7 @@ mod tests { let requests = Arc::new(AtomicUsize::new(0)); let observed = requests.clone(); let thread = thread::spawn(move || { - let end = Instant::now() + Duration::from_millis(500); + let end = Instant::now() + SERVER_WINDOW; let mut response = 0; while Instant::now() < end { match listener.accept() { @@ -741,11 +746,11 @@ mod tests { .unwrap_or_else(|error| error.into_inner()); let server = synthetic_server(vec![ (Duration::ZERO, 202, r#"{"state":"waiting"}"#), - (Duration::from_millis(500), 200, r#"{"state":"ready"}"#), + (Duration::from_millis(1500), 200, r#"{"state":"ready"}"#), ]); let root = tempfile::tempdir().unwrap(); let issuer = Issuer::create(root.path()).unwrap(); - let input = polling_input(300, 50); + let input = polling_input(1000, 50); let urls = BTreeMap::from([(Product::Breg, server.url.clone())]); let report = run(&input, &issuer, &urls, || Ok(())).unwrap(); @@ -767,7 +772,7 @@ mod tests { let server = synthetic_server(vec![(Duration::ZERO, 202, r#"{"state":"waiting"}"#)]); let root = tempfile::tempdir().unwrap(); let issuer = Issuer::create(root.path()).unwrap(); - let input = polling_input(300, 300); + let input = polling_input(1000, 1000); let urls = BTreeMap::from([(Product::Breg, server.url.clone())]); let report = run(&input, &issuer, &urls, || Ok(())).unwrap(); From c99f843a502314f227e4bf6ea0d97514e7ce2202 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Wed, 11 Nov 2026 07:43:12 +0700 Subject: [PATCH 34/34] fix(bregctl): check dev binding scopes against the OAuth limits BReg and Casework request a binding identity's grant scopes through the shared OAuth client, which refuses quoted or backslashed scopes and lists longer than 4096 bytes at startup. rstack dev check accepted both, so the run failed after provisioning. The binding check now applies the same shared predicate and reports the grant's scopes pointer. Signed-off-by: Jeremi Joslin --- .../src/project_dev/bindings.rs | 66 +++++++++++++++++++ products/platform/PROJECT-DEV.md | 5 +- 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/crates/registry-bregctl/src/project_dev/bindings.rs b/crates/registry-bregctl/src/project_dev/bindings.rs index f53cd91bb5..4a85b97016 100644 --- a/crates/registry-bregctl/src/project_dev/bindings.rs +++ b/crates/registry-bregctl/src/project_dev/bindings.rs @@ -233,6 +233,19 @@ fn require_machine_person<'a>( product.name() ), )?; + // The calling product requests these scopes through the shared OAuth + // client, which refuses the list at startup unless it passes this check. + if let Err(error) = registry_platform_httputil::validate_requested_scopes(&grant.scopes) { + let reason = match error { + registry_platform_httputil::TokenError::Configuration { reason } => reason.to_owned(), + other => other.to_string(), + }; + return plan::refuse( + &format!("/people/{alias}/grants/{}/scopes", product.name()), + &format!("A binding requests this grant's scopes over OAuth, and {reason}."), + "Use only OAuth scope characters (no spaces, quotes, or backslashes), and keep the scopes, joined by spaces, within 4096 bytes.", + ); + } Ok(person) } @@ -864,6 +877,59 @@ mod tests { value.bindings.validate(&value).unwrap(); } + #[test] + fn validation_refuses_bound_scopes_the_products_would_not_request() { + // Both products request a binding's scopes through the shared OAuth + // client, which refuses this list before the run could start them. + let too_long = (0..17) + .map(|index| format!("{index:02}{}", "s".repeat(254))) + .collect::>(); + let mut value = plan(); + value + .people + .get_mut("producer") + .unwrap() + .grants + .get_mut(&Product::Casework) + .unwrap() + .scopes + .clone_from(&too_long); + let error = value.bindings.validate(&value).unwrap_err().to_string(); + assert!( + error.contains("/people/producer/grants/casework/scopes"), + "{error}" + ); + assert!(error.contains("4096"), "{error}"); + + let mut value = plan(); + value + .people + .get_mut("context-reader") + .unwrap() + .grants + .get_mut(&Product::Breg) + .unwrap() + .scopes = vec!["registry:\"quoted\"".to_owned()]; + let error = value.bindings.validate(&value).unwrap_err().to_string(); + assert!( + error.contains("/people/context-reader/grants/breg/scopes"), + "{error}" + ); + + // A person no binding names keeps the plan's own scope rule. + let mut value = plan(); + value.bindings.casework_context_provider = None; + value + .people + .get_mut("context-reader") + .unwrap() + .grants + .get_mut(&Product::Breg) + .unwrap() + .scopes = too_long; + value.bindings.validate(&value).unwrap(); + } + #[test] fn stopping_the_token_server_does_not_wait_for_an_unfinished_request() { use std::io::Write; diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md index 9a7e1b86b9..8859aed6ab 100644 --- a/products/platform/PROJECT-DEV.md +++ b/products/platform/PROJECT-DEV.md @@ -133,7 +133,10 @@ review authority or action the BReg project does not declare, or a producer or context provider the Casework project does not declare; `start` refuses it before it provisions anything. A binding requests tokens for the other product's audience as an OAuth resource, so that audience must be an absolute -URI, such as `urn:example:review`. The run pairs the provider on a private copy +URI, such as `urn:example:review`. It requests the scopes of its identity's +grant for that product, so those scopes must be OAuth scope-tokens (no quotes or +backslashes) that fit in 4096 bytes when joined by spaces; `check` reports a +violation at the grant's `scopes`. The run pairs the provider on a private copy before packaging. A reviewer whose Casework context call forwards the bearer to BReg declares both grants, one shared client, and `caseworkContext: true`. Both grants declare the same scopes and additional claims. Casework reads the