diff --git a/.github/scripts/ci_changes.py b/.github/scripts/ci_changes.py index 6f728ce6da..4a9aaaa0d8 100644 --- a/.github/scripts/ci_changes.py +++ b/.github/scripts/ci_changes.py @@ -382,6 +382,9 @@ "docs/site/src/content/docs/configure/casework.mdx", "docs/site/src/content/docs/operate/change-a-casework-package.mdx", "docs/site/src/content/docs/tutorials/first-casework.mdx", + "products/breg/acceptance/reviewed-evidence-journey/**", + "products/platform/examples/licensing/**", + "products/platform/scripts/test-rstack-dev.py", ) # Every input the Registry Messaging tutorial gate replays or is built from. @@ -573,7 +576,7 @@ # reader makes carries a token that session issued. The clients in the Casework # shard are not on the replayed path. CASEWORK_TUTORIAL_PACKAGES = frozenset( - {"registry-casework", "registry-caseworkctl", "registry-breg", "registry-bregctl", "registry-thunderid-tooling"} + {"registry-casework", "registry-caseworkctl", "registry-breg", "registry-bregctl", "registry-rstack", "registry-thunderid-tooling"} ) # The gate builds and runs exactly messagingctl, which links the Messaging diff --git a/.github/scripts/test_ci_changes.py b/.github/scripts/test_ci_changes.py index 21ce80ec5b..1b3bd9f751 100644 --- a/.github/scripts/test_ci_changes.py +++ b/.github/scripts/test_ci_changes.py @@ -1945,6 +1945,8 @@ def test_casework_tutorial_routing(self) -> None: "docs/site/scripts/tutorial-runner/toolsets.mjs", "docs/site/src/content/docs/tutorials/first-casework.mdx", "docs/site/package.json", + "products/platform/examples/licensing/dev.yaml", + "products/platform/scripts/test-rstack-dev.py", ) for path in infrastructure: with self.subTest(path=path): diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7de868a5ce..02a92deccb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2697,6 +2697,26 @@ jobs: -p registry-breg --features registry-breg/runtime \ -p registry-bregctl -p registry-rstack --bins + - name: Install native PostgreSQL 18 and PostGIS for project development + # The Ubuntu runner's PostgreSQL 16 is below BReg's minimum. + shell: bash + run: | + set -euo pipefail + sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + sudo env DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends postgresql-18 postgresql-18-postgis-3 + /usr/lib/postgresql/18/bin/postgres --version + + - name: Verify the combined licensing development run with native PostgreSQL + shell: bash + run: | + set -euo pipefail + cargo build --locked -p registry-rstack + python3 products/platform/scripts/test-rstack-dev.py \ + --bin-dir "${GITHUB_WORKSPACE}/target/debug" \ + --postgres-bin-dir /usr/lib/postgresql/18/bin \ + --scratch-parent "${RUNNER_TEMP}" \ + --spatial + - name: Execute the Registry Casework tutorials from a reader directory shell: bash env: diff --git a/Cargo.lock b/Cargo.lock index 0b2f2c4d79..fc659c6a13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6327,6 +6327,7 @@ dependencies = [ "inquire", "jsonschema", "p256", + "postgres-protocol", "rcgen", "registry-breg", "registry-breg-client", @@ -8173,6 +8174,7 @@ dependencies = [ "registry-platform-buildinfo", "registry-platform-canonical-json", "registry-platform-config", + "registry-platform-yaml", "registry-thunderid-tooling", "rustix 1.1.5", "serde", diff --git a/Cargo.toml b/Cargo.toml index 4536e3a3bb..50d855842b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -331,6 +331,7 @@ tempfile = { version = "3" } thiserror = { version = "2" } time = { version = "0.3", features = ["serde", "formatting", "parsing", "macros"] } tokio = { version = "1", features = ["fs", "macros", "net", "rt", "rt-multi-thread", "signal", "sync", "time"] } +postgres-protocol = "=0.6.12" tokio-postgres = { version = "=0.7.18", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } tokio-postgres-rustls = { version = "=0.14.0", default-features = false, features = ["ring", "native-certs"] } tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } diff --git a/crates/registry-bregctl/Cargo.toml b/crates/registry-bregctl/Cargo.toml index e4190a4ebf..4637a037fd 100644 --- a/crates/registry-bregctl/Cargo.toml +++ b/crates/registry-bregctl/Cargo.toml @@ -52,6 +52,7 @@ hmac.workspace = true inquire.workspace = true jsonschema.workspace = true p256.workspace = true +postgres-protocol.workspace = true rcgen.workspace = true registry-platform-canonical-json.workspace = true registry-platform-config.workspace = true @@ -69,6 +70,7 @@ registry-linkml.workspace = true registry-manifest-core.workspace = true reqwest.workspace = true rustix.workspace = true +rustls.workspace = true schemars = { workspace = true, optional = true } serde.workspace = true serde_json.workspace = true @@ -79,6 +81,8 @@ tempfile.workspace = true thiserror.workspace = true time.workspace = true tokio = { workspace = true, features = ["rt"] } +tokio-postgres.workspace = true +tokio-postgres-rustls.workspace = true uuid = { workspace = true, features = ["serde"] } zeroize.workspace = true registry-platform-crypto.workspace = true @@ -87,7 +91,4 @@ url.workspace = true [dev-dependencies] registry-breg.workspace = true registry-platform-testing = { workspace = true, features = ["test-utils"] } -rustls.workspace = true -tokio-postgres.workspace = true -tokio-postgres-rustls.workspace = true wat = "1.259.0" diff --git a/crates/registry-bregctl/src/identity_facts.rs b/crates/registry-bregctl/src/identity_facts.rs index 8e30fac601..64370df215 100644 --- a/crates/registry-bregctl/src/identity_facts.rs +++ b/crates/registry-bregctl/src/identity_facts.rs @@ -197,6 +197,37 @@ impl<'a> Grant<'a> { } } +fn compiled_grants(compiled: &CompiledRegistry) -> impl Iterator> { + let entity_grants = compiled + .entities() + .values() + .flat_map(|entity| entity.access_profiles.values().map(Grant::profile)); + let dataset_grants = compiled + .statistical_datasets() + .values() + .flat_map(|dataset| dataset.access_profiles.values().map(Grant::profile)); + let action_grants = compiled.actions().actions.iter().flat_map(|action| { + let proposal = match &action.execution { + CompiledActionExecution::Proposal { proposal } => proposal.permissions.as_slice(), + CompiledActionExecution::Immediate => &[], + }; + action + .permissions + .iter() + .map(Grant::action) + .chain(proposal.iter().map(Grant::proposal)) + }); + entity_grants.chain(dataset_grants).chain(action_grants) +} + +/// Clients the runtime must list even if no fixture mints tokens for them. +pub(crate) fn named_requester_clients(compiled: &CompiledRegistry) -> BTreeSet { + compiled_grants(compiled) + .flat_map(|grant| grant.requester_clients) + .chain(compiled.recipients().clients().map(str::to_owned)) + .collect() +} + /// The claims BReg refuses a token for a grant without, apart from the /// subject every token carries and the reserved consent claims the engine /// fills from the verified client. @@ -286,26 +317,7 @@ pub(crate) fn facts( let mut profiles: BTreeMap = BTreeMap::new(); let mut omitted_agent_profiles = BTreeSet::new(); - let entity_grants = compiled - .entities() - .values() - .flat_map(|entity| entity.access_profiles.values().map(Grant::profile)); - let dataset_grants = compiled - .statistical_datasets() - .values() - .flat_map(|dataset| dataset.access_profiles.values().map(Grant::profile)); - let action_grants = compiled.actions().actions.iter().flat_map(|action| { - let proposal = match &action.execution { - CompiledActionExecution::Proposal { proposal } => proposal.permissions.as_slice(), - CompiledActionExecution::Immediate => &[], - }; - action - .permissions - .iter() - .map(Grant::action) - .chain(proposal.iter().map(Grant::proposal)) - }); - for grant in entity_grants.chain(dataset_grants).chain(action_grants) { + for grant in compiled_grants(compiled) { if grant.actor_kind == Some(ActorKindSource::Agent) { omitted_agent_profiles.insert(grant.id.to_owned()); continue; diff --git a/crates/registry-bregctl/src/lib.rs b/crates/registry-bregctl/src/lib.rs index a3c0166809..36b4de7272 100644 --- a/crates/registry-bregctl/src/lib.rs +++ b/crates/registry-bregctl/src/lib.rs @@ -62,6 +62,7 @@ mod instance_claim_lifecycle; mod module_lock_patch; mod package_inspection; mod package_lifecycle; +mod project_dev; mod proposal_retention; mod reconcile_lifecycle; mod report; @@ -225,6 +226,8 @@ enum Command { Examples(dev::examples::ExamplesArgs), #[command(name = "__dev-supervisor", hide = true)] DevSupervisor(dev::SupervisorArgs), + #[command(name = "project-dev", hide = true)] + ProjectDev(project_dev::Args), /// Explain compiled model, access, route, or event inventories. Explain(ExplainArgs), /// Compare an authoring candidate with a rederived closed package. @@ -2902,6 +2905,9 @@ where } }; } + Command::ProjectDev(args) => { + return ExitCode::from(project_dev::run(args, stdout, stderr)); + } Command::Init(args) => match (args.from, args.template.as_deref()) { (None, None) => init(&args.destination), (None, Some(id)) => starters::run(&args.destination, id), diff --git a/crates/registry-bregctl/src/project_dev/bindings.rs b/crates/registry-bregctl/src/project_dev/bindings.rs new file mode 100644 index 0000000000..4a85b97016 --- /dev/null +++ b/crates/registry-bregctl/src/project_dev/bindings.rs @@ -0,0 +1,1126 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Run-local bindings between BReg and Casework. +//! +//! The token endpoint exists only for one development run. It accepts the two +//! machine identities the plan explicitly binds, verifies their private-key +//! JWT assertions, and mints the same bounded product token a named test person +//! would receive. Keys, tokens, and assertion identifiers never enter reports +//! or logs. + +use anyhow::{ensure, Context, Result}; +use axum::{ + body::Bytes, + extract::{DefaultBodyLimit, State}, + http::{header, HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + routing::post, + Json, Router, +}; +use registry_platform_crypto::{ + generate_private_jwk, + jws::{parse_compact_jws, JwsLimits}, + parse_json_strict, GeneratedKeyAlgorithm, PrivateJwk, PublicJwk, SigningAlgorithm, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::{ + collections::{BTreeMap, BTreeSet}, + net::{Ipv4Addr, TcpListener}, + path::Path, + sync::{Arc, Mutex, Weak}, + thread, + time::Duration, +}; +use zeroize::Zeroizing; + +use super::{ + clock, + issuer::Issuer, + plan::{self, Plan, Product, TestPerson}, + write_private, RunInput, +}; + +const ASSERTION_TYPE: &str = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer"; +const FORM_LIMIT: usize = 16 * 1024; +const ASSERTION_LIMIT: usize = 12 * 1024; +const ASSERTION_LIFETIME_SECONDS: u64 = 300; +const TOKEN_LIFETIME_SECONDS: u64 = 300; +/// How long stop lets the token endpoint finish requests already in progress. +const TOKEN_SERVER_SHUTDOWN_GRACE: Duration = Duration::from_secs(2); +const CLOCK_LEEWAY_SECONDS: u64 = 5; +const MAXIMUM_REPLAY_ENTRIES: usize = 256; + +/// Run-local connections between BReg and Casework. Configuring either one requires both products, with BReg listed before Casework. +#[derive(Default, Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct Bindings { + /// Connects BReg to Casework as a review authority. BReg authenticates to Casework as a machine identity. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(super) breg_review_authority: Option, + /// Connects Casework to BReg as a context provider. Casework authenticates to BReg as a machine identity. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(super) casework_context_provider: Option, +} + +/// A BReg review authority that the run connects to Casework. +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct BregReviewAuthority { + /// Local identifier of the review authority. + #[cfg_attr(feature = "schema", schemars(with = "registry_platform_yaml::LocalId"))] + pub(super) id: String, + /// Local identifier of the review producer. + #[cfg_attr(feature = "schema", schemars(with = "registry_platform_yaml::LocalId"))] + pub(super) producer: String, + /// Name of the test person, in `people`, whose Casework grant BReg uses to authenticate. That grant needs at least one scope. + #[cfg_attr(feature = "schema", schemars(with = "registry_platform_yaml::LocalId"))] + pub(super) identity: String, +} + +/// A Casework context provider that the run connects to BReg. +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct CaseworkContextProvider { + /// Local identifier of the context provider. + #[cfg_attr(feature = "schema", schemars(with = "registry_platform_yaml::LocalId"))] + pub(super) id: String, + /// Name of the test person, in `people`, whose BReg grant Casework uses to authenticate. That grant needs at least one scope. + #[cfg_attr(feature = "schema", schemars(with = "registry_platform_yaml::LocalId"))] + pub(super) identity: String, + /// Local identifier of the BReg action that the provider pairs with. + #[cfg_attr(feature = "schema", schemars(with = "registry_platform_yaml::LocalId"))] + pub(super) action: String, +} + +impl Bindings { + pub(super) fn validate(&self, plan: &Plan) -> plan::SemanticResult<()> { + if self.breg_review_authority.is_none() && self.casework_context_provider.is_none() { + return Ok(()); + } + let breg = product_position(plan, Product::Breg); + let casework = product_position(plan, Product::Casework); + plan::require( + breg.is_some() && casework.is_some(), + "/bindings", + "Run-local bindings require both BReg and Casework products.", + "Add both products to products, or remove bindings.", + )?; + plan::require( + breg < casework, + "/products", + "Products must place BReg before Casework when bindings are configured.", + "Move the BReg product before the Casework product.", + )?; + // Each binding requests a token for the other product's audience as + // its OAuth resource, which the products accept only as an absolute URI. + if self.breg_review_authority.is_some() { + require_resource_audience(plan, casework, "bregReviewAuthority")?; + } + if self.casework_context_provider.is_some() { + require_resource_audience(plan, breg, "caseworkContextProvider")?; + } + + if let Some(binding) = &self.breg_review_authority { + plan::require( + plan::identifier(&binding.id), + "/bindings/bregReviewAuthority/id", + "The BReg review authority ID must be a local identifier.", + "Set id to a lowercase local identifier of at most 64 characters.", + )?; + plan::require( + plan::identifier(&binding.producer), + "/bindings/bregReviewAuthority/producer", + "The BReg review producer must be a local identifier.", + "Set producer to a lowercase local identifier of at most 64 characters.", + )?; + require_machine_person( + plan, + &binding.identity, + Product::Casework, + "/bindings/bregReviewAuthority/identity", + )?; + } + if let Some(binding) = &self.casework_context_provider { + plan::require( + plan::identifier(&binding.id), + "/bindings/caseworkContextProvider/id", + "The Casework context provider ID must be a local identifier.", + "Set id to a lowercase local identifier of at most 64 characters.", + )?; + plan::require( + plan::identifier(&binding.action), + "/bindings/caseworkContextProvider/action", + "The Casework context provider action must be a local identifier.", + "Set action to a lowercase local identifier of at most 64 characters.", + )?; + require_machine_person( + plan, + &binding.identity, + Product::Breg, + "/bindings/caseworkContextProvider/identity", + )?; + } + Ok(()) + } +} + +fn product_position(plan: &Plan, product: Product) -> Option { + plan.products + .iter() + .position(|entry| entry.product == product) +} + +fn require_resource_audience( + plan: &Plan, + position: Option, + binding: &str, +) -> plan::SemanticResult<()> { + let Some(index) = position else { + return Ok(()); + }; + plan::require( + registry_platform_httputil::valid_resource_uri(&plan.products[index].audience), + &format!("/products/{index}/audience"), + format!("The audience must be an absolute URI because /bindings/{binding} requests tokens for it as an OAuth resource."), + "Set audience to an absolute URI without a fragment or credentials, such as `urn:example:review`.", + ) +} + +fn require_machine_person<'a>( + plan: &'a Plan, + alias: &str, + product: Product, + field: &str, +) -> plan::SemanticResult<&'a TestPerson> { + plan::require( + plan::identifier(alias), + field, + "The binding identity must be a test person alias.", + "Set identity to a lowercase local identifier naming a test person.", + )?; + plan::require( + plan.people.contains_key(alias), + field, + "The binding identity does not name a test person.", + "Set identity to the alias of a test person in people.", + )?; + let person = &plan.people[alias]; + plan::require( + person.grants.contains_key(&product), + field, + format!( + "The binding identity must name a person with a {} grant.", + product.name() + ), + format!( + "Add a {} grant to the named test person or choose another identity.", + product.name() + ), + )?; + let grant = &person.grants[&product]; + plan::require( + !grant.scopes.is_empty(), + field, + format!( + "The binding identity must name a person whose {} grant has a scope.", + product.name() + ), + format!( + "Add a scope to the named person's {} grant or choose another identity.", + product.name() + ), + )?; + // The calling product requests these scopes through the shared OAuth + // client, which refuses the list at startup unless it passes this check. + if let Err(error) = registry_platform_httputil::validate_requested_scopes(&grant.scopes) { + let reason = match error { + registry_platform_httputil::TokenError::Configuration { reason } => reason.to_owned(), + other => other.to_string(), + }; + return plan::refuse( + &format!("/people/{alias}/grants/{}/scopes", product.name()), + &format!("A binding requests this grant's scopes over OAuth, and {reason}."), + "Use only OAuth scope characters (no spaces, quotes, or backslashes), and keep the scopes, joined by spaces, within 4096 bytes.", + ); + } + Ok(person) +} + +struct PreparedBinding { + client_id_ref: String, + client_assertion_key_ref: String, + person: String, + product: Product, +} + +struct TokenServer { + endpoint: String, + shutdown: Option>, + thread: Option>, +} + +impl Drop for TokenServer { + fn drop(&mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +#[derive(Default)] +pub(super) struct RuntimeBindings { + server: Option, + breg_review: Option, + casework_context: Option, + input: Option>, + origins: BTreeMap, + // Kept outside the server state so Issuer's Tokio runtime is dropped on + // the supervisor thread, never from inside the token server's runtime. + _issuer: Option>, +} + +impl RuntimeBindings { + pub(super) fn prepare( + root: &Path, + input: Arc, + issuer: Arc, + origins: &BTreeMap, + ) -> Result { + let bindings = &input.plan.bindings; + if bindings.breg_review_authority.is_none() && bindings.casework_context_provider.is_none() + { + return Ok(Self::default()); + } + bindings.validate(&input.plan)?; + ensure!( + origins.contains_key(&Product::Breg) && origins.contains_key(&Product::Casework), + "run-local bindings require reserved BReg and Casework origins" + ); + + let mut clients = Vec::new(); + let breg_review = bindings + .breg_review_authority + .as_ref() + .map(|binding| { + prepare_binding( + root, + &input, + &binding.identity, + Product::Casework, + "breg-review-client", + &mut clients, + ) + }) + .transpose()?; + let casework_context = bindings + .casework_context_provider + .as_ref() + .map(|binding| { + prepare_binding( + root, + &input, + &binding.identity, + Product::Breg, + "casework-context-client", + &mut clients, + ) + }) + .transpose()?; + let server = start_token_server(input.clone(), issuer.clone(), clients)?; + Ok(Self { + server: Some(server), + breg_review, + casework_context, + input: Some(input), + origins: origins.clone(), + _issuer: Some(issuer), + }) + } + + pub(super) fn configure(&self, product: Product, config: &mut Value) { + let Some(input) = self.input.as_ref() else { + return; + }; + let endpoint = &self + .server + .as_ref() + .expect("prepared token server") + .endpoint; + match product { + Product::Breg => { + let (Some(binding), Some(prepared)) = ( + input.plan.bindings.breg_review_authority.as_ref(), + self.breg_review.as_ref(), + ) else { + return; + }; + let person = &input.plan.people[&prepared.person]; + let grant = &person.grants[&prepared.product]; + config["reviewAuthorities"][binding.id.as_str()] = json!({ + "endpoint": self.origins[&Product::Casework], + "profile": grant.profile, + "privateKeyJwt": { + "tokenEndpoint": endpoint, + "clientIdRef": prepared.client_id_ref, + "clientAssertionKeyRef": prepared.client_assertion_key_ref, + "assertionAudience": input.plan.issuer, + "resource": input.plan.product(Product::Casework).audience, + "scopes": grant.scopes, + }, + "producerId": binding.producer, + "recoveryDays": 7, + }); + } + Product::Casework => { + let (Some(binding), Some(prepared)) = ( + input.plan.bindings.casework_context_provider.as_ref(), + self.casework_context.as_ref(), + ) else { + return; + }; + let person = &input.plan.people[&prepared.person]; + let grant = &person.grants[&prepared.product]; + config["contextProviders"][binding.id.as_str()] = json!({ + "baseUrl": self.origins[&Product::Breg], + "statusProfile": grant.profile, + "tokenEndpoint": endpoint, + "clientAssertionAudience": input.plan.issuer, + "resource": input.plan.product(Product::Breg).audience, + "scopes": grant.scopes, + "clientIdRef": prepared.client_id_ref, + "clientAssertionKeyRef": prepared.client_assertion_key_ref, + }); + } + } + } +} + +fn prepare_binding( + root: &Path, + input: &RunInput, + person_alias: &str, + product: Product, + filename: &str, + clients: &mut Vec, +) -> Result { + let person = require_machine_person(&input.plan, person_alias, product, "/bindings")?; + let grant = &person.grants[&product]; + let key = generate_private_jwk(GeneratedKeyAlgorithm::Es256)?; + let public_key = key.public(); + let key_name = format!("{filename}-key"); + let id_name = format!("{filename}-id"); + let encoded = encode_private_jwk(&key)?; + write_private(&root.join(&key_name), &encoded)?; + write_private(&root.join(&id_name), person.client.as_bytes())?; + clients.push(MachineClient { + client_id: person.client.clone(), + person: person_alias.to_owned(), + product, + public_key, + resource: input.plan.product(product).audience.clone(), + scope: grant.scopes.join(" "), + }); + Ok(PreparedBinding { + client_id_ref: format!("secret:file/{id_name}"), + client_assertion_key_ref: format!("secret:file/{key_name}"), + person: person_alias.to_owned(), + product, + }) +} + +#[derive(Serialize)] +struct PrivateJwkDocument<'a> { + kty: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + kid: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + alg: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + crv: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + d: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + x: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + y: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + n: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + e: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + p: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + q: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + dp: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + dq: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + qi: Option<&'a str>, +} + +pub(super) fn encode_private_jwk(key: &PrivateJwk) -> Result>> { + Ok(Zeroizing::new(serde_json::to_vec(&PrivateJwkDocument { + kty: &key.kty, + kid: key.kid.as_deref(), + alg: key.alg.as_deref(), + crv: key.crv.as_deref(), + d: key.d.as_deref(), + x: key.x.as_deref(), + y: key.y.as_deref(), + n: key.n.as_deref(), + e: key.e.as_deref(), + p: key.p.as_deref(), + q: key.q.as_deref(), + dp: key.dp.as_deref(), + dq: key.dq.as_deref(), + qi: key.qi.as_deref(), + })?)) +} + +struct MachineClient { + client_id: String, + person: String, + product: Product, + public_key: PublicJwk, + resource: String, + scope: String, +} + +struct TokenState { + input: Arc, + issuer: Weak, + clients: Vec, + audience: String, + replay: Mutex>, +} + +fn start_token_server( + input: Arc, + issuer: Arc, + clients: Vec, +) -> Result { + ensure!(!clients.is_empty(), "a binding token client is required"); + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .context("cannot bind the run-local token endpoint")?; + listener.set_nonblocking(true)?; + let origin = format!("http://{}", listener.local_addr()?); + let endpoint = format!("{origin}/oauth2/token"); + let state = Arc::new(TokenState { + audience: input.plan.issuer.clone(), + input, + issuer: Arc::downgrade(&issuer), + clients, + replay: Mutex::new(BTreeMap::new()), + }); + let app = Router::new() + .route("/oauth2/token", post(token)) + .layer(DefaultBodyLimit::max(FORM_LIMIT)) + .with_state(state); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let server_thread = thread::Builder::new() + .name("rstack-dev-token".to_owned()) + .spawn(move || { + let Ok(runtime) = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + else { + return; + }; + runtime.block_on(async move { + let Ok(listener) = tokio::net::TcpListener::from_std(listener) else { + return; + }; + let (drain_tx, drain_rx) = tokio::sync::oneshot::channel::<()>(); + let server = async move { + let _ = axum::serve(listener, app) + .with_graceful_shutdown(async move { + let _ = drain_rx.await; + }) + .await; + }; + tokio::pin!(server); + tokio::select! { + () = &mut server => return, + _ = shutdown_rx => {} + } + let _ = drain_tx.send(()); + // A client that never finishes its request must not hold up + // stop; whatever is still open after the grace is dropped. + let _ = tokio::time::timeout(TOKEN_SERVER_SHUTDOWN_GRACE, server).await; + }); + runtime.shutdown_timeout(TOKEN_SERVER_SHUTDOWN_GRACE); + })?; + Ok(TokenServer { + endpoint, + shutdown: Some(shutdown_tx), + thread: Some(server_thread), + }) +} + +async fn token(State(state): State>, headers: HeaderMap, body: Bytes) -> Response { + match tokio::time::timeout(Duration::from_secs(5), issue_token(state, headers, body)).await { + Ok(response) => response, + Err(_) => oauth_error(StatusCode::SERVICE_UNAVAILABLE, "temporarily_unavailable"), + } +} + +async fn issue_token(state: Arc, headers: HeaderMap, body: Bytes) -> Response { + if !headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value + .split(';') + .next() + .is_some_and(|media| media.trim() == "application/x-www-form-urlencoded") + }) + || body.len() > FORM_LIMIT + || !body.is_ascii() + { + return oauth_error(StatusCode::BAD_REQUEST, "invalid_request"); + } + let Some(form) = parse_form(&body) else { + return oauth_error(StatusCode::BAD_REQUEST, "invalid_request"); + }; + if form.get("grant_type").map(String::as_str) != Some("client_credentials") { + return oauth_error(StatusCode::BAD_REQUEST, "unsupported_grant_type"); + } + if form.get("client_assertion_type").map(String::as_str) != Some(ASSERTION_TYPE) { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + } + let (Some(client_id), Some(assertion), Some(resource), Some(scope)) = ( + form.get("client_id"), + form.get("client_assertion"), + form.get("resource"), + form.get("scope"), + ) else { + return oauth_error(StatusCode::BAD_REQUEST, "invalid_request"); + }; + if assertion.len() > ASSERTION_LIMIT { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + } + + let limits = JwsLimits { + max_compact_bytes: ASSERTION_LIMIT, + max_header_bytes: 1024, + max_payload_bytes: 2048, + }; + let Ok(parsed) = parse_compact_jws(assertion, &limits) else { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + }; + let Some(kid) = parsed.header().get("kid").and_then(Value::as_str) else { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + }; + let Some(client) = state.clients.iter().find(|candidate| { + candidate.client_id == client_id.as_str() + && candidate.public_key.kid.as_deref() == Some(kid) + && candidate.resource == resource.as_str() + && candidate.scope == scope.as_str() + }) else { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + }; + if parsed.header().len() != 3 + || parsed.header().get("typ").and_then(Value::as_str) != Some("JWT") + || parsed.header().get("alg").and_then(Value::as_str) != Some("ES256") + { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + } + let Ok(payload) = parsed.verify(&client.public_key, &[SigningAlgorithm::Es256]) else { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + }; + let Ok(payload) = parse_json_strict(payload) else { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + }; + let Ok(claims) = serde_json::from_value::(payload) else { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + }; + let Ok(now) = clock() else { + return oauth_error(StatusCode::SERVICE_UNAVAILABLE, "temporarily_unavailable"); + }; + if claims.iss != client.client_id + || claims.sub != client.client_id + || claims.aud != state.audience + || claims.jti.is_empty() + || claims.jti.len() > 128 + || claims + .jti + .chars() + .any(|value| value.is_control() || value.is_whitespace()) + || claims.iat > now.saturating_add(CLOCK_LEEWAY_SECONDS) + || claims.exp <= now + || !(1..=ASSERTION_LIFETIME_SECONDS).contains(&claims.exp.saturating_sub(claims.iat)) + { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + } + { + let Ok(mut replay) = state.replay.lock() else { + return oauth_error(StatusCode::SERVICE_UNAVAILABLE, "temporarily_unavailable"); + }; + replay.retain(|_, expiry| *expiry > now); + if replay.contains_key(&claims.jti) || replay.len() >= MAXIMUM_REPLAY_ENTRIES { + return oauth_error(StatusCode::UNAUTHORIZED, "invalid_client"); + } + replay.insert(claims.jti, claims.exp); + } + let Some(person) = state.input.plan.people.get(&client.person) else { + return oauth_error(StatusCode::SERVICE_UNAVAILABLE, "temporarily_unavailable"); + }; + let Some(issuer) = state.issuer.upgrade() else { + return oauth_error(StatusCode::SERVICE_UNAVAILABLE, "temporarily_unavailable"); + }; + let Ok(access_token) = issuer + .sign( + &state.input, + person, + client.product, + now.saturating_add(TOKEN_LIFETIME_SECONDS), + ) + .await + else { + return oauth_error(StatusCode::SERVICE_UNAVAILABLE, "temporarily_unavailable"); + }; + ( + StatusCode::OK, + [ + (header::CACHE_CONTROL, "no-store"), + (header::PRAGMA, "no-cache"), + ], + Json(json!({ + "access_token": &*access_token, + "token_type": "Bearer", + "expires_in": TOKEN_LIFETIME_SECONDS, + })), + ) + .into_response() +} + +fn parse_form(body: &[u8]) -> Option> { + let mut values = BTreeMap::new(); + let allowed = BTreeSet::from([ + "grant_type", + "client_id", + "client_assertion_type", + "client_assertion", + "resource", + "scope", + ]); + for (name, value) in url::form_urlencoded::parse(body) { + if !allowed.contains(name.as_ref()) + || values + .insert(name.into_owned(), value.into_owned()) + .is_some() + { + return None; + } + } + (values.len() == allowed.len()).then_some(values) +} + +fn oauth_error(status: StatusCode, error: &'static str) -> Response { + ( + status, + [(header::CACHE_CONTROL, "no-store")], + Json(json!({"error": error})), + ) + .into_response() +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct AssertionClaims { + iss: String, + sub: String, + aud: String, + iat: u64, + exp: u64, + jti: String, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::project_dev::{ + plan::{Grant, Journey, ProductPlan}, + BregAuthority, + }; + use registry_platform_crypto::{jws::sign_compact_jws, LocalJwkSigner, PrivateJwk}; + use registry_platform_httputil::{PrivateKeyJwt, PrivateKeyJwtConfig, TokenProvider}; + use serde_json::{Map, Value}; + use std::{collections::BTreeMap, path::PathBuf}; + + fn grant(profile: &str, scope: &str) -> Grant { + Grant { + profile: profile.to_owned(), + scopes: vec![scope.to_owned()], + claims: Map::new(), + } + } + + fn plan() -> Plan { + Plan { + issuer: "https://identity.example.test/realms/dev".to_owned(), + products: vec![ + ProductPlan { + product: Product::Breg, + project: PathBuf::from("registry"), + audience: "urn:dev:breg".to_owned(), + }, + ProductPlan { + product: Product::Casework, + project: PathBuf::from("casework"), + audience: "urn:dev:casework".to_owned(), + }, + ], + people: BTreeMap::from([ + ( + "producer".to_owned(), + TestPerson { + subject: "review-producer".to_owned(), + client: "review-producer-client".to_owned(), + grants: BTreeMap::from([( + Product::Casework, + grant("producer", "casework:review:produce"), + )]), + casework_context: false, + }, + ), + ( + "context-reader".to_owned(), + TestPerson { + subject: "context-reader".to_owned(), + client: "context-reader-client".to_owned(), + grants: BTreeMap::from([( + Product::Breg, + grant("casework-review-status", "registry:proposal:review-status"), + )]), + casework_context: false, + }, + ), + ]), + bindings: Bindings { + breg_review_authority: Some(BregReviewAuthority { + id: "casework".to_owned(), + producer: "registry".to_owned(), + identity: "producer".to_owned(), + }), + casework_context_provider: Some(CaseworkContextProvider { + id: "registry".to_owned(), + identity: "context-reader".to_owned(), + action: "reviewed-change".to_owned(), + }), + }, + journey: Journey { + id: "binding".to_owned(), + steps: Vec::new(), + }, + schema_test_people: BTreeMap::new(), + } + } + + fn input(plan: Plan) -> RunInput { + RunInput { + run_id: "0123456789abcdef0123456789abcdef".to_owned(), + project: PathBuf::from("/tmp/project"), + plan, + ttl_seconds: 300, + schema_people: Vec::new(), + breg_authority: Some(BregAuthority { + principal: "sub".to_owned(), + purpose: false, + requires_postgis: false, + named_clients: BTreeSet::new(), + }), + casework_principals: BTreeMap::from([("producer".to_owned(), "sub".to_owned())]), + binaries: BTreeMap::new(), + } + } + + #[test] + fn validation_refuses_missing_grants_and_wrong_product_order() { + let mut value = plan(); + value.products.reverse(); + let error = value.bindings.validate(&value).unwrap_err().to_string(); + assert!(error.contains("/products"), "{error}"); + + let mut value = plan(); + value.people.get_mut("producer").unwrap().grants.clear(); + let error = value.bindings.validate(&value).unwrap_err().to_string(); + assert!( + error.contains("/bindings/bregReviewAuthority/identity"), + "{error}" + ); + } + + #[test] + fn validation_refuses_a_bound_audience_that_is_not_a_resource_uri() { + let mut value = plan(); + value.products[1].audience = "review".to_owned(); + let error = value.bindings.validate(&value).unwrap_err().to_string(); + assert!(error.contains("/products/1/audience"), "{error}"); + assert!(error.contains("absolute URI"), "{error}"); + + let mut value = plan(); + value.products[0].audience = "registry".to_owned(); + let error = value.bindings.validate(&value).unwrap_err().to_string(); + assert!(error.contains("/products/0/audience"), "{error}"); + + let mut value = plan(); + value.bindings.casework_context_provider = None; + value.products[0].audience = "registry".to_owned(); + value.bindings.validate(&value).unwrap(); + } + + #[test] + fn validation_refuses_bound_scopes_the_products_would_not_request() { + // Both products request a binding's scopes through the shared OAuth + // client, which refuses this list before the run could start them. + let too_long = (0..17) + .map(|index| format!("{index:02}{}", "s".repeat(254))) + .collect::>(); + let mut value = plan(); + value + .people + .get_mut("producer") + .unwrap() + .grants + .get_mut(&Product::Casework) + .unwrap() + .scopes + .clone_from(&too_long); + let error = value.bindings.validate(&value).unwrap_err().to_string(); + assert!( + error.contains("/people/producer/grants/casework/scopes"), + "{error}" + ); + assert!(error.contains("4096"), "{error}"); + + let mut value = plan(); + value + .people + .get_mut("context-reader") + .unwrap() + .grants + .get_mut(&Product::Breg) + .unwrap() + .scopes = vec!["registry:\"quoted\"".to_owned()]; + let error = value.bindings.validate(&value).unwrap_err().to_string(); + assert!( + error.contains("/people/context-reader/grants/breg/scopes"), + "{error}" + ); + + // A person no binding names keeps the plan's own scope rule. + let mut value = plan(); + value.bindings.casework_context_provider = None; + value + .people + .get_mut("context-reader") + .unwrap() + .grants + .get_mut(&Product::Breg) + .unwrap() + .scopes = too_long; + value.bindings.validate(&value).unwrap(); + } + + #[test] + fn stopping_the_token_server_does_not_wait_for_an_unfinished_request() { + use std::io::Write; + + let root = tempfile::tempdir().unwrap(); + let issuer = Arc::new(Issuer::create(root.path()).unwrap()); + let origins = BTreeMap::from([ + (Product::Breg, "http://127.0.0.1:31001".to_owned()), + (Product::Casework, "http://127.0.0.1:31002".to_owned()), + ]); + let runtime = + RuntimeBindings::prepare(root.path(), Arc::new(input(plan())), issuer, &origins) + .unwrap(); + let endpoint = runtime.server.as_ref().unwrap().endpoint.clone(); + let address = endpoint + .trim_start_matches("http://") + .trim_end_matches("/oauth2/token") + .to_owned(); + let mut client = std::net::TcpStream::connect(address).unwrap(); + client + .write_all( + b"POST /oauth2/token HTTP/1.1\r\nHost: localhost\r\n\ + Content-Type: application/x-www-form-urlencoded\r\n\ + Content-Length: 100\r\n\r\ngrant_type=", + ) + .unwrap(); + thread::sleep(Duration::from_millis(200)); + + let (stopped, wait) = std::sync::mpsc::channel(); + thread::spawn(move || { + drop(runtime); + let _ = stopped.send(()); + }); + assert!( + wait.recv_timeout(Duration::from_secs(10)).is_ok(), + "the token server waited for a client that never finished its request" + ); + drop(client); + } + + async fn sign_assertion( + key: &PrivateJwk, + client: &str, + audience: &str, + issued_at: u64, + jti: &str, + ) -> String { + let signer = LocalJwkSigner::new(key.clone()).unwrap(); + let claims = serde_json::to_vec(&json!({ + "iss": client, "sub": client, "aud": audience, + "iat": issued_at, "exp": issued_at + 60, "jti": jti, + })) + .unwrap(); + sign_compact_jws( + Map::from_iter([("typ".to_owned(), Value::String("JWT".to_owned()))]), + &claims, + &signer, + ) + .await + .unwrap() + } + + #[test] + fn endpoint_accepts_one_assertion_and_refuses_replay_tampering_and_wrong_resource() { + let root = tempfile::tempdir().unwrap(); + let input = Arc::new(input(plan())); + let issuer = Arc::new(Issuer::create(root.path()).unwrap()); + let origins = BTreeMap::from([ + (Product::Breg, "http://127.0.0.1:31001".to_owned()), + (Product::Casework, "http://127.0.0.1:31002".to_owned()), + ]); + let runtime = + RuntimeBindings::prepare(root.path(), input.clone(), issuer, &origins).unwrap(); + let endpoint = runtime.server.as_ref().unwrap().endpoint.clone(); + let encoded = + Zeroizing::new(std::fs::read(root.path().join("casework-context-client-key")).unwrap()); + let key = PrivateJwk::parse(std::str::from_utf8(&encoded).unwrap()).unwrap(); + let now = clock().unwrap(); + let request_runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let provider = PrivateKeyJwt::new( + PrivateKeyJwtConfig::new( + endpoint.parse().unwrap(), + "context-reader-client", + key.clone(), + ) + .with_audience(input.plan.issuer.clone()) + .with_resource("urn:dev:breg") + .with_scopes(vec!["registry:proposal:review-status".to_owned()]), + ) + .unwrap(); + request_runtime + .block_on(provider.bearer_token()) + .expect("the maintained private-key JWT client obtains a token"); + + let assertion = request_runtime.block_on(sign_assertion( + &key, + "context-reader-client", + &input.plan.issuer, + now, + "assertion-once", + )); + let client = reqwest::Client::builder().no_proxy().build().unwrap(); + let form = [ + ("grant_type", "client_credentials"), + ("client_id", "context-reader-client"), + ("client_assertion_type", ASSERTION_TYPE), + ("client_assertion", assertion.as_str()), + ("resource", "urn:dev:breg"), + ("scope", "registry:proposal:review-status"), + ]; + let response = request_runtime + .block_on(client.post(&endpoint).form(&form).send()) + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let replay = request_runtime + .block_on(client.post(&endpoint).form(&form).send()) + .unwrap(); + assert_eq!(replay.status(), StatusCode::UNAUTHORIZED); + + let mut tampered = assertion.clone().into_bytes(); + let last = tampered.last_mut().unwrap(); + *last = if *last == b'A' { b'B' } else { b'A' }; + let tampered = String::from_utf8(tampered).unwrap(); + let invalid = [ + ("grant_type", "client_credentials"), + ("client_id", "context-reader-client"), + ("client_assertion_type", ASSERTION_TYPE), + ("client_assertion", tampered.as_str()), + ("resource", "urn:dev:breg"), + ("scope", "registry:proposal:review-status"), + ]; + let response = request_runtime + .block_on(client.post(&endpoint).form(&invalid).send()) + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + + let second = request_runtime.block_on(sign_assertion( + &key, + "context-reader-client", + &input.plan.issuer, + now, + "wrong-resource", + )); + let wrong = [ + ("grant_type", "client_credentials"), + ("client_id", "context-reader-client"), + ("client_assertion_type", ASSERTION_TYPE), + ("client_assertion", second.as_str()), + ("resource", "urn:dev:casework"), + ("scope", "registry:proposal:review-status"), + ]; + let response = request_runtime + .block_on(client.post(&endpoint).form(&wrong).send()) + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + + #[test] + fn configure_writes_native_product_bindings() { + let root = tempfile::tempdir().unwrap(); + let input = Arc::new(input(plan())); + let issuer = Arc::new(Issuer::create(root.path()).unwrap()); + let origins = BTreeMap::from([ + (Product::Breg, "http://127.0.0.1:31001".to_owned()), + (Product::Casework, "http://127.0.0.1:31002".to_owned()), + ]); + let runtime = RuntimeBindings::prepare(root.path(), input, issuer, &origins).unwrap(); + let mut breg = json!({}); + runtime.configure(Product::Breg, &mut breg); + assert_eq!( + breg["reviewAuthorities"]["casework"]["endpoint"], + "http://127.0.0.1:31002" + ); + assert_eq!( + breg["reviewAuthorities"]["casework"]["privateKeyJwt"]["resource"], + "urn:dev:casework" + ); + assert!(breg["reviewAuthorities"]["casework"]["tokenRef"].is_null()); + + let mut casework = json!({}); + runtime.configure(Product::Casework, &mut casework); + assert_eq!( + casework["contextProviders"]["registry"]["baseUrl"], + "http://127.0.0.1:31001" + ); + assert_eq!( + casework["contextProviders"]["registry"]["statusProfile"], + "casework-review-status" + ); + } +} diff --git a/crates/registry-bregctl/src/project_dev/database.rs b/crates/registry-bregctl/src/project_dev/database.rs new file mode 100644 index 0000000000..4fb99a8924 --- /dev/null +++ b/crates/registry-bregctl/src/project_dev/database.rs @@ -0,0 +1,1274 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + future::Future, + str::FromStr, + time::{Duration, Instant}, +}; + +use anyhow::{anyhow, bail, Context, Result}; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; +use rustls::pki_types::{pem::PemObject as _, CertificateDer}; +use tokio::runtime::{Builder, Runtime}; +use tokio_postgres::{ + config::{Host, SslMode}, + Client, Config, +}; +use tokio_postgres_rustls::MakeRustlsConnect; +use url::{Host as UrlHost, Url}; +use zeroize::Zeroizing; + +const BREG_SCHEMAS: [&str; 5] = [ + "registry_internal", + "registry_data", + "registry_source", + "registry_derived", + "registry_context", +]; +const CONNECTION_TIMEOUT: Duration = Duration::from_secs(10); +const STATEMENT_TIMEOUT: Duration = Duration::from_secs(10); +const OPERATION_TIMEOUT: Duration = Duration::from_secs(15); +const CLEANUP_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(30); +const KEEPALIVE_IDLE: Duration = Duration::from_secs(30); +/// Lets a role holding CREATEROLE use the roles it creates. PostgreSQL 16 and +/// later grant such a creator only ADMIN on them, which is not enough for +/// `CREATE SCHEMA ... AUTHORIZATION` or `CREATE DATABASE ... OWNER`. +const CREATEROLE_SELF_GRANT: &str = "set,inherit"; +/// Mirrors the BReg kernel's supported PostGIS floor +/// (`MIN_POSTGIS_MAJOR` and `MIN_POSTGIS_MINOR` in +/// `registry-breg/src/postgres/roles.rs`), so the runner can name it before +/// the kernel refuses with a generic connection error. +const MIN_POSTGIS: (u32, u32) = (3, 5); + +async fn bounded_connection( + connection: impl Future>, + timeout: Duration, +) -> Result { + tokio::time::timeout(timeout, connection) + .await + .map_err(|_| anyhow!("the PostgreSQL connection handshake timed out"))? + .context("the PostgreSQL connection handshake failed") +} + +#[derive(Debug, thiserror::Error)] +enum OperationFailure { + #[error("the PostgreSQL {operation} operation exceeded its I/O deadline")] + IoDeadline { operation: &'static str }, + #[error( + "the PostgreSQL {operation} operation was canceled or exceeded its statement deadline" + )] + StatementDeadline { operation: &'static str }, + #[error("the PostgreSQL {operation} operation failed")] + PostgreSql { + operation: &'static str, + #[source] + source: tokio_postgres::Error, + }, +} + +impl OperationFailure { + fn postgres(operation: &'static str, source: tokio_postgres::Error) -> Self { + if source.code() == Some(&tokio_postgres::error::SqlState::QUERY_CANCELED) { + Self::StatementDeadline { operation } + } else { + Self::PostgreSql { operation, source } + } + } + + fn effect_is_unconfirmed(&self) -> bool { + matches!(self, Self::IoDeadline { .. }) + || matches!(self, Self::PostgreSql { source, .. } if source.as_db_error().is_none()) + } +} + +async fn bounded_operation( + operation: &'static str, + future: impl Future>, + timeout: Duration, +) -> std::result::Result { + match tokio::time::timeout(timeout, future).await { + Ok(Ok(value)) => Ok(value), + Ok(Err(error)) => Err(OperationFailure::postgres(operation, error)), + Err(_) => Err(OperationFailure::IoDeadline { operation }), + } +} + +fn retain_create_acquisition( + acquisitions: &mut Vec, + acquisition: Acquisition, + created: &std::result::Result, +) { + if created.is_ok() + || created + .as_ref() + .is_err_and(OperationFailure::effect_is_unconfirmed) + { + acquisitions.push(acquisition); + } +} + +/// Startup options for every administrator session: the statement deadline and +/// the CREATEROLE self-grant. The comma in the self-grant value is not a +/// separator for the server's startup option parser, which splits on spaces. +fn apply_session_options(config: &mut Config) { + let added = format!( + "-c statement_timeout={} -c createrole_self_grant={CREATEROLE_SELF_GRANT}", + STATEMENT_TIMEOUT.as_millis() + ); + if let Some(existing) = config.get_options().filter(|value| !value.is_empty()) { + config.options(format!("{existing} {added}")); + } else { + config.options(added); + } +} + +fn postgis_version_supported(version: &str) -> bool { + let mut parts = version.split('.'); + let mut next = || parts.next().and_then(|value| value.parse::().ok()); + match (next(), next()) { + (Some(major), Some(minor)) => (major, minor) >= MIN_POSTGIS, + _ => false, + } +} + +/// Refuses a spatial BReg run when the server does not offer a supported +/// PostGIS. `available` is the `default_version` of the `postgis` row in +/// `pg_available_extensions`, or `None` when there is no such row. +fn require_postgis(available: Option<&str>) -> Result<()> { + match available { + None => bail!( + "spatial BReg needs PostGIS {}.{} or later installed on the PostgreSQL server; \ + the server does not offer the postgis extension", + MIN_POSTGIS.0, + MIN_POSTGIS.1 + ), + Some(version) if !postgis_version_supported(version) => bail!( + "spatial BReg needs PostGIS {}.{} or later installed on the PostgreSQL server; \ + the server offers {version}", + MIN_POSTGIS.0, + MIN_POSTGIS.1 + ), + Some(_) => Ok(()), + } +} + +fn remaining_timeout(deadline: Instant, maximum: Duration) -> Result { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + bail!("the PostgreSQL operation deadline expired"); + } + Ok(remaining.min(maximum)) +} + +fn ownership_lock_statement(run_id: &str) -> Result { + let prefix = run_id + .get(..16) + .context("the database run id cannot form an ownership lock")?; + let key = u64::from_str_radix(prefix, 16) + .context("the validated database run id cannot form an ownership lock")? + as i64; + Ok(format!("SELECT pg_advisory_lock({key})")) +} + +fn open_admin_session( + runtime: &Runtime, + config: &Config, + tls: &MakeRustlsConnect, + ownership_lock: &str, + deadline: Instant, +) -> Result<( + Client, + tokio::task::JoinHandle>, +)> { + let (client, connection) = runtime.block_on(bounded_connection( + config.connect(tls.clone()), + remaining_timeout(deadline, CONNECTION_TIMEOUT)?, + ))?; + let connection_task = runtime.spawn(connection); + let locked = runtime.block_on(bounded_operation( + "run ownership lock acquisition", + client.batch_execute(ownership_lock), + remaining_timeout(deadline, OPERATION_TIMEOUT)?, + )); + if let Err(error) = locked { + connection_task.abort(); + return Err(error).context("cannot acquire the PostgreSQL run ownership lock"); + } + Ok((client, connection_task)) +} + +pub(super) struct DatabaseUrls { + pub runtime: String, + pub migration: String, +} + +#[derive(Clone)] +enum Acquisition { + Database(String), + Role(String), +} + +pub(super) struct Database { + client: Client, + _connection_task: tokio::task::JoinHandle>, + admin_session_failed: bool, + tls: MakeRustlsConnect, + admin_config: Config, + ownership_lock: String, + endpoint: Endpoint, + run_id: String, + acquisitions: Vec, + runtime: Runtime, +} + +#[derive(Clone)] +struct Endpoint { + host: UrlHost, + port: Option, +} + +impl Endpoint { + fn product_url(&self, database: &str, user: &str, password: &str) -> Result { + let mut url = + Url::parse("postgresql://localhost/").expect("the fixed PostgreSQL URL base is valid"); + match &self.host { + UrlHost::Domain(host) => url.set_host(Some(host)).map_err(|_| { + anyhow!("the PostgreSQL host cannot be represented in a product URL") + })?, + UrlHost::Ipv4(address) => url.set_ip_host((*address).into()).map_err(|_| { + anyhow!("the PostgreSQL host cannot be represented in a product URL") + })?, + UrlHost::Ipv6(address) => url.set_ip_host((*address).into()).map_err(|_| { + anyhow!("the PostgreSQL host cannot be represented in a product URL") + })?, + } + url.set_port(self.port) + .map_err(|_| anyhow!("the PostgreSQL port cannot be represented in a product URL"))?; + url.set_username(user) + .map_err(|_| anyhow!("the generated PostgreSQL role cannot be represented in a URL"))?; + url.set_password(Some(password)).map_err(|_| { + anyhow!("the generated PostgreSQL password cannot be represented in a URL") + })?; + url.set_path(&format!("/{database}")); + url.query_pairs_mut().append_pair("sslmode", "require"); + Ok(url.into()) + } +} + +impl Database { + pub(super) fn connect(admin_url: &str, ca_pem: Option<&[u8]>, run_id: &str) -> Result { + validate_run_id(run_id)?; + let (mut admin_config, endpoint) = admin_config(admin_url)?; + admin_config.ssl_mode(SslMode::Require); + apply_session_options(&mut admin_config); + let tls = tls_connector(ca_pem)?; + let runtime = Builder::new_current_thread() + .enable_all() + .build() + .context("cannot initialize the PostgreSQL executor")?; + let ownership_lock = ownership_lock_statement(run_id)?; + let session_deadline = Instant::now() + CONNECTION_TIMEOUT + OPERATION_TIMEOUT; + let (client, connection_task) = open_admin_session( + &runtime, + &admin_config, + &tls, + &ownership_lock, + session_deadline, + ) + .context("cannot establish the PostgreSQL administrator session over verified TLS")?; + Ok(Self { + client, + _connection_task: connection_task, + admin_session_failed: false, + tls, + admin_config, + ownership_lock, + endpoint, + run_id: run_id.to_owned(), + acquisitions: Vec::new(), + runtime, + }) + } + + pub(super) fn provision( + &mut self, + label: &str, + breg: bool, + requires_postgis: bool, + ) -> Result { + validate_label(label)?; + if requires_postgis && !breg { + bail!("PostGIS prerequisites may be provisioned only for a BReg database"); + } + let names = Names::new(&self.run_id, label); + if self.acquisitions.iter().any(|acquisition| { + matches!(acquisition, Acquisition::Database(database) if database == &names.database) + }) { + bail!("the {label} database was already provisioned for this run"); + } + + if requires_postgis { + self.check_postgis_available()?; + } + + let migration_password = random_password()?; + let runtime_password = random_password()?; + self.create_role(&names.migration_role, &migration_password)?; + self.create_role(&names.runtime_role, &runtime_password)?; + if requires_postgis { + self.create_spatial_bbox_role(&names)?; + } + self.create_database(&names)?; + self.configure_database(&names, breg, requires_postgis)?; + + Ok(DatabaseUrls { + runtime: self.product_url(&names.database, &names.runtime_role, &runtime_password)?, + migration: self.product_url( + &names.database, + &names.migration_role, + &migration_password, + )?, + }) + } + + pub(super) fn cleanup(&mut self) -> Result<()> { + let deadline = Instant::now() + CLEANUP_ATTEMPT_TIMEOUT; + let mut failures = Vec::new(); + let mut retained = Vec::new(); + let acquisitions = self.acquisitions.clone(); + for acquisition in acquisitions.into_iter().rev() { + let result = match &acquisition { + Acquisition::Database(name) => self.execute_cleanup( + &format!( + "DROP DATABASE IF EXISTS {} WITH (FORCE)", + quote_identifier(name) + ), + deadline, + ), + Acquisition::Role(name) => self.execute_cleanup( + &format!("DROP ROLE IF EXISTS {}", quote_identifier(name)), + deadline, + ), + }; + if let Err(error) = result { + failures.push(match &acquisition { + Acquisition::Database(_) => format!("database: {error:#}"), + Acquisition::Role(_) => format!("role: {error:#}"), + }); + retained.push(acquisition); + } + } + retained.reverse(); + self.acquisitions = retained; + if failures.is_empty() { + Ok(()) + } else { + bail!( + "PostgreSQL cleanup is incomplete for {} resource(s): {}", + failures.len(), + failures.join("; ") + ) + } + } + + fn reconnect_admin(&mut self, deadline: Instant) -> Result<()> { + // The old server backend can outlive the client-side I/O timeout. Its + // session lock orders this replacement session after an unconfirmed + // CREATE has committed or rolled back, so cleanup cannot run too early. + self.retire_admin_session(); + let (client, connection_task) = open_admin_session( + &self.runtime, + &self.admin_config, + &self.tls, + &self.ownership_lock, + deadline, + ) + .context("cannot reconnect to the PostgreSQL administrator database")?; + self.client = client; + self._connection_task = connection_task; + self.admin_session_failed = false; + Ok(()) + } + + fn retire_admin_session(&mut self) { + self._connection_task.abort(); + self.admin_session_failed = true; + } + + fn ensure_admin_connection(&mut self, deadline: Instant) -> Result<()> { + if self.admin_session_failed || self.client.is_closed() { + self.reconnect_admin(deadline)?; + } + Ok(()) + } + + fn execute_cleanup(&mut self, statement: &str, deadline: Instant) -> Result<()> { + self.ensure_admin_connection(deadline)?; + let first = self.execute_admin_operation( + statement, + "cleanup", + remaining_timeout(deadline, OPERATION_TIMEOUT)?, + ); + match first { + Ok(()) => Ok(()), + Err(first) if first.effect_is_unconfirmed() => { + self.reconnect_admin(deadline)?; + self.execute_admin_operation( + statement, + "cleanup", + remaining_timeout(deadline, OPERATION_TIMEOUT)?, + ) + .with_context(|| { + format!("the PostgreSQL cleanup operation failed after reconnecting ({first})") + }) + } + Err(error) => Err(error).context("the PostgreSQL cleanup operation failed"), + } + } + + fn create_role(&mut self, role: &str, password: &str) -> Result<()> { + let verifier = Zeroizing::new(postgres_protocol::password::scram_sha_256( + password.as_bytes(), + )); + let statement = Zeroizing::new(create_role_sql(role, &verifier)); + self.create_owned_role( + role, + &statement, + "cannot create a least-privilege PostgreSQL role", + ) + } + + fn create_spatial_bbox_role(&mut self, names: &Names) -> Result<()> { + let role = spatial_bbox_role(names)?; + let statement = create_nologin_role_sql(&role); + self.create_owned_role( + &role, + &statement, + "cannot create the BReg spatial bbox role", + ) + } + + fn create_owned_role( + &mut self, + role: &str, + statement: &str, + context: &'static str, + ) -> Result<()> { + let created = self.execute_admin_operation(statement, "role creation", OPERATION_TIMEOUT); + // A transport/protocol failure or I/O deadline cannot tell us whether + // PostgreSQL committed the statement. Retaining the exact run-owned name + // lets cleanup's idempotent DROP cover both outcomes without replaying CREATE. + retain_create_acquisition( + &mut self.acquisitions, + Acquisition::Role(role.to_owned()), + &created, + ); + created.map_err(anyhow::Error::new).context(context) + } + + fn create_database(&mut self, names: &Names) -> Result<()> { + let statement = format!( + "CREATE DATABASE {} WITH ALLOW_CONNECTIONS false", + quote_identifier(&names.database) + ); + let created = + self.execute_admin_operation(&statement, "database creation", OPERATION_TIMEOUT); + retain_create_acquisition( + &mut self.acquisitions, + Acquisition::Database(names.database.clone()), + &created, + ); + created + .map_err(anyhow::Error::new) + .context("cannot create an isolated PostgreSQL database")?; + + self.execute_admin(&database_access_sql(names)) + .context("cannot restrict the PostgreSQL database to this run's roles") + } + + fn configure_database(&self, names: &Names, breg: bool, requires_postgis: bool) -> Result<()> { + let statement = database_setup_sql(names, breg); + let spatial_roles = if requires_postgis { + Some(( + registry_breg::postgres::SqlIdentifier::parse(&names.migration_role) + .map_err(|_| anyhow!("the generated migration role is invalid"))?, + registry_breg::postgres::SqlIdentifier::parse(&names.runtime_role) + .map_err(|_| anyhow!("the generated runtime role is invalid"))?, + )) + } else { + None + }; + + let mut target = self.admin_config.clone(); + target.dbname(&names.database).ssl_mode(SslMode::Require); + self.runtime + .block_on(async { + let (client, connection) = + bounded_connection(target.connect(self.tls.clone()), CONNECTION_TIMEOUT) + .await + .context("cannot connect to the provisioned PostgreSQL database")?; + let connection_task = tokio::spawn(connection); + let executed = tokio::time::timeout(OPERATION_TIMEOUT, async { + client + .batch_execute(&statement) + .await + .map_err(|error| { + OperationFailure::postgres("database configuration", error) + }) + .map_err(anyhow::Error::new) + .context("cannot configure the provisioned PostgreSQL database")?; + if let Some((migration, runtime)) = &spatial_roles { + registry_breg::postgres::provision_postgis_prerequisites( + &client, migration, runtime, + ) + .await + .context( + "cannot provision BReg PostGIS prerequisites; the PostgreSQL \ + administrator must be allowed to CREATE EXTENSION postgis, \ + which normally needs a superuser", + )?; + } + Result::<()>::Ok(()) + }) + .await + .map_err(|_| { + anyhow!( + "the PostgreSQL database configuration operation exceeded its I/O deadline" + ) + }) + .and_then(|result| result); + connection_task.abort(); + executed + }) + .context("cannot finish PostgreSQL database provisioning") + } + + /// Reads the PostGIS version the server offers, before this run acquires + /// any role or database. + fn check_postgis_available(&mut self) -> Result<()> { + let queried = self.runtime.block_on(bounded_operation( + "PostGIS availability check", + self.client.query_opt( + "SELECT default_version FROM pg_available_extensions WHERE name = 'postgis'", + &[], + ), + OPERATION_TIMEOUT, + )); + if queried + .as_ref() + .is_err_and(OperationFailure::effect_is_unconfirmed) + { + self.retire_admin_session(); + } + let row = queried + .map_err(anyhow::Error::new) + .context("cannot check that the PostgreSQL server offers PostGIS")?; + let version = row + .map(|row| row.try_get::<_, Option>(0)) + .transpose() + .context("cannot read the PostGIS version the PostgreSQL server offers")? + .flatten(); + require_postgis(version.as_deref()) + } + + fn execute_admin(&mut self, statement: &str) -> Result<()> { + self.execute_admin_operation(statement, "administrator", OPERATION_TIMEOUT) + .map_err(anyhow::Error::new) + .context("the PostgreSQL administrator operation failed") + } + + fn execute_admin_operation( + &mut self, + statement: &str, + operation: &'static str, + timeout: Duration, + ) -> std::result::Result<(), OperationFailure> { + let executed = self.runtime.block_on(bounded_operation( + operation, + self.client.batch_execute(statement), + timeout, + )); + if executed + .as_ref() + .is_err_and(OperationFailure::effect_is_unconfirmed) + { + self.retire_admin_session(); + } + executed + } + + fn product_url(&self, database: &str, user: &str, password: &str) -> Result { + self.endpoint.product_url(database, user, password) + } + + /// Names of the databases and roles this run still owns, oldest first. + pub(super) fn pending_names(&self) -> Vec { + self.acquisitions + .iter() + .map(|acquisition| match acquisition { + Acquisition::Database(name) | Acquisition::Role(name) => name.clone(), + }) + .collect() + } +} + +/// Every database and role name `provision` can create for one label of a +/// run, in creation order. Recovery after a lost supervisor uses this list, +/// since the spatial bbox role does not carry the run id. +pub(super) fn run_object_names(run_id: &str, label: &str, spatial: bool) -> Result> { + validate_run_id(run_id)?; + validate_label(label)?; + let names = Names::new(run_id, label); + let mut owned = vec![names.migration_role.clone(), names.runtime_role.clone()]; + if spatial { + owned.push(spatial_bbox_role(&names)?); + } + owned.push(names.database); + Ok(owned) +} + +struct Names { + database: String, + migration_role: String, + runtime_role: String, +} + +impl Names { + fn new(run_id: &str, label: &str) -> Self { + let database = format!("rstack_{run_id}_{label}"); + Self { + migration_role: format!("{database}_migration"), + runtime_role: format!("{database}_runtime"), + database, + } + } +} + +fn spatial_bbox_role(names: &Names) -> Result { + let runtime = registry_breg::postgres::SqlIdentifier::parse(&names.runtime_role) + .map_err(|_| anyhow!("the generated runtime role is invalid"))?; + Ok(registry_breg::postgres::spatial_bbox_role(&runtime) + .as_str() + .to_owned()) +} + +fn validate_run_id(run_id: &str) -> Result<()> { + if run_id.len() == 32 + && run_id + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) + { + Ok(()) + } else { + bail!("the database run id must be exactly 32 lowercase hexadecimal characters") + } +} + +fn validate_label(label: &str) -> Result<()> { + if matches!(label, "breg" | "breg_test" | "casework") { + Ok(()) + } else { + bail!("the database label must be breg, breg_test, or casework") + } +} + +fn database_access_sql(names: &Names) -> String { + format!( + "REVOKE ALL ON DATABASE {database} FROM PUBLIC; \ + GRANT CONNECT ON DATABASE {database} TO {migration}, {runtime}; \ + ALTER DATABASE {database} ALLOW_CONNECTIONS true", + database = quote_identifier(&names.database), + migration = quote_identifier(&names.migration_role), + runtime = quote_identifier(&names.runtime_role), + ) +} + +fn create_role_sql(role: &str, verifier: &str) -> String { + format!( + "CREATE ROLE {} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOBYPASSRLS PASSWORD {}", + quote_identifier(role), + quote_literal(verifier) + ) +} + +fn create_nologin_role_sql(role: &str) -> String { + format!( + "CREATE ROLE {} NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOBYPASSRLS", + quote_identifier(role) + ) +} + +fn database_setup_sql(names: &Names, breg: bool) -> String { + if breg { + let schemas = BREG_SCHEMAS + .iter() + .map(|schema| { + format!( + "CREATE SCHEMA {schema} AUTHORIZATION {owner}; \ + REVOKE ALL ON SCHEMA {schema} FROM PUBLIC", + schema = quote_identifier(schema), + owner = quote_identifier(&names.migration_role), + ) + }) + .collect::>() + .join("; "); + format!("CREATE EXTENSION btree_gist; {schemas}") + } else { + format!( + "REVOKE CREATE ON SCHEMA public FROM PUBLIC; \ + GRANT USAGE, CREATE ON SCHEMA public TO {}", + quote_identifier(&names.migration_role) + ) + } +} + +fn admin_config(admin_url: &str) -> Result<(Config, Endpoint)> { + let parsed = Url::parse(admin_url) + .map_err(|_| anyhow!("the PostgreSQL admin URL is not a parseable URL"))?; + if !matches!(parsed.scheme(), "postgres" | "postgresql") + || parsed.cannot_be_a_base() + || parsed.fragment().is_some() + { + bail!("the PostgreSQL admin URL must be an absolute postgres:// or postgresql:// URL"); + } + if parsed.username().is_empty() + || parsed.password().is_none_or(str::is_empty) + || parsed.host().is_none() + { + bail!("the PostgreSQL admin URL must include an explicit host, user, and password"); + } + let database = parsed.path().strip_prefix('/').unwrap_or_default(); + if database.is_empty() || database.contains('/') { + bail!("the PostgreSQL admin URL must include one explicit database name"); + } + if parsed.query_pairs().any(|(key, _)| { + matches!( + key.as_ref(), + "host" | "hostaddr" | "port" | "user" | "password" | "dbname" + ) + }) { + bail!( + "the PostgreSQL admin URL must not override its endpoint or credentials in the query" + ); + } + for (key, value) in parsed.query_pairs() { + match key.as_ref() { + "sslmode" => { + if !matches!(value.as_ref(), "require" | "verify-ca" | "verify-full") { + bail!( + "the PostgreSQL admin URL sslmode must be require, verify-ca, or \ + verify-full; the runner always uses verified TLS" + ); + } + } + "sslrootcert" => bail!( + "the PostgreSQL admin URL must not set sslrootcert; set RSTACK_DEV_DATABASE_CA \ + to a CA PEM file instead" + ), + "connect_timeout" | "application_name" => {} + other => bail!("the PostgreSQL admin URL query parameter {other} is not supported"), + } + } + + // tokio-postgres does not parse verify-ca or verify-full, and the runner + // sets its own TLS mode, so the checked sslmode is not passed on. + let mut sanitized = parsed.clone(); + let kept: Vec<(String, String)> = parsed + .query_pairs() + .filter(|(key, _)| key != "sslmode") + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect(); + if kept.is_empty() { + sanitized.set_query(None); + } else { + sanitized.query_pairs_mut().clear().extend_pairs(kept); + } + let mut config = Config::from_str(sanitized.as_str()) + .map_err(|_| anyhow!("the PostgreSQL admin URL has an invalid query parameter value"))?; + config.keepalives_idle(KEEPALIVE_IDLE); + if config.get_hosts().len() != 1 + || !matches!(config.get_hosts().first(), Some(Host::Tcp(_))) + || !config.get_hostaddrs().is_empty() + || config.get_user().is_none_or(str::is_empty) + || config.get_password().is_none_or(<[u8]>::is_empty) + || config.get_dbname().is_none_or(str::is_empty) + { + bail!("the PostgreSQL admin URL must name one TCP host, user, password, and database"); + } + + let host = parsed.host().expect("the host was checked").to_owned(); + Ok(( + config, + Endpoint { + host, + port: parsed.port(), + }, + )) +} + +fn tls_connector(ca_pem: Option<&[u8]>) -> Result { + let provider = rustls::crypto::ring::default_provider(); + if provider.install_default().is_err() + && rustls::crypto::CryptoProvider::get_default().is_none() + { + bail!("cannot initialize the PostgreSQL TLS provider"); + } + + if let Some(pem) = ca_pem { + let mut roots = rustls::RootCertStore::empty(); + let mut count = 0usize; + for certificate in CertificateDer::pem_slice_iter(pem) { + roots + .add(certificate.context("the PostgreSQL CA PEM is invalid")?) + .context("the PostgreSQL CA certificate is invalid")?; + count += 1; + } + if count == 0 { + bail!("the PostgreSQL CA PEM contains no certificates"); + } + let config = rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + Ok(MakeRustlsConnect::new(config)) + } else { + MakeRustlsConnect::with_native_certs() + .map(|value| value.0) + .map_err(|_| anyhow!("cannot load native roots for PostgreSQL TLS")) + } +} + +fn random_password() -> Result> { + let mut bytes = Zeroizing::new([0_u8; 32]); + getrandom::fill(bytes.as_mut()).context("cannot generate a PostgreSQL credential")?; + Ok(Zeroizing::new(URL_SAFE_NO_PAD.encode(bytes.as_ref()))) +} + +fn quote_identifier(value: &str) -> String { + format!("\"{}\"", value.replace('"', "\"\"")) +} + +fn quote_literal(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{future::pending, time::Instant}; + + const RUN_ID: &str = "0123456789abcdef0123456789abcdef"; + + #[test] + fn run_ids_are_exact_lowercase_hex() { + assert!(validate_run_id(RUN_ID).is_ok()); + for invalid in [ + "0123456789abcdef0123456789abcde", + "0123456789abcdef0123456789abcdef0", + "0123456789ABCDEF0123456789ABCDEF", + "0123456789abcdef0123456789abcdeg", + ] { + assert!(validate_run_id(invalid).is_err(), "accepted {invalid}"); + } + } + + #[test] + fn only_owned_database_labels_are_accepted() { + for label in ["breg", "breg_test", "casework"] { + assert!(validate_label(label).is_ok(), "refused {label}"); + } + for invalid in ["", "breg-test", "casework_test", "postgres"] { + assert!(validate_label(invalid).is_err(), "accepted {invalid}"); + } + } + + #[test] + fn generated_names_are_run_scoped_and_fit_postgresql() { + let names = Names::new(RUN_ID, "breg_test"); + assert_eq!( + names.database, + "rstack_0123456789abcdef0123456789abcdef_breg_test" + ); + assert!(names.migration_role.ends_with("_breg_test_migration")); + assert!(names.runtime_role.ends_with("_breg_test_runtime")); + assert!(names.database.len() <= 63); + assert!(names.migration_role.len() <= 63); + assert!(names.runtime_role.len() <= 63); + } + + #[test] + fn administrator_connection_attempt_has_a_deadline() { + let runtime = Builder::new_current_thread().enable_time().build().unwrap(); + let started = Instant::now(); + let result: Result<()> = runtime.block_on(bounded_connection( + pending::>(), + Duration::from_millis(20), + )); + assert!(result.unwrap_err().to_string().contains("timed out")); + assert!(started.elapsed() < Duration::from_secs(1)); + + assert_eq!( + runtime + .block_on(bounded_connection( + async { Ok::<_, tokio_postgres::Error>(7_u8) }, + Duration::from_secs(1), + )) + .unwrap(), + 7 + ); + } + + #[test] + fn post_handshake_operation_deadline_marks_the_effect_unconfirmed() { + let runtime = Builder::new_current_thread().enable_time().build().unwrap(); + let started = Instant::now(); + let result: std::result::Result<(), OperationFailure> = + runtime.block_on(bounded_operation( + "role creation", + pending::>(), + Duration::from_millis(20), + )); + let failure = result.unwrap_err(); + assert!(failure.effect_is_unconfirmed()); + assert_eq!( + failure.to_string(), + "the PostgreSQL role creation operation exceeded its I/O deadline" + ); + assert!(started.elapsed() < Duration::from_secs(1)); + } + + #[test] + fn unconfirmed_create_retains_its_exact_acquisition_once() { + let mut acquisitions = Vec::new(); + let created = Err::<(), _>(OperationFailure::IoDeadline { + operation: "role creation", + }); + retain_create_acquisition( + &mut acquisitions, + Acquisition::Role("rstack_exact_role".to_owned()), + &created, + ); + + assert_eq!(acquisitions.len(), 1); + assert!(matches!( + &acquisitions[0], + Acquisition::Role(role) if role == "rstack_exact_role" + )); + } + + #[test] + fn confirmed_statement_cancellation_is_not_tracked_as_a_creation() { + let mut acquisitions = Vec::new(); + let created = Err::<(), _>(OperationFailure::StatementDeadline { + operation: "database creation", + }); + retain_create_acquisition( + &mut acquisitions, + Acquisition::Database("rstack_refused_database".to_owned()), + &created, + ); + assert!(acquisitions.is_empty()); + } + + #[test] + fn operation_deadlines_fit_the_worker_stop_budget() { + let mut config = Config::new(); + config.options("-c application_name=rstack-dev"); + apply_session_options(&mut config); + assert_eq!( + config.get_options(), + Some( + "-c application_name=rstack-dev -c statement_timeout=10000 \ + -c createrole_self_grant=set,inherit" + ) + ); + let mut bare = Config::new(); + apply_session_options(&mut bare); + assert_eq!( + bare.get_options(), + Some("-c statement_timeout=10000 -c createrole_self_grant=set,inherit") + ); + assert!(STATEMENT_TIMEOUT < OPERATION_TIMEOUT); + assert!(OPERATION_TIMEOUT < CLEANUP_ATTEMPT_TIMEOUT); + assert!(CLEANUP_ATTEMPT_TIMEOUT < Duration::from_secs(60)); + } + + #[test] + fn run_ownership_lock_uses_the_first_64_id_bits_as_a_session_key() { + assert_eq!( + ownership_lock_statement(RUN_ID).unwrap(), + "SELECT pg_advisory_lock(81985529216486895)" + ); + assert_eq!( + ownership_lock_statement("fedcba98765432100123456789abcdef").unwrap(), + "SELECT pg_advisory_lock(-81985529216486896)" + ); + assert!(ownership_lock_statement("short").is_err()); + } + + #[test] + fn admin_urls_require_one_explicit_password_authenticated_tcp_database() { + let password = random_password().unwrap(); + let password = password.as_str(); + let endpoint = format!("postgresql://admin:{password}@db.example"); + assert!(admin_config(&format!("{endpoint}:5432/postgres?sslmode=require")).is_ok()); + assert!(admin_config(&format!("postgres://admin:{password}@[::1]:5432/postgres")).is_ok()); + let invalid = [ + format!("http://admin:{password}@db.example/postgres"), + "postgresql://db.example/postgres".to_owned(), + "postgresql://admin@db.example/postgres".to_owned(), + "postgresql://admin:@db.example/postgres".to_owned(), + endpoint.clone(), + format!("{endpoint}/one/two"), + format!("{endpoint}/postgres?host=other.example"), + format!("{endpoint}/postgres?dbname=other"), + format!("{endpoint}/postgres?hostaddr=127.0.0.1"), + "postgresql:///postgres".to_owned(), + ]; + for (case, url) in invalid.iter().enumerate() { + assert!( + admin_config(url).is_err(), + "accepted invalid admin URL case {case}" + ); + } + } + + #[test] + fn admin_url_query_keys_and_ssl_modes_are_checked_by_name() { + let password = random_password().unwrap(); + let base = format!( + "postgresql://admin:{}@db.example/postgres", + password.as_str() + ); + for accepted in [ + "sslmode=require", + "sslmode=verify-ca", + "sslmode=verify-full", + "connect_timeout=5", + "application_name=rstack", + ] { + assert!( + admin_config(&format!("{base}?{accepted}")).is_ok(), + "{accepted}" + ); + } + for (query, expected) in [ + ("sslmode=disable", "always uses verified TLS"), + ("sslmode=allow", "always uses verified TLS"), + ("sslmode=prefer", "always uses verified TLS"), + ("sslrootcert=/tmp/ca.pem", "RSTACK_DEV_DATABASE_CA"), + ( + "options=-csearch_path%3Dx", + "query parameter options is not supported", + ), + ( + "channel_binding=require", + "query parameter channel_binding is not supported", + ), + ] { + let message = admin_config(&format!("{base}?{query}")) + .err() + .unwrap_or_else(|| panic!("accepted {query}")) + .to_string(); + assert!(message.contains(expected), "{query}: {message}"); + assert!( + !message.contains(password.as_str()), + "{query} leaked the URL" + ); + } + let unparseable = admin_config("not a url").err().unwrap().to_string(); + assert!(unparseable.contains("not a parseable URL"), "{unparseable}"); + let bad_value = admin_config(&format!("{base}?connect_timeout=soon")) + .err() + .unwrap() + .to_string(); + assert!( + bad_value.contains("invalid query parameter value"), + "{bad_value}" + ); + assert!(!bad_value.contains(password.as_str())); + } + + #[test] + fn admin_config_keeps_idle_connections_alive() { + let (config, _) = admin_config("postgresql://admin:secret@db.example/postgres").unwrap(); + assert_eq!(config.get_keepalives_idle(), Duration::from_secs(30)); + } + + #[test] + fn spatial_runs_are_refused_without_a_supported_postgis() { + assert!(require_postgis(Some("3.5.2")).is_ok()); + assert!(require_postgis(Some("3.5")).is_ok()); + assert!(require_postgis(Some("3.10.0")).is_ok()); + assert!(require_postgis(Some("4.0.0")).is_ok()); + let missing = require_postgis(None).unwrap_err().to_string(); + assert!( + missing.starts_with( + "spatial BReg needs PostGIS 3.5 or later installed on the PostgreSQL server" + ), + "{missing}" + ); + for old in ["3.4.9", "2.5.5", "garbage", ""] { + let message = require_postgis(Some(old)).unwrap_err().to_string(); + assert!(message.contains("PostGIS 3.5 or later"), "{old}: {message}"); + assert!(message.contains(&format!("offers {old}")), "{message}"); + } + } + + #[test] + fn generated_product_urls_contain_only_generated_credentials_and_required_tls() { + let endpoint = Endpoint { + host: UrlHost::Domain("db.example".to_owned()), + port: Some(5543), + }; + let password = random_password().unwrap(); + let password = password.as_str(); + let url = endpoint.product_url("owned", "runtime", password).unwrap(); + assert!( + url == format!("postgresql://runtime:{password}@db.example:5543/owned?sslmode=require"), + "unexpected product connection URL" + ); + } + + #[test] + fn generated_product_urls_round_trip_ipv6_hosts_and_credentials() { + let admin_password = random_password().unwrap(); + let admin_password = admin_password.as_str(); + for address_text in ["::1", "2001:db8:1234:5678::9"] { + let address = address_text.parse().unwrap(); + let (_, endpoint) = admin_config(&format!( + "postgresql://admin:{admin_password}@[{address_text}]:5543/postgres" + )) + .unwrap(); + for (user, database) in [("runtime", "owned"), ("migration", "owned_test")] { + let password = random_password().unwrap(); + let password = password.as_str(); + let rendered = endpoint.product_url(database, user, password).unwrap(); + let parsed = Url::parse(&rendered).unwrap(); + assert_eq!(parsed.host(), Some(UrlHost::Ipv6(address))); + assert_eq!(parsed.port(), Some(5543)); + assert_eq!(parsed.username(), user); + assert!( + parsed.password() == Some(password), + "unexpected product password" + ); + assert_eq!(parsed.path(), format!("/{database}")); + assert_eq!(parsed.query(), Some("sslmode=require")); + assert!(rendered.contains(&format!("@[{address}]:5543/"))); + } + } + } + + #[test] + fn database_access_is_closed_to_public_before_connections_are_enabled() { + let names = Names::new(RUN_ID, "casework"); + let sql = database_access_sql(&names); + let revoke = sql.find("REVOKE ALL ON DATABASE").unwrap(); + let grant = sql.find("GRANT CONNECT ON DATABASE").unwrap(); + let enable = sql.find("ALLOW_CONNECTIONS true").unwrap(); + assert!(revoke < grant && grant < enable, "{sql}"); + assert!(sql.contains("e_identifier(&names.migration_role))); + assert!(sql.contains("e_identifier(&names.runtime_role))); + } + + #[test] + fn breg_setup_installs_extension_and_all_owned_schemas() { + let names = Names::new(RUN_ID, "breg"); + let sql = database_setup_sql(&names, true); + assert!(sql.starts_with("CREATE EXTENSION btree_gist;"), "{sql}"); + for schema in BREG_SCHEMAS { + assert!( + sql.contains(&format!( + "CREATE SCHEMA {} AUTHORIZATION {}", + quote_identifier(schema), + quote_identifier(&names.migration_role) + )), + "{sql}" + ); + assert!( + sql.contains(&format!( + "REVOKE ALL ON SCHEMA {} FROM PUBLIC", + quote_identifier(schema) + )), + "{sql}" + ); + } + } + + #[test] + fn spatial_breg_uses_the_governed_bbox_role_for_its_runtime() { + for label in ["breg_test", "breg"] { + let names = Names::new(RUN_ID, label); + let expected = registry_breg::postgres::spatial_bbox_role( + ®istry_breg::postgres::SqlIdentifier::parse(&names.runtime_role).unwrap(), + ); + assert_eq!(spatial_bbox_role(&names).unwrap(), expected.as_str()); + assert!(expected.as_str().starts_with("breg_bbox_")); + assert!(expected.as_str().len() <= 63); + let statement = create_nologin_role_sql(expected.as_str()); + assert_eq!( + statement, + format!( + "CREATE ROLE {} NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOBYPASSRLS", + quote_identifier(expected.as_str()) + ) + ); + assert!(!statement.contains("PASSWORD")); + } + } + + #[test] + fn run_object_names_list_every_name_provision_creates() { + for (label, spatial) in [("breg", true), ("breg_test", false), ("casework", false)] { + let names = Names::new(RUN_ID, label); + let mut expected = vec![names.migration_role.clone(), names.runtime_role.clone()]; + if spatial { + let bbox = spatial_bbox_role(&names).unwrap(); + assert!(!bbox.contains(RUN_ID), "the bbox role is not run-scoped"); + expected.push(bbox); + } + expected.push(names.database.clone()); + assert_eq!(run_object_names(RUN_ID, label, spatial).unwrap(), expected); + } + assert!(run_object_names("not-a-run", "breg", false).is_err()); + assert!(run_object_names(RUN_ID, "other", false).is_err()); + } + + #[test] + fn casework_setup_gives_only_migration_role_public_schema_creation() { + let names = Names::new(RUN_ID, "casework"); + let sql = database_setup_sql(&names, false); + assert!( + sql.starts_with("REVOKE CREATE ON SCHEMA public FROM PUBLIC;"), + "{sql}" + ); + assert!(sql.contains(&format!( + "GRANT USAGE, CREATE ON SCHEMA public TO {}", + quote_identifier(&names.migration_role) + ))); + assert!(!sql.contains(&names.runtime_role)); + } + + #[test] + fn identifiers_and_passwords_are_quoted_for_postgresql() { + assert_eq!(quote_identifier("a\"b"), "\"a\"\"b\""); + assert_eq!(quote_literal("a'b"), "'a''b'"); + } + + #[test] + fn role_statement_carries_a_postgresql_scram_verifier_not_cleartext() { + let password = random_password().unwrap(); + let verifier = postgres_protocol::password::scram_sha_256(password.as_bytes()); + let sql = create_role_sql("rstack_role", &verifier); + let expected = format!( + "CREATE ROLE \"rstack_role\" LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOBYPASSRLS PASSWORD '{verifier}'" + ); + assert!(verifier.starts_with("SCRAM-SHA-256$")); + assert!(sql == expected && !sql.contains(password.as_str())); + } + + #[test] + fn empty_or_malformed_custom_root_sets_are_refused() { + assert!(tls_connector(Some(b"")).is_err()); + assert!(tls_connector(Some(b"not a certificate")).is_err()); + } +} diff --git a/crates/registry-bregctl/src/project_dev/issuer.rs b/crates/registry-bregctl/src/project_dev/issuer.rs new file mode 100644 index 0000000000..4c093247ae --- /dev/null +++ b/crates/registry-bregctl/src/project_dev/issuer.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Local access-token issuer. Signing is available only to the owner through the CLI. + +use anyhow::{ensure, Context, Result}; +use registry_platform_crypto::{ + generate_private_jwk, jws::sign_compact_jws, GeneratedKeyAlgorithm, LocalJwkSigner, PrivateJwk, +}; +use serde_json::{json, Value}; +use std::path::Path; +use zeroize::Zeroizing; + +use super::{ + clock, + plan::{self, Product, TestPerson}, + write_private, RunInput, +}; + +/// How long a token the local issuer signs stays valid. +pub(super) const TOKEN_LIFETIME_SECONDS: u64 = 300; + +/// When a token issued at `now` expires: one lifetime later, or with the run +/// if that comes first. +pub(super) fn token_expiry(run_expires: u64, now: u64) -> u64 { + run_expires.min(now.saturating_add(TOKEN_LIFETIME_SECONDS)) +} + +pub(super) struct Issuer { + signer: LocalJwkSigner, + runtime: tokio::runtime::Runtime, +} + +impl Issuer { + pub(super) fn create(root: &Path) -> Result { + let key = generate_private_jwk(GeneratedKeyAlgorithm::Es256)?; + let encoded = super::bindings::encode_private_jwk(&key)?; + write_private(&root.join("issuer-key"), &encoded)?; + write_private( + &root.join("jwks"), + &serde_json::to_vec(&json!({ + "keys": [key.public()] + }))?, + )?; + Self::from_key(key) + } + + pub(super) fn load(root: &Path) -> Result { + let encoded = Zeroizing::new(plan::read(&root.join("issuer-key"), 16 * 1024)?); + let text = std::str::from_utf8(&encoded).context("invalid local issuer key encoding")?; + let key = + PrivateJwk::parse(text).map_err(|_| anyhow::anyhow!("invalid local issuer key"))?; + Self::from_key(key) + } + + fn from_key(key: PrivateJwk) -> Result { + Ok(Self { + signer: LocalJwkSigner::new(key)?, + runtime: tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?, + }) + } + + pub(super) fn token( + &self, + input: &RunInput, + person: &TestPerson, + product: Product, + expires: u64, + ) -> Result> { + self.runtime + .block_on(self.sign(input, person, product, expires)) + } + + pub(super) async fn sign( + &self, + input: &RunInput, + person: &TestPerson, + product: Product, + expires: u64, + ) -> Result> { + let now = clock()?; + let expires = token_expiry(expires, now); + ensure!(expires > now, "the development run has expired"); + let plan = &input.plan; + let grant = &person.grants[&product]; + let forward_context = product == Product::Casework && person.casework_context; + let mut claims = grant.claims.clone(); + let scopes = grant + .scopes + .iter() + .map(String::as_str) + .collect::>(); + let audience = if forward_context { + json!([ + plan.product(product).audience, + plan.product(Product::Breg).audience + ]) + } else { + json!(plan.product(product).audience) + }; + if product == Product::Breg || forward_context { + let authority = input.breg_authority.as_ref().expect("BReg authority"); + claims.insert(authority.principal.clone(), json!(person.subject)); + } + if product == Product::Casework { + let principal = &input.casework_principals[&grant.profile]; + if principal != "sub" { + claims.insert(principal.clone(), json!(person.subject)); + } + } + claims.extend([ + ("iss".to_owned(), json!(plan.issuer)), + ("aud".to_owned(), audience), + ("sub".to_owned(), json!(person.subject)), + ("client_id".to_owned(), json!(person.client)), + ( + "scope".to_owned(), + json!(scopes.into_iter().collect::>().join(" ")), + ), + ("iat".to_owned(), json!(now)), + ("exp".to_owned(), json!(expires)), + ("jti".to_owned(), json!(uuid::Uuid::new_v4())), + ]); + let header = json!({"typ": "at+jwt"}); + let bytes = serde_json::to_vec(&Value::Object(claims))?; + let token = sign_compact_jws( + header.as_object().expect("header object").clone(), + &bytes, + &self.signer, + ) + .await?; + Ok(Zeroizing::new(token)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_token_expires_five_minutes_after_issue_or_with_the_run() { + assert_eq!(token_expiry(10_000, 1_000), 1_000 + TOKEN_LIFETIME_SECONDS); + assert_eq!(token_expiry(1_100, 1_000), 1_100); + } +} diff --git a/crates/registry-bregctl/src/project_dev/journey.rs b/crates/registry-bregctl/src/project_dev/journey.rs new file mode 100644 index 0000000000..0cd7596abf --- /dev/null +++ b/crates/registry-bregctl/src/project_dev/journey.rs @@ -0,0 +1,971 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Execute the bounded HTTP journey declared by a development plan. + +use anyhow::{ensure, Context, Result}; +use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; +use reqwest::header::{HeaderMap, ETAG}; +use serde::Serialize; +use serde_json::Value; +use std::{ + collections::BTreeMap, + thread, + time::{Duration, Instant}, +}; + +use super::{ + clock, + issuer::Issuer, + plan::{self, Capture, CapturedHeader, Check, Product, Step}, + RunInput, +}; + +const RESPONSE_LIMIT: usize = 1024 * 1024; +const JOURNEY_TRANSFER_LIMIT: usize = 8 * 1024 * 1024; +const JOURNEY_ATTEMPT_LIMIT: usize = 512; + +#[derive(Serialize)] +pub(super) struct JourneyReport { + pub id: String, + pub result: &'static str, + pub steps: Vec, +} + +#[derive(Serialize)] +pub(super) struct StepReport { + id: String, + status: u16, + expected_status: u16, + result: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + reason: Option, +} + +const REASON_LIMIT: usize = 512; + +fn bounded_reason(text: &str) -> String { + text.chars().take(REASON_LIMIT).collect() +} + +/// What one step observed. `reason` is set when the step failed. +struct StepOutcome { + status: u16, + reason: Option, + updates: BTreeMap, +} + +struct ResponseData { + status: u16, + headers: HeaderMap, + body: Value, +} + +pub(super) fn run( + input: &RunInput, + issuer: &Issuer, + urls: &BTreeMap, + active: impl Fn() -> Result<()>, +) -> Result { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + let http = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(5)) + .build()?; + let journey = &input.plan.journey; + let mut report = JourneyReport { + id: journey.id.clone(), + result: "passed", + steps: Vec::new(), + }; + let mut captures = BTreeMap::::new(); + let mut transferred = 0usize; + let mut attempts = 0usize; + + for step in &journey.steps { + active()?; + let outcome = execute_step( + input, + issuer, + urls, + step, + &captures, + &http, + &runtime, + &active, + &mut transferred, + &mut attempts, + ); + let outcome = match outcome { + Ok(outcome) => outcome, + Err(error) => { + // A stop signal or the startup deadline interrupts the + // journey; it is not a failure of this step. + active()?; + StepOutcome { + status: 0, + reason: Some(bounded_reason(&format!("{error:#}"))), + updates: BTreeMap::new(), + } + } + }; + let passed = outcome.reason.is_none(); + if passed { + captures.extend(outcome.updates); + } + report.steps.push(StepReport { + id: step.id.clone(), + status: outcome.status, + expected_status: step.expect.status.first(), + result: if passed { "passed" } else { "failed" }, + reason: outcome.reason, + }); + if !passed { + report.result = "failed"; + break; + } + } + Ok(report) +} + +#[allow(clippy::too_many_arguments)] +fn execute_step( + input: &RunInput, + issuer: &Issuer, + urls: &BTreeMap, + step: &Step, + captures: &BTreeMap, + http: &reqwest::Client, + runtime: &tokio::runtime::Runtime, + active: &impl Fn() -> Result<()>, + transferred: &mut usize, + attempts: &mut usize, +) -> Result { + let lookup = |name: &str| { + captures + .get(name) + .cloned() + .context("missing journey capture") + }; + let path = expand_text(&step.path, lookup)?; + let mut url = request_url(&urls[&step.product], &path)?; + let person = &input.plan.people[&step.person]; + let grant = &person.grants[&step.product]; + if step.product == Product::Breg { + ensure!( + !url.query_pairs().any(|(name, _)| name == "accessProfile"), + "the plan's grant selects the BReg profile" + ); + url.query_pairs_mut() + .append_pair("accessProfile", &grant.profile); + } + + let headers = step + .headers + .iter() + .map(|(name, value)| Ok((name.clone(), expand_text(value, lookup)?))) + .collect::>>()?; + let json_body = step + .body + .as_ref() + .map(|body| expand_json(body, lookup)) + .transpose()?; + let raw_body = step + .raw + .as_ref() + .map(|body| BASE64.decode(&body.base64)) + .transpose()?; + let request_bytes = json_body + .as_ref() + .map(serde_json::to_vec) + .transpose()? + .map_or_else(|| raw_body.as_ref().map_or(0, Vec::len), |body| body.len()); + let poll_deadline = step + .poll + .as_ref() + .map(|poll| Instant::now() + Duration::from_millis(poll.timeout_ms)); + let mut last_status = 0; + let mut last_reason = None; + + loop { + active()?; + if poll_deadline.is_some_and(|deadline| Instant::now() >= deadline) { + return Ok(poll_timed_out(last_status, last_reason)); + } + add_attempt(attempts)?; + add_transfer(transferred, request_bytes)?; + let token = issuer.token(input, person, step.product, clock()? + 300)?; + let mut request = http + .request((&step.method).into(), url.clone()) + .bearer_auth(&*token); + if step.product == Product::Casework { + request = request + .header("Registry-Casework-Profile", &grant.profile) + .header("Registry-Review-Profile", &grant.profile); + } + for (name, value) in &headers { + request = request.header(name, value); + } + if let Some(body) = &json_body { + request = request.json(body); + } else if let (Some(spec), Some(body)) = (&step.raw, &raw_body) { + request = request + .header(reqwest::header::CONTENT_TYPE, &spec.content_type) + .body(body.clone()); + } + let Some(response) = + read_response_before_deadline(runtime, request, poll_deadline, transferred)? + else { + return Ok(poll_timed_out(last_status, last_reason)); + }; + last_status = response.status; + let reason = match first_mismatch(step, &response, lookup)? { + Some(reason) => reason, + None => match capture_updates(step, &response, lookup) { + Ok(updates) => { + return Ok(StepOutcome { + status: response.status, + reason: None, + updates, + }) + } + Err(error) => format!("capture failed: {error:#}"), + }, + }; + let failed = |reason: &str| StepOutcome { + status: response.status, + reason: Some(bounded_reason(reason)), + updates: BTreeMap::new(), + }; + let Some((poll, deadline)) = step.poll.as_ref().zip(poll_deadline) else { + return Ok(failed(&reason)); + }; + let now = Instant::now(); + if now >= deadline { + return Ok(failed(&reason)); + } + last_reason = Some(reason); + thread::sleep(Duration::from_millis(poll.interval_ms).min(deadline - now)); + } +} + +/// A poll whose deadline passed keeps the last completed status and says why +/// that response did not match. +fn poll_timed_out(status: u16, last_reason: Option) -> StepOutcome { + let reason = match last_reason { + Some(reason) => format!("poll deadline passed; last response: {reason}"), + None => "poll deadline passed before a response arrived".to_owned(), + }; + StepOutcome { + status, + reason: Some(bounded_reason(&reason)), + updates: BTreeMap::new(), + } +} + +fn request_url(base: &str, path: &str) -> Result { + ensure!(plan::valid_path(path), "invalid journey path"); + let base = url::Url::parse(base)?; + ensure!( + base.scheme() == "http" + && base.username().is_empty() + && base.password().is_none() + && base.query().is_none() + && base.fragment().is_none() + && base.path() == "/" + && matches!(base.host(), Some(url::Host::Ipv4(address)) if address.is_loopback()), + "journey product URL is not a numeric loopback HTTP origin" + ); + let url = base.join(path)?; + ensure!( + url.scheme() == base.scheme() + && url.host() == base.host() + && url.port_or_known_default() == base.port_or_known_default() + && url.username().is_empty() + && url.password().is_none() + && url.fragment().is_none() + && url.path().starts_with("/v1/"), + "journey path leaves the product API" + ); + Ok(url) +} + +/// Describe the first expectation the response misses, as +/// ` expected observed `. +fn first_mismatch( + step: &Step, + response: &ResponseData, + lookup: impl Fn(&str) -> Result + Copy, +) -> Result> { + let observed_at = |pointer: &str| { + response + .body + .pointer(pointer) + .map_or_else(|| "nothing".to_owned(), Value::to_string) + }; + if !step.expect.status.contains(response.status) { + return Ok(Some(format!( + "/status expected {} observed {}", + serde_json::to_string(&step.expect.status)?, + response.status + ))); + } + for (pointer, expected) in &step.expect.json { + let expected = expand_json(expected, lookup)?; + if response.body.pointer(pointer) != Some(&expected) { + return Ok(Some(format!( + "{pointer} expected {expected} observed {}", + observed_at(pointer) + ))); + } + } + if let Some(code) = &step.expect.code { + let expected = expand_text(code, lookup)?; + if response.body.pointer("/code").and_then(Value::as_str) != Some(expected.as_str()) { + return Ok(Some(format!( + "/code expected {} observed {}", + Value::String(expected), + observed_at("/code") + ))); + } + } + for check in &step.expect.checks { + if !matches_check(check, &response.body, lookup)? { + return Ok(Some(format!( + "{} expected {} observed {}", + check.pointer, + describe_check(check, lookup)?, + observed_at(&check.pointer) + ))); + } + } + Ok(None) +} + +fn describe_check(check: &Check, lookup: impl Fn(&str) -> Result + Copy) -> Result { + Ok(if let Some(expected) = &check.equals { + expand_json(expected, lookup)?.to_string() + } else if let Some(length) = check.length { + format!("length {length}") + } else if let Some(expected) = &check.contains { + format!("containing {}", expand_json(expected, lookup)?) + } else { + "a check".to_owned() + }) +} + +fn matches_check( + check: &Check, + body: &Value, + lookup: impl Fn(&str) -> Result + Copy, +) -> Result { + let Some(actual) = body.pointer(&check.pointer) else { + return Ok(false); + }; + if let Some(expected) = &check.equals { + return Ok(actual == &expand_json(expected, lookup)?); + } + if let Some(expected) = check.length { + let length = match actual { + Value::Array(value) => value.len(), + Value::Object(value) => value.len(), + Value::String(value) => value.chars().count(), + _ => return Ok(false), + }; + return Ok(length == expected); + } + if let Some(expected) = &check.contains { + let expected = expand_json(expected, lookup)?; + return Ok(match (actual, expected) { + (Value::Array(values), expected) => values.contains(&expected), + (Value::String(value), Value::String(expected)) => value.contains(&expected), + (Value::Object(values), Value::String(expected)) => values.contains_key(&expected), + _ => false, + }); + } + Ok(false) +} + +fn capture_updates( + step: &Step, + response: &ResponseData, + lookup: impl Fn(&str) -> Result + Copy, +) -> Result> { + step.capture + .iter() + .map(|(name, capture)| { + Ok(( + name.clone(), + scalar_capture(capture_value(capture, response, lookup)?)?, + )) + }) + .collect() +} + +fn capture_value( + capture: &Capture, + response: &ResponseData, + lookup: impl Fn(&str) -> Result + Copy, +) -> Result { + match capture { + Capture::Pointer { pointer } => response + .body + .pointer(pointer) + .cloned() + .context("journey capture pointer was not present"), + Capture::Header { header } => match header { + CapturedHeader::Etag => Ok(Value::String( + response + .headers + .get(ETAG) + .context("journey response had no ETag")? + .to_str() + .context("journey response ETag was not text")? + .to_owned(), + )), + }, + Capture::Value { pointer, query } => { + let value = response + .body + .pointer(pointer) + .cloned() + .context("journey capture pointer was not present")?; + if let Some(query) = query { + let text = value + .as_str() + .context("journey query capture did not select text")?; + let url = url::Url::parse(text) + .or_else(|_| url::Url::parse("http://127.0.0.1/")?.join(text))?; + let value = url + .query_pairs() + .find_map(|(name, value)| (name == query.as_str()).then(|| value.into_owned())) + .context("journey capture query parameter was not present")?; + Ok(Value::String(value)) + } else { + Ok(value) + } + } + Capture::Find { + find, + r#where, + pointer, + } => { + let expected = expand_json(&r#where.equals, lookup)?; + let items = response + .body + .pointer(find) + .and_then(Value::as_array) + .context("journey find capture did not select an array")?; + let mut matches = items + .iter() + .filter(|item| item.pointer(&r#where.pointer) == Some(&expected)); + let item = matches + .next() + .context("journey find capture found no item")?; + ensure!( + matches.next().is_none(), + "journey find capture matched more than one item" + ); + item.pointer(pointer) + .cloned() + .context("journey find capture pointer was not present") + } + } +} + +fn scalar_capture(value: Value) -> Result { + ensure!( + matches!(value, Value::String(_) | Value::Number(_) | Value::Bool(_)) + && serde_json::to_vec(&value).is_ok_and(|bytes| bytes.len() <= 4096), + "journey captures must be bounded scalar values" + ); + Ok(value) +} + +fn expand_json(value: &Value, lookup: impl Fn(&str) -> Result + Copy) -> Result { + match value { + Value::String(value) => { + if let Some(name) = exact_reference(value) { + lookup(name) + } else { + Ok(Value::String(expand_text(value, lookup)?)) + } + } + Value::Array(values) => values + .iter() + .map(|value| expand_json(value, lookup)) + .collect(), + Value::Object(values) => values + .iter() + .map(|(key, value)| Ok((key.clone(), expand_json(value, lookup)?))) + .collect(), + value => Ok(value.clone()), + } +} + +fn expand_text(value: &str, lookup: impl Fn(&str) -> Result + Copy) -> Result { + plan::expand(value, |name| scalar_text(&lookup(name)?)) +} + +fn exact_reference(value: &str) -> Option<&str> { + let name = value.strip_prefix("{{")?.strip_suffix("}}")?; + (!name.is_empty() && !name.contains(['{', '}'])).then_some(name) +} + +fn scalar_text(value: &Value) -> Result { + match value { + Value::String(value) => Ok(value.clone()), + Value::Number(value) => Ok(value.to_string()), + Value::Bool(value) => Ok(value.to_string()), + _ => anyhow::bail!("journey interpolation requires a scalar capture"), + } +} + +fn add_transfer(transferred: &mut usize, bytes: usize) -> Result<()> { + *transferred = transferred + .checked_add(bytes) + .context("journey transfer bound overflowed")?; + ensure!( + *transferred <= JOURNEY_TRANSFER_LIMIT, + "journey transferred too much data" + ); + Ok(()) +} + +fn add_attempt(attempts: &mut usize) -> Result<()> { + *attempts = attempts + .checked_add(1) + .context("journey attempt bound overflowed")?; + ensure!( + *attempts <= JOURNEY_ATTEMPT_LIMIT, + "journey made too many requests" + ); + Ok(()) +} + +fn read_response_before_deadline( + runtime: &tokio::runtime::Runtime, + request: reqwest::RequestBuilder, + deadline: Option, + transferred: &mut usize, +) -> Result> { + let response = read_response(request, transferred); + let Some(deadline) = deadline else { + return runtime.block_on(response).map(Some); + }; + if Instant::now() >= deadline { + return Ok(None); + } + let timeout = tokio::time::Instant::from_std(deadline); + match runtime.block_on(async { tokio::time::timeout_at(timeout, response).await }) { + Ok(response) if Instant::now() < deadline => response.map(Some), + Ok(_) => Ok(None), + Err(_) => Ok(None), + } +} + +async fn read_response( + request: reqwest::RequestBuilder, + transferred: &mut usize, +) -> Result { + let mut response = request.send().await?; + let status = response.status().as_u16(); + let headers = response.headers().clone(); + let mut bytes = Vec::new(); + while let Some(chunk) = response.chunk().await? { + ensure!( + bytes.len() + chunk.len() <= RESPONSE_LIMIT, + "journey response is too large" + ); + bytes.extend_from_slice(&chunk); + add_transfer(transferred, chunk.len())?; + } + let body = serde_json::from_slice(&bytes).unwrap_or(Value::Null); + Ok(ResponseData { + status, + headers, + body, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::project_dev::plan::ExpectedStatus; + use serde_json::json; + use std::{ + io::{Read, Write}, + net::{Ipv4Addr, TcpListener, TcpStream}, + path::PathBuf, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, Mutex, + }, + }; + + static POLL_TEST_LOCK: Mutex<()> = Mutex::new(()); + + fn no_capture(_: &str) -> Result { + anyhow::bail!("unexpected capture") + } + + struct SyntheticServer { + url: String, + requests: Arc, + thread: thread::JoinHandle<()>, + } + + impl SyntheticServer { + fn finish(self) -> usize { + self.thread.join().unwrap(); + self.requests.load(Ordering::SeqCst) + } + } + + /// How long a synthetic server accepts requests. A journey run spends a few + /// hundred milliseconds before its first request, so the poll budgets below + /// leave room for a busy machine. + const SERVER_WINDOW: Duration = Duration::from_secs(2); + + fn synthetic_server(responses: Vec<(Duration, u16, &'static str)>) -> SyntheticServer { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + listener.set_nonblocking(true).unwrap(); + let url = format!("http://{}/", listener.local_addr().unwrap()); + let requests = Arc::new(AtomicUsize::new(0)); + let observed = requests.clone(); + let thread = thread::spawn(move || { + let end = Instant::now() + SERVER_WINDOW; + let mut response = 0; + while Instant::now() < end { + match listener.accept() { + Ok((mut stream, _)) => { + observed.fetch_add(1, Ordering::SeqCst); + stream.set_nonblocking(false).unwrap(); + read_request(&mut stream); + let &(delay, status, body) = responses + .get(response) + .unwrap_or_else(|| responses.last().unwrap()); + response += 1; + let reason = if status == 200 { "OK" } else { "Accepted" }; + let headers = format!( + "HTTP/1.1 {status} {reason}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + if write_to_client(&mut stream, headers.as_bytes()) { + thread::sleep(delay); + write_to_client(&mut stream, body.as_bytes()); + } + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(1)); + } + Err(error) => panic!("synthetic server accept failed: {error}"), + } + } + }); + SyntheticServer { + url, + requests, + thread, + } + } + + fn read_request(stream: &mut TcpStream) { + stream + .set_read_timeout(Some(Duration::from_secs(1))) + .unwrap(); + let mut request = Vec::new(); + let mut chunk = [0u8; 2048]; + while request.len() < 16 * 1024 && !request.windows(4).any(|part| part == b"\r\n\r\n") { + match stream.read(&mut chunk) { + Ok(0) | Err(_) => break, + Ok(length) => request.extend_from_slice(&chunk[..length]), + } + } + assert!(request.windows(4).any(|part| part == b"\r\n\r\n")); + } + + fn write_to_client(stream: &mut TcpStream, bytes: &[u8]) -> bool { + match stream.write_all(bytes) { + Ok(()) => true, + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::NotConnected + ) => + { + false + } + Err(error) => panic!("synthetic server write failed: {error}"), + } + } + + fn polling_input(maximum_wait: u64, interval: u64) -> RunInput { + let plan = serde_json::from_value(json!({ + "issuer": "https://issuer.example.invalid", + "products": [{"product": "breg", "project": ".", "audience": "registry"}], + "people": {"reader": { + "subject": "reader", "client": "reader-client", + "grants": {"breg": {"profile": "reader", "scopes": ["registry:read"]}} + }}, + "journey": {"id": "poll", "steps": [{ + "id": "wait", "product": "breg", "as": "reader", "method": "GET", + "path": "/v1/result", "expect": {"status": 200, "json": {"/state": "ready"}}, + "poll": { + "maximumWaitMilliseconds": maximum_wait, + "pollIntervalMilliseconds": interval + } + }]} + })) + .unwrap(); + RunInput { + run_id: "0123456789abcdef0123456789abcdef".to_owned(), + project: PathBuf::from("/tmp/project"), + plan, + ttl_seconds: 300, + schema_people: Vec::new(), + breg_authority: Some(super::super::BregAuthority { + principal: "sub".to_owned(), + purpose: false, + requires_postgis: false, + named_clients: Default::default(), + }), + casework_principals: BTreeMap::new(), + binaries: BTreeMap::new(), + } + } + + #[test] + fn poll_refuses_a_matching_response_that_finishes_after_its_deadline() { + let _lock = POLL_TEST_LOCK + .lock() + .unwrap_or_else(|error| error.into_inner()); + let server = synthetic_server(vec![ + (Duration::ZERO, 202, r#"{"state":"waiting"}"#), + (Duration::from_millis(1500), 200, r#"{"state":"ready"}"#), + ]); + let root = tempfile::tempdir().unwrap(); + let issuer = Issuer::create(root.path()).unwrap(); + let input = polling_input(1000, 50); + let urls = BTreeMap::from([(Product::Breg, server.url.clone())]); + + let report = run(&input, &issuer, &urls, || Ok(())).unwrap(); + + assert_eq!(report.result, "failed"); + assert_eq!(report.steps[0].status, 202); + assert!(report.steps[0] + .reason + .as_deref() + .is_some_and(|reason| reason.starts_with("poll deadline passed; last response: "))); + assert_eq!(server.finish(), 2); + } + + #[test] + fn poll_starts_no_retry_after_its_deadline() { + let _lock = POLL_TEST_LOCK + .lock() + .unwrap_or_else(|error| error.into_inner()); + let server = synthetic_server(vec![(Duration::ZERO, 202, r#"{"state":"waiting"}"#)]); + let root = tempfile::tempdir().unwrap(); + let issuer = Issuer::create(root.path()).unwrap(); + let input = polling_input(1000, 1000); + let urls = BTreeMap::from([(Product::Breg, server.url.clone())]); + + let report = run(&input, &issuer, &urls, || Ok(())).unwrap(); + + assert_eq!(report.result, "failed"); + assert_eq!(report.steps[0].status, 202); + assert!(report.steps[0] + .reason + .as_deref() + .is_some_and(|reason| reason.starts_with("poll deadline passed; last response: "))); + assert_eq!(server.finish(), 1); + } + + #[test] + fn poll_sends_nothing_with_zero_remaining_budget() { + let _lock = POLL_TEST_LOCK + .lock() + .unwrap_or_else(|error| error.into_inner()); + let server = synthetic_server(vec![(Duration::ZERO, 200, r#"{"state":"ready"}"#)]); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let request = reqwest::Client::new().get(format!("{}v1/result", server.url)); + let mut transferred = 0; + + let response = read_response_before_deadline( + &runtime, + request, + Some(Instant::now()), + &mut transferred, + ) + .unwrap(); + + assert!(response.is_none()); + assert_eq!(transferred, 0); + assert_eq!(server.finish(), 0); + } + + #[test] + fn exact_capture_reference_preserves_scalar_type() { + let captures = BTreeMap::from([("revision", json!(7))]); + let expanded = expand_json( + &json!({ + "revision": "{{revision}}", "label": "revision-{{revision}}" + }), + |name| Ok(captures[name].clone()), + ) + .unwrap(); + assert_eq!(expanded, json!({"revision": 7, "label": "revision-7"})); + } + + #[test] + fn request_url_stays_on_the_numeric_loopback_v1_origin() { + let url = request_url("http://127.0.0.1:31415/", "/v1/records?id=one").unwrap(); + assert_eq!(url.as_str(), "http://127.0.0.1:31415/v1/records?id=one"); + assert!(request_url("http://localhost:31415/", "/v1/records").is_err()); + assert!(request_url("http://127.0.0.1:31415/", "/v1/../admin").is_err()); + } + + #[test] + fn checks_compare_typed_values_lengths_and_membership() { + let body = json!({"state": true, "items": [1, 2], "label": "ready-now"}); + for check in [ + Check { + pointer: "/state".into(), + equals: Some(json!(true)), + length: None, + contains: None, + }, + Check { + pointer: "/items".into(), + equals: None, + length: Some(2), + contains: None, + }, + Check { + pointer: "/items".into(), + equals: None, + length: None, + contains: Some(json!(2)), + }, + Check { + pointer: "/label".into(), + equals: None, + length: None, + contains: Some(json!("now")), + }, + ] { + assert!(matches_check(&check, &body, no_capture).unwrap()); + } + } + + #[test] + fn expectation_accepts_any_declared_status_and_problem_code() { + let step: Step = serde_json::from_value(json!({ + "id": "read", "product": "breg", "as": "person", "method": "GET", + "path": "/v1/records", "expect": {"status": [404, 409], "code": "missing"} + })) + .unwrap(); + let response = ResponseData { + status: 409, + headers: HeaderMap::new(), + body: json!({"code": "missing"}), + }; + assert!(first_mismatch(&step, &response, no_capture) + .unwrap() + .is_none()); + assert_eq!(step.expect.status.first(), 404); + } + + #[test] + fn mismatch_names_the_pointer_expected_and_observed_values() { + let step: Step = serde_json::from_value(json!({ + "id": "read", "product": "breg", "as": "person", "method": "GET", + "path": "/v1/records", "expect": {"status": 200, "json": {"/state": "open"}} + })) + .unwrap(); + let mut response = ResponseData { + status: 500, + headers: HeaderMap::new(), + body: json!({"state": "closed"}), + }; + assert_eq!( + first_mismatch(&step, &response, no_capture) + .unwrap() + .unwrap(), + "/status expected 200 observed 500" + ); + response.status = 200; + assert_eq!( + first_mismatch(&step, &response, no_capture) + .unwrap() + .unwrap(), + "/state expected \"open\" observed \"closed\"" + ); + response.body = json!({"state": "open"}); + assert!(first_mismatch(&step, &response, no_capture) + .unwrap() + .is_none()); + } + + #[test] + fn reasons_are_bounded_and_omitted_when_the_step_passes() { + assert_eq!(bounded_reason(&"é".repeat(2000)).chars().count(), 512); + let report = StepReport { + id: "a".into(), + status: 200, + expected_status: 200, + result: "passed", + reason: None, + }; + assert!(serde_json::to_value(&report) + .unwrap() + .get("reason") + .is_none()); + } + + #[test] + fn find_capture_requires_one_matching_item() { + let capture: Capture = serde_json::from_value(json!({ + "type": "find", + "find": "/items", "where": {"pointer": "/id", "equals": "two"}, + "pointer": "/revision" + })) + .unwrap(); + let response = ResponseData { + status: 200, + headers: HeaderMap::new(), + body: json!({"items": [ + {"id": "one", "revision": 1}, {"id": "two", "revision": 2} + ]}), + }; + assert_eq!( + capture_value(&capture, &response, no_capture).unwrap(), + json!(2) + ); + } + + #[test] + fn expected_status_helpers_keep_the_first_for_reports() { + let status: ExpectedStatus = serde_json::from_value(json!([202, 200])).unwrap(); + assert_eq!(status.first(), 202); + assert!(status.contains(200)); + assert_eq!(serde_json::to_value(&status).unwrap(), json!([202, 200])); + } + + #[test] + fn total_request_work_is_bounded_across_steps() { + let mut attempts = 0; + for _ in 0..JOURNEY_ATTEMPT_LIMIT { + add_attempt(&mut attempts).unwrap(); + } + assert!(add_attempt(&mut attempts).is_err()); + } +} diff --git a/crates/registry-bregctl/src/project_dev/mod.rs b/crates/registry-bregctl/src/project_dev/mod.rs new file mode 100644 index 0000000000..31b4ac4a24 --- /dev/null +++ b/crates/registry-bregctl/src/project_dev/mod.rs @@ -0,0 +1,1375 @@ +// SPDX-License-Identifier: Apache-2.0 +//! The private runner behind the rstack launcher. The bootstrap binary does +//! not link the database, HTTP server, or signing libraries. + +mod bindings; +mod database; +mod issuer; +mod journey; +pub(crate) mod plan; +mod products; + +use anyhow::{bail, ensure, Context, Result}; +use registry_platform_yaml::{Diagnostic, Report}; +use rustix::fs::{flock, FlockOperation}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::{ + collections::{BTreeMap, BTreeSet}, + env, + fs::{self, File, OpenOptions}, + io::{self, BufRead, BufReader, Read, Write}, + os::unix::{ + fs::{DirBuilderExt, MetadataExt, OpenOptionsExt}, + process::CommandExt, + }, + path::{Path, PathBuf}, + process::{Command, Stdio}, + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc, Arc, + }, + thread, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; +use uuid::Uuid; +use zeroize::Zeroizing; + +use self::{ + database::Database, + issuer::Issuer, + plan::{Plan, Product}, + products::{ProductRunner, SchemaPerson}, +}; + +const SCHEMA: &str = "registry.rstack.dev.v1"; +const REQUEST: &str = "RSTACK_PROJECT_DEV_REQUEST"; +const DATABASE_URL: &str = "RSTACK_DEV_DATABASE_URL"; +const CA_FILE: &str = "RSTACK_DEV_DATABASE_CA"; +const REPORT: &str = "report.json"; +const PENDING: &str = "cleanup-pending.json"; +const CLEANUP_CAP: Duration = Duration::from_secs(30 * 60); +const POLL: Duration = Duration::from_millis(100); +// A capture can contain the 1 MiB development plan after JSON escaping, +// selections derived from a second 1 MiB journeys document, facts selected +// from the bounded 4 MiB Casework authoring report, and the remaining paths +// and authority metadata. Keep headroom for those representations while +// retaining one hard bound for both writing and restoring the private input. +const MAXIMUM_CAPTURED_INPUT_BYTES: u64 = + 16 * registry_platform_yaml::MAXIMUM_DOCUMENT_BYTES as u64; +const MAXIMUM_RUN_REPORT_BYTES: u64 = 512 * 1024; + +#[derive(clap::Args, Debug)] +pub(crate) struct Args { + #[arg(long, hide = true)] + worker: Option, +} + +#[derive(Deserialize)] +#[serde(tag = "action", rename_all = "snake_case", deny_unknown_fields)] +enum Request { + Check { + project: PathBuf, + deny_warnings: bool, + json: bool, + }, + Start { + project: PathBuf, + ttl: Option, + json: bool, + }, + Status { + run_id: String, + json: bool, + }, + Token { + run_id: String, + person: String, + json: bool, + }, + Stop { + run_id: String, + json: bool, + }, +} + +impl Request { + fn json(&self) -> bool { + match self { + Self::Check { json, .. } + | Self::Start { json, .. } + | Self::Status { json, .. } + | Self::Token { json, .. } + | Self::Stop { json, .. } => *json, + } + } +} + +#[derive(Deserialize, Serialize)] +struct RunInput { + run_id: String, + project: PathBuf, + plan: Plan, + ttl_seconds: u64, + schema_people: Vec, + breg_authority: Option, + casework_principals: BTreeMap, + binaries: BTreeMap, +} + +#[derive(Deserialize, Serialize)] +struct BregAuthority { + principal: String, + purpose: bool, + requires_postgis: bool, + named_clients: BTreeSet, +} + +#[derive(Deserialize, Serialize)] +struct Binaries { + tool: PathBuf, + runtime: PathBuf, +} + +#[derive(Deserialize, Serialize)] +#[serde(tag = "outcome", rename_all = "snake_case")] +enum WorkerReply { + Ready { report: Value }, + Failed { error: String }, +} + +#[derive(Debug, thiserror::Error)] +#[error("run {run_id} still needs cleanup: {detail}")] +struct CleanupPending { + run_id: String, + detail: String, +} + +/// A run that cannot finish by itself: its supervisor is gone, or cleanup +/// gave up. `facts` carries what an operator needs to release it by hand. +#[derive(Debug, thiserror::Error)] +#[error("{}", recovery_message(run_id, facts))] +struct RecoveryNeeded { + run_id: String, + facts: Value, +} + +fn recovery_message(run_id: &str, facts: &Value) -> String { + let mut text = match facts["kind"].as_str() { + Some("cleanup-pending") => { + format!("run {run_id} gave up cleanup; release these resources by hand") + } + _ => format!( + "run {run_id} lost its supervisor before cleanup; release these resources by hand" + ), + }; + let mut line = |entry: String| { + text.push_str("\n "); + text.push_str(&entry); + }; + if let Some(directory) = facts["run_directory"].as_str() { + line(format!("run directory {directory}")); + } + for process in facts["processes"].as_array().into_iter().flatten() { + line(format!( + "process {} pid {} pgid {}", + process["product"], process["pid"], process["pgid"] + )); + } + for object in facts["objects"].as_array().into_iter().flatten() { + let roles: Vec<_> = object["roles"] + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .collect(); + line(format!( + "{} database {} roles {}", + object["label"].as_str().unwrap_or("?"), + object["database"].as_str().unwrap_or("?"), + roles.join(", ") + )); + } + for name in facts["databases_and_roles"] + .as_array() + .into_iter() + .flatten() + { + line(format!("database or role {}", name.as_str().unwrap_or("?"))); + } + if let Some(reason) = facts["unavailable"].as_str() { + line(format!("some facts are unavailable: {reason}")); + } + text +} + +/// Everything a lost supervisor leaves behind: the processes it recorded and +/// every database and role its plan provisions. +fn lost_supervisor_facts(root: &Path, run_id: &str) -> Value { + let mut facts = json!({ + "kind": "lost-supervisor", + "run_directory": root, + "processes": recorded_processes(root), + "objects": [], + }); + match provisioned_objects(root, run_id) { + Ok(objects) => facts["objects"] = json!(objects), + Err(error) => facts["unavailable"] = json!(format!("{error:#}")), + } + facts +} + +fn recorded_processes(root: &Path) -> Value { + read_json::(&root.join("processes.json"), MAXIMUM_RUN_REPORT_BYTES) + .ok() + .filter(Value::is_array) + .unwrap_or_else(|| json!([])) +} + +fn provisioned_objects(root: &Path, run_id: &str) -> Result> { + let input: RunInput = read_json(&root.join("input.json"), MAXIMUM_CAPTURED_INPUT_BYTES) + .context("cannot read the captured development plan")?; + let spatial = input + .breg_authority + .as_ref() + .is_some_and(|authority| authority.requires_postgis); + let mut labels = Vec::new(); + for entry in &input.plan.products { + match entry.product { + Product::Breg => { + labels.push(("breg_test", spatial)); + labels.push(("breg", spatial)); + } + Product::Casework => labels.push(("casework", false)), + } + } + labels + .into_iter() + .map(|(label, spatial)| { + let mut names = database::run_object_names(run_id, label, spatial)?; + let database = names.pop().context("run has no database name")?; + Ok(json!({"label": label, "database": database, "roles": names})) + }) + .collect() +} + +fn pending_report(run_id: &str, names: Vec, processes: Value) -> Value { + json!({"run_id": run_id, "databases_and_roles": names, "processes": processes}) +} + +/// The delays between cleanup attempts: 2 s doubling to 30 s, until their +/// sum reaches the cap. +fn cleanup_schedule(cap: Duration) -> impl Iterator { + let mut next = Duration::from_secs(2); + let mut total = Duration::ZERO; + std::iter::from_fn(move || { + if total >= cap { + return None; + } + let delay = next; + total += delay; + next = (next * 2).min(Duration::from_secs(30)); + Some(delay) + }) +} + +/// The wall clock decides expiry, because the monotonic clock stops while a +/// laptop sleeps. +fn expired(now_unix: u64, expires_at: u64) -> bool { + now_unix >= expires_at +} + +fn startup_error(root: &Path, run_id: &str, error: anyhow::Error) -> anyhow::Error { + if root.exists() { + CleanupPending { + run_id: run_id.to_owned(), + detail: format!("{error:#}"), + } + .into() + } else { + error + } +} + +pub(crate) fn run(args: Args, stdout: &mut dyn Write, stderr: &mut dyn Write) -> u8 { + if let Some(root) = args.worker { + // The CLI panic hook exits immediately. In the supervisor, unwind so + // its startup boundary can release the run without exposing payloads. + std::panic::set_hook(Box::new(|_| {})); + return match worker(&root, stdout) { + Ok(()) => 0, + Err(error) => { + let _ = reply( + stdout, + &WorkerReply::Failed { + error: format!("{error:#}"), + }, + ); + 3 + } + }; + } + let request = env::var(REQUEST) + .ok() + .and_then(|text| serde_json::from_str::(&text).ok()); + let Some(request) = request else { + let _ = writeln!( + stderr, + "run project development commands through rstack dev" + ); + return 2; + }; + let machine = request.json(); + let starting = matches!(&request, Request::Start { .. }); + let outcome = match request { + Request::Check { + project, + deny_warnings, + .. + } => return check(&project, deny_warnings, machine, stdout, stderr), + Request::Start { project, ttl, .. } => start(&project, ttl.as_deref()), + Request::Status { run_id, .. } => status(&run_id), + Request::Token { run_id, person, .. } => token(&run_id, &person), + Request::Stop { run_id, .. } => stop(&run_id), + }; + let (result, code) = match outcome { + Ok(value) => { + let code = u8::from(starting && value["journey"]["result"] == "failed"); + (emit(stdout, &value, machine), code) + } + Err(error) if machine => { + let mut report = json!({ + "schema_version": SCHEMA, "status": "error", "error": format!("{error:#}") + }); + if let Some(pending) = error.downcast_ref::() { + report["run_id"] = json!(pending.run_id); + report["cleanup_pending"] = json!(true); + } + if let Some(recovery) = error.downcast_ref::() { + report["run_id"] = json!(recovery.run_id); + report["recovery"] = recovery.facts.clone(); + } + (writeln!(stdout, "{report}"), 1) + } + Err(error) => (writeln!(stderr, "rstack dev: {error:#}"), 1), + }; + if result.is_ok() { + code + } else { + 3 + } +} + +fn emit(output: &mut dyn Write, report: &Value, machine: bool) -> std::io::Result<()> { + if machine { + return writeln!(output, "{report}"); + } + let run = report["run_id"].as_str().expect("run report"); + let status = report["status"].as_str().expect("run status"); + writeln!(output, "Run {run} {status}")?; + if status == "cleanup-pending" { + let message = recovery_message(run, &report["recovery"]); + for line in message.lines().skip(1) { + writeln!(output, "{line}")?; + } + } + if let Some(products) = report["products"].as_object() { + for (product, url) in products { + writeln!(output, " {product} {}", url.as_str().expect("product URL"))?; + } + } + if let Some(result) = report["journey"]["result"].as_str() { + writeln!( + output, + " Journey {} {result}", + report["journey"]["id"].as_str().expect("journey ID") + )?; + if let Some(steps) = report["journey"]["steps"].as_array() { + for step in steps.iter().filter(|step| step["result"] == "failed") { + writeln!( + output, + " {} expected HTTP {}, observed {}", + step["id"], step["expected_status"], step["status"] + )?; + if let Some(reason) = step["reason"].as_str() { + writeln!(output, " {reason}")?; + } + } + } + } + if let Some(logs) = report["logs"].as_str() { + writeln!(output, " Logs {logs}")?; + } + if let Some(people) = report["people"].as_array() { + for name in people { + writeln!( + output, + " rstack dev token {run} --as {}", + name.as_str().expect("person name") + )?; + } + } + if let Some(credentials) = report["credentials"].as_array() { + for credential in credentials { + writeln!( + output, + " {} headers {}", + credential["product"].as_str().expect("product"), + credential["header_file"].as_str().expect("header path") + )?; + } + } + Ok(()) +} + +fn check( + project: &Path, + deny_warnings: bool, + machine: bool, + stdout: &mut dyn Write, + stderr: &mut dyn Write, +) -> u8 { + let outcome = project + .canonicalize() + .context("cannot open the development project directory") + .and_then(|project| plan::check(&project)); + let (report, exit) = match outcome { + Ok(report) => { + let refused = report.has_errors() || (deny_warnings && report.warning_count() > 0); + (report, u8::from(refused)) + } + Err(error) => match error.downcast::() { + Ok(report) => (report, crate::DOMAIN_REFUSAL_EXIT), + Err(error) => { + if let Some(unavailable) = error.downcast_ref::() { + let report = Report::new(vec![Diagnostic::error( + "breg.project-dev.check-unavailable", + unavailable.field(), + unavailable.reason(), + unavailable.remedy(), + )]); + return write_check_report( + report, + crate::OPERATIONAL_FAILURE_EXIT, + machine, + stdout, + stderr, + ); + } + let (code, action, exit) = if error.is::() { + ( + "breg.project-dev.check-unavailable", + "Name a project directory containing a readable dev.yaml.", + crate::OPERATIONAL_FAILURE_EXIT, + ) + } else { + ( + "breg.project-dev.invalid-plan", + "Correct the development plan at the reported field.", + crate::DOMAIN_REFUSAL_EXIT, + ) + }; + ( + Report::new(vec![Diagnostic::error(code, "", error.to_string(), action)]), + exit, + ) + } + }, + }; + write_check_report(report, exit, machine, stdout, stderr) +} + +fn write_check_report( + report: Report, + exit: u8, + machine: bool, + stdout: &mut dyn Write, + stderr: &mut dyn Write, +) -> u8 { + let result = if machine { + let payload = json!({ + "ok": exit == 0, + "command": "dev-check", + "status": registry_cli_report::Status::from_exit(exit).as_str(), + "diagnostics": report.to_json_value(), + }); + crate::write_ctl_report(stdout, &payload) + .map_err(io::Error::other) + .and_then(|()| writeln!(stdout)) + } else if exit == 0 { + writeln!(stdout, "Development plan is valid") + .and_then(|()| write!(stdout, "{}", report.render_human())) + } else { + write!(stderr, "{}", report.render_human()) + }; + if result.is_ok() { + exit + } else { + let _ = writeln!(stderr, "rstack dev check: output could not be written"); + crate::OPERATIONAL_FAILURE_EXIT + } +} + +fn start(project: &Path, ttl: Option<&str>) -> Result { + let project = project + .canonicalize() + .context("cannot open the development project directory")?; + let plan = plan::load(&project)?; + let schema_people = products::schema_people(&project, &plan, true)?; + let signals = signals()?; + let inspection = products::inspect(&project, &plan, &signals)?; + let ttl_seconds = parse_ttl(ttl.unwrap_or("30m"))?; + ensure!( + env::var_os(DATABASE_URL).is_some_and(|value| !value.is_empty()), + "supply --database-url or {DATABASE_URL} for a TLS PostgreSQL development server" + ); + let binaries = plan + .products + .iter() + .map(|entry| Ok((entry.product, binaries(entry.product)?))) + .collect::>()?; + let run_id = Uuid::new_v4().simple().to_string(); + let base = state_dir(); + ensure_private(&base, true)?; + let root = base.canonicalize()?.join(&run_id); + private_directory(&root)?; + let input = RunInput { + run_id, + project, + plan, + ttl_seconds, + schema_people, + breg_authority: inspection.breg_authority, + casework_principals: inspection.casework_principals, + binaries, + }; + let preparation = (|| { + write_private(&root.join("input.json"), &capture_input(&input)?)?; + let supervisor_log = open_supervisor_log(&root)?; + Command::new(env::current_exe()?) + .arg("project-dev") + .arg("--worker") + .arg(&root) + .env_remove(REQUEST) + .process_group(0) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::from(supervisor_log)) + .spawn() + .context("cannot start the development run supervisor") + })(); + let mut child = match preparation { + Ok(child) => child, + Err(error) => { + let kept = remove_failed_run(&root, &input.run_id) + .context("cannot remove unstarted run directory")?; + return Err(note_kept_logs(error, kept)); + } + }; + let stream = child.stdout.take().expect("startup channel"); + let (sender, receiver) = mpsc::sync_channel(1); + thread::spawn(move || { + let mut line = String::new(); + let result = BufReader::new(stream) + .take(MAXIMUM_RUN_REPORT_BYTES) + .read_line(&mut line) + .context("cannot read startup report") + .and_then(|_| { + serde_json::from_str::(&line) + .context("missing or invalid startup report") + }); + let _ = sender.send(result); + }); + let deadline = Instant::now() + Duration::from_secs(360); + loop { + match receiver.recv_timeout(POLL) { + Ok(Ok(WorkerReply::Ready { report })) => return Ok(report), + Ok(Ok(WorkerReply::Failed { error })) => { + let error = anyhow::anyhow!(error); + let kept_logs = failed_logs_dir(&base, &input.run_id); + if root.join("worker.lock").exists() { + if let Err(cleanup) = stop(&input.run_id) { + return Err(startup_error( + &root, + &input.run_id, + error.context(format!( + "automatic cleanup could not finish: {cleanup:#}" + )), + )); + } + } else if root.exists() { + remove_failed_run(&root, &input.run_id) + .context("cannot remove unstarted run directory")?; + } + return Err(note_kept_logs( + error, + kept_logs.exists().then_some(kept_logs), + )); + } + Ok(Err(error)) => { + if child.try_wait()?.is_some() + && root.exists() + && !root.join("worker.lock").exists() + { + remove_failed_run(&root, &input.run_id) + .context("cannot remove unstarted run directory")?; + } + return Err(startup_error(&root, &input.run_id, error)); + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + return Err(startup_error( + &root, + &input.run_id, + anyhow::anyhow!("the supervisor lost its startup report"), + )); + } + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + if signals.load(Ordering::Relaxed) || Instant::now() >= deadline { + stop(&input.run_id).map_err(|error| startup_error(&root, &input.run_id, error))?; + bail!("development run start was interrupted or timed out"); + } + } +} + +fn worker(root: &Path, output: &mut dyn Write) -> Result<()> { + let name = root + .file_name() + .and_then(|value| value.to_str()) + .context("invalid worker run directory")?; + let expected = run_dir(name)?; + ensure!( + expected.canonicalize()? == root, + "worker run directory differs from the allocated run" + ); + ensure_private(root, false)?; + let input = Arc::new( + read_json::(&root.join("input.json"), MAXIMUM_CAPTURED_INPUT_BYTES) + .context("cannot restore the captured development plan")?, + ); + ensure!( + input.run_id == name, + "captured development plan has a different run ID" + ); + let signals = signals()?; + let mut products = ProductRunner::new(input.clone(), root, &signals)?; + let lock = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(root.join("worker.lock"))?; + flock(&lock, FlockOperation::LockExclusive)?; + let mut database = None; + let mut expiry = None; + let prepared = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + products.check()?; + let ca = env::var_os(CA_FILE) + .map(|path| { + let bytes = plan::read(Path::new(&path), 1024 * 1024) + .context("cannot read the PostgreSQL CA file")?; + write_private(&root.join("postgres-ca"), &bytes)?; + Ok::<_, anyhow::Error>(bytes) + }) + .transpose()?; + let admin = Zeroizing::new( + env::var(DATABASE_URL) + .map_err(|_| anyhow::anyhow!("missing PostgreSQL administrator URL"))?, + ); + let issuer = Arc::new(Issuer::create(root)?); + let database = database.insert(Database::connect(&admin, ca.as_deref(), &input.run_id)?); + let urls = products.start(database, issuer.clone())?; + let journey = products.journey(&issuer, &urls)?; + ensure!( + products.services_alive()?, + "a product exited during the journey" + ); + let expires_at = clock()? + input.ttl_seconds; + expiry = Some(expires_at); + let report = json!({ + "schema_version": SCHEMA, "run_id": input.run_id, "status": "ready", + "products": urls, "people": input.plan.people.keys().collect::>(), + "journey": journey, "expires_at": expires_at, "logs": root.join("logs"), + }); + write_private(&root.join(REPORT), &capture_report(&report)?)?; + Ok::<_, anyhow::Error>(report) + })) + .unwrap_or_else(|_| Err(anyhow::anyhow!("development run startup failed internally"))); + + let mut failure = None; + match prepared { + Ok(report) => { + if let Err(error) = reply(output, &WorkerReply::Ready { report }) { + failure = Some(format!("cannot report development run readiness: {error}")); + } else { + let deadline = Instant::now() + Duration::from_secs(input.ttl_seconds); + while Instant::now() < deadline + && !stopping(root, &signals) + && !expiry.is_some_and(|at| clock().map_or(true, |now| expired(now, at))) + { + if !products.services_alive().unwrap_or(false) { + break; + } + thread::sleep(POLL); + } + } + } + Err(error) => failure = Some(format!("{error:#}")), + } + let stopped = simple_report(&input.run_id, "stopping"); + let mark_stopping = write_private(&root.join(REPORT), &serde_json::to_vec(&stopped)?); + let mut told_parent = false; + let mut delays = cleanup_schedule(CLEANUP_CAP); + loop { + // Keep the exact ownership list until both the supervised processes + // and their database resources have been released. + let cleanup = products + .stop() + .and_then(|()| database.as_mut().map_or(Ok(()), Database::cleanup)); + if cleanup.is_ok() { + break; + } + if let Some(error) = failure.as_ref().filter(|_| !told_parent) { + let _ = reply( + output, + &WorkerReply::Failed { + error: error.clone(), + }, + ); + told_parent = true; + } + let Some(delay) = delays.next() else { + let names = database + .as_ref() + .map_or_else(Vec::new, Database::pending_names); + let pending = pending_report(&input.run_id, names, recorded_processes(root)); + write_private(&root.join(PENDING), &serde_json::to_vec(&pending)?)?; + bail!( + "run {} gave up cleanup after {} minutes; run rstack dev status {} for the resources to release", + input.run_id, + CLEANUP_CAP.as_secs() / 60, + input.run_id + ); + }; + thread::sleep(delay); + } + // A failed run keeps its logs outside the run directory. + let removed = if failure.is_some() { + keep_logs(root, &input.run_id).map(drop) + } else { + Ok(()) + } + .and_then(|()| remove_run_dir(root)); + if let Some(error) = failure.filter(|_| !told_parent) { + let error = match removed.as_ref() { + Ok(()) => error, + Err(cleanup) => format!("{error}; run {} {cleanup}", input.run_id), + }; + reply(output, &WorkerReply::Failed { error })?; + } + removed?; + // If the root is gone, no stale ready report could have survived. + mark_stopping?; + Ok(()) +} + +fn reply(output: &mut dyn Write, message: &WorkerReply) -> Result<()> { + serde_json::to_writer(&mut *output, message)?; + writeln!(output)?; + output.flush()?; + Ok(()) +} + +fn status(run_id: &str) -> Result { + let root = run_dir(run_id)?; + if !root.exists() { + return Ok(simple_report(run_id, "stopped")); + } + let observed = (|| { + if let Ok(pending) = read_json::(&root.join(PENDING), MAXIMUM_RUN_REPORT_BYTES) { + return Ok(pending_status(&root, run_id, pending)); + } + if supervisor_gone(&root)? { + return Err(RecoveryNeeded { + run_id: run_id.to_owned(), + facts: lost_supervisor_facts(&root, run_id), + } + .into()); + } + let report: Value = read_json(&root.join(REPORT), MAXIMUM_RUN_REPORT_BYTES) + .context("the development run has not reported readiness")?; + if report["expires_at"] + .as_u64() + .is_some_and(|expiry| clock().is_ok_and(|now| now >= expiry)) + { + return Ok(simple_report(run_id, "stopping")); + } + Ok(report) + })(); + if observed.is_err() && !root.exists() { + return Ok(simple_report(run_id, "stopped")); + } + observed +} + +fn pending_status(root: &Path, run_id: &str, pending: Value) -> Value { + let mut report = simple_report(run_id, "cleanup-pending"); + report["recovery"] = json!({ + "kind": "cleanup-pending", + "run_directory": root, + "databases_and_roles": pending["databases_and_roles"], + "processes": pending["processes"], + }); + report +} + +fn token(run_id: &str, person: &str) -> Result { + ensure!(plan::identifier(person), "invalid test person name"); + let report = status(run_id)?; + ensure!( + report["status"] == "ready", + "development run {run_id} is not ready" + ); + let root = run_dir(run_id)?; + let input: RunInput = read_json(&root.join("input.json"), MAXIMUM_CAPTURED_INPUT_BYTES)?; + let signer = Issuer::load(&root)?; + let run_expires = report["expires_at"] + .as_u64() + .context("run expiry is missing")?; + // Report the expiry the tokens carry, which is earlier than the run's. + let expires = issuer::token_expiry(run_expires, clock()?); + let subject = input + .plan + .people + .get(person) + .context("test person is not in the run's plan")?; + let mut credentials = Vec::new(); + for (product, grant) in &subject.grants { + let token = signer.token(&input, subject, *product, expires)?; + let path = root.join(format!( + "headers-{}-{person}-{}", + product.name(), + Uuid::new_v4().simple() + )); + let headers = match product { + Product::Breg => format!("Authorization: Bearer {}\n", &*token), + Product::Casework => format!("Authorization: Bearer {}\nRegistry-Casework-Profile: {}\nRegistry-Review-Profile: {}\n", + &*token, grant.profile, grant.profile), + }; + write_private(&path, Zeroizing::new(headers).as_bytes())?; + credentials + .push(json!({"product": product, "profile": grant.profile, "header_file": path})); + } + Ok(json!({ + "schema_version": SCHEMA, "run_id": run_id, "status": "token", + "alias": person, "credentials": credentials, "expires_at": expires, + })) +} + +fn stop(run_id: &str) -> Result { + let root = run_dir(run_id)?; + if !root.exists() { + return Ok(simple_report(run_id, "stopped")); + } + if let Err(error) = ensure_private(&root, false) { + // The supervisor can finish and remove the run between the checks. + if !root.exists() { + return Ok(simple_report(run_id, "stopped")); + } + return Err(error); + } + match OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(root.join("stop")) + { + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound && !root.exists() => { + return Ok(simple_report(run_id, "stopped")); + } + Err(error) => return Err(error).context("cannot signal the run to stop"), + } + let deadline = Instant::now() + Duration::from_secs(60); + while root.exists() { + ensure!( + Instant::now() < deadline, + "run {run_id} is still cleaning up or its cleanup has stalled; run rstack dev status {run_id} to see its state" + ); + thread::sleep(POLL); + if !root.exists() { + break; + } + if let Ok(pending) = read_json::(&root.join(PENDING), MAXIMUM_RUN_REPORT_BYTES) { + return Err(RecoveryNeeded { + run_id: run_id.to_owned(), + facts: pending_status(&root, run_id, pending)["recovery"].clone(), + } + .into()); + } + let gone = supervisor_gone(&root); + if root.exists() && gone? { + return Err(RecoveryNeeded { + run_id: run_id.to_owned(), + facts: lost_supervisor_facts(&root, run_id), + } + .into()); + } + } + Ok(simple_report(run_id, "stopped")) +} + +fn simple_report(run_id: &str, status: &str) -> Value { + json!({"schema_version": SCHEMA, "run_id": run_id, "status": status}) +} + +/// Whether a supervisor that took its lock has since vanished. The lock file +/// appears just before the lock is taken, so a first sighting is confirmed +/// after one poll before it counts. +fn supervisor_gone(root: &Path) -> Result { + for attempt in 0..2 { + if !root.join("worker.lock").exists() || supervisor_running(root)? { + return Ok(false); + } + if attempt == 0 { + thread::sleep(POLL); + } + } + Ok(true) +} + +fn supervisor_running(root: &Path) -> Result { + ensure_private(root, false)?; + let lock = + File::open(root.join("worker.lock")).context("run supervisor lock is unavailable")?; + match flock(&lock, FlockOperation::NonBlockingLockExclusive) { + Ok(()) => Ok(false), + Err(rustix::io::Errno::WOULDBLOCK) => Ok(true), + Err(error) => Err(error).context("cannot check run supervisor lock"), + } +} + +fn signals() -> Result> { + let terminate = Arc::new(AtomicBool::new(false)); + for signal in [signal_hook::consts::SIGTERM, signal_hook::consts::SIGINT] { + signal_hook::flag::register(signal, terminate.clone())?; + } + Ok(terminate) +} + +fn stopping(root: &Path, signals: &AtomicBool) -> bool { + signals.load(Ordering::Relaxed) || root.join("stop").exists() +} + +fn binary(product: Product, name: &str) -> Result { + let path = env::var_os(name).map(PathBuf::from).with_context(|| { + format!( + "{name} is not set; run rstack install {} or set {name}", + product.name() + ) + })?; + let path = path + .canonicalize() + .with_context(|| format!("cannot resolve {name}"))?; + ensure!(path.is_file(), "{name} must be a binary file"); + Ok(path) +} + +fn product_tool(product: Product) -> Result { + binary( + product, + match product { + Product::Breg => "RSTACK_BREGCTL_BIN", + Product::Casework => "RSTACK_CASEWORKCTL_BIN", + }, + ) +} + +fn binaries(product: Product) -> Result { + Ok(match product { + Product::Breg => Binaries { + tool: product_tool(product)?, + runtime: binary(product, "RSTACK_BREG_BIN")?, + }, + Product::Casework => Binaries { + tool: product_tool(product)?, + runtime: binary(product, "RSTACK_CASEWORK_BIN")?, + }, + }) +} + +fn parse_ttl(value: &str) -> Result { + let error = || anyhow::anyhow!("--ttl must be between 1s and 24h, with an s, m, or h suffix"); + let (number, unit) = value + .split_at_checked(value.len().saturating_sub(1)) + .ok_or_else(error)?; + let seconds = match unit { + "s" => 1, + "m" => 60, + "h" => 3600, + _ => return Err(error()), + }; + number + .parse::() + .ok() + .and_then(|value| value.checked_mul(seconds)) + .filter(|seconds| (1..=86400).contains(seconds)) + .ok_or_else(error) +} + +fn state_dir() -> PathBuf { + env::var_os("RSTACK_DEV_STATE_DIR").map_or_else( + || env::temp_dir().join(format!("rstack-dev-{}", rustix::process::getuid().as_raw())), + PathBuf::from, + ) +} + +fn run_dir(run_id: &str) -> Result { + ensure!( + run_id.len() == 32 + && run_id + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)), + "invalid run ID; use the ID printed by rstack dev start" + ); + Ok(state_dir().join(run_id)) +} + +fn private_directory(path: &Path) -> std::io::Result<()> { + fs::DirBuilder::new().mode(0o700).create(path) +} + +fn ensure_private(path: &Path, create: bool) -> Result<()> { + if create { + match private_directory(path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => { + return Err(error).context("cannot create the development state directory") + } + } + } + let info = + fs::symlink_metadata(path).context("cannot inspect the development state directory")?; + ensure!( + info.is_dir() + && info.uid() == rustix::process::getuid().as_raw() + && info.mode() & 0o077 == 0, + "the development state directory must be owner-only and must not be a symlink" + ); + Ok(()) +} + +fn capture_input(input: &RunInput) -> Result> { + let bytes = serde_json::to_vec(input)?; + ensure!( + bytes.len() as u64 <= MAXIMUM_CAPTURED_INPUT_BYTES, + "captured development input exceeds the internal size limit" + ); + Ok(bytes) +} + +fn capture_report(report: &Value) -> Result> { + let bytes = serde_json::to_vec(report)?; + ensure!( + bytes.len() as u64 <= MAXIMUM_RUN_REPORT_BYTES, + "development run report exceeds the internal size limit" + ); + Ok(bytes) +} + +/// Create `/logs` and the owner-only supervisor log inside it. +fn open_supervisor_log(root: &Path) -> Result { + match private_directory(&root.join("logs")) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error).context("cannot create the development log directory"), + } + OpenOptions::new() + .create(true) + .append(true) + .mode(0o600) + .open(root.join("logs").join("supervisor.log")) + .context("cannot open the supervisor log") +} + +fn failed_logs_dir(base: &Path, run_id: &str) -> PathBuf { + base.join("failed").join(run_id).join("logs") +} + +fn has_log_content(logs: &Path) -> Result { + let entries = match fs::read_dir(logs) { + Ok(entries) => entries, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error).context("cannot read the development log directory"), + }; + for entry in entries { + if entry?.metadata()?.len() > 0 { + return Ok(true); + } + } + Ok(false) +} + +/// Move a failed run's non-empty logs to `/failed//logs`. +fn keep_logs(root: &Path, run_id: &str) -> Result> { + let logs = root.join("logs"); + if !has_log_content(&logs)? { + return Ok(None); + } + let base = root.parent().context("run directory has no parent")?; + let target = failed_logs_dir(base, run_id); + let run = target.parent().expect("failed run directory"); + let failed = run.parent().expect("failed directory"); + ensure_private(failed, true)?; + match private_directory(run) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error).context("cannot create the failed run log directory"), + } + ensure_private(run, false)?; + fs::rename(&logs, &target).context("cannot keep the failed run logs")?; + Ok(Some(target)) +} + +fn remove_failed_run(root: &Path, run_id: &str) -> Result> { + let kept = keep_logs(root, run_id)?; + remove_run_dir(root)?; + Ok(kept) +} + +/// Where a run directory goes while it is deleted. The name is not a run ID, +/// so `status` and `stop` never read it as a run. +fn removing_dir(root: &Path) -> Result { + let name = root.file_name().context("run directory has no name")?; + Ok(root.with_file_name(format!(".{}.removing", name.to_string_lossy()))) +} + +/// Renames the run directory away before deleting it, so `status` sees either +/// the whole run or no run, never a run whose report is already deleted. +fn remove_run_dir(root: &Path) -> Result<()> { + let doomed = removing_dir(root)?; + fs::rename(root, &doomed).context("cannot remove run scratch files")?; + fs::remove_dir_all(&doomed) + .with_context(|| format!("cannot remove run scratch files at {}", doomed.display())) +} + +fn note_kept_logs(error: anyhow::Error, kept: Option) -> anyhow::Error { + match kept { + Some(path) => anyhow::anyhow!("{error:#}; logs kept at {}", path.display()), + None => error, + } +} + +fn read_json(path: &Path, limit: u64) -> Result { + Ok(serde_json::from_slice(&plan::read(path, limit)?)?) +} + +fn write_private(path: &Path, bytes: &[u8]) -> Result<()> { + let mut file = + tempfile::NamedTempFile::new_in(path.parent().context("file has no directory")?)?; + file.write_all(bytes)?; + file.persist(path).map_err(|error| error.error)?; + Ok(()) +} + +fn clock() -> Result { + Ok(SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const RUN_ID: &str = "0123456789abcdef0123456789abcdef"; + + fn fake_run(root: &Path, products: Value, postgis: bool) { + let input = json!({ + "run_id": RUN_ID, "project": "/project", "ttl_seconds": 60, + "plan": { + "issuer": "https://issuer.example.invalid", + "products": products, + "people": {"operator": { + "subject": "operator", "client": "operator-client", + "grants": {"breg": {"profile": "operator", "scopes": []}} + }}, + "journey": {"id": "smoke", "steps": [{ + "id": "read", "product": "breg", "as": "operator", + "method": "GET", "path": "/v1/records", + "expect": {"status": 200} + }]} + }, + "schema_people": [], + "breg_authority": { + "principal": "p", "purpose": false, + "requires_postgis": postgis, "named_clients": [] + }, + "casework_principals": {}, + "binaries": {} + }); + fs::write(root.join("input.json"), input.to_string()).unwrap(); + } + + #[test] + fn ttl_is_bounded_and_requires_a_unit() { + assert_eq!(parse_ttl("1s").unwrap(), 1); + assert_eq!(parse_ttl("30m").unwrap(), 1800); + assert_eq!(parse_ttl("24h").unwrap(), 86400); + for invalid in ["", "0s", "3600", "25h", "18446744073709551615h", "é"] { + assert!(parse_ttl(invalid).is_err(), "{invalid}"); + } + } + + #[test] + fn expiry_follows_the_wall_clock() { + assert!(!expired(99, 100)); + assert!(expired(100, 100)); + assert!(expired(101, 100)); + } + + #[test] + fn cleanup_backs_off_from_two_to_thirty_seconds_and_stops_at_the_cap() { + let delays: Vec<_> = cleanup_schedule(CLEANUP_CAP).map(|d| d.as_secs()).collect(); + assert_eq!(delays[..6], [2, 4, 8, 16, 30, 30]); + let total: u64 = delays.iter().sum(); + assert!(total >= CLEANUP_CAP.as_secs()); + assert!(total - delays.last().unwrap() < CLEANUP_CAP.as_secs()); + assert_eq!(cleanup_schedule(Duration::ZERO).count(), 0); + } + + #[test] + fn lost_supervisor_facts_name_processes_databases_and_roles() { + let scratch = tempfile::tempdir().unwrap(); + let root = scratch.path(); + fake_run( + root, + json!([ + {"product": "breg", "project": ".", "audience": "registry"}, + {"product": "casework", "project": ".", "audience": "casework"} + ]), + true, + ); + fs::write( + root.join("processes.json"), + json!([{"product": "breg", "pid": 4242, "pgid": 4242}]).to_string(), + ) + .unwrap(); + let facts = lost_supervisor_facts(root, RUN_ID); + assert_eq!(facts["kind"], "lost-supervisor"); + assert_eq!(facts["run_directory"], json!(root)); + assert_eq!(facts["processes"][0]["pgid"], 4242); + let labels: Vec<_> = facts["objects"] + .as_array() + .unwrap() + .iter() + .map(|object| object["label"].as_str().unwrap()) + .collect(); + assert_eq!(labels, ["breg_test", "breg", "casework"]); + let breg = &facts["objects"][1]; + assert_eq!(breg["database"], format!("rstack_{RUN_ID}_breg")); + // Migration, runtime and the spatial bbox role. + assert_eq!(breg["roles"].as_array().unwrap().len(), 3); + assert_eq!(facts["objects"][2]["roles"].as_array().unwrap().len(), 2); + let text = recovery_message(RUN_ID, &facts); + assert!( + text.contains("process \"breg\" pid 4242 pgid 4242"), + "{text}" + ); + assert!( + text.contains(&format!("breg database rstack_{RUN_ID}_breg")), + "{text}" + ); + } + + #[test] + fn lost_supervisor_facts_survive_missing_files() { + let scratch = tempfile::tempdir().unwrap(); + let facts = lost_supervisor_facts(scratch.path(), RUN_ID); + assert_eq!(facts["processes"], json!([])); + assert!(facts["unavailable"].is_string()); + } + + #[test] + fn a_dead_supervisor_is_detected_only_once_its_lock_file_exists() { + let scratch = tempfile::tempdir().unwrap(); + let root = scratch.path().canonicalize().unwrap(); + fs::set_permissions(&root, std::os::unix::fs::PermissionsExt::from_mode(0o700)).unwrap(); + assert!(!supervisor_gone(&root).unwrap()); + let lock = File::create(root.join("worker.lock")).unwrap(); + assert!(supervisor_gone(&root).unwrap()); + flock(&lock, FlockOperation::NonBlockingLockExclusive).unwrap(); + assert!(!supervisor_gone(&root).unwrap()); + } + + #[test] + fn pending_cleanup_reports_resources_and_processes() { + let report = pending_report(RUN_ID, vec!["db".into(), "role".into()], json!([])); + assert_eq!(report["databases_and_roles"], json!(["db", "role"])); + let status = pending_status(Path::new("/run"), RUN_ID, report); + assert_eq!(status["status"], "cleanup-pending"); + assert_eq!(status["recovery"]["databases_and_roles"][1], "role"); + let text = recovery_message(RUN_ID, &status["recovery"]); + assert!(text.contains("database or role role"), "{text}"); + } + + #[test] + fn a_run_directory_disappears_in_one_step_and_leaves_nothing() { + let scratch = tempfile::tempdir().unwrap(); + let base = scratch.path().canonicalize().unwrap(); + let root = base.join(RUN_ID); + private_directory(&root).unwrap(); + fs::write(root.join(REPORT), "{}").unwrap(); + private_directory(&root.join("logs")).unwrap(); + fs::write(root.join("logs").join("breg-check.log"), "boom").unwrap(); + let doomed = removing_dir(&root).unwrap(); + assert_eq!(doomed.parent(), Some(base.as_path())); + assert!(run_dir(&doomed.file_name().unwrap().to_string_lossy()).is_err()); + remove_run_dir(&root).unwrap(); + assert!(!root.exists()); + assert_eq!(fs::read_dir(&base).unwrap().count(), 0); + } + + #[test] + fn failed_run_logs_move_out_of_the_run_directory() { + let scratch = tempfile::tempdir().unwrap(); + let base = scratch.path().canonicalize().unwrap(); + let root = base.join(RUN_ID); + private_directory(&root).unwrap(); + open_supervisor_log(&root).unwrap(); + assert_eq!(keep_logs(&root, RUN_ID).unwrap(), None, "empty logs stay"); + fs::write(root.join("logs").join("breg-check.log"), "boom").unwrap(); + let kept = remove_failed_run(&root, RUN_ID).unwrap().unwrap(); + assert_eq!(kept, failed_logs_dir(&base, RUN_ID)); + assert!(!root.exists()); + assert_eq!( + fs::read_to_string(kept.join("breg-check.log")).unwrap(), + "boom" + ); + assert_eq!( + fs::metadata(base.join("failed")).unwrap().mode() & 0o777, + 0o700 + ); + assert_eq!( + fs::metadata(kept.join("supervisor.log")).unwrap().mode() & 0o777, + 0o600 + ); + // `failed` is not a run. + assert!(run_dir("failed").is_err()); + } + + #[test] + fn run_reports_are_bounded() { + assert!(capture_report(&json!({"a": 1})).is_ok()); + let big = json!({"a": "x".repeat(MAXIMUM_RUN_REPORT_BYTES as usize)}); + assert!(capture_report(&big).is_err()); + } + + #[test] + fn a_missing_binary_names_the_install_command() { + let error = binary(Product::Casework, "RSTACK_DEV_TEST_UNSET_BIN") + .unwrap_err() + .to_string(); + assert!(error.contains("rstack install casework"), "{error}"); + assert!(error.contains("RSTACK_DEV_TEST_UNSET_BIN"), "{error}"); + } +} diff --git a/crates/registry-bregctl/src/project_dev/plan.rs b/crates/registry-bregctl/src/project_dev/plan.rs new file mode 100644 index 0000000000..44154e410e --- /dev/null +++ b/crates/registry-bregctl/src/project_dev/plan.rs @@ -0,0 +1,1486 @@ +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::{Context, Result}; +use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; +use registry_breg::literal_text::{LiteralText, WRITE_THE_VALUE}; +use registry_platform_yaml::{ + escape_pointer_segment, ApiVersion, BoundedU32, Decoded, EnvelopeRule, Expect, FormatSpec, + Identified, LocalId, Reader, Report, Severity, UniqueIdList, UniqueList, +}; +#[cfg(feature = "schema")] +use registry_platform_yaml::{ExternalId, Url as PlatformUrl}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + io::Read, + path::{Component, Path, PathBuf}, +}; +use url::Url as ParsedUrl; + +pub(crate) const API_VERSION: &str = "id.registrystack.org/formats/breg/project-dev/v1alpha1"; +pub(crate) const KIND: &str = "BRegProjectDev"; + +pub(crate) const FORMAT: FormatSpec<'static> = FormatSpec { + kind: KIND, + envelope: EnvelopeRule::ApiVersionKind { + api_versions: &[ApiVersion::current(API_VERSION)], + retired_api_versions: &[], + }, + removed_keys: &[], +}; + +#[cfg(feature = "schema")] +pub(crate) fn schema() -> schemars::Schema { + schemars::schema_for!(Plan) +} + +#[cfg(feature = "schema")] +fn keyed_schema( + generator: &mut schemars::SchemaGenerator, +) -> schemars::Schema { + schemars::json_schema!({ + "type": "object", + "propertyNames": generator.subschema_for::(), + "additionalProperties": generator.subschema_for::(), + }) +} + +#[cfg(feature = "schema")] +struct DerivedId; + +#[cfg(feature = "schema")] +impl schemars::JsonSchema for DerivedId { + fn schema_name() -> std::borrow::Cow<'static, str> { + "DerivedId".into() + } + + fn json_schema(_generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "string", + "maxLength": 64, + "pattern": "^[a-z][a-z0-9_-]{0,63}(\\.[a-z][a-z0-9_-]{0,63})+$", + "description": "A dotted development journey step identifier whose segments are local identifiers.", + }) + } +} + +#[cfg(feature = "schema")] +fn step_id_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "anyOf": [ + generator.subschema_for::(), + generator.subschema_for::(), + ], + }) +} + +#[cfg(feature = "schema")] +fn capture_map_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + let names = schemars::json_schema!({ + "allOf": [ + generator.subschema_for::(), + { + "type": "string", + "maxLength": 64, + "pattern": "^[A-Za-z][A-Za-z0-9-]{0,63}$", + }, + ], + }); + schemars::json_schema!({ + "type": "object", + "propertyNames": names, + "additionalProperties": generator.subschema_for::(), + }) +} + +#[cfg(feature = "schema")] +fn schema_test_people_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + let steps = schemars::json_schema!({ + "type": "object", + "propertyNames": step_id_schema(generator), + "additionalProperties": generator.subschema_for::(), + }); + schemars::json_schema!({ + "type": "object", + "propertyNames": generator.subschema_for::(), + "additionalProperties": steps, + }) +} + +#[cfg(feature = "schema")] +fn claims_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "object", + "propertyNames": generator.subschema_for::(), + "additionalProperties": true, + "x-registry-passthrough": "Claim values are written into the development token as given.", + }) +} + +#[cfg(feature = "schema")] +fn scopes_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "array", + "items": generator.subschema_for::(), + "maxItems": 32, + "uniqueItems": true, + }) +} + +#[cfg(feature = "schema")] +fn expected_json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "object", + "propertyNames": generator.subschema_for::(), + "additionalProperties": true, + "x-registry-passthrough": "Expected values are compared with the product response at the named JSON pointers.", + }) +} + +/// A product the development run can start. +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "lowercase")] +pub(super) enum Product { + Breg, + Casework, +} + +impl Product { + pub(super) fn name(self) -> &'static str { + match self { + Self::Breg => "breg", + Self::Casework => "casework", + } + } +} + +/// The development plan: the products to start, the test people who call them, and the journey to run. +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct Plan { + /// HTTPS URL, at most 512 characters and without credentials, query or fragment, that the run's tokens name as their issuer. + #[cfg_attr(feature = "schema", schemars(with = "PlatformUrl"))] + pub issuer: String, + /// The products to start, each with its project directory and token audience. List each product at most once. + pub products: Vec, + /// The test people, keyed by a lowercase local identifier. Define 1 to 32, and give at least one grant to every started product. + #[cfg_attr( + feature = "schema", + schemars(schema_with = "keyed_schema::") + )] + pub people: BTreeMap, + /// The ordered sequence of HTTP requests the run sends to the started products. + pub journey: Journey, + /// Run-local connections between BReg and Casework. Leave empty when the products do not call each other. + #[serde(default)] + pub bindings: super::bindings::Bindings, + /// Chooses the test person for a BReg fixture step in tests/journeys.yaml, keyed by journey identifier and then step identifier. Needed only when several people could run the step; every named person needs a BReg grant. + #[serde(default)] + #[cfg_attr( + feature = "schema", + schemars(schema_with = "schema_test_people_schema") + )] + pub schema_test_people: BTreeMap>, +} + +/// One product to start and where its project lives. +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct ProductPlan { + /// Which product to start. + pub product: Product, + /// Relative path from the plan's directory to the product project, or `.` for the plan's own directory. It must contain registry.yaml for BReg or casework.yaml for Casework. + pub project: PathBuf, + /// Token audience this product accepts: 1 to 256 characters without whitespace or control characters, distinct from the other products' audiences. + #[cfg_attr(feature = "schema", schemars(with = "ExternalId"))] + pub audience: String, +} + +/// A named test identity that signs in to the started products. +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct TestPerson { + /// Subject claim of this person's tokens: 1 to 256 characters without whitespace or control characters. + #[cfg_attr(feature = "schema", schemars(with = "ExternalId"))] + pub subject: String, + /// Client identifier of this person's tokens: 1 to 256 characters without whitespace or control characters. + #[cfg_attr(feature = "schema", schemars(with = "ExternalId"))] + pub client: String, + /// The authority this person holds in each product, keyed by `breg` or `casework`. Give at least one grant, only for products listed in `products`. + #[serde(deserialize_with = "deserialize_grants")] + pub grants: BTreeMap, + /// Forward the same authority to both products. Both grants must declare the same scopes and additional claims. + #[serde(default, rename = "caseworkContext")] + pub casework_context: bool, +} + +/// The authority a test person holds in one product. +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct Grant { + /// Local identifier of the product profile this grant uses. + #[cfg_attr(feature = "schema", schemars(with = "LocalId"))] + pub profile: String, + /// Distinct scopes written into the token, at most 32, each 1 to 256 characters without whitespace or control characters. + #[serde(deserialize_with = "deserialize_scopes")] + #[cfg_attr(feature = "schema", schemars(schema_with = "scopes_schema"))] + pub scopes: Vec, + /// Additional token claims, keyed by claim name. Names must not be reserved token claims such as `iss`, `aud`, `sub`, `scope` or any `registry_grant_` claim. + #[serde(default)] + #[cfg_attr(feature = "schema", schemars(schema_with = "claims_schema"))] + pub claims: Map, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct GrantsDocument { + breg: Option, + casework: Option, +} + +fn deserialize_grants<'de, D>( + deserializer: D, +) -> std::result::Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let grants = GrantsDocument::deserialize(deserializer)?; + Ok([ + grants.breg.map(|grant| (Product::Breg, grant)), + grants.casework.map(|grant| (Product::Casework, grant)), + ] + .into_iter() + .flatten() + .collect()) +} + +fn deserialize_scopes<'de, D>(deserializer: D) -> std::result::Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + UniqueList::::deserialize(deserializer).map(UniqueList::into_vec) +} + +fn deserialize_steps<'de, D>(deserializer: D) -> std::result::Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + UniqueIdList::::deserialize(deserializer).map(UniqueIdList::into_vec) +} + +/// An ordered list of requests the run sends and checks. +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct Journey { + /// Local identifier naming this journey. + #[cfg_attr(feature = "schema", schemars(with = "LocalId"))] + pub id: String, + /// The steps, run in order. Include 1 to 32 with distinct identifiers. A step may refer to captures from earlier steps only. + #[serde(deserialize_with = "deserialize_steps")] + pub steps: Vec, +} + +/// One HTTP request, what to expect back, and what to remember from the response. +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct Step { + /// Unique step identifier: a local identifier, or a dotted sequence of local identifiers, at most 64 characters. + #[cfg_attr(feature = "schema", schemars(schema_with = "step_id_schema"))] + pub id: String, + /// The product that receives the request. + pub product: Product, + /// The test person who sends the request. The person needs a grant for this step's product. + #[serde(rename = "as")] + #[cfg_attr(feature = "schema", schemars(with = "LocalId"))] + pub person: String, + /// The HTTP method. + pub method: Method, + /// Request path, starting with `/v1/`, at most 2048 characters, without whitespace, control characters, backslashes or fragments. It may contain `{{name}}` references to earlier captures. + pub path: String, + /// JSON request body, at most 1 MiB once encoded. String values may contain `{{name}}` references to earlier captures. Do not combine with `raw`. + pub body: Option, + /// Non-JSON request body. Allowed only for methods other than GET. Do not combine with `body`. + pub raw: Option, + /// Extra request headers, keyed by header name. Only `If-Match`, `Idempotency-Key` and `Registry-Source-Profile` are supported. Values may contain `{{name}}` references to earlier captures. + #[serde(default)] + #[cfg_attr( + feature = "schema", + schemars(schema_with = "keyed_schema::") + )] + pub headers: BTreeMap, + /// What the response must look like for the step to pass. + pub expect: Expected, + /// Values to remember from the response, keyed by a capture name. Later steps refer to a capture as `{{name}}` in paths, headers, bodies and expectations. Names are unique across the journey. + #[serde(default)] + #[cfg_attr(feature = "schema", schemars(schema_with = "capture_map_schema"))] + pub capture: BTreeMap, + /// Repeat a GET request until it passes or the wait runs out. Allowed only for GET. + pub poll: Option, +} + +impl Identified for Step { + fn id(&self) -> &str { + &self.id + } +} + +/// An HTTP request method. +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "UPPERCASE")] +pub(super) enum Method { + Get, + Post, + Put, + Patch, + Delete, +} + +impl From<&Method> for reqwest::Method { + fn from(method: &Method) -> Self { + match method { + Method::Get => Self::GET, + Method::Post => Self::POST, + Method::Put => Self::PUT, + Method::Patch => Self::PATCH, + Method::Delete => Self::DELETE, + } + } +} + +/// The conditions a response must meet. +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct Expected { + /// Accepted HTTP status: one status, or a list of 1 to 16 distinct statuses, each from 200 to 599. + pub status: ExpectedStatus, + /// Exact values the response body must hold, keyed by JSON pointer (empty for the whole body, otherwise starting with `/`). String values may contain `{{name}}` references to earlier captures. + #[serde(default)] + #[cfg_attr(feature = "schema", schemars(schema_with = "expected_json_schema"))] + pub json: BTreeMap, + /// Further assertions about the response body, at most 32. Each one tests a single JSON pointer. + #[serde(default)] + pub checks: Vec, + /// Problem code the error response must carry, 1 to 256 characters without whitespace or control characters. It may contain `{{name}}` references to earlier captures. + pub code: Option, +} + +/// One accepted HTTP status or a list of them. +#[derive(Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(untagged)] +#[cfg_attr(feature = "schema", schemars(untagged))] +pub(super) enum ExpectedStatus { + /// A single accepted status from 200 to 599. + One(BoundedU32<200, 599>), + /// Several accepted statuses, each from 200 to 599; the response may carry any one of them. + Many(Vec>), +} + +registry_platform_yaml::shape_union!(ExpectedStatus { + scalar => One, + list => Many, +}); + +impl ExpectedStatus { + pub(super) fn first(&self) -> u16 { + match self { + Self::One(status) => status.get() as u16, + Self::Many(statuses) => statuses[0].get() as u16, + } + } + + pub(super) fn contains(&self, status: u16) -> bool { + match self { + Self::One(expected) => expected.get() == u32::from(status), + Self::Many(expected) => expected + .iter() + .any(|expected| expected.get() == u32::from(status)), + } + } + + fn values(&self) -> Vec { + match self { + Self::One(status) => vec![status.get() as u16], + Self::Many(statuses) => statuses.iter().map(|status| status.get() as u16).collect(), + } + } +} + +/// An assertion about one value in the response body. Set exactly one of `equals`, `length` or `contains`. +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct Check { + /// JSON pointer to the value to test: empty for the whole body, otherwise starting with `/`. + pub pointer: String, + /// The value at the pointer must equal this. String values may contain `{{name}}` references to earlier captures. + pub equals: Option, + /// The array, object or string at the pointer must have this many items, entries or characters, at most 1048576. + #[cfg_attr(feature = "schema", schemars(range(min = 0, max = 1_048_576)))] + pub length: Option, + /// The array at the pointer must hold this item, the string must contain this text, or the object must have this key. String values may contain `{{name}}` references to earlier captures. + pub contains: Option, +} + +/// A value remembered from a response, selected by `type`. +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(tag = "type", rename_all = "kebab-case", deny_unknown_fields)] +pub(super) enum Capture { + /// Remember the value at a JSON pointer in the response body. + Pointer { + /// JSON pointer into the response body: empty for the whole body, otherwise starting with `/`. + pointer: String, + }, + /// Remember a response header. + Header { + /// The response header to remember. + header: CapturedHeader, + }, + /// Remember a value from the one array item that matches a condition. The step fails if no item or more than one item matches. + Find { + /// JSON pointer to the array in the response body to search. + find: String, + /// The condition an item must meet. + r#where: FindWhere, + /// JSON pointer, relative to the matching item, of the value to remember. + pointer: String, + }, + /// Remember the value at a JSON pointer, optionally reading one query parameter from a URL found there. + Value { + /// JSON pointer into the response body: empty for the whole body, otherwise starting with `/`. + pointer: String, + /// Name of a query parameter. When set, the value at the pointer must be a URL or relative reference, and the parameter's value is remembered instead. At most 256 characters without whitespace or control characters. + query: Option, + }, +} + +/// A response header a capture can remember. +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "lowercase")] +pub(super) enum CapturedHeader { + /// The entity tag, usable as an `If-Match` value in a later step. + Etag, +} + +/// The condition a `find` capture uses to pick an array item. +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub(super) struct FindWhere { + /// JSON pointer, relative to each array item, of the value to compare. + pub pointer: String, + /// The value an item must hold at the pointer to match. String values may contain `{{name}}` references to earlier captures. + pub equals: Value, +} + +/// How long and how often to repeat a GET request. The wait divided by the interval, rounded up, must be below 250 attempts. +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct Poll { + /// Longest total wait in milliseconds, from 1 to 60000. + #[serde(rename = "maximumWaitMilliseconds")] + #[cfg_attr(feature = "schema", schemars(range(min = 1, max = 60_000)))] + pub timeout_ms: u64, + /// Pause between attempts in milliseconds, from 50 to 5000 and not above the maximum wait. + #[serde(rename = "pollIntervalMilliseconds")] + #[cfg_attr(feature = "schema", schemars(range(min = 50, max = 5_000)))] + pub interval_ms: u64, +} + +/// A request body sent as given, without JSON encoding. +#[derive(Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct RawBody { + /// HTTP media type for the Content-Type header, containing a slash, 1 to 256 ASCII characters. + pub content_type: String, + /// The body as standard base64, at most 1 MiB once decoded. + pub base64: String, +} + +pub(super) fn identifier(value: &str) -> bool { + LocalId::new(value).is_ok() +} + +fn bounded_atom(value: &str) -> bool { + (1..=256).contains(&value.len()) && !value.chars().any(|c| c.is_control() || c.is_whitespace()) +} + +pub(super) fn reserved_claim(claim: &str) -> bool { + matches!( + claim, + "iss" + | "aud" + | "sub" + | "client_id" + | "azp" + | "exp" + | "iat" + | "nbf" + | "jti" + | "scope" + | "registry_principal" + | "registry_assertion_issuer" + ) || claim.starts_with("registry_grant_") +} + +#[derive(Debug, thiserror::Error)] +#[error("dev.yaml {field}: {reason} Next: {remedy}")] +pub(super) struct SemanticRefusal { + field: String, + reason: String, + remedy: String, +} + +impl SemanticRefusal { + pub(super) fn new( + field: impl Into, + reason: impl Into, + remedy: impl Into, + ) -> Self { + Self { + field: field.into(), + reason: reason.into(), + remedy: remedy.into(), + } + } +} + +pub(super) type SemanticResult = std::result::Result; + +pub(super) fn require( + condition: bool, + field: &str, + reason: impl Into, + remedy: impl Into, +) -> SemanticResult<()> { + if !condition { + return Err(SemanticRefusal::new(field, reason, remedy)); + } + Ok(()) +} + +pub(super) fn refuse(field: &str, reason: &str, remedy: &str) -> SemanticResult { + Err(SemanticRefusal::new(field, reason, remedy)) +} + +#[derive(Debug, thiserror::Error)] +#[error("capture reference must name an earlier capture")] +struct EarlierCapture; + +fn validate_expansion(result: Result, field: &str) -> SemanticResult { + match result { + Ok(value) => Ok(value), + Err(error) if error.downcast_ref::().is_some() => refuse( + field, + "The capture reference must name an earlier capture.", + "Move the referenced capture to an earlier step or correct the capture name.", + ), + Err(_) => refuse( + field, + "The field has an unclosed capture reference.", + "Close the capture reference with `}}`.", + ), + } +} + +pub(super) fn read(path: &Path, limit: u64) -> Result> { + let mut bytes = Vec::new(); + fs::File::open(path)? + .take(limit + 1) + .read_to_end(&mut bytes)?; + anyhow::ensure!(bytes.len() as u64 <= limit, "file exceeds the size limit"); + Ok(bytes) +} + +fn read_plan(root: &Path) -> Result> { + let file = root.join("dev.yaml"); + let bytes = read( + &file, + registry_platform_yaml::MAXIMUM_DOCUMENT_BYTES as u64 + 1, + ) + .context("cannot read dev.yaml")?; + Ok(Reader::new(file.display().to_string()) + .with_hook(&mut LiteralText { + remedy: WRITE_THE_VALUE, + }) + .decode::(&bytes, &Expect::one(&FORMAT))?) +} + +fn refusal_report(decoded: &Decoded, refusal: &SemanticRefusal) -> Report { + let diagnostic = decoded.document.diagnostic_at_value( + Severity::Error, + "breg.project-dev.invalid-plan", + &refusal.field, + &refusal.reason, + &refusal.remedy, + ); + decoded.document.report_with([diagnostic]) +} + +fn validate_plan(root: &Path, decoded: &Decoded) -> Result<()> { + decoded + .value + .validate(root) + .map_err(|refusal| refusal_report(decoded, &refusal).into()) +} + +pub(crate) fn check(root: &Path) -> Result { + let decoded = read_plan(root)?; + validate_plan(root, &decoded)?; + if let Err(refusal) = super::products::schema_people(root, &decoded.value, false) { + return Err(refusal_report(&decoded, &refusal).into()); + } + let terminate = super::signals()?; + if let Err(error) = super::products::inspect(root, &decoded.value, &terminate) { + return match error.downcast::() { + Ok(refusal) => Err(refusal_report(&decoded, &refusal).into()), + Err(error) => Err(error), + }; + } + Ok(decoded.document.report_with(Vec::new())) +} + +pub(super) fn load(root: &Path) -> Result { + let decoded = read_plan(root)?; + validate_plan(root, &decoded)?; + Ok(decoded.value) +} + +impl Plan { + pub(super) fn product(&self, product: Product) -> &ProductPlan { + self.products + .iter() + .find(|entry| entry.product == product) + .expect("validated product") + } + + fn validate(&self, root: &Path) -> SemanticResult<()> { + require( + ParsedUrl::parse(&self.issuer).is_ok_and(|url| { + url.scheme() == "https" + && url.host().is_some() + && url.username().is_empty() + && url.password().is_none() + && url.query().is_none() + && url.fragment().is_none() + && self.issuer.len() <= 512 + }), + "/issuer", + "The issuer must be the project's bounded HTTPS URL without credentials, query, or fragment.", + "Set issuer to an HTTPS URL with a host and no credentials, query, or fragment.", + )?; + require( + !self.products.is_empty(), + "/products", + "The plan must name at least one product.", + "Add a BReg or Casework product to products.", + )?; + let mut products = BTreeSet::new(); + let mut audiences = BTreeSet::new(); + for (index, entry) in self.products.iter().enumerate() { + let field = format!("/products/{index}"); + require( + products.insert(entry.product), + &format!("{field}/product"), + "The product is repeated.", + "Remove the duplicate product entry.", + )?; + require( + bounded_atom(&entry.audience) && audiences.insert(&entry.audience), + &format!("{field}/audience"), + "The audience must be distinct, bounded text without whitespace or control characters.", + "Set audience to a distinct identifier of at most 256 characters without whitespace or control characters.", + )?; + require( + entry.project == Path::new(".") + || (!entry.project.as_os_str().is_empty() + && entry + .project + .components() + .all(|part| matches!(part, Component::Normal(_)))), + &format!("{field}/project"), + "The product project must be a relative path inside the development project directory.", + "Set project to `.` or a relative path made only of normal path components.", + )?; + let manifest = match entry.product { + Product::Breg => "registry.yaml", + Product::Casework => "casework.yaml", + }; + require( + root.join(&entry.project) + .join(manifest) + .canonicalize() + .is_ok_and(|path| path.starts_with(root) && path.is_file()), + &format!("{field}/project"), + format!("The product project must contain {manifest} inside the development project directory."), + format!("Create {manifest} at this project path inside the development project directory."), + )?; + } + require( + (1..=32).contains(&self.people.len()), + "/people", + "The plan must define 1 to 32 test people.", + "Add or remove test people until people contains between 1 and 32 entries.", + )?; + for (name, person) in &self.people { + require( + identifier(name), + "/people", + "A test person name is not a local identifier.", + "Rename the test person with a lowercase local identifier of at most 64 characters.", + )?; + let field = format!("/people/{name}"); + require( + bounded_atom(&person.subject), + &format!("{field}/subject"), + "The test person's subject must be bounded text without whitespace or control characters.", + "Set subject to at most 256 characters without whitespace or control characters.", + )?; + require( + bounded_atom(&person.client), + &format!("{field}/client"), + "The test person's client must be a bounded client identifier.", + "Set client to at most 256 characters without whitespace or control characters.", + )?; + require( + !person.grants.is_empty(), + &format!("{field}/grants"), + "The test person must have at least one product grant.", + "Add a BReg or Casework grant for this test person.", + )?; + for (product, grant) in &person.grants { + let field = format!("{field}/grants/{}", product.name()); + require( + products.contains(product), + &field, + "The grant names a product that is not started.", + "Add the grant's product to products, or remove this grant.", + )?; + require( + identifier(&grant.profile), + &format!("{field}/profile"), + "The grant profile must be a local identifier.", + "Set profile to a lowercase local identifier of at most 64 characters.", + )?; + let scopes = grant.scopes.iter().collect::>(); + require( + scopes.len() == grant.scopes.len() + && scopes.len() <= 32 + && scopes.iter().all(|scope| bounded_atom(scope)), + &format!("{field}/scopes"), + "The grant must contain at most 32 distinct bounded scopes.", + "Remove duplicate or excess scopes, and keep each scope within 256 characters without whitespace or control characters.", + )?; + for claim in grant.claims.keys() { + require( + bounded_atom(claim) && !reserved_claim(claim), + &format!("{field}/claims"), + "The grant contains a reserved or invalid additional claim name.", + "Remove the claim or rename it to bounded text that is not reserved by the token contract.", + )?; + } + } + if person.casework_context { + require( + self.bindings.casework_context_provider.is_some(), + &format!("{field}/caseworkContext"), + "Casework context forwarding requires /bindings/caseworkContextProvider.", + "Configure bindings.caseworkContextProvider or remove caseworkContext from this person.", + )?; + let breg = person.grants.get(&Product::Breg); + let casework = person.grants.get(&Product::Casework); + require( + breg.is_some() && casework.is_some(), + &format!("{field}/caseworkContext"), + "Casework context forwarding requires both BReg and Casework grants.", + "Add both product grants to this person or remove caseworkContext.", + )?; + let (breg, casework) = ( + breg.expect("checked grant"), + casework.expect("checked grant"), + ); + require( + breg.claims == casework.claims + && breg.scopes.iter().collect::>() + == casework.scopes.iter().collect(), + &format!("{field}/caseworkContext"), + "Casework context forwarding requires the same scopes and additional claims in both grants.", + "Make the BReg and Casework grant scopes and additional claims identical.", + )?; + } + } + for (journey, steps) in &self.schema_test_people { + for (step, person) in steps { + let field = format!( + "/schemaTestPeople/{}/{}", + escape_pointer_segment(journey), + escape_pointer_segment(step) + ); + require( + self.people + .get(person) + .is_some_and(|person| person.grants.contains_key(&Product::Breg)), + &field, + "The schema test assignment must name a person with a BReg grant.", + "Set this assignment to a test person that has a BReg grant.", + )?; + } + } + for product in products { + require( + self.people + .values() + .any(|person| person.grants.contains_key(&product)), + "/people", + format!( + "The plan must define at least one grant for {}.", + product.name() + ), + format!( + "Add a {} grant to at least one test person.", + product.name() + ), + )?; + } + self.bindings.validate(self)?; + self.journey.validate(self) + } +} + +impl Journey { + fn validate(&self, plan: &Plan) -> SemanticResult<()> { + require( + identifier(&self.id), + "/journey/id", + "The journey ID must be a local identifier.", + "Set id to a lowercase local identifier of at most 64 characters.", + )?; + require( + (1..=32).contains(&self.steps.len()), + "/journey/steps", + "The journey must contain 1 to 32 steps.", + "Add or remove steps until the journey contains between 1 and 32 steps.", + )?; + let mut ids = BTreeSet::new(); + let mut captures = BTreeSet::new(); + for (index, step) in self.steps.iter().enumerate() { + let field = format!("/journey/steps/{index}"); + require( + step_identifier(&step.id) && ids.insert(&step.id), + &format!("{field}/id"), + "The step ID must be a unique local or dotted derived identifier.", + "Set id to a unique lowercase local identifier or dotted sequence of local identifiers.", + )?; + require( + plan.people + .get(&step.person) + .is_some_and(|person| person.grants.contains_key(&step.product)), + &format!("{field}/as"), + "The step must name a test person with a grant for its product.", + "Set as to a test person that has a grant for this step's product.", + )?; + require( + valid_path(&step.path), + &format!("{field}/path"), + "The request path must be a relative product API path starting with /v1/.", + "Set path to a bounded /v1/ path without whitespace, control characters, backslashes, or fragments.", + )?; + let statuses = step.expect.status.values(); + require( + (1..=16).contains(&statuses.len()) + && statuses.iter().all(|status| (200..=599).contains(status)) + && statuses.iter().collect::>().len() == statuses.len(), + &format!("{field}/expect/status"), + "Expected status must contain 1 to 16 distinct HTTP statuses from 200 to 599.", + "List between 1 and 16 distinct HTTP status numbers from 200 to 599.", + )?; + require( + step.body.is_none() || step.raw.is_none(), + &field, + "A journey step must not contain both body and raw.", + "Remove body or raw so the step declares only one request body form.", + )?; + require( + step.raw.is_none() || !matches!(step.method, Method::Get), + &format!("{field}/raw"), + "A raw request body is supported only for non-GET requests.", + "Remove raw or change method to a non-GET request method.", + )?; + if let Some(body) = &step.body { + require( + serde_json::to_vec(body).is_ok_and(|bytes| bytes.len() <= 1024 * 1024), + &format!("{field}/body"), + "The JSON request body must encode to at most 1 MiB.", + "Reduce body until its encoded JSON is at most 1 MiB.", + )?; + } + if let Some(raw) = &step.raw { + require( + (1..=256).contains(&raw.content_type.len()) + && raw.content_type.contains('/') + && raw + .content_type + .bytes() + .all(|byte| byte.is_ascii() && byte != b'\r' && byte != b'\n'), + &format!("{field}/raw/contentType"), + "The raw content type must be a bounded ASCII HTTP media type.", + "Set contentType to an ASCII media type with a slash and at most 256 characters.", + )?; + require( + raw.base64.len() <= 1_398_104 + && BASE64 + .decode(&raw.base64) + .is_ok_and(|bytes| bytes.len() <= 1024 * 1024), + &format!("{field}/raw/base64"), + "The raw body must be valid base64 encoding of at most 1 MiB.", + "Encode at most 1 MiB of raw request data as base64.", + )?; + } + if let Some(poll) = &step.poll { + require( + matches!(step.method, Method::Get), + &format!("{field}/poll"), + "Polling is supported only for GET requests.", + "Remove poll or change method to GET.", + )?; + require( + (1..=60_000).contains(&poll.timeout_ms) + && (50..=5_000).contains(&poll.interval_ms) + && poll.timeout_ms.div_ceil(poll.interval_ms) < 250, + &format!("{field}/poll"), + "Polling must use a 1 to 60000 ms maximum wait, a 50 to 5000 ms interval, and at most 250 attempts.", + "Set maximumWaitMilliseconds and pollIntervalMilliseconds within those bounds and limit the resulting attempts to at most 250.", + )?; + require( + poll.interval_ms <= poll.timeout_ms, + &format!("{field}/poll/pollIntervalMilliseconds"), + "The poll interval must not exceed the maximum wait.", + "Set pollIntervalMilliseconds to no more than maximumWaitMilliseconds.", + )?; + } + for header in step.headers.keys() { + require( + matches!( + header.as_str(), + "If-Match" | "Idempotency-Key" | "Registry-Source-Profile" + ), + &format!("{field}/headers"), + "The request contains a header outside the supported list.", + "Remove the header or use only If-Match, Idempotency-Key, or Registry-Source-Profile.", + )?; + } + let verify_reference = |value: &str| { + captures + .contains(value) + .then(|| value.to_owned()) + .ok_or_else(|| EarlierCapture.into()) + }; + validate_expansion( + expand(&step.path, verify_reference), + &format!("{field}/path"), + )?; + for value in step.headers.values() { + validate_expansion(expand(value, verify_reference), &format!("{field}/headers"))?; + } + if let Some(body) = &step.body { + validate_expansion( + expand_json(body, verify_reference), + &format!("{field}/body"), + )?; + } + require( + step.expect.json.keys().all(|value| pointer(value)), + &format!("{field}/expect/json"), + "Expected JSON keys must be JSON pointers.", + "Replace each expected JSON key with an empty pointer or one starting with `/`.", + )?; + for value in step.expect.json.values() { + validate_expansion( + expand_json(value, verify_reference), + &format!("{field}/expect/json"), + )?; + } + require( + step.expect.checks.len() <= 32, + &format!("{field}/expect/checks"), + "Expected checks must contain at most 32 entries.", + "Remove checks until at most 32 remain.", + )?; + for check in &step.expect.checks { + require( + pointer(&check.pointer), + &format!("{field}/expect/checks"), + "Each expected check must use a JSON pointer.", + "Set each check pointer to an empty pointer or one starting with `/`.", + )?; + require( + usize::from(check.equals.is_some()) + + usize::from(check.length.is_some()) + + usize::from(check.contains.is_some()) + == 1, + &format!("{field}/expect/checks"), + "Each expected check must contain exactly one of equals, length, or contains.", + "Keep exactly one assertion member in each expected check.", + )?; + require( + check.length.is_none_or(|length| length <= 1024 * 1024), + &format!("{field}/expect/checks"), + "An expected length exceeds 1 MiB.", + "Set length to at most 1048576.", + )?; + for value in check.equals.iter().chain(&check.contains) { + validate_expansion( + expand_json(value, verify_reference), + &format!("{field}/expect/checks"), + )?; + } + } + if let Some(code) = &step.expect.code { + require( + bounded_atom(code), + &format!("{field}/expect/code"), + "The expected problem code must be bounded text without whitespace or control characters.", + "Set code to at most 256 characters without whitespace or control characters.", + )?; + validate_expansion( + expand(code, verify_reference), + &format!("{field}/expect/code"), + )?; + } + for (name, capture) in &step.capture { + require( + capture_identifier(name) && !captures.contains(name.as_str()), + &format!("{field}/capture"), + "Capture names must be unique bounded ASCII identifiers.", + "Rename the capture with a unique identifier of at most 64 ASCII letters, digits, or hyphens, starting with a letter.", + )?; + match capture { + Capture::Pointer { + pointer: pointer_value, + } => require( + pointer(pointer_value), + &format!("{field}/capture/{name}"), + "The capture must use a JSON pointer.", + "Set pointer to an empty pointer or one starting with `/`.", + )?, + Capture::Header { .. } => {} + Capture::Value { pointer, query } => { + require( + self::pointer(pointer), + &format!("{field}/capture/{name}/pointer"), + "The value capture pointer must be a JSON pointer.", + "Set pointer to an empty pointer or one starting with `/`.", + )?; + if let Some(query) = query { + require( + bounded_atom(query), + &format!("{field}/capture/{name}/query"), + "The value capture query must name a bounded query parameter.", + "Set query to at most 256 characters without whitespace or control characters.", + )?; + } + } + Capture::Find { + find, + r#where, + pointer: selected, + } => { + require( + pointer(find) && pointer(&r#where.pointer) && pointer(selected), + &format!("{field}/capture/{name}"), + "The find capture must use JSON pointers.", + "Set find, where.pointer, and pointer to empty pointers or pointers starting with `/`.", + )?; + validate_expansion( + expand_json(&r#where.equals, |value| { + captures + .contains(value) + .then(|| value.to_owned()) + .ok_or_else(|| EarlierCapture.into()) + }), + &format!("{field}/capture/{name}/find"), + )?; + } + } + } + // A step's captures are all read from its own response, so they + // become available only to later steps. + captures.extend(step.capture.keys().cloned()); + } + Ok(()) + } +} + +fn step_identifier(value: &str) -> bool { + (1..=64).contains(&value.len()) + && (identifier(value) || (value.contains('.') && value.split('.').all(identifier))) +} + +fn capture_identifier(value: &str) -> bool { + (1..=64).contains(&value.len()) + && value.as_bytes()[0].is_ascii_alphabetic() + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') +} + +fn pointer(value: &str) -> bool { + value.is_empty() || value.starts_with('/') +} + +pub(super) fn valid_path(value: &str) -> bool { + value.len() <= 2048 + && value.starts_with("/v1/") + && !value.contains(['\\', '#']) + && !value.chars().any(|c| c.is_control() || c.is_whitespace()) +} + +pub(super) fn expand(value: &str, lookup: impl Fn(&str) -> Result) -> Result { + let mut result = String::new(); + let mut remainder = value; + while let Some((prefix, after)) = remainder.split_once("{{") { + result.push_str(prefix); + let (name, tail) = after + .split_once("}}") + .context("has an unclosed capture reference")?; + result.push_str(&lookup(name)?); + remainder = tail; + } + result.push_str(remainder); + Ok(result) +} + +pub(super) fn expand_json( + value: &Value, + lookup: impl Fn(&str) -> Result + Copy, +) -> Result { + match value { + Value::String(value) => Ok(Value::String(expand(value, lookup)?)), + Value::Array(values) => values + .iter() + .map(|item| expand_json(item, lookup)) + .collect(), + Value::Object(values) => values + .iter() + .map(|(key, item)| Ok((key.clone(), expand_json(item, lookup)?))) + .collect(), + value => Ok(value.clone()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn plan(step: Value) -> (tempfile::TempDir, Plan) { + let root = tempfile::tempdir().unwrap(); + fs::write(root.path().join("registry.yaml"), "fixture").unwrap(); + let plan = serde_json::from_value(json!({ + "issuer": "https://issuer.example.invalid", + "products": [{ + "product": "breg", "project": ".", "audience": "registry" + }], + "people": {"operator": { + "subject": "operator", "client": "operator-client", + "grants": {"breg": {"profile": "operator", "scopes": []}} + }}, + "journey": {"id": "smoke", "steps": [step]} + })) + .unwrap(); + (root, plan) + } + + fn canonical(root: &tempfile::TempDir) -> PathBuf { + root.path().canonicalize().unwrap() + } + + #[test] + fn studio_journey_wire_variants_deserialize() { + let (_, plan) = plan(json!({ + "id": "create.v1", "product": "breg", "as": "operator", + "method": "POST", "path": "/v1/records", + "raw": {"contentType": "application/octet-stream", "base64": "YQ=="}, + "expect": { + "status": [200, 201], "code": "request.accepted", + "checks": [ + {"pointer": "/items", "length": 1}, + {"pointer": "/state", "equals": "ready"}, + {"pointer": "/tags", "contains": "open"} + ] + }, + "capture": { + "requestId": {"type": "value", "pointer": "/requestId", "query": "id"}, + "etag": {"type": "header", "header": "etag"}, + "selected-item": { + "type": "find", + "find": "/items", "where": {"pointer": "/id", "equals": "one"}, + "pointer": "/value" + } + } + })); + assert_eq!(plan.journey.steps[0].expect.status.first(), 200); + assert!(plan.journey.steps[0].expect.status.contains(201)); + assert!(matches!( + plan.journey.steps[0].capture["etag"], + Capture::Header { .. } + )); + assert!(!plan.people["operator"].casework_context); + } + + #[test] + fn validation_accepts_dotted_steps_and_camel_case_captures() { + let (root, plan) = plan(json!({ + "id": "read.v1", "product": "breg", "as": "operator", + "method": "GET", "path": "/v1/records", + "expect": {"status": [200, 202]}, + "capture": {"draftEtag": {"type": "pointer", "pointer": "/recordIdentifier"}}, + "poll": {"maximumWaitMilliseconds": 500, "pollIntervalMilliseconds": 50} + })); + let mut value = serde_json::to_value(plan).unwrap(); + let person = value["people"] + .as_object_mut() + .unwrap() + .remove("operator") + .unwrap(); + value["people"]["date_of_birth"] = person; + value["people"]["date_of_birth"]["grants"]["breg"]["profile"] = json!("date_of_birth"); + value["journey"]["id"] = json!("review_flow"); + value["journey"]["steps"][0]["as"] = json!("date_of_birth"); + value["journey"]["steps"] + .as_array_mut() + .unwrap() + .push(json!({ + "id": "use.v1", "product": "breg", "as": "date_of_birth", + "method": "GET", "path": "/v1/records/{{draftEtag}}", + "expect": {"status": 200} + })); + value["schemaTestPeople"] = json!({ + "review_flow": {"read.v1": "date_of_birth"} + }); + let plan: Plan = serde_json::from_value(value).unwrap(); + + plan.validate(&canonical(&root)).unwrap(); + } + + #[test] + fn identifier_grammars_refuse_values_the_schema_refuses() { + for accepted in ["date-of-birth", "date_of_birth"] { + assert!(identifier(accepted), "local identifier: {accepted}"); + } + for refused in ["-person", "1person", "Person", "person.name"] { + assert!(!identifier(refused), "local identifier: {refused}"); + } + + for accepted in ["read", "read_v1", "read.v1", "read_record.v1_beta"] { + assert!(step_identifier(accepted), "step identifier: {accepted}"); + } + for refused in [ + "-read", "1read", ".read", "read.", "read..v1", "read.-v1", "Read.v1", + ] { + assert!(!step_identifier(refused), "step identifier: {refused}"); + } + + for accepted in ["draftEtag", "DraftEtag", "draft-etag"] { + assert!( + capture_identifier(accepted), + "capture identifier: {accepted}" + ); + } + for refused in ["1draft", "draft_etag", "draft.etag"] { + assert!( + !capture_identifier(refused), + "capture identifier: {refused}" + ); + } + } + + #[test] + fn validation_refuses_a_product_project_outside_the_development_project() { + let (root, mut plan) = plan(json!({ + "id": "read", "product": "breg", "as": "operator", + "method": "GET", "path": "/v1/records", "expect": {"status": 200} + })); + plan.products[0].project = PathBuf::from("../outside"); + + let error = plan.validate(&canonical(&root)).unwrap_err().to_string(); + + assert!(error.contains("/products/0/project"), "{error}"); + } + + #[test] + fn validation_refuses_a_product_project_link_outside_the_development_project() { + let (root, mut plan) = plan(json!({ + "id": "read", "product": "breg", "as": "operator", + "method": "GET", "path": "/v1/records", "expect": {"status": 200} + })); + let outside = tempfile::tempdir().unwrap(); + fs::write(outside.path().join("registry.yaml"), "fixture").unwrap(); + std::os::unix::fs::symlink(outside.path(), root.path().join("registry")).unwrap(); + plan.products[0].project = PathBuf::from("registry"); + + let error = plan.validate(&canonical(&root)).unwrap_err().to_string(); + + assert!(error.contains("/products/0/project"), "{error}"); + } + + #[test] + fn validation_refuses_polling_a_mutation() { + let (root, plan) = plan(json!({ + "id": "create", "product": "breg", "as": "operator", + "method": "POST", "path": "/v1/records", "body": {}, + "expect": {"status": 201}, + "poll": {"maximumWaitMilliseconds": 500, "pollIntervalMilliseconds": 50} + })); + let error = plan.validate(&canonical(&root)).unwrap_err().to_string(); + assert!(error.contains("supported only for GET"), "{error}"); + } + + #[test] + fn validation_refuses_forward_capture_references() { + let (root, plan) = plan(json!({ + "id": "read", "product": "breg", "as": "operator", + "method": "GET", "path": "/v1/records/{{later}}", + "expect": {"status": 200}, + "capture": {"later": {"type": "pointer", "pointer": "/recordIdentifier"}} + })); + let error = format!("{:#}", plan.validate(&canonical(&root)).unwrap_err()); + assert!(error.contains("earlier capture"), "{error}"); + } + + #[test] + fn validation_refuses_a_find_capture_that_refers_to_its_own_step() { + let (root, plan) = plan(json!({ + "id": "list", "product": "breg", "as": "operator", + "method": "GET", "path": "/v1/records", + "expect": {"status": 200}, + "capture": { + "first": {"type": "pointer", "pointer": "/items/0/recordIdentifier"}, + "second": { + "type": "find", "find": "/items", + "where": {"pointer": "/recordIdentifier", "equals": "{{first}}"}, + "pointer": "/status" + } + } + })); + let error = format!("{:#}", plan.validate(&canonical(&root)).unwrap_err()); + assert!(error.contains("earlier capture"), "{error}"); + assert!(error.contains("/capture/second/find"), "{error}"); + } + + #[test] + fn validation_refuses_unbounded_poll_attempts() { + let (root, plan) = plan(json!({ + "id": "read", "product": "breg", "as": "operator", + "method": "GET", "path": "/v1/records", + "expect": {"status": 200}, + "poll": {"maximumWaitMilliseconds": 30000, "pollIntervalMilliseconds": 50} + })); + let error = plan.validate(&canonical(&root)).unwrap_err().to_string(); + assert!(error.contains("at most 250 attempts"), "{error}"); + } + + #[test] + fn validation_accepts_the_studio_review_poll() { + let (root, plan) = plan(json!({ + "id": "read", "product": "breg", "as": "operator", + "method": "GET", "path": "/v1/records", "expect": {"status": 200}, + "poll": {"maximumWaitMilliseconds": 60000, "pollIntervalMilliseconds": 250} + })); + plan.validate(&canonical(&root)).unwrap(); + } + + #[test] + fn casework_context_requires_the_binding_and_both_grants() { + let (root, plan) = plan(json!({ + "id": "read", "product": "breg", "as": "operator", + "method": "GET", "path": "/v1/records", "expect": {"status": 200} + })); + let mut value = serde_json::to_value(plan).unwrap(); + value["people"]["operator"]["caseworkContext"] = json!(true); + let plan: Plan = serde_json::from_value(value).unwrap(); + let error = plan.validate(&canonical(&root)).unwrap_err().to_string(); + assert!( + error.contains("requires /bindings/caseworkContextProvider"), + "{error}" + ); + } + + #[test] + fn casework_context_requires_both_grants_to_declare_all_forwarded_authority() { + let root = tempfile::tempdir().unwrap(); + fs::write(root.path().join("registry.yaml"), "fixture").unwrap(); + fs::write(root.path().join("casework.yaml"), "fixture").unwrap(); + let value = |casework_jurisdiction: &str, casework_scopes: &[&str]| { + json!({ + "issuer": "https://issuer.example.invalid", + "products": [ + {"product": "breg", "project": ".", "audience": "urn:example:registry"}, + {"product": "casework", "project": ".", "audience": "urn:example:review"} + ], + "people": { + "context-reader": { + "subject": "context-reader", "client": "context-client", + "grants": {"breg": { + "profile": "context-reader", "scopes": ["registry:read"] + }} + }, + "reviewer": { + "subject": "reviewer", "client": "reviewer-client", + "caseworkContext": true, + "grants": { + "breg": { + "profile": "reviewer", "scopes": ["registry:read", "casework:review"], + "claims": {"jurisdiction": "north"} + }, + "casework": { + "profile": "reviewer", "scopes": casework_scopes, + "claims": {"jurisdiction": casework_jurisdiction} + } + } + } + }, + "bindings": {"caseworkContextProvider": { + "id": "registry", "identity": "context-reader", "action": "review-record" + }}, + "journey": {"id": "review", "steps": [{ + "id": "read", "product": "breg", "as": "reviewer", + "method": "GET", "path": "/v1/records", "expect": {"status": 200} + }]} + }) + }; + let shared_scopes = ["casework:review", "registry:read"]; + let valid: Plan = serde_json::from_value(value("north", &shared_scopes)).unwrap(); + valid.validate(&canonical(&root)).unwrap(); + + let mut one_sided_claim = value("north", &shared_scopes); + one_sided_claim["people"]["reviewer"]["grants"]["breg"]["claims"]["extra"] = json!(true); + for invalid in [ + value("south", &shared_scopes), + value("north", &["casework:review"]), + one_sided_claim, + ] { + let invalid: Plan = serde_json::from_value(invalid).unwrap(); + let error = invalid.validate(&canonical(&root)).unwrap_err().to_string(); + assert!( + error.contains("same scopes and additional claims"), + "{error}" + ); + } + } +} diff --git a/crates/registry-bregctl/src/project_dev/products.rs b/crates/registry-bregctl/src/project_dev/products.rs new file mode 100644 index 0000000000..79ce78d7c1 --- /dev/null +++ b/crates/registry-bregctl/src/project_dev/products.rs @@ -0,0 +1,1983 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Activate ordinary packages and serve them on the run's private bindings. + +use anyhow::{bail, ensure, Context, Result}; +use registry_breg::fixtures::journey_step_profiles; +use rustix::process::{kill_process_group, waitid, Pid, Signal, WaitId, WaitIdOptions}; +use serde::Serialize; +use serde_json::{json, Value}; +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + io::{Read, Seek, SeekFrom}, + net::{Ipv4Addr, TcpListener}, + os::unix::{fs::OpenOptionsExt, process::CommandExt}, + path::{Path, PathBuf}, + process::{Child, Command, ExitStatus, Stdio}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + thread, + time::{Duration, Instant}, +}; + +use super::{ + bindings::{Bindings, RuntimeBindings}, + clock, + database::{Database, DatabaseUrls}, + issuer::Issuer, + plan::{self, Plan, Product}, + private_directory, stopping, write_private, BregAuthority, RunInput, POLL, +}; + +pub(super) struct ProductRunner<'a> { + input: Arc, + root: &'a Path, + terminate: &'a AtomicBool, + deadline: Instant, + runtime: tokio::runtime::Runtime, + http: reqwest::Client, + children: Vec<(Product, Child)>, + bindings: Option, +} + +impl<'a> ProductRunner<'a> { + pub(super) fn new( + input: Arc, + root: &'a Path, + terminate: &'a AtomicBool, + ) -> Result { + Ok(Self { + input, + root, + terminate, + deadline: Instant::now() + Duration::from_secs(300), + runtime: tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?, + http: reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(5)) + .build()?, + children: Vec::new(), + bindings: None, + }) + } + + pub(super) fn check(&self) -> Result<()> { + for entry in &self.input.plan.products { + if entry.product == Product::Casework + && self.input.plan.bindings.casework_context_provider.is_some() + { + // The owning tool validates the paired provider export before + // packaging; the published source is left intact. + continue; + } + self.tool( + entry.product, + "check", + &[self.input.project.join(&entry.project).as_os_str()], + )?; + } + Ok(()) + } + + pub(super) fn start( + &mut self, + database: &mut Database, + issuer: Arc, + ) -> Result> { + // Keep the reservations for later products until each is ready to bind. + let ports: Vec<_> = self + .input + .plan + .products + .iter() + .map(|_| TcpListener::bind((Ipv4Addr::LOCALHOST, 0))) + .collect::>() + .context("reserve loopback listeners")?; + let urls = self + .input + .plan + .products + .iter() + .zip(&ports) + .map(|(entry, listener)| { + Ok((entry.product, format!("http://{}", listener.local_addr()?))) + }) + .collect::>>()?; + self.bindings = Some(RuntimeBindings::prepare( + self.root, + self.input.clone(), + issuer.clone(), + &urls, + )?); + let mut breg_package = None; + let input = self.input.clone(); + for (entry, reservation) in input.plan.products.iter().zip(ports) { + self.ensure_active()?; + let product = entry.product; + let project = self.input.project.join(&entry.project); + let url = &urls[&product]; + let bind = url.strip_prefix("http://").expect("local URL"); + let config = match product { + Product::Breg => { + let (config, package) = + self.activate_breg(database, &issuer, &project, bind)?; + breg_package = Some(package); + config + } + Product::Casework => { + self.activate_casework(database, &project, bind, breg_package.as_deref())? + } + }; + let binaries = &self.input.binaries[&product]; + let mut command = child_command(&binaries.runtime); + command.arg("--runtime-config").arg(config); + if product == Product::Casework { + command.arg("serve"); + } + let log = open_log(self.root, &format!("{}-runtime", product.name()))?; + command.stdout(log.try_clone()?).stderr(log); + // Released runtimes own their listeners, so the reservation is + // released before the spawn. A competing process can take the port + // in that gap; the runtime then fails to bind and exits, which the + // readiness wait reports along with the end of its log. + drop(reservation); + let child = command + .spawn() + .with_context(|| format!("cannot start {}", product.name()))?; + self.children.push((product, child)); + self.record_processes()?; + self.ready(product, url)?; + } + Ok(urls) + } + + fn activate_breg( + &self, + database: &mut Database, + issuer: &Issuer, + project: &Path, + bind: &str, + ) -> Result<(PathBuf, PathBuf)> { + let requires_postgis = self + .input + .breg_authority + .as_ref() + .expect("BReg authority") + .requires_postgis; + let rehearsal = database.provision("breg_test", true, requires_postgis)?; + let empty_package = self.root.join("breg-test-package"); + fs::create_dir(&empty_package)?; + let config = self.config( + Product::Breg, + "breg-test", + "127.0.0.1:0", + &empty_package, + rehearsal, + )?; + let credentials = self.schema_credentials(issuer)?; + let receipt = self.root.join("schema-test-receipt.json"); + self.tool( + Product::Breg, + "test", + &[ + project.as_os_str(), + "--runtime-config".as_ref(), + config.as_os_str(), + "--credentials".as_ref(), + credentials.as_os_str(), + "--output".as_ref(), + receipt.as_os_str(), + ], + )?; + let build = self.root.join("breg-build"); + self.tool( + Product::Breg, + "package", + &[ + project.as_os_str(), + "--test-receipt".as_ref(), + receipt.as_os_str(), + "--output".as_ref(), + build.as_os_str(), + ], + )?; + let package = build.join("package"); + let urls = database.provision("breg", true, requires_postgis)?; + let config = self.config(Product::Breg, "breg", bind, &package, urls)?; + self.tool( + Product::Breg, + "apply", + &[ + "--runtime-config".as_ref(), + config.as_os_str(), + "--package".as_ref(), + package.as_os_str(), + "--initial".as_ref(), + ], + )?; + self.tool( + Product::Breg, + "verify", + &["--runtime-config".as_ref(), config.as_os_str()], + )?; + Ok((config, package)) + } + + fn schema_credentials(&self, issuer: &Issuer) -> Result { + let mut bindings = Vec::with_capacity(self.input.schema_people.len()); + for (index, binding) in self.input.schema_people.iter().enumerate() { + let token = issuer.token( + &self.input, + &self.input.plan.people[&binding.person], + Product::Breg, + clock()? + 300, + )?; + let filename = format!("breg-schema-token-{index}"); + write_private(&self.root.join(&filename), token.as_bytes())?; + bindings.push(json!({ + "journeyId": binding.journey, "stepId": binding.step, + "credential": {"type": "bearer", "tokenRef": format!("secret:file/{filename}")} + })); + } + let path = self.root.join("schema-test-credentials.json"); + write_private( + &path, + &serde_json::to_vec(&json!({ + "apiVersion": crate::test_lifecycle::CREDENTIALS_API_VERSION, + "kind": crate::test_lifecycle::CREDENTIALS_KIND, + "bindings": bindings + }))?, + )?; + Ok(path) + } + + fn activate_casework( + &self, + database: &mut Database, + project: &Path, + bind: &str, + breg_package: Option<&Path>, + ) -> Result { + let paired; + let project = if let Some(binding) = &self.input.plan.bindings.casework_context_provider { + let source = + breg_package.context("the Casework context provider needs the BReg package")?; + paired = self.root.join("casework-paired"); + copy_project(project, &paired)?; + self.tool( + Product::Casework, + "provider", + &[ + "pair".as_ref(), + paired.as_os_str(), + "--provider-id".as_ref(), + binding.id.as_ref(), + "--breg-package".as_ref(), + source.as_os_str(), + "--action-id".as_ref(), + binding.action.as_ref(), + "--bregctl-bin".as_ref(), + self.input.binaries[&Product::Breg].tool.as_os_str(), + "--apply".as_ref(), + ], + )?; + self.tool(Product::Casework, "check", &[paired.as_os_str()])?; + &paired + } else { + project + }; + let package = self.root.join("casework-package"); + self.tool( + Product::Casework, + "package", + &[ + project.as_os_str(), + "--output".as_ref(), + package.as_os_str(), + ], + )?; + let urls = database.provision("casework", false, false)?; + let config = self.config(Product::Casework, "casework", bind, &package, urls)?; + let arguments = ["--runtime-config".as_ref(), config.as_os_str()]; + self.tool(Product::Casework, "apply", &arguments)?; + self.tool(Product::Casework, "doctor", &arguments)?; + Ok(config) + } + + fn config( + &self, + product: Product, + name: &str, + bind: &str, + package: &Path, + urls: DatabaseUrls, + ) -> Result { + let mut clients: BTreeSet<_> = self + .input + .plan + .people + .values() + .filter(|person| person.grants.contains_key(&product)) + .map(|person| person.client.as_str()) + .collect(); + if product == Product::Breg { + let authority = self.input.breg_authority.as_ref().expect("BReg authority"); + clients.extend(authority.named_clients.iter().map(String::as_str)); + } + let mut oidc = json!({ + "issuer": self.input.plan.issuer, + "audience": self.input.plan.product(product).audience, + "allowedClients": clients, + "scopeClaim": "scope", + "jwksSource": {"type": "static", "documentRef": "secret:file/jwks"} + }); + let mut database = json!({ + "runtimeUrlRef": self.secret(&format!("{name}-runtime-url"), urls.runtime.as_bytes())?, + "migrationUrlRef": self.secret(&format!("{name}-migration-url"), urls.migration.as_bytes())? + }); + if self.root.join("postgres-ca").is_file() { + database["trustedRootCertificateRef"] = json!("secret:file/postgres-ca"); + } + let audit = json!({ + "hashKeyRef": self.secret(&format!("{name}-audit-key"), random_secret()?.as_bytes())?, + "path": self.root.join(format!("{name}-audit")).join("audit.jsonl"), + }); + let identity = format!("dev-{}-{name}", self.input.run_id); + let mut config = json!({ + "package": {"root": package}, + "secretProviders": {"file": {"root": self.root}}, + "database": database, + "audit": audit, + }); + match product { + Product::Breg => { + oidc["allowedAlgorithm"] = json!("ES256"); + oidc["accessTokenType"] = json!("at+jwt"); + oidc["scopeSeparator"] = json!(" "); + oidc["maximumTokenLifetimeSeconds"] = json!(300); + oidc["leewayMilliseconds"] = json!(1000); + oidc["jwksCache"] = json!({ + "cacheTtlSeconds": 600, "negativeCacheTtlSeconds": 60, + "refreshCooldownSeconds": 30, "maximumDocumentBytes": 65536, + "attemptTimeoutMilliseconds": 5000, "outageToleranceSeconds": 0 + }); + config["apiVersion"] = json!("id.registrystack.org/formats/breg/runtime/v1alpha1"); + config["kind"] = json!("BRegRuntimeConfig"); + config["database"]["pool"] = json!({"maximumConnections": 4}); + config["database"]["roles"] = json!({ + "runtime": url::Url::parse(&urls.runtime)?.username(), + "migration": url::Url::parse(&urls.migration)?.username(), + }); + config["audit"]["destination"] = json!("file"); + config["listener"] = json!({"bind": bind}); + let authority = self.input.breg_authority.as_ref().expect("BReg authority"); + if name == "breg" && authority.requires_postgis { + config["listener"]["publicOrigin"] = json!(format!("http://{bind}")); + } + config["identity"] = json!({ + "environment": "development", "instanceId": identity, + "databaseId": identity, "databaseInitializationEnvironment": "development" + }); + let mut authority_claims = json!({"principal": authority.principal}); + if authority.purpose { + authority_claims["purpose"] = json!("registry_purpose"); + } + config["authentication"] = json!({ + "oidc": oidc, + "authorityClaims": authority_claims + }); + config["cursor"] = json!({ + "secretRef": self.secret(&format!("{name}-cursor-key"), random_secret()?.as_bytes())?, "maximumAgeSeconds": 300 + }); + config["destinations"] = json!({"hooks": {}}); + config["operationalTimeouts"] = json!({ + "httpRequestMilliseconds": 10000, "shutdownGraceMilliseconds": 1000, + "recordLockMilliseconds": 5000, "migrationLockMilliseconds": 30000, + "migrationStatementMilliseconds": 60000 + }); + } + Product::Casework => { + config["apiVersion"] = + json!("id.registrystack.org/formats/casework/runtime/v1alpha1"); + config["kind"] = json!("CaseworkRuntimeConfig"); + config["listener"] = + json!({"bind": bind, "tlsTermination": "development-loopback"}); + config["identity"] = json!({"databaseId": identity}); + config["authentication"] = json!({"oidc": oidc}); + } + } + if let Some(bindings) = &self.bindings { + bindings.configure(product, &mut config); + } + let path = self.root.join(format!("{name}-runtime.yaml")); + write_private(&path, &serde_json::to_vec(&config)?)?; + Ok(path) + } + + fn secret(&self, name: &str, bytes: &[u8]) -> Result { + write_private(&self.root.join(name), bytes)?; + Ok(format!("secret:file/{name}")) + } + + fn tool(&self, product: Product, stage: &str, args: &[&std::ffi::OsStr]) -> Result<()> { + self.ensure_active()?; + let mut arguments = vec![std::ffi::OsStr::new(stage)]; + arguments.extend_from_slice(args); + arguments.extend(["--format", "json"].map(std::ffi::OsStr::new)); + run_logged_tool( + &self.input.binaries[&product].tool, + &arguments, + self.root, + (product.name(), stage), + || self.ensure_active(), + ) + } + + /// Record every live child's process group so a later `stop` can reach + /// processes whose supervisor died. + fn record_processes(&self) -> Result<()> { + let entries: Vec<_> = self + .children + .iter() + .map(|(product, child)| (*product, child.id())) + .collect(); + write_processes(self.root, &entries) + } + + fn ready(&mut self, product: Product, base: &str) -> Result<()> { + let end = Instant::now() + Duration::from_secs(30); + let url = format!("{base}/ready"); + loop { + self.ensure_active()?; + self.ensure_alive(product)?; + if self + .runtime + .block_on(async { self.http.get(&url).send().await }) + .is_ok_and(|response| response.status().is_success()) + { + self.ensure_alive(product)?; + return Ok(()); + } + ensure!( + Instant::now() < end, + "{} did not become ready{}", + product.name(), + self.runtime_log_tail(product) + ); + thread::sleep(POLL); + } + } + + /// Fail while waiting on `waiting`, naming whichever child exited. + fn ensure_alive(&self, waiting: Product) -> Result<()> { + for (product, child) in &self.children { + if child_exited(child)? { + let detail = self.runtime_log_tail(*product); + if *product == waiting { + bail!("{} exited before readiness{detail}", product.name()); + } + bail!( + "{} exited while {} was starting{detail}", + product.name(), + waiting.name() + ); + } + } + Ok(()) + } + + fn runtime_log_tail(&self, product: Product) -> String { + failure_detail(&log_path(self.root, &format!("{}-runtime", product.name()))) + } + + pub(super) fn services_alive(&self) -> Result { + for (_, child) in &self.children { + if child_exited(child)? { + return Ok(false); + } + } + Ok(true) + } + + pub(super) fn ensure_active(&self) -> Result<()> { + ensure!( + !stopping(self.root, self.terminate), + "development run was stopped" + ); + ensure!( + Instant::now() < self.deadline, + "development run did not start within five minutes" + ); + Ok(()) + } + + pub(super) fn stop(&mut self) -> Result<()> { + let mut failures = Vec::new(); + self.children.retain_mut(|(product, child)| { + if stop_child(child).is_err() { + failures.push(product.name()); + true + } else { + false + } + }); + // Drop reaped groups from the record so a later recovery cannot signal + // a reused process group ID. + self.record_processes()?; + ensure!(failures.is_empty(), "cannot stop {}", failures.join(", ")); + drop(self.bindings.take()); + Ok(()) + } + + pub(super) fn journey( + &self, + issuer: &Issuer, + urls: &BTreeMap, + ) -> Result { + super::journey::run(&self.input, issuer, urls, || self.ensure_active()) + } +} + +fn log_path(root: &Path, name: &str) -> PathBuf { + root.join("logs").join(format!("{name}.log")) +} + +/// Open the owner-only log for one product stage, appending to earlier output. +fn open_log(root: &Path, name: &str) -> Result { + let directory = root.join("logs"); + match private_directory(&directory) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error).context("cannot create the development log directory"), + } + fs::OpenOptions::new() + .create(true) + .append(true) + .mode(0o600) + .open(log_path(root, name)) + .with_context(|| format!("cannot open the {name} log")) +} + +const LOG_TAIL_LINES: usize = 20; +const LOG_TAIL_BYTES: u64 = 4096; + +/// The last lines of a log, at most 4 KiB, decoded lossily. An unreadable +/// log yields an empty tail; the caller reports the path either way. +fn log_tail(path: &Path) -> String { + let Ok(mut file) = fs::File::open(path) else { + return String::new(); + }; + let mut bytes = Vec::new(); + let read = file + .metadata() + .map(|meta| meta.len()) + .and_then(|length| file.seek(SeekFrom::Start(length.saturating_sub(LOG_TAIL_BYTES)))) + .and_then(|_| file.take(LOG_TAIL_BYTES).read_to_end(&mut bytes)); + if read.is_err() { + return String::new(); + } + let text = String::from_utf8_lossy(&bytes); + let lines: Vec<_> = text.lines().collect(); + lines[lines.len().saturating_sub(LOG_TAIL_LINES)..] + .join("\n") + .trim() + .to_owned() +} + +/// The suffix appended to a stage failure: the log path and its last lines. +fn failure_detail(path: &Path) -> String { + let tail = log_tail(path); + if tail.is_empty() { + format!(" (log: {})", path.display()) + } else { + format!(" (log: {}); last lines:\n{tail}", path.display()) + } +} + +/// Run a product tool with its stderr in `/logs/.log`. Stdout is +/// the JSON report, which supplies the diagnostic codes of a failure. +fn run_logged_tool( + binary: &Path, + args: &[&std::ffi::OsStr], + root: &Path, + (product, stage): (&str, &str), + active: impl Fn() -> Result<()>, +) -> Result<()> { + let name = format!("{product}-{stage}"); + let mut output = tempfile::tempfile_in(root)?; + let mut child = child_command(binary) + .args(args) + .stdout(output.try_clone()?) + .stderr(open_log(root, &name)?) + .spawn() + .with_context(|| format!("cannot start {product} {stage}"))?; + if wait_for_tool(&mut child, active)?.success() { + return Ok(()); + } + output.rewind()?; + let codes = report_codes(output)?; + bail!( + "{product} {stage} failed{codes}{}", + failure_detail(&log_path(root, &name)) + ); +} + +/// Record each child's process group. Children start as group leaders, so the +/// group ID is the PID. +fn write_processes(root: &Path, children: &[(Product, u32)]) -> Result<()> { + let entries: Vec<_> = children + .iter() + .map(|(product, pid)| json!({"product": product.name(), "pid": pid, "pgid": pid})) + .collect(); + write_private(&root.join("processes.json"), &serde_json::to_vec(&entries)?) +} + +fn random_secret() -> Result> { + use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; + let mut bytes = zeroize::Zeroizing::new([0u8; 32]); + getrandom::fill(bytes.as_mut()).map_err(|_| anyhow::anyhow!("cannot generate a local key"))?; + let mut encoded = zeroize::Zeroizing::new(String::new()); + URL_SAFE_NO_PAD.encode_string(bytes.as_slice(), &mut encoded); + Ok(encoded) +} + +fn copy_project(source: &Path, target: &Path) -> Result<()> { + fn copy_dir( + source: &Path, + target: &Path, + depth: usize, + count: &mut usize, + bytes: &mut u64, + ) -> Result<()> { + ensure!( + depth <= 16, + "Casework project directories exceed the depth limit" + ); + fs::create_dir(target)?; + for entry in fs::read_dir(source)? { + let entry = entry?; + if entry.file_name() == ".git" { + continue; + } + *count += 1; + ensure!( + *count <= 256, + "Casework project exceeds the file count limit" + ); + let info = fs::symlink_metadata(entry.path())?; + let destination = target.join(entry.file_name()); + if info.is_dir() { + copy_dir(&entry.path(), &destination, depth + 1, count, bytes)?; + } else { + ensure!( + info.is_file(), + "Casework project contains a symlink or non-file entry" + ); + *bytes = bytes.saturating_add(info.len()); + ensure!( + *bytes <= 32 * 1024 * 1024, + "Casework project exceeds the copy size limit" + ); + fs::copy(entry.path(), destination)?; + } + } + Ok(()) + } + copy_dir(source, target, 0, &mut 0, &mut 0) +} + +pub(super) struct ProductInspection { + pub(super) breg_authority: Option, + pub(super) casework_principals: BTreeMap, +} + +#[derive(Debug, thiserror::Error)] +#[error("{reason}")] +pub(super) struct InspectionUnavailable { + field: String, + reason: String, + remedy: String, +} + +impl InspectionUnavailable { + fn new(field: &str, reason: &str, remedy: &str) -> Self { + Self { + field: field.to_owned(), + reason: reason.to_owned(), + remedy: remedy.to_owned(), + } + } + + pub(super) fn field(&self) -> &str { + &self.field + } + + pub(super) fn reason(&self) -> &str { + &self.reason + } + + pub(super) fn remedy(&self) -> &str { + &self.remedy + } +} + +struct SelectedProduct { + field: String, + root: PathBuf, +} + +pub(super) fn inspect( + root: &Path, + plan: &Plan, + terminate: &AtomicBool, +) -> Result { + let breg_authority = breg_authority(root, plan)?; + let casework_principals = casework_principals(root, plan, breg_authority.as_ref(), terminate)?; + Ok(ProductInspection { + breg_authority, + casework_principals, + }) +} + +fn product_root( + root: &Path, + plan: &Plan, + product: Product, +) -> plan::SemanticResult> { + let Some((index, entry)) = plan + .products + .iter() + .enumerate() + .find(|(_, entry)| entry.product == product) + else { + return Ok(None); + }; + let field = format!("/products/{index}/project"); + let project = root.join(&entry.project).canonicalize().map_err(|_| { + plan::SemanticRefusal::new( + &field, + "The selected product project directory cannot be resolved.", + "Restore the product project directory inside the development project.", + ) + })?; + plan::require( + project.starts_with(root) && project.is_dir(), + &field, + "The selected product project directory is outside the development project.", + "Select a product project directory inside the development project.", + )?; + let manifest_name = match product { + Product::Breg => "registry.yaml", + Product::Casework => "casework.yaml", + }; + let manifest = project.join(manifest_name).canonicalize().map_err(|_| { + plan::SemanticRefusal::new( + &field, + "The selected product manifest cannot be resolved.", + format!("Restore {manifest_name} inside the selected product project."), + ) + })?; + plan::require( + manifest.starts_with(&project) && manifest.is_file(), + &field, + "The selected product manifest is outside the selected product project.", + format!("Place {manifest_name} inside the selected product project."), + )?; + Ok(Some(SelectedProduct { + field, + root: project, + })) +} + +fn breg_authority(root: &Path, plan: &Plan) -> Result> { + let Some(project) = product_root(root, plan, Product::Breg)? else { + return Ok(None); + }; + let registry = crate::compile(&project.root, crate::ProfileArg::Production, "project-dev") + .map_err(|_| { + plan::SemanticRefusal::new( + &project.field, + "The BReg project did not pass its owning validation.", + "Run `rstack breg check` for authored diagnostics, then correct the BReg project.", + ) + })?; + refuse_encrypted_fields(®istry, &project.field)?; + let review_authorities = registry + .review_authorities() + .iter() + .map(|authority| authority.id.as_str()) + .collect(); + let actions = registry + .actions() + .actions + .iter() + .map(|action| action.id.as_str()) + .collect(); + validate_breg_binding_ids(&plan.bindings, &review_authorities, &actions)?; + check_breg_grants(®istry, &plan.people)?; + let inventory = registry_breg::authority::authority_inventory(®istry)?; + let principal = breg_principal(plan, &project.field, &inventory.principal_claims)?; + Ok(Some(BregAuthority { + principal, + purpose: inventory.purpose_required, + named_clients: crate::identity_facts::named_requester_clients(®istry), + requires_postgis: breg_requires_postgis(®istry), + })) +} + +fn breg_principal( + plan: &Plan, + product_field: &str, + principal_claims: &BTreeSet, +) -> plan::SemanticResult { + plan::require( + principal_claims.len() <= 1, + product_field, + "The BReg access profiles select more than one principal claim.", + "Use one principal claim across the BReg project's access profiles.", + )?; + let principal = principal_claims + .iter() + .next() + .cloned() + .unwrap_or_else(|| "sub".to_owned()); + for (name, person) in &plan.people { + if let Some(grant) = person.grants.get(&Product::Breg) { + plan::require( + !grant.claims.contains_key(&principal), + &format!( + "/people/{}/grants/breg/claims", + registry_platform_yaml::escape_pointer_segment(name) + ), + "The BReg grant claims cannot replace the principal claim selected by the BReg project.", + "Remove the selected principal claim from this grant's additional claims.", + )?; + } + } + Ok(principal) +} + +fn validate_breg_binding_ids( + bindings: &Bindings, + review_authorities: &BTreeSet<&str>, + actions: &BTreeSet<&str>, +) -> plan::SemanticResult<()> { + if let Some(binding) = &bindings.breg_review_authority { + plan::require( + review_authorities.contains(binding.id.as_str()), + "/bindings/bregReviewAuthority/id", + "The BReg review authority ID does not name a review authority in the BReg project.", + "Set id to a review authority defined by the BReg project.", + )?; + } + if let Some(binding) = &bindings.casework_context_provider { + plan::require( + actions.contains(binding.action.as_str()), + "/bindings/caseworkContextProvider/action", + "The Casework context provider action does not name an action in the BReg project.", + "Set action to an action defined by the BReg project.", + )?; + } + Ok(()) +} + +/// The generated runtime configuration has no field-encryption provider, so a +/// project with encrypted fields would only fail late and obscurely. +fn refuse_encrypted_fields( + registry: ®istry_breg::CompiledRegistry, + product_field: &str, +) -> plan::SemanticResult<()> { + for entity in registry.entities().values() { + for field in entity.fields.values() { + plan::require( + field.encryption.is_none(), + product_field, + format!( + "The BReg project encrypts {}.{}, and rstack dev does not configure field \ + encryption yet.", + entity.id, field.id + ), + "Run this BReg project with `rstack breg` and a field-encryption provider, or \ + select a project without encrypted fields.", + )?; + } + } + Ok(()) +} + +/// Every BReg grant must name an access profile the project declares, +/// including profiles that carry only proposal permissions and so belong to no +/// entity. Scopes are not checked here: a profile's required scopes are a +/// per-entity rule the runtime applies, and a mismatch surfaces when the +/// journey runs. +fn check_breg_grants( + registry: ®istry_breg::CompiledRegistry, + people: &BTreeMap, +) -> plan::SemanticResult<()> { + let profiles: BTreeSet<_> = registry + .access_profile_displays() + .iter() + .map(|profile| profile.id.as_str()) + .collect(); + for (name, person) in people { + if let Some(grant) = person.grants.get(&Product::Breg) { + plan::require( + profiles.contains(grant.profile.as_str()), + &format!( + "/people/{}/grants/breg/profile", + registry_platform_yaml::escape_pointer_segment(name) + ), + "The BReg grant profile does not name an access profile in the BReg project.", + "Set profile to an access profile defined by the BReg project.", + )?; + } + } + Ok(()) +} + +fn breg_requires_postgis(registry: ®istry_breg::CompiledRegistry) -> bool { + registry.ddl().requires_postgis +} + +fn casework_principals( + root: &Path, + plan: &Plan, + breg: Option<&BregAuthority>, + terminate: &AtomicBool, +) -> Result> { + let Some(project) = product_root(root, plan, Product::Casework)? else { + return Ok(BTreeMap::new()); + }; + let tool = super::product_tool(Product::Casework).map_err(|_| { + InspectionUnavailable::new( + &project.field, + "The Casework project cannot be inspected because its owning tool is unavailable.", + "Install the matching Casework tools or provide RSTACK_CASEWORKCTL_BIN, then run the check again.", + ) + })?; + let facts = match explain_casework_facts(&tool, &project.root, terminate) { + Ok(facts) => facts, + Err(error) if error.is::() => { + return Err(plan::SemanticRefusal::new( + &project.field, + "The Casework project did not pass validation by its owning tool.", + "Run `rstack casework check` for authored diagnostics, then correct the Casework project.", + ).into()); + } + Err(_) => { + return Err(InspectionUnavailable::new( + &project.field, + "The selected `caseworkctl` helper could not inspect the Casework project.", + "Install the matching Casework tools or provide RSTACK_CASEWORKCTL_BIN, then run the check again.", + ).into()); + } + }; + validate_casework_binding_ids(&plan.bindings, &facts)?; + Ok(casework_principals_from_facts(plan, breg, &facts)?) +} + +fn casework_principals_from_facts( + plan: &Plan, + breg: Option<&BregAuthority>, + facts: &CaseworkFacts, +) -> plan::SemanticResult> { + let mut principals = BTreeMap::new(); + for (name, person) in &plan.people { + let Some(grant) = person.grants.get(&Product::Casework) else { + continue; + }; + let escaped = registry_platform_yaml::escape_pointer_segment(name); + let claim = facts.profiles.get(&grant.profile).map(String::as_str).ok_or_else(|| { + plan::SemanticRefusal::new( + format!("/people/{escaped}/grants/casework/profile"), + "The Casework grant profile does not name an access profile in the Casework project.", + "Set profile to an access profile defined by the Casework project.", + ) + })?; + plan::require( + matches!(claim, "sub" | "registry_principal") + || (!plan::reserved_claim(claim) + && !matches!(claim, "registry_actor_kind" | "registry_purpose")), + &format!("/people/{escaped}/grants/casework/profile"), + "The Casework access profile selects an incompatible principal claim.", + "Choose a Casework access profile whose principal claim is `sub`, `registry_principal`, or a non-reserved custom claim.", + )?; + plan::require( + !grant.claims.contains_key(claim), + &format!("/people/{escaped}/grants/casework/claims"), + "The Casework grant claims cannot replace the principal claim selected by its access profile.", + "Remove the selected principal claim from this grant's additional claims.", + )?; + if person.casework_context { + plan::require( + claim == "sub" || breg.is_some_and(|authority| authority.principal == claim), + &format!("/people/{escaped}/caseworkContext"), + "Casework context forwarding requires Casework to read `sub` or the BReg principal claim.", + "Choose a compatible Casework access profile or remove caseworkContext.", + )?; + } + principals.insert(grant.profile.clone(), claim.to_owned()); + } + Ok(principals) +} + +fn explain_casework_facts( + tool: &Path, + project: &Path, + terminate: &AtomicBool, +) -> Result { + const MAX_REPORT: u64 = 4 * 1024 * 1024; + + let mut child = child_command(tool) + .args(["--format", "json", "explain"]) + .arg(project) + .stdout(Stdio::piped()) + .spawn() + .context("cannot start caseworkctl explain")?; + let stdout = child.stdout.take().expect("the explain output was piped"); + let reader = thread::spawn(move || { + let mut bytes = Vec::new(); + stdout.take(MAX_REPORT + 1).read_to_end(&mut bytes)?; + Ok::<_, std::io::Error>(bytes) + }); + let deadline = Instant::now() + Duration::from_secs(30); + let status = wait_for_tool(&mut child, || { + ensure!( + !terminate.load(Ordering::Relaxed), + "development product inspection was interrupted" + ); + ensure!(Instant::now() < deadline, "caseworkctl explain timed out"); + Ok(()) + })?; + let bytes = reader + .join() + .map_err(|_| anyhow::anyhow!("cannot collect the Casework authoring report"))??; + ensure!( + bytes.len() as u64 <= MAX_REPORT, + "the Casework authoring report exceeds the size limit" + ); + let report: Value = serde_json::from_slice(&bytes) + .map_err(|_| anyhow::anyhow!("caseworkctl explain did not emit a JSON report"))?; + if status.success() { + return casework_report_facts(&report); + } + if status.code() == Some(1) && casework_domain_refusal(&report) { + return Err(InvalidCaseworkProject.into()); + } + anyhow::bail!("caseworkctl explain did not complete successfully") +} + +fn casework_domain_refusal(report: &Value) -> bool { + report["apiVersion"] == "id.registrystack.org/formats/casework/report/v1alpha2" + && report["kind"] == "CaseworkReport" + && report["command"] == "explain" + && report["status"] == "domain-refusal" + && report["ok"] == false + && report["diagnostics"].as_array().is_some_and(|diagnostics| { + diagnostics.iter().any(|diagnostic| { + diagnostic["severity"] == "error" + && diagnostic["code"] + .as_str() + .is_some_and(|value| !value.is_empty()) + && diagnostic["path"].is_string() + && diagnostic["message"] + .as_str() + .is_some_and(|value| !value.is_empty()) + && diagnostic["suggestedAction"] + .as_str() + .is_some_and(|value| !value.is_empty()) + }) + }) +} + +#[derive(Debug, thiserror::Error)] +#[error("the Casework project did not pass its owning validation")] +struct InvalidCaseworkProject; + +#[derive(Debug, Eq, PartialEq)] +struct CaseworkFacts { + profiles: BTreeMap, + trusted_producers: BTreeSet, + context_providers: BTreeSet, +} + +fn casework_report_facts(report: &Value) -> Result { + ensure!( + report["apiVersion"] == "id.registrystack.org/formats/casework/report/v1alpha2" + && report["kind"] == "CaseworkReport" + && report["command"] == "explain" + && report["status"] == "complete" + && report["ok"] == true, + "caseworkctl explain returned an incompatible report" + ); + let profiles = report + .pointer("/accessProfiles") + .and_then(Value::as_array) + .context("caseworkctl explain did not report access profiles")?; + let mut principals = BTreeMap::new(); + for profile in profiles { + let id = profile["id"] + .as_str() + .context("an explained Casework profile has no ID")?; + let claim = profile["principalClaim"] + .as_str() + .filter(|claim| !claim.is_empty()) + .context("an explained Casework profile has no principal claim")?; + ensure!( + principals.insert(id.to_owned(), claim.to_owned()).is_none(), + "caseworkctl explain returned duplicate access profiles" + ); + } + Ok(CaseworkFacts { + profiles: principals, + trusted_producers: casework_report_ids(report, "trustedProducers")?, + context_providers: casework_report_ids(report, "contextProviders")?, + }) +} + +fn casework_report_ids(report: &Value, member: &str) -> Result> { + let entries = report[member] + .as_array() + .with_context(|| format!("caseworkctl explain did not report {member}"))?; + let mut ids = BTreeSet::new(); + for entry in entries { + let id = entry["id"] + .as_str() + .with_context(|| format!("a caseworkctl explain {member} entry has no ID"))?; + ensure!( + ids.insert(id.to_owned()), + "caseworkctl explain returned duplicate {member} IDs" + ); + } + Ok(ids) +} + +fn validate_casework_binding_ids( + bindings: &Bindings, + facts: &CaseworkFacts, +) -> plan::SemanticResult<()> { + if let Some(binding) = &bindings.breg_review_authority { + plan::require( + facts.trusted_producers.contains(&binding.producer), + "/bindings/bregReviewAuthority/producer", + "The BReg review producer does not name a trusted producer in the Casework project.", + "Set producer to a trusted producer defined by the Casework project.", + )?; + } + if let Some(binding) = &bindings.casework_context_provider { + plan::require( + facts.context_providers.contains(&binding.id), + "/bindings/caseworkContextProvider/id", + "The Casework context provider ID does not name a context provider in the Casework project.", + "Set id to a context provider defined by the Casework project.", + )?; + } + Ok(()) +} + +impl Drop for ProductRunner<'_> { + fn drop(&mut self) { + // The normal exit reports stop failures explicitly. + for (_, child) in &mut self.children { + let _ = stop_child(child); + } + } +} + +pub(super) fn child_command(binary: &Path) -> Command { + let mut command = Command::new(binary); + command + .env_remove("RSTACK_DEV_DATABASE_URL") + .env_remove("RSTACK_PROJECT_DEV_REQUEST") + .process_group(0) + .stdin(Stdio::null()) + .stderr(Stdio::null()); + command +} + +fn child_pid(child: &Child) -> Result { + Pid::from_raw(i32::try_from(child.id())?).context("invalid owned process ID") +} + +fn child_exited(child: &Child) -> Result { + let pid = child_pid(child)?; + // Keep the leader's PID until the group has received the final signal. + // Reaping the leader first would allow reuse of the group ID. + let options = WaitIdOptions::EXITED | WaitIdOptions::NOHANG | WaitIdOptions::NOWAIT; + loop { + match waitid(WaitId::Pid(pid), options) { + Ok(status) => return Ok(status.is_some()), + Err(rustix::io::Errno::INTR) => continue, + Err(error) => return Err(error.into()), + } + } +} + +fn signal_group(child: &Child, signal: Signal) -> Result<()> { + match kill_process_group(child_pid(child)?, signal) { + Ok(()) | Err(rustix::io::Errno::SRCH) => Ok(()), + // Darwin excludes zombies from killpg and reports EPERM when there + // are no signalable members. An owned, same-user group with an + // exited leader can already be empty of live processes. + Err(rustix::io::Errno::PERM) if cfg!(target_os = "macos") && child_exited(child)? => Ok(()), + Err(error) => Err(error.into()), + } +} + +fn stop_child(child: &mut Child) -> Result { + if !child_exited(child)? { + signal_group(child, Signal::TERM)?; + let deadline = Instant::now() + Duration::from_secs(2); + while Instant::now() < deadline && !child_exited(child)? { + thread::sleep(POLL); + } + } + signal_group(child, Signal::KILL)?; + Ok(child.wait()?) +} + +fn wait_for_tool(child: &mut Child, active: impl Fn() -> Result<()>) -> Result { + loop { + if let Err(error) = active() { + stop_child(child)?; + return Err(error); + } + if child_exited(child)? { + return stop_child(child); + } + thread::sleep(POLL); + } +} + +fn report_codes(output: impl Read) -> Result { + let mut bytes = Vec::new(); + output.take(256 * 1024).read_to_end(&mut bytes)?; + let Ok(value) = serde_json::from_slice::(&bytes) else { + return Ok(String::new()); + }; + let codes: BTreeSet<_> = value["diagnostics"] + .as_array() + .into_iter() + .flatten() + .filter_map(|item| { + let code = item["code"].as_str().filter(|code| { + code.len() <= 128 + && code + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"-_.".contains(&byte)) + })?; + let path = item["path"] + .as_str() + .or_else(|| item["pointer"].as_str()) + .filter(|path| { + !path.is_empty() + && path.len() <= 128 + && path + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"-_./~".contains(&byte)) + }); + Some(path.map_or_else(|| code.to_owned(), |path| format!("{code} at {path}"))) + }) + .take(6) + .collect(); + if codes.is_empty() { + Ok(String::new()) + } else { + Ok(format!( + " ({})", + codes.into_iter().collect::>().join(", ") + )) + } +} + +#[derive(serde::Deserialize, Serialize)] +pub(super) struct SchemaPerson { + journey: String, + step: String, + person: String, +} + +fn breg_entry(plan: &Plan) -> Option<(usize, &plan::ProductPlan)> { + plan.products + .iter() + .enumerate() + .find(|(_, entry)| entry.product == Product::Breg) +} + +fn schema_assignment_pointer(journey: &str, step: &str) -> String { + format!( + "/schemaTestPeople/{}/{}", + registry_platform_yaml::escape_pointer_segment(journey), + registry_platform_yaml::escape_pointer_segment(step) + ) +} + +fn fixture_file_refusal(plan: &Plan, field: &str) -> plan::SemanticResult { + if plan.schema_test_people.is_empty() { + plan::refuse( + field, + "The BReg fixture journeys file must be a readable regular file inside the BReg product project.", + "Place tests/journeys.yaml inside the BReg product project without linking it outside that project.", + ) + } else { + plan::refuse( + "/schemaTestPeople", + "The BReg fixture journeys file is missing, unreadable, or outside the BReg product project.", + "Create a readable tests/journeys.yaml inside the BReg product project, or remove schemaTestPeople.", + ) + } +} + +pub(super) fn schema_people( + root: &Path, + plan: &Plan, + required: bool, +) -> plan::SemanticResult> { + let Some((product_index, entry)) = breg_entry(plan) else { + if !plan.schema_test_people.is_empty() { + return plan::refuse( + "/schemaTestPeople", + "Schema test assignments require a BReg product.", + "Add BReg to products, or remove schemaTestPeople.", + ); + } + return Ok(Vec::new()); + }; + let field = format!("/products/{product_index}/project"); + let product_root = match root.join(&entry.project).canonicalize() { + Ok(path) if path.starts_with(root) && path.is_dir() => path, + Ok(_) | Err(_) => return fixture_file_refusal(plan, &field), + }; + let path = product_root.join("tests/journeys.yaml"); + match fs::symlink_metadata(&path) { + Ok(_) => {} + Err(error) + if error.kind() == std::io::ErrorKind::NotFound + && !required + && plan.schema_test_people.is_empty() => + { + return Ok(Vec::new()); + } + Err(_) => return fixture_file_refusal(plan, &field), + } + let canonical = match path.canonicalize() { + Ok(path) if path.starts_with(&product_root) && path.is_file() => path, + Ok(_) | Err(_) => return fixture_file_refusal(plan, &field), + }; + let bytes = match plan::read( + &canonical, + registry_platform_yaml::MAXIMUM_DOCUMENT_BYTES as u64, + ) { + Ok(bytes) => bytes, + Err(_) => { + return plan::refuse( + if plan.schema_test_people.is_empty() { + &field + } else { + "/schemaTestPeople" + }, + "The BReg fixture journeys file is unreadable or exceeds the 1 MiB limit.", + "Make tests/journeys.yaml readable and no larger than 1 MiB, then run the check again.", + ); + } + }; + let steps = match journey_step_profiles(&bytes) { + Ok(steps) => steps, + Err(_) => { + return plan::refuse( + if plan.schema_test_people.is_empty() { + &field + } else { + "/schemaTestPeople" + }, + "The BReg fixture journeys file is not valid.", + "Run `rstack breg check` for authored diagnostics, then correct tests/journeys.yaml.", + ); + } + }; + let mut fixture_steps = BTreeMap::<&str, BTreeSet<&str>>::new(); + for step in &steps { + fixture_steps + .entry(&step.journey_id) + .or_default() + .insert(&step.step_id); + } + for (journey, assignments) in &plan.schema_test_people { + for step in assignments.keys() { + let field = schema_assignment_pointer(journey, step); + if !fixture_steps.contains_key(journey.as_str()) { + return plan::refuse( + &field, + "The schema test assignment names a BReg fixture journey that does not exist.", + "Set the journey key to a journey defined in tests/journeys.yaml, or remove this assignment.", + ); + } + if !fixture_steps[journey.as_str()].contains(step.as_str()) { + return plan::refuse( + &field, + "The schema test assignment names a BReg fixture step that does not exist in the named journey.", + "Set the step key to a step defined in that fixture journey, or remove this assignment.", + ); + } + } + } + let mut bindings = Vec::new(); + for step in steps { + let eligible = |person: &plan::TestPerson| { + person.grants.get(&Product::Breg).is_some_and(|grant| { + grant.profile == step.access_profile + && step.scopes.as_ref().is_none_or(|scopes| { + scopes.iter().collect::>() == grant.scopes.iter().collect() + }) + && step.purpose.as_ref().is_none_or(|purpose| { + grant.claims.get("registry_purpose").and_then(Value::as_str) + == Some(purpose) + }) + }) + }; + let explicit = plan + .schema_test_people + .get(&step.journey_id) + .and_then(|steps| steps.get(&step.step_id)); + let aliases: Vec<_> = plan + .people + .iter() + .filter(|(name, person)| { + explicit.is_none_or(|selected| selected == *name) && eligible(person) + }) + .map(|(name, _)| name) + .collect(); + let [person] = aliases.as_slice() else { + let field = schema_assignment_pointer(&step.journey_id, &step.step_id); + return plan::refuse( + &field, + "A BReg fixture step must resolve to exactly one test person with matching profile, scopes, and purpose.", + "Add or update a matching BReg grant, or select the matching person in schemaTestPeople.", + ); + }; + bindings.push(SchemaPerson { + journey: step.journey_id, + step: step.step_id, + person: (*person).clone(), + }); + } + Ok(bindings) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::project_dev::bindings::{Bindings, BregReviewAuthority, CaseworkContextProvider}; + use std::{ + io::{BufRead, BufReader}, + os::unix::fs::PermissionsExt, + sync::mpsc, + }; + + #[test] + fn postgis_requirement_comes_from_the_compiled_breg_inventory() { + let products = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../products/breg"); + for (project, expected) in [ + ("acceptance/spatial-service-sites", true), + ("examples/minimal", false), + ] { + let compiled = crate::compile( + &products.join(project), + crate::ProfileArg::Production, + "project-dev-test", + ) + .unwrap_or_else(|_| panic!("{project} must compile")); + assert_eq!(breg_requires_postgis(&compiled), expected, "{project}"); + } + } + + #[test] + fn the_casework_cli_report_owns_profile_principals() { + let mut report = json!({ + "apiVersion": "id.registrystack.org/formats/casework/report/v1alpha2", + "kind": "CaseworkReport", "command": "explain", "status": "complete", "ok": true, + "reviewPolicies": [], + "accessProfiles": [ + {"id": "producer", "principalClaim": "producer_identity"}, + {"id": "reviewer", "principalClaim": "sub"} + ], + "trustedProducers": [], + "contextProviders": [] + }); + assert_eq!( + casework_report_facts(&report).unwrap().profiles, + BTreeMap::from([ + ("producer".to_owned(), "producer_identity".to_owned()), + ("reviewer".to_owned(), "sub".to_owned()), + ]) + ); + report["accessProfiles"] + .as_array_mut() + .unwrap() + .push(json!({"id": "reviewer", "principalClaim": "different"})); + assert!(casework_report_facts(&report) + .unwrap_err() + .to_string() + .contains("duplicate")); + report["command"] = json!("check"); + assert!(casework_report_facts(&report) + .unwrap_err() + .to_string() + .contains("incompatible report")); + } + + fn authority_plan(breg_claims: Value, casework_claims: Value, context: bool) -> Plan { + serde_json::from_value(json!({ + "issuer": "https://issuer.example.invalid", + "products": [ + {"product": "breg", "project": "breg", "audience": "registry"}, + {"product": "casework", "project": "casework", "audience": "review"} + ], + "people": {"operator": { + "subject": "operator", "client": "operator-client", + "grants": { + "breg": {"profile": "operator", "scopes": [], "claims": breg_claims}, + "casework": {"profile": "reviewer", "scopes": [], "claims": casework_claims} + }, + "caseworkContext": context + }}, + "journey": {"id": "smoke", "steps": []} + })) + .unwrap() + } + + #[test] + fn breg_authority_restrictions_are_typed_and_located() { + let plan = authority_plan(json!({}), json!({}), false); + let error = breg_principal( + &plan, + "/products/0/project", + &BTreeSet::from(["registry_principal".to_owned(), "sub".to_owned()]), + ) + .unwrap_err() + .to_string(); + assert!(error.contains("/products/0/project"), "{error}"); + assert!( + error.contains("The BReg access profiles select more than one principal claim."), + "{error}" + ); + assert!( + error.contains("Use one principal claim across the BReg project's access profiles."), + "{error}" + ); + + let plan = authority_plan( + json!({"registry_principal": "private-claim-marker"}), + json!({}), + false, + ); + let error = breg_principal( + &plan, + "/products/0/project", + &BTreeSet::from(["registry_principal".to_owned()]), + ) + .unwrap_err() + .to_string(); + assert!( + error.contains("/people/operator/grants/breg/claims"), + "{error}" + ); + assert!( + error.contains("cannot replace the principal claim"), + "{error}" + ); + assert!(!error.contains("private-claim-marker"), "{error}"); + } + + #[test] + fn casework_authority_restrictions_are_typed_and_located() { + let facts = |principal: &str| CaseworkFacts { + profiles: BTreeMap::from([("reviewer".to_owned(), principal.to_owned())]), + trusted_producers: BTreeSet::new(), + context_providers: BTreeSet::new(), + }; + + let plan = authority_plan(json!({}), json!({}), false); + let error = casework_principals_from_facts(&plan, None, &facts("registry_actor_kind")) + .unwrap_err() + .to_string(); + assert!( + error.contains("/people/operator/grants/casework/profile"), + "{error}" + ); + assert!(error.contains("incompatible principal claim"), "{error}"); + + let plan = authority_plan(json!({}), json!({"sub": "private-claim-marker"}), false); + let error = casework_principals_from_facts(&plan, None, &facts("sub")) + .unwrap_err() + .to_string(); + assert!( + error.contains("/people/operator/grants/casework/claims"), + "{error}" + ); + assert!( + error.contains("cannot replace the principal claim"), + "{error}" + ); + assert!(!error.contains("private-claim-marker"), "{error}"); + + let plan = authority_plan(json!({}), json!({}), true); + let breg = BregAuthority { + principal: "registry_principal".to_owned(), + purpose: false, + requires_postgis: false, + named_clients: BTreeSet::new(), + }; + let error = casework_principals_from_facts(&plan, Some(&breg), &facts("custom_principal")) + .unwrap_err() + .to_string(); + assert!( + error.contains("/people/operator/caseworkContext"), + "{error}" + ); + assert!( + error.contains("requires Casework to read `sub` or the BReg principal claim"), + "{error}" + ); + } + + fn product_bindings() -> Bindings { + Bindings { + breg_review_authority: Some(BregReviewAuthority { + id: "review-authority".to_owned(), + producer: "review-producer".to_owned(), + identity: "reviewer".to_owned(), + }), + casework_context_provider: Some(CaseworkContextProvider { + id: "proposal-context".to_owned(), + identity: "context-reader".to_owned(), + action: "review-action".to_owned(), + }), + } + } + + #[test] + fn breg_binding_ids_are_resolved_with_typed_value_free_refusals() { + let bindings = product_bindings(); + let authorities = BTreeSet::from(["review-authority"]); + let actions = BTreeSet::from(["review-action"]); + validate_breg_binding_ids(&bindings, &authorities, &actions).unwrap(); + + for (authorities, actions, field, reason, action) in [ + ( + BTreeSet::new(), + actions.clone(), + "/bindings/bregReviewAuthority/id", + "The BReg review authority ID does not name a review authority in the BReg project.", + "Set id to a review authority defined by the BReg project.", + ), + ( + authorities.clone(), + BTreeSet::new(), + "/bindings/caseworkContextProvider/action", + "The Casework context provider action does not name an action in the BReg project.", + "Set action to an action defined by the BReg project.", + ), + ] { + let error = validate_breg_binding_ids(&bindings, &authorities, &actions) + .unwrap_err() + .to_string(); + assert!(error.contains(field), "{error}"); + assert!(error.contains(reason), "{error}"); + assert!(error.contains(action), "{error}"); + assert!(!error.contains("review-authority"), "{error}"); + assert!(!error.contains("review-action"), "{error}"); + } + } + + #[test] + fn casework_binding_ids_use_the_existing_explain_facts() { + let bindings = product_bindings(); + let mut report = json!({ + "apiVersion": "id.registrystack.org/formats/casework/report/v1alpha2", + "kind": "CaseworkReport", "command": "explain", "status": "complete", "ok": true, + "reviewPolicies": [], + "accessProfiles": [{"id": "reviewer", "principalClaim": "sub"}], + "trustedProducers": [{"id": "review-producer"}], + "contextProviders": [{"id": "proposal-context"}] + }); + let facts = casework_report_facts(&report).unwrap(); + validate_casework_binding_ids(&bindings, &facts).unwrap(); + + for (member, field, reason, action) in [ + ( + "trustedProducers", + "/bindings/bregReviewAuthority/producer", + "The BReg review producer does not name a trusted producer in the Casework project.", + "Set producer to a trusted producer defined by the Casework project.", + ), + ( + "contextProviders", + "/bindings/caseworkContextProvider/id", + "The Casework context provider ID does not name a context provider in the Casework project.", + "Set id to a context provider defined by the Casework project.", + ), + ] { + report[member] = json!([]); + let facts = casework_report_facts(&report).unwrap(); + let error = validate_casework_binding_ids(&bindings, &facts) + .unwrap_err() + .to_string(); + assert!(error.contains(field), "{error}"); + assert!(error.contains(reason), "{error}"); + assert!(error.contains(action), "{error}"); + assert!(!error.contains("review-producer"), "{error}"); + assert!(!error.contains("proposal-context"), "{error}"); + report[member] = match member { + "trustedProducers" => json!([{"id": "review-producer"}]), + "contextProviders" => json!([{"id": "proposal-context"}]), + _ => unreachable!(), + }; + } + } + + #[test] + fn a_report_without_the_access_profile_list_is_refused() { + let report = json!({ + "apiVersion": "id.registrystack.org/formats/casework/report/v1alpha2", + "kind": "CaseworkReport", "command": "explain", "status": "complete", "ok": true, + "reviewPolicies": [{"dependencies": {"accessProfiles": [ + {"id": "reviewer", "principalClaim": "sub"} + ]}}] + }); + assert!(casework_report_facts(&report) + .unwrap_err() + .to_string() + .contains("did not report access profiles")); + } + + #[test] + fn a_failed_tool_leaves_its_stderr_in_an_owner_only_log_and_the_error() { + let scratch = tempfile::tempdir().unwrap(); + let root = scratch.path(); + let error = run_logged_tool( + Path::new("/bin/sh"), + &["-c".as_ref(), "echo boom >&2; exit 3".as_ref()], + root, + ("breg", "test"), + || Ok(()), + ) + .unwrap_err() + .to_string(); + assert!(error.contains("breg test failed"), "{error}"); + assert!(error.contains("boom"), "{error}"); + let logs = root.join("logs"); + let log = logs.join("breg-test.log"); + assert_eq!(fs::read_to_string(&log).unwrap(), "boom\n"); + assert!(error.contains(&log.display().to_string()), "{error}"); + assert_eq!( + fs::metadata(&logs).unwrap().permissions().mode() & 0o777, + 0o700 + ); + assert_eq!( + fs::metadata(&log).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + + #[test] + fn a_successful_tool_leaves_no_failure_text() { + let scratch = tempfile::tempdir().unwrap(); + run_logged_tool( + Path::new("/bin/sh"), + &["-c".as_ref(), "echo fine >&2".as_ref()], + scratch.path(), + ("breg", "test"), + || Ok(()), + ) + .unwrap(); + } + + #[test] + fn the_log_tail_keeps_the_last_twenty_lines_within_four_kib() { + let scratch = tempfile::tempdir().unwrap(); + let path = scratch.path().join("log"); + let lines: Vec<_> = (1..=30).map(|number| format!("line {number}")).collect(); + fs::write(&path, lines.join("\n")).unwrap(); + let tail = log_tail(&path); + assert_eq!(tail.lines().count(), 20); + assert!(tail.starts_with("line 11") && tail.ends_with("line 30")); + + fs::write(&path, format!("{}\nlast", "x".repeat(10_000))).unwrap(); + let tail = log_tail(&path); + assert!(tail.len() <= 4096 && tail.ends_with("last")); + + fs::write(&path, [b'a', 0xff, b'\n', b'b']).unwrap(); + assert_eq!(log_tail(&path), "a\u{fffd}\nb"); + } + + #[test] + fn spawned_processes_are_recorded_as_product_pid_and_group() { + let scratch = tempfile::tempdir().unwrap(); + write_processes( + scratch.path(), + &[(Product::Breg, 4242), (Product::Casework, 4343)], + ) + .unwrap(); + let path = scratch.path().join("processes.json"); + assert_eq!( + serde_json::from_slice::(&fs::read(&path).unwrap()).unwrap(), + json!([ + {"product": "breg", "pid": 4242, "pgid": 4242}, + {"product": "casework", "pid": 4343, "pgid": 4343} + ]) + ); + assert_eq!( + fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + + fn minimal_registry(edit: impl FnOnce(&str) -> String) -> registry_breg::CompiledRegistry { + let source = + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../products/breg/examples/minimal"); + let scratch = tempfile::tempdir().unwrap(); + let project = scratch.path().join("project"); + copy_project(&source, &project).unwrap(); + let registry = project.join("registry.yaml"); + fs::write(®istry, edit(&fs::read_to_string(®istry).unwrap())).unwrap(); + crate::compile(&project, crate::ProfileArg::Production, "project-dev-test") + .unwrap_or_else(|_| panic!("the edited project must compile")) + } + + fn breg_person(profile: &str) -> plan::TestPerson { + serde_json::from_value(json!({ + "subject": "someone", "client": "client", + "grants": {"breg": {"profile": profile, "scopes": []}} + })) + .unwrap() + } + + #[test] + fn a_breg_grant_naming_an_unknown_profile_is_refused_at_its_pointer() { + let registry = minimal_registry(str::to_owned); + let people = BTreeMap::from([ + ("editor".to_owned(), breg_person("directory-editor")), + ("typo".to_owned(), breg_person("directory-edtor")), + ]); + let error = check_breg_grants(®istry, &people) + .unwrap_err() + .to_string(); + assert!( + error.contains("/people/typo/grants/breg/profile"), + "{error}" + ); + assert!( + error.ends_with("Next: Set profile to an access profile defined by the BReg project."), + "{error}" + ); + assert!(!error.contains("directory-edtor"), "{error}"); + let valid = BTreeMap::from([("editor".to_owned(), breg_person("directory-editor"))]); + check_breg_grants(®istry, &valid).unwrap(); + } + + #[test] + fn a_breg_grant_may_name_a_profile_with_only_proposal_permissions() { + let source = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../products/breg/acceptance/reviewed-evidence-journey/registry"); + let scratch = tempfile::tempdir().unwrap(); + let project = scratch.path().join("project"); + copy_project(&source, &project).unwrap(); + let registry = crate::compile(&project, crate::ProfileArg::Production, "project-dev-test") + .unwrap_or_else(|_| panic!("the reviewed evidence project must compile")); + let people = BTreeMap::from([("applier".to_owned(), breg_person("applier"))]); + check_breg_grants(®istry, &people).unwrap(); + } + + #[test] + fn encrypted_fields_are_refused_before_provisioning() { + let registry = minimal_registry(|source| { + let source = source.replacen( + "mutationMode: mutable\n display: {label: Contact", + "mutationMode: mutable\n classification: restricted\n display: {label: Contact", + 1, + ); + source.replacen( + "{id: email, type: string, required: true, minimumLength: 1, maximumLength: 254, classification: internal}\n - {id: jurisdiction", + "{id: email, type: string, required: true, minimumLength: 1, maximumLength: 254, classification: restricted, encrypted: true}\n - {id: jurisdiction", + 1, + ) + }); + assert_eq!( + refuse_encrypted_fields(®istry, "/products/0/project") + .unwrap_err() + .to_string(), + "dev.yaml /products/0/project: The BReg project encrypts contact.email, and rstack \ + dev does not configure field encryption yet. Next: Run this BReg project with \ + `rstack breg` and a field-encryption provider, or select a project without \ + encrypted fields." + ); + refuse_encrypted_fields(&minimal_registry(str::to_owned), "/products/0/project").unwrap(); + } + + #[test] + fn cancellation_closes_the_pipe_of_a_nested_tool() { + closes_nested_tool_pipe(false); + } + + #[test] + fn cancellation_cleans_up_a_nested_tool_after_its_leader_exits() { + closes_nested_tool_pipe(true); + } + + #[test] + fn completed_tool_without_descendants_keeps_its_exit_status() { + let mut child = child_command(Path::new("/bin/sh")) + .args(["-c", "exit 7"]) + .spawn() + .unwrap(); + wait_for_leader(&child); + assert_eq!(stop_child(&mut child).unwrap().code(), Some(7)); + } + + fn wait_for_leader(child: &Child) { + let deadline = Instant::now() + Duration::from_secs(2); + while !child_exited(child).unwrap() { + assert!(Instant::now() < deadline, "the tool leader did not exit"); + thread::sleep(POLL); + } + } + + fn closes_nested_tool_pipe(leader_exits: bool) { + let leader_end = if leader_exits { "exit 0" } else { "wait" }; + let mut child = child_command(Path::new("/bin/sh")) + .arg("-c") + .arg(format!( + "sh -c 'trap \"\" TERM; printf \"ready\\n\"; exec sleep 60' & {leader_end}" + )) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let pid = child_pid(&child).unwrap(); + assert_eq!(rustix::process::getpgid(Some(pid)).unwrap(), pid); + let mut pipe = BufReader::new(child.stdout.take().unwrap()); + let mut ready = String::new(); + pipe.read_line(&mut ready).unwrap(); + assert_eq!(ready, "ready\n"); + if leader_exits { + wait_for_leader(&child); + } + stop_child(&mut child).unwrap(); + + let (sender, receiver) = mpsc::sync_channel(1); + let reader = thread::spawn(move || { + let _ = sender.send(pipe.read(&mut [0])); + }); + let eof = receiver.recv_timeout(Duration::from_secs(3)); + if eof.is_err() { + let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL); + } + reader.join().unwrap(); + assert_eq!(eof.unwrap().unwrap(), 0); + } +} diff --git a/crates/registry-bregctl/src/tool_schema.rs b/crates/registry-bregctl/src/tool_schema.rs index 731a8fa938..803c044193 100644 --- a/crates/registry-bregctl/src/tool_schema.rs +++ b/crates/registry-bregctl/src/tool_schema.rs @@ -13,7 +13,7 @@ use registry_breg::{fixtures, migration_plan}; use registry_platform_yaml::{EnvelopeRule, FormatSpec, VersionStatus}; use serde_json::{json, Map, Value}; -use crate::{dev, init_from_model, test_lifecycle}; +use crate::{dev, init_from_model, project_dev, test_lifecycle}; const SCHEMA_DIALECT: &str = "https://json-schema.org/draft/2020-12/schema"; const SCHEMA_ID_BASE: &str = "https://id.registrystack.org/schemas/"; @@ -27,7 +27,7 @@ struct Tool { members: fn() -> schemars::Schema, } -fn tools() -> [Tool; 7] { +fn tools() -> [Tool; 8] { [ Tool { format: "journeys", @@ -53,6 +53,12 @@ fn tools() -> [Tool; 7] { spec: dev::config::DEV_CLIENTS_FORMAT, members: dev::config::clients_schema, }, + Tool { + format: "project-dev", + title: "Registry Stack project development plan", + spec: project_dev::plan::FORMAT, + members: project_dev::plan::schema, + }, Tool { format: "example-scenarios", title: "Base Registry Engine example scenarios", diff --git a/crates/registry-bregctl/tests/project_dev.rs b/crates/registry-bregctl/tests/project_dev.rs new file mode 100644 index 0000000000..f895516d1a --- /dev/null +++ b/crates/registry-bregctl/tests/project_dev.rs @@ -0,0 +1,1066 @@ +// SPDX-License-Identifier: Apache-2.0 + +use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; +use std::{fs, os::unix::fs::PermissionsExt, process::Command}; + +fn project(root: &std::path::Path, extra: &str) { + fs::write( + root.join("registry.yaml"), + include_str!( + "../../../products/platform/conformance/projects/breg-project-dev/registry/registry.yaml" + ), + ) + .unwrap(); + fs::write( + root.join("dev.yaml"), + format!( + "# yaml-language-server: $schema=https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json\n\ +apiVersion: id.registrystack.org/formats/breg/project-dev/v1alpha1\n\ +kind: BRegProjectDev\n\ +issuer: https://issuer.example.invalid\n\ +products:\n - {{product: breg, project: ., audience: registry}}\n\ +people:\n operator:\n subject: operator\n client: operator-client\n grants:\n breg: {{profile: operator, scopes: []}}\n\ +journey:\n id: smoke\n steps:\n - {{id: read, product: breg, as: operator, method: GET, path: /v1/records, expect: {{status: 200}}}}\n\ +{extra}" + ), + ) + .unwrap(); +} + +fn semantic_project(root: &std::path::Path, grant: &str, step_extra: &str) { + fs::write(root.join("registry.yaml"), "fixture\n").unwrap(); + fs::write( + root.join("dev.yaml"), + format!( + "apiVersion: id.registrystack.org/formats/breg/project-dev/v1alpha1\n\ +kind: BRegProjectDev\n\ +issuer: https://issuer.example.invalid\n\ +products:\n - {{product: breg, project: ., audience: private-audience-marker}}\n\ +people:\n operator:\n subject: private-subject-marker\n client: private-client-marker\n grants:\n {grant}: {{profile: private-profile-marker, scopes: []}}\n\ +journey:\n id: smoke\n steps:\n - id: read\n product: breg\n as: operator\n method: GET\n path: /v1/records\n expect: {{status: 200}}\n{step_extra}" + ), + ) + .unwrap(); +} + +fn fixture_project(root: &std::path::Path, extra: &str) { + project(root, extra); + fs::create_dir(root.join("tests")).unwrap(); + fs::write( + root.join("tests/journeys.yaml"), + concat!( + "apiVersion: id.registrystack.org/formats/breg/journeys/v1\n", + "kind: BRegJourneys\n", + "journeys:\n", + " - id: real-journey\n", + " steps:\n", + " - id: real-step\n", + " recordType: record\n", + " accessProfile: operator\n", + " claims: {principal: fixture-operator}\n", + " request: {type: list}\n", + " expect: {outcome: success, status: 200, count: 0}\n", + ), + ) + .unwrap(); +} + +fn check(root: &std::path::Path, state: &std::path::Path, json: bool) -> std::process::Output { + let request = serde_json::json!({ + "action": "check", "project": root, "deny_warnings": true, "json": json, + }); + Command::new(env!("CARGO_BIN_EXE_bregctl")) + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request.to_string()) + .env("RSTACK_DEV_STATE_DIR", state) + .env_remove("RSTACK_DEV_DATABASE_URL") + .output() + .unwrap() +} + +fn start(root: &std::path::Path, state: &std::path::Path) -> std::process::Output { + let request = serde_json::json!({ + "action": "start", "project": root, "json": true, + }); + Command::new(env!("CARGO_BIN_EXE_bregctl")) + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request.to_string()) + .env("RSTACK_DEV_STATE_DIR", state) + .env_remove("RSTACK_DEV_DATABASE_URL") + .output() + .unwrap() +} + +fn check_with_casework_tool( + root: &std::path::Path, + state: &std::path::Path, + tool: &std::path::Path, +) -> std::process::Output { + let request = serde_json::json!({ + "action": "check", "project": root, "deny_warnings": true, "json": true, + }); + Command::new(env!("CARGO_BIN_EXE_bregctl")) + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request.to_string()) + .env("RSTACK_DEV_STATE_DIR", state) + .env("RSTACK_CASEWORKCTL_BIN", tool) + .env_remove("RSTACK_CASEWORK_BIN") + .env_remove("RSTACK_DEV_DATABASE_URL") + .output() + .unwrap() +} + +fn start_with_casework_tool( + root: &std::path::Path, + state: &std::path::Path, + tool: &std::path::Path, +) -> std::process::Output { + let request = serde_json::json!({ + "action": "start", "project": root, "json": true, + }); + Command::new(env!("CARGO_BIN_EXE_bregctl")) + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request.to_string()) + .env("RSTACK_DEV_STATE_DIR", state) + .env("RSTACK_CASEWORKCTL_BIN", tool) + .env_remove("RSTACK_CASEWORK_BIN") + .env_remove("RSTACK_DEV_DATABASE_URL") + .output() + .unwrap() +} + +fn fake_casework_tool(root: &std::path::Path) -> std::path::PathBuf { + let tool = root.join("caseworkctl"); + fs::write( + &tool, + concat!( + "#!/usr/bin/env bash\n", + "set -euo pipefail\n", + "[[ $# -eq 4 ]]\n", + "[[ $1 == --format && $2 == json && $3 == explain ]]\n", + "[[ -f $4/casework.yaml ]]\n", + "printf '%s\\n' '", + "{\"apiVersion\":\"id.registrystack.org/formats/casework/report/v1alpha2\",", + "\"kind\":\"CaseworkReport\",\"command\":\"explain\",\"status\":\"complete\",\"ok\":true,", + "\"accessProfiles\":[{\"id\":\"reviewer\",\"principalClaim\":\"sub\"}],", + "\"reviewPolicies\":[{\"dependencies\":{\"accessProfiles\":[{\"id\":\"reviewer\",\"principalClaim\":\"sub\"}]}}],", + "\"trustedProducers\":[{\"id\":\"review-producer\"}],", + "\"contextProviders\":[{\"id\":\"proposal-context\"}]}", + "'\n", + ), + ) + .unwrap(); + let mut permissions = fs::metadata(&tool).unwrap().permissions(); + permissions.set_mode(0o700); + fs::set_permissions(&tool, permissions).unwrap(); + tool +} + +fn casework_reference_project( + root: &std::path::Path, + casework_profile: &str, + bindings: &str, +) -> std::path::PathBuf { + fs::write( + root.join("registry.yaml"), + include_str!( + "../../../products/platform/conformance/projects/breg-project-dev/registry/registry.yaml" + ), + ) + .unwrap(); + fs::create_dir(root.join("casework")).unwrap(); + fs::write(root.join("casework/casework.yaml"), "fixture\n").unwrap(); + fs::create_dir(root.join("tests")).unwrap(); + fs::write( + root.join("tests/journeys.yaml"), + concat!( + "apiVersion: id.registrystack.org/formats/breg/journeys/v1\n", + "kind: BRegJourneys\n", + "journeys:\n", + " - id: real-journey\n", + " steps:\n", + " - id: real-step\n", + " recordType: record\n", + " accessProfile: operator\n", + " claims: {principal: fixture-operator}\n", + " request: {type: list}\n", + " expect: {outcome: success, status: 200, count: 0}\n", + ), + ) + .unwrap(); + fs::write( + root.join("dev.yaml"), + format!( + "apiVersion: id.registrystack.org/formats/breg/project-dev/v1alpha1\n\ +kind: BRegProjectDev\n\ +issuer: https://issuer.example.invalid\n\ +products:\n - {{product: breg, project: ., audience: \"urn:example:registry\"}}\n - {{product: casework, project: casework, audience: \"urn:example:review\"}}\n\ +people:\n operator:\n subject: operator\n client: operator-client\n grants:\n breg: {{profile: operator, scopes: [registry:read]}}\n reviewer:\n subject: reviewer\n client: reviewer-client\n grants:\n casework: {{profile: {casework_profile}, scopes: [casework:review]}}\n\ +journey:\n id: smoke\n steps:\n - {{id: read, product: breg, as: operator, method: GET, path: /v1/records, expect: {{status: 200}}}}\n\ +{bindings}" + ), + ) + .unwrap(); + fake_casework_tool(root) +} + +fn assert_dangling_schema_assignment( + extra: &str, + path: &str, + message: &str, + suggested_action: &str, +) { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + fixture_project(scratch.path(), extra); + + let checked = check(scratch.path(), &state, true); + + assert_eq!(checked.status.code(), Some(1), "{checked:?}"); + let report: serde_json::Value = serde_json::from_slice(&checked.stdout).unwrap(); + assert_eq!(report["kind"], "BRegCtlReport"); + assert_eq!(report["command"], "dev-check"); + assert_eq!(report["status"], "domain-refusal"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["code"], "breg.project-dev.invalid-plan"); + assert_eq!(diagnostic["path"], path); + assert_eq!(diagnostic["message"], message); + assert_eq!(diagnostic["suggestedAction"], suggested_action); + assert!(!state.exists()); + + let started = start(scratch.path(), &state); + + assert_eq!(started.status.code(), Some(1), "{started:?}"); + let report: serde_json::Value = serde_json::from_slice(&started.stdout).unwrap(); + let error = report["error"].as_str().unwrap(); + assert!(error.contains(path), "{error}"); + assert!(error.contains(message), "{error}"); + assert!(error.contains(suggested_action), "{error}"); + assert!(!state.exists()); +} + +#[test] +fn check_validates_the_closed_plan_without_allocating_run_state() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + project(scratch.path(), ""); + + let result = check(scratch.path(), &state, true); + + assert!(result.status.success(), "{result:?}"); + assert!(result.stderr.is_empty(), "{result:?}"); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + assert_eq!(report["kind"], "BRegCtlReport"); + assert_eq!(report["command"], "dev-check"); + assert_eq!(report["status"], "complete"); + assert_eq!(report["ok"], true); + assert_eq!(report["diagnostics"], serde_json::json!([])); + assert!(!state.exists()); +} + +#[test] +fn check_and_start_reject_an_unknown_fixture_journey_without_allocating_state() { + assert_dangling_schema_assignment( + "schemaTestPeople:\n \"private/journey~marker\":\n real-step: operator\n", + "/schemaTestPeople/private~1journey~0marker/real-step", + "The schema test assignment names a BReg fixture journey that does not exist.", + "Set the journey key to a journey defined in tests/journeys.yaml, or remove this assignment.", + ); +} + +#[test] +fn check_and_start_reject_an_unknown_fixture_step_without_allocating_state() { + assert_dangling_schema_assignment( + "schemaTestPeople:\n real-journey:\n \"private/step~marker\": operator\n", + "/schemaTestPeople/real-journey/private~1step~0marker", + "The schema test assignment names a BReg fixture step that does not exist in the named journey.", + "Set the step key to a step defined in that fixture journey, or remove this assignment.", + ); +} + +#[test] +fn check_accepts_a_valid_explicit_fixture_assignment_without_allocating_state() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + fixture_project( + scratch.path(), + "schemaTestPeople:\n real-journey:\n real-step: operator\n", + ); + + let result = check(scratch.path(), &state, true); + + assert!(result.status.success(), "{result:?}"); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + assert_eq!(report["status"], "complete"); + assert_eq!(report["diagnostics"], serde_json::json!([])); + assert!(!state.exists()); +} + +#[test] +fn implicit_fixture_person_refusals_keep_the_intended_assignment_pointer() { + for ambiguous in [false, true] { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + fixture_project(scratch.path(), ""); + let plan = fs::read_to_string(scratch.path().join("dev.yaml")).unwrap(); + let plan = if ambiguous { + plan.replace( + "journey:\n", + " second:\n subject: second\n client: second-client\n grants:\n breg: {profile: operator, scopes: []}\njourney:\n", + ) + } else { + plan.replace("profile: operator", "profile: other") + }; + fs::write(scratch.path().join("dev.yaml"), plan).unwrap(); + + let result = check(scratch.path(), &state, true); + + assert_eq!(result.status.code(), Some(1), "{result:?}"); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + let diagnostic = &report["diagnostics"][0]; + assert_eq!( + diagnostic["path"], + "/schemaTestPeople/real-journey/real-step" + ); + assert_eq!( + diagnostic["message"], + "A BReg fixture step must resolve to exactly one test person with matching profile, scopes, and purpose." + ); + assert_eq!( + diagnostic["suggestedAction"], + "Add or update a matching BReg grant, or select the matching person in schemaTestPeople." + ); + assert!(!state.exists()); + } +} + +#[test] +fn check_and_start_report_an_explicit_assignment_with_no_fixture_file() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + project( + scratch.path(), + "schemaTestPeople:\n real-journey:\n real-step: operator\n", + ); + + let checked = check(scratch.path(), &state, true); + + assert_eq!(checked.status.code(), Some(1), "{checked:?}"); + let report: serde_json::Value = serde_json::from_slice(&checked.stdout).unwrap(); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/schemaTestPeople"); + assert_eq!( + diagnostic["message"], + "The BReg fixture journeys file is missing, unreadable, or outside the BReg product project." + ); + assert_eq!( + diagnostic["suggestedAction"], + "Create a readable tests/journeys.yaml inside the BReg product project, or remove schemaTestPeople." + ); + assert!(!state.exists()); + + let started = start(scratch.path(), &state); + + assert_eq!(started.status.code(), Some(1), "{started:?}"); + let report: serde_json::Value = serde_json::from_slice(&started.stdout).unwrap(); + let error = report["error"].as_str().unwrap(); + assert!(error.contains("/schemaTestPeople"), "{error}"); + assert!( + error.contains("fixture journeys file is missing"), + "{error}" + ); + assert!( + error.contains("Create a readable tests/journeys.yaml"), + "{error}" + ); + assert!(!state.exists()); +} + +#[test] +fn check_and_start_refuse_a_fixture_symlink_outside_the_breg_project() { + let scratch = tempfile::tempdir().unwrap(); + let sibling = tempfile::tempdir_in(scratch.path()).unwrap(); + let state = scratch.path().join("state"); + fixture_project(sibling.path(), ""); + project(scratch.path(), ""); + fs::create_dir(scratch.path().join("breg")).unwrap(); + fs::write(scratch.path().join("breg/registry.yaml"), "fixture\n").unwrap(); + let plan = fs::read_to_string(scratch.path().join("dev.yaml")) + .unwrap() + .replace("product: breg, project: .", "product: breg, project: breg"); + fs::write(scratch.path().join("dev.yaml"), plan).unwrap(); + fs::create_dir(scratch.path().join("breg/tests")).unwrap(); + std::os::unix::fs::symlink( + sibling.path().join("tests/journeys.yaml"), + scratch.path().join("breg/tests/journeys.yaml"), + ) + .unwrap(); + + let checked = check(scratch.path(), &state, true); + + assert_eq!(checked.status.code(), Some(1), "{checked:?}"); + let report: serde_json::Value = serde_json::from_slice(&checked.stdout).unwrap(); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/products/0/project"); + assert_eq!( + diagnostic["message"], + "The BReg fixture journeys file must be a readable regular file inside the BReg product project." + ); + assert_eq!( + diagnostic["suggestedAction"], + "Place tests/journeys.yaml inside the BReg product project without linking it outside that project." + ); + assert!(!state.exists()); + + let started = start(scratch.path(), &state); + + assert_eq!(started.status.code(), Some(1), "{started:?}"); + let report: serde_json::Value = serde_json::from_slice(&started.stdout).unwrap(); + let error = report["error"].as_str().unwrap(); + assert!(error.contains("/products/0/project"), "{error}"); + assert!(error.contains("fixture journeys file must be"), "{error}"); + assert!(error.contains("without linking it outside"), "{error}"); + assert!(!state.exists()); +} + +#[test] +fn check_reports_an_invalid_fixture_as_an_actionable_plan_refusal() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + project( + scratch.path(), + "schemaTestPeople:\n real-journey:\n real-step: operator\n", + ); + fs::create_dir(scratch.path().join("tests")).unwrap(); + fs::write( + scratch.path().join("tests/journeys.yaml"), + "not a fixture\n", + ) + .unwrap(); + + let result = check(scratch.path(), &state, true); + + assert_eq!(result.status.code(), Some(1), "{result:?}"); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/schemaTestPeople"); + assert_eq!( + diagnostic["message"], + "The BReg fixture journeys file is not valid." + ); + assert_eq!( + diagnostic["suggestedAction"], + "Run `rstack breg check` for authored diagnostics, then correct tests/journeys.yaml." + ); + assert!(!state.exists()); +} + +#[test] +fn check_reports_an_oversized_fixture_as_an_actionable_plan_refusal() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + project( + scratch.path(), + "schemaTestPeople:\n real-journey:\n real-step: operator\n", + ); + fs::create_dir(scratch.path().join("tests")).unwrap(); + fs::write( + scratch.path().join("tests/journeys.yaml"), + vec![b' '; 1024 * 1024 + 1], + ) + .unwrap(); + + let result = check(scratch.path(), &state, true); + + assert_eq!(result.status.code(), Some(1), "{result:?}"); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/schemaTestPeople"); + assert_eq!( + diagnostic["message"], + "The BReg fixture journeys file is unreadable or exceeds the 1 MiB limit." + ); + assert_eq!( + diagnostic["suggestedAction"], + "Make tests/journeys.yaml readable and no larger than 1 MiB, then run the check again." + ); + assert!(!state.exists()); +} + +#[test] +fn check_refuses_a_breg_project_directory_that_resolves_outside_the_project() { + let scratch = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + project(scratch.path(), ""); + fs::create_dir(outside.path().join("tests")).unwrap(); + fs::write( + outside.path().join("tests/journeys.yaml"), + include_str!("../../../products/breg/examples/minimal/tests/journeys.yaml"), + ) + .unwrap(); + std::os::unix::fs::symlink( + scratch.path().join("registry.yaml"), + outside.path().join("registry.yaml"), + ) + .unwrap(); + std::os::unix::fs::symlink(outside.path(), scratch.path().join("linked-breg")).unwrap(); + let plan = fs::read_to_string(scratch.path().join("dev.yaml")) + .unwrap() + .replace( + "product: breg, project: .", + "product: breg, project: linked-breg", + ); + fs::write(scratch.path().join("dev.yaml"), plan).unwrap(); + + let result = check(scratch.path(), &state, true); + + assert_eq!(result.status.code(), Some(1), "{result:?}"); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/products/0/project"); + assert_eq!( + diagnostic["message"], + "The BReg fixture journeys file must be a readable regular file inside the BReg product project." + ); + assert!(!state.exists()); +} + +#[test] +fn check_refuses_an_unknown_plan_field_without_allocating_run_state() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + project(scratch.path(), "unexpected: true\n"); + + let result = check(scratch.path(), &state, true); + + assert!(!result.status.success(), "{result:?}"); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + assert_eq!(report["status"], "domain-refusal"); + assert_eq!(report["diagnostics"][0]["code"], "config.unknown-key"); + assert_eq!(report["diagnostics"][0]["path"], "/unexpected"); + assert!(!state.exists()); +} + +#[test] +fn check_renders_both_locations_of_a_duplicate_key() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + project(scratch.path(), "issuer: https://second.example.invalid\n"); + + let result = check(scratch.path(), &state, false); + + assert_eq!(result.status.code(), Some(1)); + assert!(result.stdout.is_empty()); + let human = String::from_utf8(result.stderr).unwrap(); + assert!(human.starts_with("error[yaml.duplicate-key] "), "{human}"); + assert!(human.contains("dev.yaml:17:1 /issuer"), "{human}"); + assert!(human.contains("dev.yaml:4:1 /issuer"), "{human}"); + assert!( + human.contains("next: Keep one definition of the key."), + "{human}" + ); + assert!(!human.contains("second.example.invalid")); + assert!(!state.exists()); +} + +#[test] +fn check_distinguishes_unavailable_input_from_an_invalid_plan() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let result = check(scratch.path(), &state, true); + + assert_eq!(result.status.code(), Some(3)); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + assert_eq!(report["status"], "operational-failure"); + assert_eq!( + report["diagnostics"][0]["code"], + "breg.project-dev.check-unavailable" + ); + assert!(!state.exists()); +} + +#[test] +fn check_reports_an_actionable_value_free_poll_relationship() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + semantic_project( + scratch.path(), + "breg", + " poll: {maximumWaitMilliseconds: 173, pollIntervalMilliseconds: 211}\n", + ); + + let result = check(scratch.path(), &state, true); + + assert_eq!(result.status.code(), Some(1)); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["code"], "breg.project-dev.invalid-plan"); + assert_eq!( + diagnostic["path"], + "/journey/steps/0/poll/pollIntervalMilliseconds" + ); + assert_eq!( + diagnostic["message"], + "The poll interval must not exceed the maximum wait." + ); + assert_eq!( + diagnostic["suggestedAction"], + "Set pollIntervalMilliseconds to no more than maximumWaitMilliseconds." + ); + let output = String::from_utf8(result.stdout).unwrap(); + for supplied in [ + "173", + "211", + "private-audience-marker", + "private-client-marker", + ] { + assert!(!output.contains(supplied), "leaked {supplied}: {output}"); + } + assert!(!state.exists()); +} + +#[test] +fn semantic_diagnostic_escapes_authored_mapping_keys_only_in_its_pointer() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + project( + scratch.path(), + "schemaTestPeople:\n \"private/journey~marker\":\n \"private/step~marker\": private-person-marker\n", + ); + + let result = check(scratch.path(), &state, true); + + assert_eq!(result.status.code(), Some(1)); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + let diagnostic = &report["diagnostics"][0]; + assert_eq!( + diagnostic["path"], + "/schemaTestPeople/private~1journey~0marker/private~1step~0marker" + ); + assert_eq!( + diagnostic["message"], + "The schema test assignment must name a person with a BReg grant." + ); + assert_eq!( + diagnostic["suggestedAction"], + "Set this assignment to a test person that has a BReg grant." + ); + for member in ["message", "suggestedAction"] { + let text = diagnostic[member].as_str().unwrap(); + for supplied in [ + "private/journey~marker", + "private/step~marker", + "private-person-marker", + ] { + assert!(!text.contains(supplied), "leaked {supplied}: {text}"); + } + } + assert!(!state.exists()); +} + +#[test] +fn start_reports_an_actionable_value_free_unstarted_grant() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + semantic_project(scratch.path(), "casework", ""); + let request = serde_json::json!({ + "action": "start", "project": scratch.path(), "json": true, + }); + + let result = Command::new(env!("CARGO_BIN_EXE_bregctl")) + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request.to_string()) + .env("RSTACK_DEV_STATE_DIR", &state) + .env_remove("RSTACK_DEV_DATABASE_URL") + .output() + .unwrap(); + + assert_eq!(result.status.code(), Some(1)); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + let error = report["error"].as_str().unwrap(); + assert!( + error.contains("/people/operator/grants/casework"), + "{error}" + ); + assert!( + error.contains("The grant names a product that is not started."), + "{error}" + ); + assert!( + error.contains("Add the grant's product to products, or remove this grant."), + "{error}" + ); + for supplied in [ + "private-audience-marker", + "private-subject-marker", + "private-client-marker", + "private-profile-marker", + ] { + assert!(!error.contains(supplied), "leaked {supplied}: {error}"); + } + assert!(!state.exists()); +} + +#[test] +fn start_rejects_a_dangling_breg_grant_profile_before_allocating_state() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + fs::write( + scratch.path().join("registry.yaml"), + include_str!("../../../products/breg/examples/minimal/registry.yaml"), + ) + .unwrap(); + fs::create_dir(scratch.path().join("tests")).unwrap(); + fs::write( + scratch.path().join("tests/journeys.yaml"), + include_str!("../../../products/breg/examples/minimal/tests/journeys.yaml"), + ) + .unwrap(); + fs::write( + scratch.path().join("dev.yaml"), + "apiVersion: id.registrystack.org/formats/breg/project-dev/v1alpha1\n\ +kind: BRegProjectDev\n\ +issuer: https://issuer.example.invalid\n\ +products:\n - {product: breg, project: ., audience: registry}\n\ +people:\n editor:\n subject: editor\n client: editor-client\n grants:\n breg:\n profile: directory-editor\n scopes: [registry:directory:write]\n claims: {registry_purpose: directory-administration}\n unused:\n subject: unused\n client: unused-client\n grants:\n breg: {profile: private-profile-marker, scopes: []}\n\ +journey:\n id: smoke\n steps:\n - {id: read, product: breg, as: editor, method: GET, path: /v1/records, expect: {status: 200}}\n", + ) + .unwrap(); + + let checked = check(scratch.path(), &state, true); + + assert_eq!(checked.status.code(), Some(1), "{checked:?}"); + let report: serde_json::Value = serde_json::from_slice(&checked.stdout).unwrap(); + assert_eq!(report["status"], "domain-refusal"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/people/unused/grants/breg/profile"); + assert_eq!( + diagnostic["message"], + "The BReg grant profile does not name an access profile in the BReg project." + ); + assert_eq!( + diagnostic["suggestedAction"], + "Set profile to an access profile defined by the BReg project." + ); + assert!(!checked + .stdout + .windows(22) + .any(|value| value == b"private-profile-marker")); + assert!(!state.exists()); + + let result = start(scratch.path(), &state); + + assert_eq!(result.status.code(), Some(1), "{result:?}"); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + let error = report["error"].as_str().unwrap(); + assert!( + error.contains("/people/unused/grants/breg/profile"), + "{error}" + ); + assert!( + error.contains( + "The BReg grant profile does not name an access profile in the BReg project." + ), + "{error}" + ); + assert!( + error.contains("Set profile to an access profile defined by the BReg project."), + "{error}" + ); + assert!(!error.contains("private-profile-marker"), "{error}"); + assert!(!state.exists()); +} + +#[test] +fn check_and_start_resolve_casework_profiles_and_bindings_with_the_owning_report() { + for (profile, bindings, path, message, action, supplied) in [ + ( + "private-profile-marker", + "", + "/people/reviewer/grants/casework/profile", + "The Casework grant profile does not name an access profile in the Casework project.", + "Set profile to an access profile defined by the Casework project.", + "private-profile-marker", + ), + ( + "reviewer", + "bindings:\n bregReviewAuthority: {id: local-review, producer: private-producer-marker, identity: reviewer}\n", + "/bindings/bregReviewAuthority/producer", + "The BReg review producer does not name a trusted producer in the Casework project.", + "Set producer to a trusted producer defined by the Casework project.", + "private-producer-marker", + ), + ( + "reviewer", + "bindings:\n caseworkContextProvider: {id: private-provider-marker, identity: operator, action: review-action}\n", + "/bindings/caseworkContextProvider/id", + "The Casework context provider ID does not name a context provider in the Casework project.", + "Set id to a context provider defined by the Casework project.", + "private-provider-marker", + ), + ] { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let tool = casework_reference_project(scratch.path(), profile, bindings); + + let checked = check_with_casework_tool(scratch.path(), &state, &tool); + + assert_eq!(checked.status.code(), Some(1), "{checked:?}"); + let report: serde_json::Value = serde_json::from_slice(&checked.stdout).unwrap(); + assert_eq!(report["status"], "domain-refusal"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], path); + assert_eq!(diagnostic["message"], message); + assert_eq!(diagnostic["suggestedAction"], action); + assert!(!diagnostic.to_string().contains(supplied)); + assert!(!state.exists()); + + let started = start_with_casework_tool(scratch.path(), &state, &tool); + + assert_eq!(started.status.code(), Some(1), "{started:?}"); + let report: serde_json::Value = serde_json::from_slice(&started.stdout).unwrap(); + let error = report["error"].as_str().unwrap(); + assert!(error.contains(path), "{error}"); + assert!(error.contains(message), "{error}"); + assert!(error.contains(action), "{error}"); + assert!(!error.contains(supplied), "{error}"); + assert!(!state.exists()); + } +} + +#[test] +fn check_distinguishes_an_unavailable_casework_helper_from_an_invalid_project() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + let tool = casework_reference_project(scratch.path(), "reviewer", ""); + let request = serde_json::json!({ + "action": "check", "project": scratch.path(), "deny_warnings": true, "json": true, + }); + + let unavailable = Command::new(env!("CARGO_BIN_EXE_bregctl")) + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request.to_string()) + .env("RSTACK_DEV_STATE_DIR", &state) + .env_remove("RSTACK_CASEWORKCTL_BIN") + .env_remove("RSTACK_CASEWORK_BIN") + .env_remove("RSTACK_DEV_DATABASE_URL") + .output() + .unwrap(); + + assert_eq!(unavailable.status.code(), Some(3), "{unavailable:?}"); + let report: serde_json::Value = serde_json::from_slice(&unavailable.stdout).unwrap(); + assert_eq!(report["status"], "operational-failure"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/products/1/project"); + assert_eq!( + diagnostic["message"], + "The Casework project cannot be inspected because its owning tool is unavailable." + ); + assert_eq!( + diagnostic["suggestedAction"], + "Install the matching Casework tools or provide RSTACK_CASEWORKCTL_BIN, then run the check again." + ); + assert!(!state.exists()); + + fs::write( + &tool, + concat!( + "#!/usr/bin/env bash\n", + "set -euo pipefail\n", + "printf '%s\\n' '", + "{\"ok\":false,\"command\":\"explain\",\"status\":\"operational-failure\",", + "\"apiVersion\":\"id.registrystack.org/formats/casework/report/v1alpha2\",", + "\"kind\":\"CaseworkReport\",\"version\":\"0.46.1-dev\",", + "\"diagnostics\":[{\"severity\":\"error\",\"code\":\"caseworkctl.operational-failure\",", + "\"path\":\"runtime\",\"message\":\"A dependency was unavailable.\",", + "\"suggestedAction\":\"Restore the dependency, then retry.\"}]}'\n", + "exit 3\n", + ), + ) + .unwrap(); + let helper_failure = check_with_casework_tool(scratch.path(), &state, &tool); + + assert_eq!(helper_failure.status.code(), Some(3), "{helper_failure:?}"); + let report: serde_json::Value = serde_json::from_slice(&helper_failure.stdout).unwrap(); + assert_eq!(report["status"], "operational-failure"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/products/1/project"); + assert_eq!( + diagnostic["message"], + "The selected `caseworkctl` helper could not inspect the Casework project." + ); + assert!(!state.exists()); + + fs::write( + &tool, + concat!( + "#!/usr/bin/env bash\n", + "set -euo pipefail\n", + "printf '%s\\n' '", + "{\"ok\":false,\"command\":\"explain\",\"status\":\"domain-refusal\",", + "\"apiVersion\":\"id.registrystack.org/formats/casework/report/v1alpha2\",", + "\"kind\":\"CaseworkReport\",\"version\":\"0.46.1-dev\",", + "\"diagnostics\":[{\"severity\":\"error\",\"code\":\"casework.project.invalid\",", + "\"path\":\"/reviewPolicies\",\"message\":\"The authored project was refused.\",", + "\"suggestedAction\":\"Correct the Casework project.\"}]}'\n", + "exit 1\n", + ), + ) + .unwrap(); + let invalid = check_with_casework_tool(scratch.path(), &state, &tool); + + assert_eq!(invalid.status.code(), Some(1), "{invalid:?}"); + let report: serde_json::Value = serde_json::from_slice(&invalid.stdout).unwrap(); + assert_eq!(report["status"], "domain-refusal"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["path"], "/products/1/project"); + assert_eq!( + diagnostic["message"], + "The Casework project did not pass validation by its owning tool." + ); + assert_eq!( + diagnostic["suggestedAction"], + "Run `rstack casework check` for authored diagnostics, then correct the Casework project." + ); + assert!(!state.exists()); +} + +#[test] +fn a_malformed_plan_names_its_field_before_allocating_run_state() { + let scratch = tempfile::tempdir().unwrap(); + let state = scratch.path().join("state"); + fs::write( + scratch.path().join("dev.yaml"), + "apiVersion: id.registrystack.org/formats/breg/project-dev/v1alpha1\nkind: BRegProjectDev\nissuer: https://example.test\nproducts: [{product: coordinator, project: registry, audience: test}]\npeople: {}\njourney: {id: licensing, steps: []}\n", + ) + .unwrap(); + let request = serde_json::json!({ + "action": "start", "project": scratch.path(), "json": true, + }); + let result = Command::new(env!("CARGO_BIN_EXE_bregctl")) + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request.to_string()) + .env("RSTACK_DEV_STATE_DIR", &state) + .env_remove("RSTACK_DEV_DATABASE_URL") + .output() + .unwrap(); + assert!(!result.status.success()); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + assert_eq!(report["schema_version"], "registry.rstack.dev.v1"); + assert!(report.get("diagnostics").is_none()); + assert!( + report["error"] + .as_str() + .unwrap() + .contains("/products/0/product"), + "{report}" + ); + assert!(!state.exists()); +} + +#[test] +fn a_run_id_cannot_name_a_path() { + let scratch = tempfile::tempdir().unwrap(); + let request = serde_json::json!({ + "action": "stop", "run_id": "../other-run", "json": true, + }); + let result = Command::new(env!("CARGO_BIN_EXE_bregctl")) + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request.to_string()) + .env("RSTACK_DEV_STATE_DIR", scratch.path()) + .output() + .unwrap(); + assert!(!result.status.success()); + let report: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + assert!(report["error"].as_str().unwrap().contains("run ID")); + assert_eq!(fs::read_dir(scratch.path()).unwrap().count(), 0); +} + +#[test] +fn worker_restores_a_valid_plan_whose_captured_input_exceeds_512_kib() { + let scratch = tempfile::tempdir().unwrap(); + let project_root = scratch.path().join("project"); + fs::create_dir(&project_root).unwrap(); + fs::write( + project_root.join("registry.yaml"), + include_str!( + "../../../products/platform/conformance/projects/breg-project-dev/registry/registry.yaml" + ), + ) + .unwrap(); + let raw = BASE64.encode(vec![b'x'; 400 * 1024]); + let mut plan = serde_json::json!({ + "apiVersion": "id.registrystack.org/formats/breg/project-dev/v1alpha1", + "kind": "BRegProjectDev", + "issuer": "https://issuer.example.invalid", + "products": [{"product": "breg", "project": ".", "audience": "registry"}], + "people": {"operator": { + "subject": "operator", "client": "operator-client", + "grants": {"breg": {"profile": "operator", "scopes": []}} + }}, + "journey": {"id": "smoke", "steps": [{ + "id": "send-raw", "product": "breg", "as": "operator", "method": "POST", + "path": "/v1/records", + "raw": {"contentType": "application/octet-stream", "base64": raw}, + "expect": {"status": 200} + }]} + }); + let document = serde_json::to_vec(&plan).unwrap(); + assert!( + document.len() <= registry_platform_yaml::MAXIMUM_DOCUMENT_BYTES, + "the regression fixture must remain a valid-size development plan" + ); + fs::write(project_root.join("dev.yaml"), &document).unwrap(); + + let checked = check(&project_root, &scratch.path().join("check-state"), true); + assert!( + checked.status.success(), + "the substantial development plan was refused" + ); + + let members = plan.as_object_mut().unwrap(); + members.remove("apiVersion"); + members.remove("kind"); + let run_id = "0123456789abcdef0123456789abcdef"; + let state = scratch.path().join("state"); + let root = state.join(run_id); + fs::create_dir_all(&root).unwrap(); + fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap(); + let root = root.canonicalize().unwrap(); + let false_binary = ["/usr/bin/false", "/bin/false"] + .into_iter() + .map(std::path::Path::new) + .find(|path| path.is_file()) + .unwrap() + .canonicalize() + .unwrap(); + let input = serde_json::to_vec(&serde_json::json!({ + "run_id": run_id, + "project": project_root, + "plan": plan, + "ttl_seconds": 1, + "schema_people": [], + "breg_authority": { + "principal": "sub", "purpose": false, "requires_postgis": false, + "named_clients": [] + }, + "casework_principals": {}, + "binaries": {"breg": {"tool": false_binary, "runtime": false_binary}} + })) + .unwrap(); + assert!(input.len() > 512 * 1024, "captured input: {}", input.len()); + fs::write(root.join("input.json"), input).unwrap(); + + let result = Command::new(env!("CARGO_BIN_EXE_bregctl")) + .args(["project-dev", "--worker"]) + .arg(&root) + .env("RSTACK_DEV_STATE_DIR", &state) + .env_remove("RSTACK_DEV_DATABASE_URL") + .output() + .unwrap(); + let reply: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + let error = reply["error"].as_str().unwrap(); + assert!(error.contains("breg check failed"), "{error}"); + assert!(!error.contains("cannot restore the captured development plan")); +} diff --git a/crates/registry-bregctl/tests/tool_schema.rs b/crates/registry-bregctl/tests/tool_schema.rs index b4802bebfa..88c7637111 100644 --- a/crates/registry-bregctl/tests/tool_schema.rs +++ b/crates/registry-bregctl/tests/tool_schema.rs @@ -31,6 +31,10 @@ const EXAMPLES: &[(&str, &str)] = &[ "dev-clients.v1alpha1.schema.json", "products/breg/acceptance/facility/dev-clients.yaml", ), + ( + "project-dev.v1alpha1.schema.json", + "products/platform/examples/licensing/dev.yaml", + ), ( "example-scenarios.v1alpha1.schema.json", "products/breg/examples/minimal/examples/scenarios.json", @@ -83,6 +87,92 @@ fn every_committed_example_satisfies_its_generated_schema() { } } +fn project_dev_with_studio_identifiers() -> Value { + let mut plan = example("products/platform/examples/licensing/dev.yaml"); + let applicant = plan["people"] + .as_object_mut() + .expect("people is a mapping") + .remove("applicant") + .expect("the licensing example has an applicant"); + plan["people"]["date_of_birth"] = applicant; + plan["people"]["date_of_birth"]["grants"]["breg"]["profile"] = json!("date_of_birth"); + plan["journey"]["id"] = json!("review_flow"); + plan["journey"]["steps"][0]["id"] = json!("read.v1"); + plan["journey"]["steps"][0]["as"] = json!("date_of_birth"); + plan["journey"]["steps"][0]["capture"] = + json!({"draftEtag": {"type": "pointer", "pointer": "/etag"}}); + plan["schemaTestPeople"] = json!({ + "review_flow": {"read.v1": "date_of_birth"} + }); + plan +} + +#[test] +fn project_dev_schema_accepts_the_readers_local_derived_and_capture_identifiers() { + let documents = documents().expect("the tool schemas generate"); + let schema = compile(&documents["project-dev.v1alpha1.schema.json"]); + + assert!(schema.is_valid(&project_dev_with_studio_identifiers())); +} + +#[test] +fn project_dev_schema_refuses_identifiers_the_reader_refuses() { + let documents = documents().expect("the tool schemas generate"); + let schema = compile(&documents["project-dev.v1alpha1.schema.json"]); + let valid = project_dev_with_studio_identifiers(); + + for step in ["-read", "1read", ".read", "read.", "read..v1", "read.-v1"] { + let mut changed = valid.clone(); + changed["journey"]["steps"][0]["id"] = json!(step); + assert!(!schema.is_valid(&changed), "step identifier: {step}"); + } + for capture in ["1draft", "draft_etag", "draft.etag"] { + let mut changed = valid.clone(); + let value = changed["journey"]["steps"][0]["capture"] + .as_object_mut() + .unwrap() + .remove("draftEtag") + .unwrap(); + changed["journey"]["steps"][0]["capture"][capture] = value; + assert!(!schema.is_valid(&changed), "capture identifier: {capture}"); + } + for person in ["-person", "1person", "Person", "person.name"] { + let mut changed = valid.clone(); + let value = changed["people"] + .as_object_mut() + .unwrap() + .remove("date_of_birth") + .unwrap(); + changed["people"][person] = value; + assert!(!schema.is_valid(&changed), "person identifier: {person}"); + } + + let mut bad_step_reference = valid.clone(); + bad_step_reference["schemaTestPeople"]["review_flow"] = json!({"read..v1": "date_of_birth"}); + assert!(!schema.is_valid(&bad_step_reference)); + + let mut bad_journey_reference = valid.clone(); + let people = bad_journey_reference["schemaTestPeople"] + .as_object_mut() + .unwrap() + .remove("review_flow") + .unwrap(); + bad_journey_reference["schemaTestPeople"]["review.flow"] = people; + assert!(!schema.is_valid(&bad_journey_reference)); + + let mut bad_person_reference = valid.clone(); + bad_person_reference["schemaTestPeople"]["review_flow"] = json!({"read.v1": "date.of.birth"}); + assert!(!schema.is_valid(&bad_person_reference)); + + let mut bad_person_alias = valid.clone(); + bad_person_alias["journey"]["steps"][0]["as"] = json!("date.of.birth"); + assert!(!schema.is_valid(&bad_person_alias)); + + let mut bad_journey_id = valid; + bad_journey_id["journey"]["id"] = json!("review.flow"); + assert!(!schema.is_valid(&bad_journey_id)); +} + #[test] fn example_scenario_schema_uses_bounded_display_text() { let documents = documents().expect("the tool schemas generate"); @@ -401,13 +491,14 @@ fn a_wrong_header_a_missing_header_and_an_explicit_null_are_refused() { } /// The identifiers the catalog records, in file-name order. -const IDENTIFIERS: [&str; 7] = [ +const IDENTIFIERS: [&str; 8] = [ "https://id.registrystack.org/schemas/breg/backup-binding/backup-binding.v1alpha1.schema.json", "https://id.registrystack.org/schemas/breg/dev-clients/dev-clients.v1alpha1.schema.json", "https://id.registrystack.org/schemas/breg/example-scenarios/example-scenarios.v1alpha1.schema.json", "https://id.registrystack.org/schemas/breg/journeys/journeys.v1.schema.json", "https://id.registrystack.org/schemas/breg/migration-descriptor/migration-descriptor.v1alpha1.schema.json", "https://id.registrystack.org/schemas/breg/model-selection/model-selection.v1alpha1.schema.json", + "https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json", "https://id.registrystack.org/schemas/breg/schema-test-credentials/schema-test-credentials.v1.schema.json", ]; diff --git a/crates/registry-caseworkctl/src/project.rs b/crates/registry-caseworkctl/src/project.rs index 9ab977f186..e5076aea17 100644 --- a/crates/registry-caseworkctl/src/project.rs +++ b/crates/registry-caseworkctl/src/project.rs @@ -778,8 +778,17 @@ pub(super) fn explain(project: &Path) -> Result { "refusals": ["untrusted producer", "policy binding mismatch", "concealed source", "changed subject or contract", "stale task revision", "ineligible reviewer"], })) }).collect::,registry_casework_core::ReviewPolicyError>>()?; + let mut access_profiles = policy + .access_profiles + .iter() + .map(|profile| crate::report::ExplainAccessProfile { + id: profile.id.clone(), + principal_claim: profile.principal_claim.clone(), + }) + .collect::>(); + access_profiles.sort_by(|a, b| a.id.cmp(&b.id)); Ok( - json!({"ok":true,"command":"explain","project":policy.project,"reviewPolicies":reviews,"contextProviders":policy.context_providers,"clocks":policy.clocks,"calendars":policy.calendars,"trustedProducers":policy.trusted_producers,"staffing":{"next":"due ordered eligible visible tasks","holdings":"claimed or assigned tasks under current visibility","caseload":"atomic preview and revision-fenced apply","routing":"once per task or stage from compiled minimized fields","clockRecompute":"pinned policy closure"},"authority":"producer, human reviewer and downstream executor authenticate independently"}), + json!({"ok":true,"command":"explain","project":policy.project,"accessProfiles":access_profiles,"reviewPolicies":reviews,"contextProviders":policy.context_providers,"clocks":policy.clocks,"calendars":policy.calendars,"trustedProducers":policy.trusted_producers,"staffing":{"next":"due ordered eligible visible tasks","holdings":"claimed or assigned tasks under current visibility","caseload":"atomic preview and revision-fenced apply","routing":"once per task or stage from compiled minimized fields","clockRecompute":"pinned policy closure"},"authority":"producer, human reviewer and downstream executor authenticate independently"}), ) } pub(super) fn test(project: &Path, deny_warnings: bool) -> Result { diff --git a/crates/registry-caseworkctl/src/report.rs b/crates/registry-caseworkctl/src/report.rs index 4ed3641f9c..601805f6d2 100644 --- a/crates/registry-caseworkctl/src/report.rs +++ b/crates/registry-caseworkctl/src/report.rs @@ -345,8 +345,14 @@ payload!(ExplainStaffing { routing: String, clock_recompute: String }); +payload!(ExplainAccessProfile { + #[cfg_attr(feature = "schema", schemars(with = "registry_platform_yaml::LocalId"))] + id: String, + principal_claim: String +}); payload!(ExplainData { project: ProjectIdentity, + access_profiles: Vec, review_policies: Vec, context_providers: Vec, clocks: Vec, diff --git a/crates/registry-caseworkctl/tests/report_examples.rs b/crates/registry-caseworkctl/tests/report_examples.rs index c79bc7a381..4550d71f56 100644 --- a/crates/registry-caseworkctl/tests/report_examples.rs +++ b/crates/registry-caseworkctl/tests/report_examples.rs @@ -205,3 +205,18 @@ fn stage_outcomes_example_passes_offline_authoring_commands() { assert_eq!(report["status"], "complete"); } } + +#[test] +fn explain_lists_every_access_profile_at_the_top_level() { + let root = repo_root(); + let (report, _) = output(&root, &["explain", STANDALONE], 0); + assert_eq!( + report["accessProfiles"], + serde_json::json!([ + {"id": "administrator", "principalClaim": "sub"}, + {"id": "requester", "principalClaim": "sub"}, + {"id": "staff", "principalClaim": "sub"}, + {"id": "supervisor", "principalClaim": "sub"} + ]) + ); +} diff --git a/crates/registry-cli-docs/catalog.json b/crates/registry-cli-docs/catalog.json index 34fbd4c741..8002f85a12 100644 --- a/crates/registry-cli-docs/catalog.json +++ b/crates/registry-cli-docs/catalog.json @@ -25860,6 +25860,510 @@ "subcommands": [] } ] + }, + { + "name": "dev", + "invocation": "rstack dev", + "about": "Run and inspect a local Registry Stack development environment", + "long_about": null, + "usage": "rstack dev [OPTIONS] ", + "arguments": [], + "options": [ + { + "display": "--format ", + "description": "Report encoding. Defaults to human even when stdout is piped. Evidence test and fixtures run also accept junit", + "always_required": false, + "repeatable": true, + "default_values": [ + "human" + ], + "possible_values": [ + "human", + "json" + ], + "environment": null + }, + { + "display": "--install-root ", + "description": "Installation generation storage directory", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--bin-dir ", + "description": "Directory for the single public rstack executable link", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--asset-dir ", + "description": "Directory of already downloaded, verified release assets", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "-h, --help", + "description": "Print help", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "constraints": [], + "subcommands": [ + { + "name": "check", + "invocation": "rstack dev check", + "about": "Validate a development plan without starting services", + "long_about": null, + "usage": "rstack dev check [OPTIONS] ", + "arguments": [ + { + "display": "", + "description": "Project directory containing dev.yaml", + "always_required": true, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "options": [ + { + "display": "--deny-warnings", + "description": "Refuse an unaccepted warning", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--json", + "description": "Emit the check report as JSON", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--format ", + "description": "Report encoding. Defaults to human even when stdout is piped. Evidence test and fixtures run also accept junit", + "always_required": false, + "repeatable": true, + "default_values": [ + "human" + ], + "possible_values": [ + "human", + "json" + ], + "environment": null + }, + { + "display": "--install-root ", + "description": "Installation generation storage directory", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--bin-dir ", + "description": "Directory for the single public rstack executable link", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--asset-dir ", + "description": "Directory of already downloaded, verified release assets", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "-h, --help", + "description": "Print help", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "constraints": [], + "subcommands": [] + }, + { + "name": "start", + "invocation": "rstack dev start", + "about": "Start a development environment for a registry project", + "long_about": null, + "usage": "rstack dev start [OPTIONS] ", + "arguments": [ + { + "display": "", + "description": "Project directory containing dev.yaml", + "always_required": true, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "options": [ + { + "display": "--database-url ", + "description": "TLS PostgreSQL administrator URL. Defaults to RSTACK_DEV_DATABASE_URL", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--ttl ", + "description": "Lifetime after readiness, from 1s to 24h. Defaults to 30m", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--json", + "description": "Emit the runner's JSON report", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--format ", + "description": "Report encoding. Defaults to human even when stdout is piped. Evidence test and fixtures run also accept junit", + "always_required": false, + "repeatable": true, + "default_values": [ + "human" + ], + "possible_values": [ + "human", + "json" + ], + "environment": null + }, + { + "display": "--install-root ", + "description": "Installation generation storage directory", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--bin-dir ", + "description": "Directory for the single public rstack executable link", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--asset-dir ", + "description": "Directory of already downloaded, verified release assets", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "-h, --help", + "description": "Print help", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "constraints": [], + "subcommands": [] + }, + { + "name": "status", + "invocation": "rstack dev status", + "about": "Report the state of a development environment", + "long_about": null, + "usage": "rstack dev status [OPTIONS] ", + "arguments": [ + { + "display": "", + "description": "Run ID returned by rstack dev start", + "always_required": true, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "options": [ + { + "display": "--json", + "description": "Emit the runner's JSON report", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--format ", + "description": "Report encoding. Defaults to human even when stdout is piped. Evidence test and fixtures run also accept junit", + "always_required": false, + "repeatable": true, + "default_values": [ + "human" + ], + "possible_values": [ + "human", + "json" + ], + "environment": null + }, + { + "display": "--install-root ", + "description": "Installation generation storage directory", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--bin-dir ", + "description": "Directory for the single public rstack executable link", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--asset-dir ", + "description": "Directory of already downloaded, verified release assets", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "-h, --help", + "description": "Print help", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "constraints": [], + "subcommands": [] + }, + { + "name": "token", + "invocation": "rstack dev token", + "about": "Issue a development token for one named synthetic person", + "long_about": null, + "usage": "rstack dev token [OPTIONS] --as ", + "arguments": [ + { + "display": "", + "description": "Run ID returned by rstack dev start", + "always_required": true, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "options": [ + { + "display": "--as ", + "description": "Synthetic person configured by the project", + "always_required": true, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--json", + "description": "Emit the runner's JSON report", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--format ", + "description": "Report encoding. Defaults to human even when stdout is piped. Evidence test and fixtures run also accept junit", + "always_required": false, + "repeatable": true, + "default_values": [ + "human" + ], + "possible_values": [ + "human", + "json" + ], + "environment": null + }, + { + "display": "--install-root ", + "description": "Installation generation storage directory", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--bin-dir ", + "description": "Directory for the single public rstack executable link", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--asset-dir ", + "description": "Directory of already downloaded, verified release assets", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "-h, --help", + "description": "Print help", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "constraints": [], + "subcommands": [] + }, + { + "name": "stop", + "invocation": "rstack dev stop", + "about": "Stop a development environment", + "long_about": null, + "usage": "rstack dev stop [OPTIONS] ", + "arguments": [ + { + "display": "", + "description": "Run ID returned by rstack dev start", + "always_required": true, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "options": [ + { + "display": "--json", + "description": "Emit the runner's JSON report", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--format ", + "description": "Report encoding. Defaults to human even when stdout is piped. Evidence test and fixtures run also accept junit", + "always_required": false, + "repeatable": true, + "default_values": [ + "human" + ], + "possible_values": [ + "human", + "json" + ], + "environment": null + }, + { + "display": "--install-root ", + "description": "Installation generation storage directory", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--bin-dir ", + "description": "Directory for the single public rstack executable link", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--asset-dir ", + "description": "Directory of already downloaded, verified release assets", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "-h, --help", + "description": "Print help", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "constraints": [], + "subcommands": [] + } + ] } ] }, diff --git a/crates/registry-rstack/Cargo.toml b/crates/registry-rstack/Cargo.toml index 2c427270fb..6ee9fdebfa 100644 --- a/crates/registry-rstack/Cargo.toml +++ b/crates/registry-rstack/Cargo.toml @@ -22,6 +22,7 @@ hex.workspace = true registry-platform-buildinfo.workspace = true registry-platform-canonical-json.workspace = true registry-platform-config.workspace = true +registry-platform-yaml.workspace = true registry-thunderid-tooling.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/crates/registry-rstack/README.md b/crates/registry-rstack/README.md index a4034dd569..2c471ce436 100644 --- a/crates/registry-rstack/README.md +++ b/crates/registry-rstack/README.md @@ -100,6 +100,12 @@ from the products' own configuration and one operator deployment file, and `rstack identity check FILE` checks that file or a product's identity facts offline. See [Generate ThunderID configuration](../../docs/site/src/content/docs/operate/advanced/generate-identity-configuration.mdx). +## Develop a composed project + +Run BReg and Casework together with `rstack dev start PROJECT` against a +dedicated TLS PostgreSQL server. The project's `dev.yaml` declares the +products, test people and journey. See [Run a project for development](../../products/platform/PROJECT-DEV.md). + ## Command migration | Previous local command | Managed command | diff --git a/crates/registry-rstack/src/cli.rs b/crates/registry-rstack/src/cli.rs index ba3f526af0..cda7a57406 100644 --- a/crates/registry-rstack/src/cli.rs +++ b/crates/registry-rstack/src/cli.rs @@ -5,6 +5,7 @@ use crate::install::{self, BootstrapOptions}; use crate::store::{GenerationStore, InvocationLease}; use crate::ReleaseManifest; use clap::{ArgAction, CommandFactory, Parser, Subcommand, ValueEnum}; +use serde::Serialize; use serde_json::{json, Value}; use std::ffi::OsString; use std::fs; @@ -153,10 +154,105 @@ enum Command { #[command(subcommand)] command: IdentityCommand, }, + /// Run and inspect a local Registry Stack development environment. + Dev { + #[command(subcommand)] + command: DevCommand, + }, #[command(external_subcommand)] Product(Vec), } +#[derive(Subcommand)] +enum DevCommand { + /// Validate a development plan without starting services. + Check { + /// Project directory containing dev.yaml. + #[arg(value_name = "PROJECT")] + project: PathBuf, + /// Refuse an unaccepted warning. + #[arg(long)] + deny_warnings: bool, + /// Emit the check report as JSON. + #[arg(long)] + json: bool, + }, + /// Start a development environment for a registry project. + Start { + /// Project directory containing dev.yaml. + #[arg(value_name = "PROJECT")] + project: PathBuf, + /// TLS PostgreSQL administrator URL. Defaults to RSTACK_DEV_DATABASE_URL. + #[arg(long, value_name = "URL")] + database_url: Option, + /// Lifetime after readiness, from 1s to 24h. Defaults to 30m. + #[arg(long, value_name = "DURATION")] + ttl: Option, + /// Emit the runner's JSON report. + #[arg(long)] + json: bool, + }, + /// Report the state of a development environment. + Status { + /// Run ID returned by rstack dev start. + #[arg(value_name = "RUN_ID")] + run_id: String, + /// Emit the runner's JSON report. + #[arg(long)] + json: bool, + }, + /// Issue a development token for one named synthetic person. + Token { + /// Run ID returned by rstack dev start. + #[arg(value_name = "RUN_ID")] + run_id: String, + /// Synthetic person configured by the project. + #[arg(long = "as", value_name = "NAME")] + person: String, + /// Emit the runner's JSON report. + #[arg(long)] + json: bool, + }, + /// Stop a development environment. + Stop { + /// Run ID returned by rstack dev start. + #[arg(value_name = "RUN_ID")] + run_id: String, + /// Emit the runner's JSON report. + #[arg(long)] + json: bool, + }, +} + +#[derive(Serialize)] +#[serde(tag = "action", rename_all = "snake_case")] +enum DevRequest<'a> { + Check { + project: &'a std::path::Path, + deny_warnings: bool, + json: bool, + }, + Start { + project: &'a std::path::Path, + #[serde(skip_serializing_if = "Option::is_none")] + ttl: Option<&'a str>, + json: bool, + }, + Status { + run_id: &'a str, + json: bool, + }, + Token { + run_id: &'a str, + person: &'a str, + json: bool, + }, + Stop { + run_id: &'a str, + json: bool, + }, +} + #[derive(Subcommand)] enum IdentityCommand { /// Generate identity-provider configuration from product identity facts. @@ -288,6 +384,7 @@ pub fn run() -> i32 { let arguments: Vec = std::env::args_os().collect(); let (duplicate_format, json_requested, format_supplied, requested_format) = report_arguments(&arguments); + let json_requested = json_requested || dev_json_requested(&arguments); if duplicate_format { return emit_failure(2, "--format may be supplied only once", json_requested); } @@ -357,6 +454,16 @@ fn report_arguments(arguments: &[OsString]) -> (bool, bool, bool, Option 1, json, occurrences > 0, requested) } +/// Whether a `dev` subcommand carries its own `--json` before any `--`. +fn dev_json_requested(arguments: &[OsString]) -> bool { + command_tokens(arguments).first() == Some(&"dev") + && arguments + .iter() + .skip(1) + .take_while(|argument| *argument != "--") + .any(|argument| argument == "--json") +} + fn requested_report_format(value: &OsString) -> RequestedReportFormat { match value.to_str() { Some("human") => RequestedReportFormat::Human, @@ -548,6 +655,7 @@ fn execute(cli: Cli, format_supplied: bool, evidence_junit: bool) -> Result<(), ReportFormat::Human => { reportln!("rstack [options]"); reportln!("Use rstack help for offline product help."); + reportln!("Use rstack dev --help for local development commands."); reportln!("Use rstack components list to inspect installation availability."); reportln!("Products: {}", PRODUCTS.join(", ")); } @@ -691,6 +799,7 @@ fn execute(cli: Cli, format_supplied: bool, evidence_junit: bool) -> Result<(), PRODUCTS.join(" ") ); } + Some(Command::Dev { command }) => dispatch_dev(&cli, command, format)?, Some(Command::Identity { command }) => match command { IdentityCommand::Generate { target: IdentityTarget::Thunderid(arguments), @@ -1239,6 +1348,219 @@ fn first_path_rstack() -> Option { None } +/// What a development action needs from the installed components. Only +/// `start` runs services; Casework is needed only when the plan lists it. +#[derive(Default)] +struct Needs { + services: bool, + casework: bool, +} + +impl Needs { + fn components(&self) -> Vec<&'static str> { + if self.casework { + vec!["breg", "casework"] + } else { + vec!["breg"] + } + } + + fn variables(&self) -> Vec<&'static str> { + let mut variables = vec!["RSTACK_BREGCTL_BIN"]; + if self.services { + variables.push("RSTACK_BREG_BIN"); + } + if self.services && self.casework { + variables.push("RSTACK_CASEWORKCTL_BIN"); + variables.push("RSTACK_CASEWORK_BIN"); + } + variables + } +} + +/// Whether the project's dev.yaml lists a casework product under +/// `/products/*/product`. A missing, unreadable, oversized or unparsable plan, +/// or one without a products list, counts as using Casework, so the launcher +/// never skips a binary the helper might need; the helper reports the real +/// plan error. +fn plan_uses_casework(project: &std::path::Path) -> bool { + use registry_platform_yaml::{NodeValue, Reader, MAXIMUM_DOCUMENT_BYTES}; + use std::io::Read; + let limit = MAXIMUM_DOCUMENT_BYTES as u64; + let file = project.join("dev.yaml"); + let mut bytes = Vec::new(); + let read = std::fs::File::open(&file) + .and_then(|handle| handle.take(limit + 1).read_to_end(&mut bytes)); + if read.is_err() || bytes.len() as u64 > limit { + return true; + } + let Ok(Some(root)) = Reader::new(file.display().to_string()).scan(&bytes) else { + return true; + }; + let Some(NodeValue::Sequence(products)) = root.pointer("/products").map(|node| &node.value) + else { + return true; + }; + products.iter().any(|entry| { + !matches!( + entry.pointer("/product").map(|node| &node.value), + Some(NodeValue::String(text)) if text.text != "casework" + ) + }) +} + +fn dispatch_dev(cli: &Cli, action: &DevCommand, format: ReportFormat) -> Result<(), CliFailure> { + let launcher_json = matches!(format, ReportFormat::Json); + let (request, database_url, needs) = match action { + DevCommand::Check { + project, + deny_warnings, + json, + } => ( + DevRequest::Check { + project, + deny_warnings: *deny_warnings, + json: *json || launcher_json, + }, + None, + Needs::default(), + ), + DevCommand::Start { + project, + database_url, + ttl, + json, + } => ( + DevRequest::Start { + project, + ttl: ttl.as_deref(), + json: *json || launcher_json, + }, + database_url.as_deref(), + Needs { + services: true, + casework: plan_uses_casework(project), + }, + ), + DevCommand::Status { run_id, json } => ( + DevRequest::Status { + run_id, + json: *json || launcher_json, + }, + None, + Needs::default(), + ), + DevCommand::Token { + run_id, + person, + json, + } => ( + DevRequest::Token { + run_id, + person, + json: *json || launcher_json, + }, + None, + Needs::default(), + ), + DevCommand::Stop { run_id, json } => ( + DevRequest::Stop { + run_id, + json: *json || launcher_json, + }, + None, + Needs::default(), + ), + }; + let request = serde_json::to_string(&request).map_err(|error| { + CliFailure::operational(format!("cannot encode development request: {error}")) + })?; + + let selected = selected_state(cli)?; + let (mut command, lease) = if let Some((lease, selection, manifest)) = selected { + for component in needs.components() { + if !selection.components.iter().any(|name| name == component) { + return Err(CliFailure::domain(format!( + "component {component} is missing; run rstack install {component}" + ))); + } + } + let executable = lease.path().join("components/breg/bregctl/bregctl"); + if !executable.is_file() { + return Err(CliFailure::operational( + "selected BReg helper is missing".into(), + )); + } + let mut command = ProcessCommand::new(executable); + set_selected_generation_environment(&mut command, lease.path(), &selection, &manifest)?; + let required_variables = needs.variables(); + let missing: Vec<_> = required_variables + .iter() + .filter(|variable| { + !command.get_envs().any(|(name, value)| { + name == std::ffi::OsStr::new(**variable) && value.is_some() + }) + }) + .copied() + .collect(); + if !missing.is_empty() { + return Err(CliFailure::operational(format!( + "selected release lacks development binaries for {}", + missing.join(", ") + ))); + } + (command, Some(lease)) + } else { + let variables = needs.variables(); + let mut paths = Vec::with_capacity(variables.len()); + let mut missing = Vec::new(); + for variable in variables { + let path = std::env::var_os(variable) + .filter(|value| !value.is_empty()) + .map(PathBuf::from); + match path { + Some(path) if path.is_file() => paths.push((variable, path)), + _ => missing.push(variable), + } + } + if !missing.is_empty() { + return Err(CliFailure::domain(format!( + "without a managed installation, run rstack install {}, or set existing source binary overrides: {}", + needs + .components() + .into_iter() + .filter(|component| { + missing + .iter() + .any(|variable| variable.starts_with(&format!("RSTACK_{}", component.to_ascii_uppercase()))) + }) + .collect::>() + .join(" "), + missing.join(", ") + ))); + } + let mut command = ProcessCommand::new(&paths[0].1); + for (variable, path) in paths { + command.env(variable, path); + } + (command, None) + }; + + command + .arg("project-dev") + .env("RSTACK_PROJECT_DEV_REQUEST", request); + if let Some(database_url) = database_url { + command.env("RSTACK_DEV_DATABASE_URL", database_url); + } + if let Some(lease) = &lease { + inherit_invocation_lease(lease)?; + } + Err(CliFailure::operational(format!( + "cannot launch development helper: {}", + command.exec() + ))) +} + fn dispatch_product( cli: &Cli, product: &str, @@ -1312,6 +1634,14 @@ fn dispatch_product( } else { command.args(arguments); } + inherit_invocation_lease(&lease)?; + Err(CliFailure::operational(format!( + "cannot launch {product}: {}", + command.exec() + ))) +} + +fn inherit_invocation_lease(lease: &InvocationLease) -> Result<(), CliFailure> { let mut flags = rustix::io::fcntl_getfd(lease.lease_file()).map_err(|error| { CliFailure::operational(format!("cannot inspect invocation lease: {error}")) })?; @@ -1319,10 +1649,7 @@ fn dispatch_product( rustix::io::fcntl_setfd(lease.lease_file(), flags).map_err(|error| { CliFailure::operational(format!("cannot inherit invocation lease: {error}")) })?; - Err(CliFailure::operational(format!( - "cannot launch {product}: {}", - command.exec() - ))) + Ok(()) } /// Put a launcher report format before the first native Evidence runtime token diff --git a/crates/registry-rstack/tests/dev.rs b/crates/registry-rstack/tests/dev.rs new file mode 100644 index 0000000000..360a1b3a8f --- /dev/null +++ b/crates/registry-rstack/tests/dev.rs @@ -0,0 +1,651 @@ +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +const CURRENT_VERSION: &str = env!("CARGO_PKG_VERSION"); + +fn scratch() -> tempfile::TempDir { + if let Ok(target) = std::env::var("CARGO_TARGET_DIR") { + tempfile::tempdir_in(target).unwrap() + } else { + tempfile::tempdir().unwrap() + } +} + +fn executable(path: &Path, body: &str) { + fs::write(path, body).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); +} + +fn digest(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) +} + +fn platform() -> &'static str { + match (std::env::consts::OS, std::env::consts::ARCH) { + ("linux", "x86_64") => "linux-amd64", + ("linux", "aarch64") => "linux-arm64", + ("macos", "aarch64") => "macos-arm64", + other => panic!("unsupported test platform: {other:?}"), + } +} + +fn release_asset(directory: &Path, binary: &str, body: &str, fips: bool) -> (String, Vec) { + let release = format!("v{CURRENT_VERSION}"); + let platform = platform(); + let archived = platform == "macos-arm64"; + let asset_name = format!( + "{binary}-{release}-{platform}{}", + if archived { ".tar.gz" } else { "" } + ); + let asset = directory.join(&asset_name); + if archived { + let contents = directory.join(format!("contents-{binary}")); + fs::create_dir(&contents).unwrap(); + let archived_binary = asset_name.trim_end_matches(".tar.gz"); + executable(&contents.join(archived_binary), body); + fs::write(contents.join("THIRD_PARTY_NOTICES"), "fixture\n").unwrap(); + let mut entries = vec![archived_binary, "THIRD_PARTY_NOTICES"]; + if fips { + fs::write(contents.join("libaws_lc_fips_test.dylib"), "fixture\n").unwrap(); + entries.push("libaws_lc_fips_test.dylib"); + } + let output = Command::new("tar") + .arg("-czf") + .arg(&asset) + .arg("-C") + .arg(&contents) + .args(entries) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + } else { + executable(&asset, body); + } + let bytes = fs::read(asset).unwrap(); + (asset_name, bytes) +} + +fn versioned_tool(binary: &str, body: &str) -> String { + format!( + "#!/usr/bin/env bash\nset -euo pipefail\nif [[ \"${{1:-}}\" == '--version' ]]; then printf '{binary} {CURRENT_VERSION}\\n'; exit 0; fi\n{body}" + ) +} + +struct ManagedInstall { + _scratch: tempfile::TempDir, + root: PathBuf, + bin: PathBuf, + project: PathBuf, +} + +fn managed_install() -> ManagedInstall { + let scratch = scratch(); + let assets = scratch.path().join("assets"); + fs::create_dir(&assets).unwrap(); + let executable = env!("CARGO_BIN_EXE_rstack").replace('\'', "'\"'\"'"); + let launcher_body = format!( + "#!/usr/bin/env bash\nset -euo pipefail\nif [[ \"${{1:-}}\" == '--version' ]]; then printf 'rstack {CURRENT_VERSION}\\n'; exit 0; fi\nif [[ -n \"${{REGISTRY_CARGO_RUNTIME_LIBRARY_PATH:-}}\" ]]; then export DYLD_FALLBACK_LIBRARY_PATH=\"$REGISTRY_CARGO_RUNTIME_LIBRARY_PATH\"; fi\nexec '{executable}' \"$@\"\n" + ); + let helper_body = versioned_tool( + "bregctl", + "printf 'arg:%s\\n' \"$@\"\nprintf 'request:%s\\n' \"${RSTACK_PROJECT_DEV_REQUEST:?}\"\n[[ \"${RSTACK_DEV_DATABASE_URL:-}\" == \"${EXPECTED_DATABASE_URL:?}\" ]]\nprintf 'database:matched\\n'\nprintf 'generation:%s\\n' \"${RSTACK_GENERATION_DIR:?}\"\nprintf 'release:%s\\n' \"${RSTACK_RELEASE_TAG:?}\"\nprintf 'bregctl:%s\\n' \"${RSTACK_BREGCTL_BIN:?}\"\nprintf 'breg:%s\\n' \"${RSTACK_BREG_BIN:?}\"\nprintf 'caseworkctl:%s\\n' \"${RSTACK_CASEWORKCTL_BIN:?}\"\nprintf 'casework:%s\\n' \"${RSTACK_CASEWORK_BIN:?}\"\n", + ); + let (launcher_name, launcher_bytes) = release_asset(&assets, "rstack", &launcher_body, false); + let (bregctl_name, bregctl_bytes) = release_asset(&assets, "bregctl", &helper_body, true); + let (breg_name, breg_bytes) = + release_asset(&assets, "breg", &versioned_tool("breg", "exit 0\n"), true); + let (caseworkctl_name, caseworkctl_bytes) = release_asset( + &assets, + "caseworkctl", + &versioned_tool("caseworkctl", "exit 0\n"), + true, + ); + let (casework_name, casework_bytes) = release_asset( + &assets, + "casework", + &versioned_tool("casework", "exit 0\n"), + true, + ); + let release = format!("v{CURRENT_VERSION}"); + let platform = platform(); + let payload = |name: &str, bytes: &[u8]| { + serde_json::json!({ + "name":name, + "kind":"binary", + "size":bytes.len(), + "sha256":digest(bytes) + }) + }; + let component_asset = |name: &str, bytes: &[u8]| serde_json::json!({"name":name,"size":bytes.len(),"sha256":digest(bytes)}); + let manifest_name = format!("registry-stack-{release}-release-manifest.json"); + let manifest_path = assets.join(&manifest_name); + let manifest = serde_json::json!({ + "schema_version":"registry-stack.release-candidate.v2", + "repository":"registrystack/registry-stack", + "release":{"tag":release.as_str(),"version":CURRENT_VERSION,"source_sha":"a".repeat(40),"release_id":"test"}, + "payloads":[ + payload(&launcher_name, &launcher_bytes), + payload(&bregctl_name, &bregctl_bytes), + payload(&breg_name, &breg_bytes), + payload(&caseworkctl_name, &caseworkctl_bytes), + payload(&casework_name, &casework_bytes) + ], + "components":{ + "schema_version":"registry-stack.components.v1", + "release":release.as_str(), + "launcher":{(platform):component_asset(&launcher_name, &launcher_bytes)}, + "components":{ + "breg":{ + "entrypoint":"bregctl", + "dependencies":[], + "platforms":{(platform):{ + "assets":[ + component_asset(&bregctl_name, &bregctl_bytes), + component_asset(&breg_name, &breg_bytes) + ], + "capabilities":{"tooling":true,"runtime":true} + }} + }, + "casework":{ + "entrypoint":"caseworkctl", + "dependencies":["breg"], + "platforms":{(platform):{ + "assets":[ + component_asset(&caseworkctl_name, &caseworkctl_bytes), + component_asset(&casework_name, &casework_bytes) + ], + "capabilities":{"tooling":true,"runtime":true} + }} + } + } + } + }); + let manifest_bytes = serde_json::to_vec(&manifest).unwrap(); + fs::write(&manifest_path, &manifest_bytes).unwrap(); + let checksums = assets.join("SHA256SUMS"); + let mut checksum_text = String::new(); + for (name, bytes) in [ + (&launcher_name, &launcher_bytes), + (&bregctl_name, &bregctl_bytes), + (&breg_name, &breg_bytes), + (&caseworkctl_name, &caseworkctl_bytes), + (&casework_name, &casework_bytes), + ] { + checksum_text.push_str(&format!("{} {name}\n", digest(bytes))); + } + checksum_text.push_str(&format!("{} {manifest_name}\n", digest(&manifest_bytes))); + fs::write(&checksums, checksum_text).unwrap(); + + let root = scratch.path().join("lib"); + let bin = scratch.path().join("bin"); + let installed = Command::new(env!("CARGO_BIN_EXE_rstack")) + .args(["bootstrap", "--release-manifest"]) + .arg(&manifest_path) + .arg("--checksum-file") + .arg(&checksums) + .arg("--launcher-asset") + .arg(assets.join(&launcher_name)) + .arg("--asset-dir") + .arg(&assets) + .arg("--install-root") + .arg(&root) + .arg("--bin-dir") + .arg(&bin) + .args(["breg", "casework"]) + .output() + .unwrap(); + assert!(installed.status.success(), "{installed:?}"); + let project = scratch.path().join("project"); + fs::create_dir(&project).unwrap(); + ManagedInstall { + _scratch: scratch, + root, + bin, + project, + } +} + +struct SourceTools { + _scratch: tempfile::TempDir, + project: PathBuf, + install_root: PathBuf, + bin_dir: PathBuf, + bregctl: PathBuf, + breg: PathBuf, + caseworkctl: PathBuf, + casework: PathBuf, +} + +impl SourceTools { + fn new() -> Self { + let scratch = scratch(); + let project = scratch.path().join("registry project"); + fs::create_dir(&project).unwrap(); + let install_root = scratch.path().join("uninstalled"); + let bin_dir = scratch.path().join("bin"); + let bregctl = scratch.path().join("bregctl"); + executable( + &bregctl, + "#!/usr/bin/env bash\nset -euo pipefail\nprintf 'arg:%s\\n' \"$@\"\nprintf 'request:%s\\n' \"${RSTACK_PROJECT_DEV_REQUEST:?}\"\nif [[ -n \"${EXPECTED_DATABASE_URL:-}\" ]]; then\n [[ \"${RSTACK_DEV_DATABASE_URL:-}\" == \"$EXPECTED_DATABASE_URL\" ]]\n printf 'database:matched\\n'\nelse\n printf 'database:%s\\n' \"${RSTACK_DEV_DATABASE_URL:+set}\"\nfi\nprintf 'bregctl:%s\\n' \"${RSTACK_BREGCTL_BIN:-}\"\nprintf 'breg:%s\\n' \"${RSTACK_BREG_BIN:-}\"\nprintf 'caseworkctl:%s\\n' \"${RSTACK_CASEWORKCTL_BIN:-}\"\nprintf 'casework:%s\\n' \"${RSTACK_CASEWORK_BIN:-}\"\n", + ); + let breg = scratch.path().join("breg"); + let caseworkctl = scratch.path().join("caseworkctl"); + let casework = scratch.path().join("casework"); + for path in [&breg, &caseworkctl, &casework] { + executable(path, "#!/usr/bin/env bash\nset -euo pipefail\nexit 0\n"); + } + Self { + _scratch: scratch, + project, + install_root, + bin_dir, + bregctl, + breg, + caseworkctl, + casework, + } + } + + fn empty_command(&self) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_rstack")); + command + .args(["--install-root", self.install_root.to_str().unwrap()]) + .args(["--bin-dir", self.bin_dir.to_str().unwrap()]) + .env_remove("RSTACK_DEV_DATABASE_URL") + .env_remove("RSTACK_BREGCTL_BIN") + .env_remove("RSTACK_BREG_BIN") + .env_remove("RSTACK_CASEWORKCTL_BIN") + .env_remove("RSTACK_CASEWORK_BIN"); + command + } + + fn command(&self) -> Command { + let mut command = self.empty_command(); + command + .env("RSTACK_BREGCTL_BIN", &self.bregctl) + .env("RSTACK_BREG_BIN", &self.breg) + .env("RSTACK_CASEWORKCTL_BIN", &self.caseworkctl) + .env("RSTACK_CASEWORK_BIN", &self.casework); + command + } +} + +fn request(output: &Output) -> Value { + let stdout = String::from_utf8(output.stdout.clone()).unwrap(); + let request = stdout + .lines() + .find_map(|line| line.strip_prefix("request:")) + .unwrap_or_else(|| panic!("missing request in {stdout:?}")); + serde_json::from_str(request).unwrap() +} + +fn output_field<'a>(output: &'a str, name: &str) -> &'a str { + output + .lines() + .find_map(|line| line.strip_prefix(&format!("{name}:"))) + .unwrap_or_else(|| panic!("missing {name} in {output}")) +} + +#[test] +fn managed_start_uses_one_release_generation_on_supported_platforms() { + let install = managed_install(); + let database_url = "postgresql://example.invalid/managed-secret"; + let output = Command::new(install.bin.join("rstack")) + .arg("--install-root") + .arg(&install.root) + .arg("--bin-dir") + .arg(&install.bin) + .args(["dev", "start"]) + .arg(&install.project) + .args(["--database-url", database_url]) + .env("EXPECTED_DATABASE_URL", database_url) + .output() + .unwrap(); + + assert!(output.status.success(), "{output:?}"); + assert!(output.stderr.is_empty(), "{output:?}"); + let stdout = String::from_utf8(output.stdout.clone()).unwrap(); + assert!(!stdout.contains(database_url), "{stdout}"); + assert_eq!( + stdout + .lines() + .filter(|line| line.starts_with("arg:")) + .collect::>(), + ["arg:project-dev"] + ); + assert_eq!( + request(&output), + serde_json::json!({ + "action":"start", + "project":install.project, + "json":false + }) + ); + let generation = PathBuf::from(output_field(&stdout, "generation")); + assert_eq!( + output_field(&stdout, "release"), + format!("v{CURRENT_VERSION}") + ); + for (name, relative) in [ + ("bregctl", "components/breg/bregctl/bregctl"), + ("breg", "components/breg/breg/breg"), + ("caseworkctl", "components/casework/caseworkctl/caseworkctl"), + ("casework", "components/casework/casework/casework"), + ] { + assert_eq!( + PathBuf::from(output_field(&stdout, name)), + generation.join(relative) + ); + } +} + +#[test] +fn source_start_sends_one_json_request_and_binary_set() { + let tools = SourceTools::new(); + let database_url = "postgresql://example.invalid/dev-secret"; + let output = tools + .command() + .args(["dev", "start"]) + .arg(&tools.project) + .args(["--ttl", "45m", "--database-url", database_url, "--json"]) + .env("EXPECTED_DATABASE_URL", database_url) + .output() + .unwrap(); + + assert!(output.status.success(), "{output:?}"); + assert!(output.stderr.is_empty(), "{output:?}"); + let stdout = String::from_utf8(output.stdout.clone()).unwrap(); + assert_eq!( + stdout + .lines() + .filter(|line| line.starts_with("arg:")) + .collect::>(), + ["arg:project-dev"] + ); + assert_eq!( + request(&output), + serde_json::json!({ + "action":"start", + "project":tools.project, + "ttl":"45m", + "json":true + }) + ); + assert!(stdout.contains("database:matched\n")); + assert!(stdout.contains(&format!("bregctl:{}\n", tools.bregctl.display()))); + assert!(stdout.contains(&format!("breg:{}\n", tools.breg.display()))); + assert!(stdout.contains(&format!("caseworkctl:{}\n", tools.caseworkctl.display()))); + assert!(stdout.contains(&format!("casework:{}\n", tools.casework.display()))); + assert!(!stdout + .lines() + .any(|line| line == format!("arg:{database_url}"))); + assert!(!request(&output).to_string().contains(database_url)); + assert!(!stdout.contains(database_url)); +} + +#[test] +fn dev_actions_encode_the_public_contract_and_launcher_json_format() { + let tools = SourceTools::new(); + for (arguments, expected) in [ + ( + vec![ + "dev", + "check", + tools.project.to_str().unwrap(), + "--deny-warnings", + "--json", + ], + serde_json::json!({ + "action":"check", + "project":tools.project, + "deny_warnings":true, + "json":true + }), + ), + ( + vec!["--format", "json", "dev", "status", "run-1"], + serde_json::json!({"action":"status","run_id":"run-1","json":true}), + ), + ( + vec!["dev", "token", "run-2", "--as", "reviewer", "--json"], + serde_json::json!({ + "action":"token", + "run_id":"run-2", + "person":"reviewer", + "json":true + }), + ), + ( + vec!["dev", "stop", "run-3"], + serde_json::json!({"action":"stop","run_id":"run-3","json":false}), + ), + ( + vec!["dev", "stop", "run-4", "--json"], + serde_json::json!({"action":"stop","run_id":"run-4","json":true}), + ), + ] { + let output = tools + .empty_command() + .args(arguments) + .env("RSTACK_BREGCTL_BIN", &tools.bregctl) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + assert_eq!(request(&output), expected); + } +} + +#[test] +fn start_requires_the_complete_source_binary_set_without_an_installation() { + let tools = SourceTools::new(); + let output = tools + .empty_command() + .args(["dev", "start"]) + .arg(&tools.project) + .env("RSTACK_BREGCTL_BIN", &tools.bregctl) + .output() + .unwrap(); + + assert_eq!(output.status.code(), Some(1), "{output:?}"); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(stderr.contains("RSTACK_BREG_BIN"), "{stderr}"); + assert!(stderr.contains("RSTACK_CASEWORKCTL_BIN"), "{stderr}"); + assert!(stderr.contains("RSTACK_CASEWORK_BIN"), "{stderr}"); +} + +#[test] +fn root_and_dev_help_need_no_installation_or_binary_overrides() { + let tools = SourceTools::new(); + for (arguments, expected) in [ + (&["--help"][..], &["dev"][..]), + ( + &["dev", "--help"][..], + &["check", "start", "status", "token", "stop"][..], + ), + ( + &["dev", "check", "--help"][..], + &["PROJECT", "--deny-warnings", "--json"][..], + ), + ( + &["dev", "start", "--help"][..], + &["--database-url", "--ttl", "--json"][..], + ), + ] { + let output = tools.empty_command().args(arguments).output().unwrap(); + assert!(output.status.success(), "{arguments:?}: {output:?}"); + assert!(output.stderr.is_empty(), "{arguments:?}: {output:?}"); + let stdout = String::from_utf8(output.stdout).unwrap(); + for text in expected { + assert!(stdout.contains(text), "{arguments:?}: {stdout}"); + } + } +} + +fn write_plan(project: &Path, products: &[&str]) { + let entries: Vec = products + .iter() + .map(|product| { + format!(" - {{product: {product}, project: {product}, audience: \"urn:example:{product}\"}}\n") + }) + .collect(); + fs::write( + project.join("dev.yaml"), + format!( + "issuer: https://issuer.example\nproducts:\n{}", + entries.concat() + ), + ) + .unwrap(); +} + +#[test] +fn dev_subcommand_json_makes_launcher_failures_json() { + let tools = SourceTools::new(); + let output = tools + .empty_command() + .args(["dev", "start", "/missing", "--json"]) + .output() + .unwrap(); + + assert_eq!(output.status.code(), Some(1), "{output:?}"); + assert!(output.stderr.is_empty(), "{output:?}"); + let report: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(report["schema_version"], "registry.rstack.error.v1"); + assert_eq!(report["status"], "refused"); + assert_eq!(report["error"]["class"], "domain"); + + let usage = tools + .empty_command() + .args(["dev", "token", "run-1", "--json"]) + .output() + .unwrap(); + assert_eq!(usage.status.code(), Some(2), "{usage:?}"); + let report: Value = serde_json::from_slice(&usage.stdout).unwrap(); + assert_eq!(report["schema_version"], "registry.rstack.error.v1"); + assert_eq!(report["error"]["class"], "usage"); + + let human = tools + .empty_command() + .args(["dev", "start", "/missing"]) + .output() + .unwrap(); + assert_eq!(human.status.code(), Some(1), "{human:?}"); + assert!(human.stdout.is_empty(), "{human:?}"); + assert!(String::from_utf8(human.stderr) + .unwrap() + .starts_with("rstack: ")); +} + +#[test] +fn breg_only_plan_does_not_require_casework_binaries() { + let tools = SourceTools::new(); + write_plan(&tools.project, &["breg"]); + let output = tools + .empty_command() + .args(["dev", "start"]) + .arg(&tools.project) + .env("RSTACK_BREGCTL_BIN", &tools.bregctl) + .env("RSTACK_BREG_BIN", &tools.breg) + .output() + .unwrap(); + + assert!(output.status.success(), "{output:?}"); + let stdout = String::from_utf8(output.stdout.clone()).unwrap(); + assert_eq!(output_field(&stdout, "caseworkctl"), ""); + assert_eq!(output_field(&stdout, "casework"), ""); +} + +#[test] +fn casework_plan_passes_casework_binaries_and_names_missing_installs() { + let tools = SourceTools::new(); + write_plan(&tools.project, &["breg", "casework"]); + let output = tools + .command() + .args(["dev", "start"]) + .arg(&tools.project) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + let stdout = String::from_utf8(output.stdout).unwrap(); + assert_eq!( + output_field(&stdout, "casework"), + tools.casework.to_str().unwrap() + ); + + let missing = tools + .empty_command() + .args(["dev", "start"]) + .arg(&tools.project) + .env("RSTACK_BREGCTL_BIN", &tools.bregctl) + .env("RSTACK_BREG_BIN", &tools.breg) + .output() + .unwrap(); + assert_eq!(missing.status.code(), Some(1), "{missing:?}"); + let stderr = String::from_utf8(missing.stderr).unwrap(); + assert!(stderr.contains("rstack install casework"), "{stderr}"); + assert!(!stderr.contains("rstack install breg"), "{stderr}"); + assert!(stderr.contains("RSTACK_CASEWORKCTL_BIN"), "{stderr}"); +} + +#[test] +fn missing_breg_names_the_install_command_before_the_overrides() { + let tools = SourceTools::new(); + write_plan(&tools.project, &["breg", "casework"]); + let output = tools + .empty_command() + .args(["dev", "start"]) + .arg(&tools.project) + .output() + .unwrap(); + + assert_eq!(output.status.code(), Some(1), "{output:?}"); + let stderr = String::from_utf8(output.stderr).unwrap(); + let install = stderr + .find("rstack install breg casework") + .unwrap_or_else(|| panic!("{stderr}")); + let overrides = stderr + .find("RSTACK_BREGCTL_BIN") + .unwrap_or_else(|| panic!("{stderr}")); + assert!(install < overrides, "{stderr}"); +} + +#[test] +fn casework_text_outside_the_products_list_does_not_require_casework() { + let tools = SourceTools::new(); + fs::write( + tools.project.join("dev.yaml"), + "# product: casework\nissuer: https://issuer.example\nproducts:\n - {product: breg, project: breg, audience: \"urn:example:breg\"}\njourney:\n steps:\n - name: \"product: casework\"\n", + ) + .unwrap(); + let output = tools + .empty_command() + .args(["dev", "start"]) + .arg(&tools.project) + .env("RSTACK_BREGCTL_BIN", &tools.bregctl) + .env("RSTACK_BREG_BIN", &tools.breg) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); +} + +#[test] +fn unparsable_plan_still_requires_casework() { + let tools = SourceTools::new(); + fs::write(tools.project.join("dev.yaml"), "products: [\n").unwrap(); + let output = tools + .empty_command() + .args(["dev", "start"]) + .arg(&tools.project) + .env("RSTACK_BREGCTL_BIN", &tools.bregctl) + .env("RSTACK_BREG_BIN", &tools.breg) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(1), "{output:?}"); + assert!(String::from_utf8(output.stderr) + .unwrap() + .contains("rstack install casework")); +} diff --git a/docs/site/scripts/generate-breg-configuration.mjs b/docs/site/scripts/generate-breg-configuration.mjs index ab2a744945..ebf775c385 100644 --- a/docs/site/scripts/generate-breg-configuration.mjs +++ b/docs/site/scripts/generate-breg-configuration.mjs @@ -25,8 +25,14 @@ export const CONTRACTS = [ // Files `bregctl` reads beside a project: adopter tooling, published by the // `tool-schema` example under products/breg/generated/tools. Listed in the -// order an adopter meets them, which is the order the reference page shows. +// order an adopter meets them; the generated inventory preserves this order. export const TOOL_CONTRACTS = [ + { + id: 'project-dev', + title: 'dev.yaml', + file: 'tools/project-dev.v1alpha1.schema.json', + reference: 'reference/breg-configuration.mdx', + }, { id: 'dev-clients', title: 'dev-clients.yaml', diff --git a/docs/site/scripts/generate-breg-configuration.test.mjs b/docs/site/scripts/generate-breg-configuration.test.mjs index 1ca4f42cdb..e1d5ceca29 100644 --- a/docs/site/scripts/generate-breg-configuration.test.mjs +++ b/docs/site/scripts/generate-breg-configuration.test.mjs @@ -59,8 +59,8 @@ test('BReg covers every committed tool schema, in the order an adopter meets the const root = new URL('../../../products/breg/generated/tools/', import.meta.url); assert.deepEqual((await readdir(root)).map((name) => `tools/${name}`).sort(), TOOL_CONTRACTS.map((entry) => entry.file.split('/generated/')[1]).sort()); - assert.deepEqual(TOOL_CONTRACTS.map((entry) => entry.id), ['dev-clients', 'example-scenarios', 'journeys', - 'schema-test-credentials', 'migration-descriptor', 'backup-binding', 'model-selection']); + assert.deepEqual(TOOL_CONTRACTS.map((entry) => entry.id), ['project-dev', 'dev-clients', 'example-scenarios', + 'journeys', 'schema-test-credentials', 'migration-descriptor', 'backup-binding', 'model-selection']); }); test('BReg tool and service references read the committed schemas they name', async () => { @@ -71,6 +71,10 @@ test('BReg tool and service references read the committed schemas they name', as for (const entry of [...TOOL_CONTRACTS, ...SERVICE_CONTRACTS]) { const schema = JSON.parse(await readFile(new URL(entry.file.split('/generated/')[1], root), 'utf8')); assert.ok(typeof schema.title === 'string' && schema.title.length > 0, entry.file); + if (entry.id === 'project-dev') { + assert.equal(schema.$id, + 'https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json'); + } } for (const entry of [...document.tools, ...document.services]) { assert.ok(['tooling', 'beta'].includes(entry.status), entry.id); @@ -79,6 +83,11 @@ test('BReg tool and service references read the committed schemas they name', as const fields = (group, id) => new Map(document[group].find((entry) => entry.id === id) .fields.map((field) => [field.key_path, field])); + const projectDev = fields('tools', 'project-dev'); + assert.deepEqual(projectDev.get('kind').values, ['BRegProjectDev']); + assert.ok(projectDev.has('products[].product')); + assert.ok(projectDev.has('journey.steps[].expect.status')); + const descriptor = fields('tools', 'migration-descriptor'); assert.deepEqual(descriptor.get('steps[].type').values, ['chunked-backfill', 'field-encryption-backfill', 'transactional-sql']); diff --git a/editors/configure.py b/editors/configure.py index dd0c63242f..3dd2c7fb26 100644 --- a/editors/configure.py +++ b/editors/configure.py @@ -60,6 +60,7 @@ ), ("products/breg/generated/tools/backup-binding.v1alpha1.schema.json", "*-binding.json"), ("products/breg/generated/tools/dev-clients.v1alpha1.schema.json", "dev-clients.yaml"), + ("products/breg/generated/tools/project-dev.v1alpha1.schema.json", "dev.yaml"), ("products/breg/generated/tools/example-scenarios.v1alpha1.schema.json", "examples/scenarios.json"), ( "products/breg/generated/tools/migration-descriptor.v1alpha1.schema.json", diff --git a/editors/tests/test_configure.py b/editors/tests/test_configure.py index 062ecc4fc8..7944b07897 100644 --- a/editors/tests/test_configure.py +++ b/editors/tests/test_configure.py @@ -97,9 +97,11 @@ def test_breg_schema_snapshots_tasks_and_idempotent_merge(self): settings = json.loads(settings_path.read_text()) self.assertEqual(settings["editor.tabSize"], 4) schemas = settings["yaml.schemas"] - self.assertEqual(len(schemas), 8) + self.assertEqual(len(schemas), 9) dev_clients_schema = (project / ".registry-stack-editor/schemas/dev-clients.v1alpha1.schema.json").as_uri() self.assertEqual(schemas[dev_clients_schema], [str(project / "dev-clients.yaml")]) + project_dev_schema = (project / ".registry-stack-editor/schemas/project-dev.v1alpha1.schema.json").as_uri() + self.assertEqual(schemas[project_dev_schema], [str(project / "dev.yaml")]) project_schema = (project / ".registry-stack-editor/schemas/registry-project.schema.json").as_uri() self.assertEqual(schemas[project_schema], [str(project / "registry.yaml")]) zed = json.loads((self.workspace / ".zed/settings.json").read_text()) @@ -381,7 +383,7 @@ def test_two_products_share_workspace_without_schema_or_task_collision(self): configure.configure("casework", casework, self.workspace, None) configure.configure("breg", breg, self.workspace, None) settings = json.loads(settings_path.read_text()) - self.assertEqual(len(settings["yaml.schemas"]), 15) + self.assertEqual(len(settings["yaml.schemas"]), 16) breg_dev_clients = (breg / ".registry-stack-editor/schemas/dev-clients.v1alpha1.schema.json").as_uri() self.assertEqual(settings["yaml.schemas"][breg_dev_clients], [str(breg / "dev-clients.yaml")]) self.assertEqual(settings["yaml.schemas"][unrelated_uri], [str(self.workspace / "unrelated/registry.yaml")]) diff --git a/products/breg/CHANGELOG.md b/products/breg/CHANGELOG.md index 0efb10562f..a6b6783269 100644 --- a/products/breg/CHANGELOG.md +++ b/products/breg/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +- `bregctl project-dev` runs BReg projects, including spatial projects, from a + `breg/project-dev` v1alpha1 plan. It is an alpha format and may change. + `rstack dev` is its entry point. + ## 0.46.0 - 2026-11-08 - A `record_timer.worker.iteration_failed` warning carries a closed `error` diff --git a/products/breg/SECURITY-MODEL.md b/products/breg/SECURITY-MODEL.md index 63e29e8dfd..13500fc101 100644 --- a/products/breg/SECURITY-MODEL.md +++ b/products/breg/SECURITY-MODEL.md @@ -461,6 +461,12 @@ mitigation, and related matrix IDs. - `database.testOnlyPlaintext` exists only in test builds; a release binary refuses it with `breg.runtime.plaintext-database`. +`rstack dev` can instead activate BReg as part of an isolated project run. +It uses the normal package and database activation checks, distinct TLS +PostgreSQL credentials, and a loopback listener. The runtime trusts the +run's static signing key under the declared issuer name. See +[Run isolation](../platform/PROJECT-DEV.md#run-isolation). + ## 5. Reporting Report vulnerabilities privately as described in [SECURITY.md](../../SECURITY.md). diff --git a/products/breg/acceptance/reviewed-evidence-journey/dev.yaml b/products/breg/acceptance/reviewed-evidence-journey/dev.yaml new file mode 100644 index 0000000000..520452630c --- /dev/null +++ b/products/breg/acceptance/reviewed-evidence-journey/dev.yaml @@ -0,0 +1,180 @@ +# yaml-language-server: $schema=https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json +apiVersion: id.registrystack.org/formats/breg/project-dev/v1alpha1 +kind: BRegProjectDev +issuer: https://issuer.example.test +products: + - {product: breg, project: registry, audience: "urn:journey:breg"} + - {product: casework, project: casework, audience: "urn:journey:casework"} +bindings: + bregReviewAuthority: {id: casework, producer: registry, identity: producer} + caseworkContextProvider: {id: registry, identity: context-status, action: correct-status} +people: + operator: + subject: generic-registry-operator + client: operator + grants: + breg: + profile: operator + scopes: ["registry:generic:operate"] + claims: {registry_actor_kind: human, registry_purpose: registry-operations} + source: + subject: evidence-source + client: source + grants: + breg: + profile: evidence-source + scopes: ["registry:evidence:lookup"] + claims: {registry_actor_kind: service, registry_purpose: evidence-source-read} + applier: + subject: applier + client: applier + grants: + breg: + profile: applier + scopes: ["registry:apply"] + claims: {registry_actor_kind: human, registry_purpose: registry-operations} + reviewer: + subject: staff + client: staff + caseworkContext: true + grants: + breg: + profile: reviewer + scopes: ["registry:review", "casework:staff"] + claims: {registry_actor_kind: human, registry_purpose: registry-operations} + casework: + profile: staff + scopes: ["registry:review", "casework:staff"] + claims: {registry_actor_kind: human, registry_purpose: registry-operations} + administrator: + subject: administrator + client: administrator + grants: + casework: + profile: administrator + scopes: ["casework:admin"] + claims: {registry_actor_kind: human} + producer: + subject: producer + client: producer + grants: + casework: + profile: requester + scopes: ["casework:request"] + claims: {registry_actor_kind: service} + context-status: + subject: context-status + client: context-status + grants: + breg: + profile: review-status + scopes: ["registry:review-status"] + claims: {registry_actor_kind: service, registry_purpose: registry-operations} +journey: + id: native-review-handoff + steps: + - id: staff-queue + product: casework + as: administrator + method: POST + path: /v1/directory/bootstrap + headers: {If-Match: '"0"', Idempotency-Key: staff-queue} + body: + teamId: reviewers + staff: [{issuer: https://issuer.example.test, subject: staff}] + supervisors: [] + queueId: reviews + expect: {status: 200} + - id: create-record + product: breg + as: operator + method: POST + path: /v1/records/records + headers: {Idempotency-Key: create-record} + body: + data: {code: DEV-PROOF, registrationNumber: DEV-REG, status: retired, label: Synthetic record} + expect: {status: 201} + capture: {record: {type: pointer, pointer: /data/recordIdentifier}} + - id: start-proposal + product: breg + as: operator + method: POST + path: /v1/actions/correct-status/proposals + headers: {Idempotency-Key: start-proposal} + body: {input: {subjectId: "{{record}}", status: active}} + expect: {status: 201} + capture: + proposal: {type: pointer, pointer: /id} + draftEtag: {type: header, header: etag} + - id: submit-proposal + product: breg + as: operator + method: POST + path: "/v1/proposals/{{proposal}}/submit" + headers: {If-Match: "{{draftEtag}}", Idempotency-Key: submit-proposal} + body: {} + expect: {status: 200, json: {/state: submitted}} + capture: + submissionVersion: {type: pointer, pointer: /submissionVersion} + effectDigest: {type: pointer, pointer: /effectDigest} + - id: find-task + product: casework + as: reviewer + method: GET + path: /v1/review-tasks?queue=reviews + headers: {Registry-Source-Profile: reviewer} + expect: {status: 200} + poll: {maximumWaitMilliseconds: 60000, pollIntervalMilliseconds: 500} + capture: + task: {type: find, find: /items, where: {pointer: /subject/id, equals: "{{proposal}}"}, pointer: /taskId} + request: {type: find, find: /items, where: {pointer: /subject/id, equals: "{{proposal}}"}, pointer: /requestId} + taskRevision: {type: find, find: /items, where: {pointer: /subject/id, equals: "{{proposal}}"}, pointer: /revision} + - id: read-context-from-breg + product: casework + as: reviewer + method: GET + path: "/v1/review-tasks/{{task}}/context" + headers: {Registry-Source-Profile: reviewer} + expect: + status: 200 + json: + /context/projection/display/currentStatus: retired + /context/projection/display/proposedStatus: active + - id: claim-task + product: casework + as: reviewer + method: POST + path: "/v1/review-tasks/{{task}}/claim" + headers: {If-Match: '"{{taskRevision}}"', Idempotency-Key: claim-task, Registry-Source-Profile: reviewer} + expect: {status: 200} + capture: {claimRevision: {type: pointer, pointer: /revision}} + - id: approve-correction + product: casework + as: reviewer + method: POST + path: "/v1/review-tasks/{{task}}/decisions" + headers: {If-Match: '"{{claimRevision}}"', Idempotency-Key: approve-correction, Registry-Source-Profile: reviewer} + body: {decision: {type: approve, outcome: approved}} + expect: {status: 204} + - id: read-review-result + product: casework + as: producer + method: GET + path: "/v1/review-requests/{{request}}/result" + expect: {status: 200, json: {/subject/id: "{{proposal}}", /status: approved}} + - id: await-source-decision + product: breg + as: applier + method: GET + path: "/v1/proposals/{{proposal}}" + expect: {status: 200, json: {/decision/status: approved}} + poll: {maximumWaitMilliseconds: 60000, pollIntervalMilliseconds: 500} + capture: {approvalEtag: {type: header, header: etag}} + - id: apply-correction + product: breg + as: applier + method: POST + path: "/v1/proposals/{{proposal}}/apply" + headers: {If-Match: "{{approvalEtag}}", Idempotency-Key: apply-correction} + body: {submissionVersion: "{{submissionVersion}}", effectDigest: "{{effectDigest}}"} + expect: {status: 200, json: {/state: applied}} diff --git a/products/breg/generated/tools/project-dev.v1alpha1.schema.json b/products/breg/generated/tools/project-dev.v1alpha1.schema.json new file mode 100644 index 0000000000..028109f595 --- /dev/null +++ b/products/breg/generated/tools/project-dev.v1alpha1.schema.json @@ -0,0 +1,644 @@ +{ + "$defs": { + "Bindings": { + "additionalProperties": false, + "description": "Run-local connections between BReg and Casework. Configuring either one requires both products, with BReg listed before Casework.", + "properties": { + "bregReviewAuthority": { + "$ref": "#/$defs/BregReviewAuthority", + "description": "Connects BReg to Casework as a review authority. BReg authenticates to Casework as a machine identity." + }, + "caseworkContextProvider": { + "$ref": "#/$defs/CaseworkContextProvider", + "description": "Connects Casework to BReg as a context provider. Casework authenticates to BReg as a machine identity." + } + }, + "type": "object" + }, + "BregReviewAuthority": { + "additionalProperties": false, + "description": "A BReg review authority that the run connects to Casework.", + "properties": { + "id": { + "$ref": "#/$defs/LocalId", + "description": "Local identifier of the review authority." + }, + "identity": { + "$ref": "#/$defs/LocalId", + "description": "Name of the test person, in `people`, whose Casework grant BReg uses to authenticate. That grant needs at least one scope." + }, + "producer": { + "$ref": "#/$defs/LocalId", + "description": "Local identifier of the review producer." + } + }, + "required": [ + "id", + "producer", + "identity" + ], + "type": "object" + }, + "Capture": { + "description": "A value remembered from a response, selected by `type`.", + "oneOf": [ + { + "additionalProperties": false, + "description": "Remember the value at a JSON pointer in the response body.", + "properties": { + "pointer": { + "description": "JSON pointer into the response body: empty for the whole body, otherwise starting with `/`.", + "type": "string" + }, + "type": { + "const": "pointer", + "type": "string" + } + }, + "required": [ + "type", + "pointer" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Remember a response header.", + "properties": { + "header": { + "$ref": "#/$defs/CapturedHeader", + "description": "The response header to remember." + }, + "type": { + "const": "header", + "type": "string" + } + }, + "required": [ + "type", + "header" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Remember a value from the one array item that matches a condition. The step fails if no item or more than one item matches.", + "properties": { + "find": { + "description": "JSON pointer to the array in the response body to search.", + "type": "string" + }, + "pointer": { + "description": "JSON pointer, relative to the matching item, of the value to remember.", + "type": "string" + }, + "type": { + "const": "find", + "type": "string" + }, + "where": { + "$ref": "#/$defs/FindWhere", + "description": "The condition an item must meet." + } + }, + "required": [ + "type", + "find", + "where", + "pointer" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Remember the value at a JSON pointer, optionally reading one query parameter from a URL found there.", + "properties": { + "pointer": { + "description": "JSON pointer into the response body: empty for the whole body, otherwise starting with `/`.", + "type": "string" + }, + "query": { + "description": "Name of a query parameter. When set, the value at the pointer must be a URL or relative reference, and the parameter's value is remembered instead. At most 256 characters without whitespace or control characters.", + "type": "string" + }, + "type": { + "const": "value", + "type": "string" + } + }, + "required": [ + "type", + "pointer" + ], + "type": "object" + } + ] + }, + "CapturedHeader": { + "const": "etag", + "description": "The entity tag, usable as an `If-Match` value in a later step.", + "type": "string" + }, + "CaseworkContextProvider": { + "additionalProperties": false, + "description": "A Casework context provider that the run connects to BReg.", + "properties": { + "action": { + "$ref": "#/$defs/LocalId", + "description": "Local identifier of the BReg action that the provider pairs with." + }, + "id": { + "$ref": "#/$defs/LocalId", + "description": "Local identifier of the context provider." + }, + "identity": { + "$ref": "#/$defs/LocalId", + "description": "Name of the test person, in `people`, whose BReg grant Casework uses to authenticate. That grant needs at least one scope." + } + }, + "required": [ + "id", + "identity", + "action" + ], + "type": "object" + }, + "Check": { + "additionalProperties": false, + "description": "An assertion about one value in the response body. Set exactly one of `equals`, `length` or `contains`.", + "properties": { + "contains": { + "description": "The array at the pointer must hold this item, the string must contain this text, or the object must have this key. String values may contain `{{name}}` references to earlier captures." + }, + "equals": { + "description": "The value at the pointer must equal this. String values may contain `{{name}}` references to earlier captures." + }, + "length": { + "description": "The array, object or string at the pointer must have this many items, entries or characters, at most 1048576.", + "format": "uint", + "maximum": 1048576, + "minimum": 0, + "type": "integer" + }, + "pointer": { + "description": "JSON pointer to the value to test: empty for the whole body, otherwise starting with `/`.", + "type": "string" + } + }, + "required": [ + "pointer" + ], + "type": "object" + }, + "DerivedId": { + "description": "A dotted development journey step identifier whose segments are local identifiers.", + "maxLength": 64, + "pattern": "^[a-z][a-z0-9_-]{0,63}(\\.[a-z][a-z0-9_-]{0,63})+$", + "type": "string" + }, + "Expected": { + "additionalProperties": false, + "description": "The conditions a response must meet.", + "properties": { + "checks": { + "default": [], + "description": "Further assertions about the response body, at most 32. Each one tests a single JSON pointer.", + "items": { + "$ref": "#/$defs/Check" + }, + "type": "array" + }, + "code": { + "description": "Problem code the error response must carry, 1 to 256 characters without whitespace or control characters. It may contain `{{name}}` references to earlier captures.", + "type": "string" + }, + "json": { + "additionalProperties": true, + "default": {}, + "description": "Exact values the response body must hold, keyed by JSON pointer (empty for the whole body, otherwise starting with `/`). String values may contain `{{name}}` references to earlier captures.", + "propertyNames": { + "$ref": "#/$defs/ExternalId" + }, + "type": "object", + "x-registry-passthrough": "Expected values are compared with the product response at the named JSON pointers." + }, + "status": { + "$ref": "#/$defs/ExpectedStatus", + "description": "Accepted HTTP status: one status, or a list of 1 to 16 distinct statuses, each from 200 to 599." + } + }, + "required": [ + "status" + ], + "type": "object" + }, + "ExpectedStatus": { + "anyOf": [ + { + "description": "A single accepted status from 200 to 599.", + "format": "uint32", + "maximum": 599, + "minimum": 200, + "type": "integer" + }, + { + "description": "Several accepted statuses, each from 200 to 599; the response may carry any one of them.", + "items": { + "format": "uint32", + "maximum": 599, + "minimum": 200, + "type": "integer" + }, + "type": "array" + } + ], + "description": "One accepted HTTP status or a list of them." + }, + "ExternalId": { + "description": "An identifier another system issues, written exactly as its issuer gives it.", + "maxLength": 512, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "FindWhere": { + "additionalProperties": false, + "description": "The condition a `find` capture uses to pick an array item.", + "properties": { + "equals": { + "description": "The value an item must hold at the pointer to match. String values may contain `{{name}}` references to earlier captures." + }, + "pointer": { + "description": "JSON pointer, relative to each array item, of the value to compare.", + "type": "string" + } + }, + "required": [ + "pointer", + "equals" + ], + "type": "object" + }, + "Grant": { + "additionalProperties": false, + "description": "The authority a test person holds in one product.", + "properties": { + "claims": { + "additionalProperties": true, + "default": {}, + "description": "Additional token claims, keyed by claim name. Names must not be reserved token claims such as `iss`, `aud`, `sub`, `scope` or any `registry_grant_` claim.", + "propertyNames": { + "$ref": "#/$defs/ExternalId" + }, + "type": "object", + "x-registry-passthrough": "Claim values are written into the development token as given." + }, + "profile": { + "$ref": "#/$defs/LocalId", + "description": "Local identifier of the product profile this grant uses." + }, + "scopes": { + "description": "Distinct scopes written into the token, at most 32, each 1 to 256 characters without whitespace or control characters.", + "items": { + "$ref": "#/$defs/ExternalId" + }, + "maxItems": 32, + "type": "array", + "uniqueItems": true + } + }, + "required": [ + "profile", + "scopes" + ], + "type": "object" + }, + "Journey": { + "additionalProperties": false, + "description": "An ordered list of requests the run sends and checks.", + "properties": { + "id": { + "$ref": "#/$defs/LocalId", + "description": "Local identifier naming this journey." + }, + "steps": { + "description": "The steps, run in order. Include 1 to 32 with distinct identifiers. A step may refer to captures from earlier steps only.", + "items": { + "$ref": "#/$defs/Step" + }, + "type": "array" + } + }, + "required": [ + "id", + "steps" + ], + "type": "object" + }, + "LocalId": { + "description": "A local identifier: a lowercase letter, then up to 63 lowercase letters, digits, `_`, or `-`.", + "pattern": "^[a-z][a-z0-9_-]{0,63}$", + "type": "string" + }, + "Method": { + "description": "An HTTP request method.", + "enum": [ + "GET", + "POST", + "PUT", + "PATCH", + "DELETE" + ], + "type": "string" + }, + "Poll": { + "additionalProperties": false, + "description": "How long and how often to repeat a GET request. The wait divided by the interval, rounded up, must be below 250 attempts.", + "properties": { + "maximumWaitMilliseconds": { + "description": "Longest total wait in milliseconds, from 1 to 60000.", + "format": "uint64", + "maximum": 60000, + "minimum": 1, + "type": "integer" + }, + "pollIntervalMilliseconds": { + "description": "Pause between attempts in milliseconds, from 50 to 5000 and not above the maximum wait.", + "format": "uint64", + "maximum": 5000, + "minimum": 50, + "type": "integer" + } + }, + "required": [ + "maximumWaitMilliseconds", + "pollIntervalMilliseconds" + ], + "type": "object" + }, + "Product": { + "description": "A product the development run can start.", + "enum": [ + "breg", + "casework" + ], + "type": "string" + }, + "ProductPlan": { + "additionalProperties": false, + "description": "One product to start and where its project lives.", + "properties": { + "audience": { + "$ref": "#/$defs/ExternalId", + "description": "Token audience this product accepts: 1 to 256 characters without whitespace or control characters, distinct from the other products' audiences." + }, + "product": { + "$ref": "#/$defs/Product", + "description": "Which product to start." + }, + "project": { + "description": "Relative path from the plan's directory to the product project, or `.` for the plan's own directory. It must contain registry.yaml for BReg or casework.yaml for Casework.", + "type": "string" + } + }, + "required": [ + "product", + "project", + "audience" + ], + "type": "object" + }, + "RawBody": { + "additionalProperties": false, + "description": "A request body sent as given, without JSON encoding.", + "properties": { + "base64": { + "description": "The body as standard base64, at most 1 MiB once decoded.", + "type": "string" + }, + "contentType": { + "description": "HTTP media type for the Content-Type header, containing a slash, 1 to 256 ASCII characters.", + "type": "string" + } + }, + "required": [ + "contentType", + "base64" + ], + "type": "object" + }, + "Step": { + "additionalProperties": false, + "description": "One HTTP request, what to expect back, and what to remember from the response.", + "properties": { + "as": { + "$ref": "#/$defs/LocalId", + "description": "The test person who sends the request. The person needs a grant for this step's product." + }, + "body": { + "description": "JSON request body, at most 1 MiB once encoded. String values may contain `{{name}}` references to earlier captures. Do not combine with `raw`." + }, + "capture": { + "additionalProperties": { + "$ref": "#/$defs/Capture" + }, + "default": {}, + "description": "Values to remember from the response, keyed by a capture name. Later steps refer to a capture as `{{name}}` in paths, headers, bodies and expectations. Names are unique across the journey.", + "propertyNames": { + "allOf": [ + { + "$ref": "#/$defs/ExternalId" + }, + { + "maxLength": 64, + "pattern": "^[A-Za-z][A-Za-z0-9-]{0,63}$", + "type": "string" + } + ] + }, + "type": "object" + }, + "expect": { + "$ref": "#/$defs/Expected", + "description": "What the response must look like for the step to pass." + }, + "headers": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra request headers, keyed by header name. Only `If-Match`, `Idempotency-Key` and `Registry-Source-Profile` are supported. Values may contain `{{name}}` references to earlier captures.", + "propertyNames": { + "$ref": "#/$defs/ExternalId" + }, + "type": "object" + }, + "id": { + "anyOf": [ + { + "$ref": "#/$defs/LocalId" + }, + { + "$ref": "#/$defs/DerivedId" + } + ], + "description": "Unique step identifier: a local identifier, or a dotted sequence of local identifiers, at most 64 characters." + }, + "method": { + "$ref": "#/$defs/Method", + "description": "The HTTP method." + }, + "path": { + "description": "Request path, starting with `/v1/`, at most 2048 characters, without whitespace, control characters, backslashes or fragments. It may contain `{{name}}` references to earlier captures.", + "type": "string" + }, + "poll": { + "$ref": "#/$defs/Poll", + "description": "Repeat a GET request until it passes or the wait runs out. Allowed only for GET." + }, + "product": { + "$ref": "#/$defs/Product", + "description": "The product that receives the request." + }, + "raw": { + "$ref": "#/$defs/RawBody", + "description": "Non-JSON request body. Allowed only for methods other than GET. Do not combine with `body`." + } + }, + "required": [ + "id", + "product", + "as", + "method", + "path", + "expect" + ], + "type": "object" + }, + "TestPerson": { + "additionalProperties": false, + "description": "A named test identity that signs in to the started products.", + "properties": { + "caseworkContext": { + "default": false, + "description": "Forward the same authority to both products. Both grants must declare the same scopes and additional claims.", + "type": "boolean" + }, + "client": { + "$ref": "#/$defs/ExternalId", + "description": "Client identifier of this person's tokens: 1 to 256 characters without whitespace or control characters." + }, + "grants": { + "additionalProperties": false, + "description": "The authority this person holds in each product, keyed by `breg` or `casework`. Give at least one grant, only for products listed in `products`.", + "properties": { + "breg": { + "$ref": "#/$defs/Grant" + }, + "casework": { + "$ref": "#/$defs/Grant" + } + }, + "type": "object" + }, + "subject": { + "$ref": "#/$defs/ExternalId", + "description": "Subject claim of this person's tokens: 1 to 256 characters without whitespace or control characters." + } + }, + "required": [ + "subject", + "client", + "grants" + ], + "type": "object" + }, + "Url": { + "description": "An absolute http or https URL with a host preserved as written, no user information, and no backslash, whitespace or control character. DNS case and equivalent IPv6 literals are permitted.", + "format": "uri", + "maxLength": 2048, + "pattern": "^[Hh][Tt][Tt][Pp][Ss]?://[^/?#@\\u005C\\u0000-\\u0020\\u007F-\\u00A0\\u1680\\u2000-\\u200A\\u2028\\u2029\\u202F\\u205F\\u3000]+([/?#][^\\u005C\\u0000-\\u0020\\u007F-\\u00A0\\u1680\\u2000-\\u200A\\u2028\\u2029\\u202F\\u205F\\u3000]*)?$", + "type": "string" + } + }, + "$id": "https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "The development plan: the products to start, the test people who call them, and the journey to run.", + "properties": { + "apiVersion": { + "const": "id.registrystack.org/formats/breg/project-dev/v1alpha1", + "description": "The format and version this file is written in.", + "type": "string" + }, + "bindings": { + "$ref": "#/$defs/Bindings", + "default": {}, + "description": "Run-local connections between BReg and Casework. Leave empty when the products do not call each other." + }, + "issuer": { + "$ref": "#/$defs/Url", + "description": "HTTPS URL, at most 512 characters and without credentials, query or fragment, that the run's tokens name as their issuer." + }, + "journey": { + "$ref": "#/$defs/Journey", + "description": "The ordered sequence of HTTP requests the run sends to the started products." + }, + "kind": { + "const": "BRegProjectDev", + "description": "The kind of document this file is.", + "type": "string" + }, + "people": { + "additionalProperties": { + "$ref": "#/$defs/TestPerson" + }, + "description": "The test people, keyed by a lowercase local identifier. Define 1 to 32, and give at least one grant to every started product.", + "propertyNames": { + "$ref": "#/$defs/LocalId" + }, + "type": "object" + }, + "products": { + "description": "The products to start, each with its project directory and token audience. List each product at most once.", + "items": { + "$ref": "#/$defs/ProductPlan" + }, + "type": "array" + }, + "schemaTestPeople": { + "additionalProperties": { + "additionalProperties": { + "$ref": "#/$defs/LocalId" + }, + "propertyNames": { + "anyOf": [ + { + "$ref": "#/$defs/LocalId" + }, + { + "$ref": "#/$defs/DerivedId" + } + ] + }, + "type": "object" + }, + "default": {}, + "description": "Chooses the test person for a BReg fixture step in tests/journeys.yaml, keyed by journey identifier and then step identifier. Needed only when several people could run the step; every named person needs a BReg grant.", + "propertyNames": { + "$ref": "#/$defs/LocalId" + }, + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "issuer", + "products", + "people", + "journey" + ], + "title": "Registry Stack project development plan", + "type": "object" +} diff --git a/products/casework/CHANGELOG.md b/products/casework/CHANGELOG.md index d01a406c6b..925bbb49a4 100644 --- a/products/casework/CHANGELOG.md +++ b/products/casework/CHANGELOG.md @@ -7,6 +7,10 @@ `caseworkctl explain` lists them. A stage that omits the list offers every policy outcome. +- `caseworkctl explain` adds a top-level `accessProfiles` list, sorted by `id`, + with each access profile's `id` and `principalClaim`, so a caller no longer + reads them from a review policy's `dependencies`. The member is additive. + ## 0.46.0 - 2026-11-08 - Each access profile may carry an optional `label` and `description`. Neither is matched against a caller, a profile that writes neither reads as before, and rewording them leaves a policy's behavioral binding unchanged. Casework keeps both in the review policy description, which any caller who may read that description, requesters included, can see. diff --git a/products/casework/SECURITY-MODEL.md b/products/casework/SECURITY-MODEL.md index 57d6c6b913..ff0fac46a2 100644 --- a/products/casework/SECURITY-MODEL.md +++ b/products/casework/SECURITY-MODEL.md @@ -192,6 +192,13 @@ Deployment behavior is in the [README](README.md), keys with a static `jwksSource`. - **Related:** CASEWORK-SEC-01. +`rstack dev` uses a loopback listener and a private static key pin for each +development run. The shared issuer gives a named synthetic person only +the scopes and additional claims declared in their Casework grant. A +reviewer token forwarded to BReg uses authority explicitly declared in both +grants. Casework applies its usual client, profile, and human-identity checks. See +[Run isolation](../platform/PROJECT-DEV.md#run-isolation). + ### CASEWORK-RES-09: key refusal is by `kid`, locally, at startup - **Condition:** `authentication.oidc.revokedKeyIds` lists a compromised or diff --git a/products/casework/contracts/cli/report.schema.json b/products/casework/contracts/cli/report.schema.json index ab9637319a..e063563756 100644 --- a/products/casework/contracts/cli/report.schema.json +++ b/products/casework/contracts/cli/report.schema.json @@ -803,6 +803,22 @@ ], "type": "object" }, + "ExplainAccessProfile": { + "additionalProperties": false, + "properties": { + "id": { + "$ref": "#/$defs/LocalId" + }, + "principalClaim": { + "type": "string" + } + }, + "required": [ + "id", + "principalClaim" + ], + "type": "object" + }, "ExplainPolicy": { "additionalProperties": false, "properties": { @@ -2737,6 +2753,11 @@ "ok" ] }, + { + "required": [ + "accessProfiles" + ] + }, { "required": [ "authority" @@ -2782,6 +2803,7 @@ "then": { "required": [ "project", + "accessProfiles", "reviewPolicies", "contextProviders", "clocks", @@ -2794,6 +2816,12 @@ } ], "properties": { + "accessProfiles": { + "items": { + "$ref": "#/$defs/ExplainAccessProfile" + }, + "type": "array" + }, "apiVersion": { "const": "id.registrystack.org/formats/casework/report/v1alpha2", "type": "string" diff --git a/products/identifiers/generated/catalog.v1.json b/products/identifiers/generated/catalog.v1.json index 8c6a02f161..0a8544ba4c 100644 --- a/products/identifiers/generated/catalog.v1.json +++ b/products/identifiers/generated/catalog.v1.json @@ -4776,7 +4776,7 @@ "description": "Base Registry Engine tool-file JSON Schemas at version v1alpha1.", "source": { "path": "crates/registry-bregctl/src/tool_schema.rs", - "sha256": "28567d54120dd29d734ddcd3e10c4c6ee128eb7b2d195e4c77bd1cd6c9bc5cfd" + "sha256": "8197938479291503ba0975ae3c595b580a2f686bed6ae3cf4fe57cf01318c94d" }, "artifact": { "path": "products/breg/generated/tools/backup-binding.v1alpha1.schema.json", @@ -4794,7 +4794,7 @@ "description": "Base Registry Engine tool-file JSON Schemas at version v1alpha1.", "source": { "path": "crates/registry-bregctl/src/tool_schema.rs", - "sha256": "28567d54120dd29d734ddcd3e10c4c6ee128eb7b2d195e4c77bd1cd6c9bc5cfd" + "sha256": "8197938479291503ba0975ae3c595b580a2f686bed6ae3cf4fe57cf01318c94d" }, "artifact": { "path": "products/breg/generated/tools/dev-clients.v1alpha1.schema.json", @@ -4848,7 +4848,7 @@ "description": "Base Registry Engine tool-file JSON Schemas at version v1alpha1.", "source": { "path": "crates/registry-bregctl/src/tool_schema.rs", - "sha256": "28567d54120dd29d734ddcd3e10c4c6ee128eb7b2d195e4c77bd1cd6c9bc5cfd" + "sha256": "8197938479291503ba0975ae3c595b580a2f686bed6ae3cf4fe57cf01318c94d" }, "artifact": { "path": "products/breg/generated/tools/example-scenarios.v1alpha1.schema.json", @@ -4866,7 +4866,7 @@ "description": "Base Registry Engine tool-file JSON Schemas at version v1.", "source": { "path": "crates/registry-bregctl/src/tool_schema.rs", - "sha256": "28567d54120dd29d734ddcd3e10c4c6ee128eb7b2d195e4c77bd1cd6c9bc5cfd" + "sha256": "8197938479291503ba0975ae3c595b580a2f686bed6ae3cf4fe57cf01318c94d" }, "artifact": { "path": "products/breg/generated/tools/journeys.v1.schema.json", @@ -4920,7 +4920,7 @@ "description": "Base Registry Engine tool-file JSON Schemas at version v1alpha1.", "source": { "path": "crates/registry-bregctl/src/tool_schema.rs", - "sha256": "28567d54120dd29d734ddcd3e10c4c6ee128eb7b2d195e4c77bd1cd6c9bc5cfd" + "sha256": "8197938479291503ba0975ae3c595b580a2f686bed6ae3cf4fe57cf01318c94d" }, "artifact": { "path": "products/breg/generated/tools/migration-descriptor.v1alpha1.schema.json", @@ -4956,7 +4956,7 @@ "description": "Base Registry Engine tool-file JSON Schemas at version v1alpha1.", "source": { "path": "crates/registry-bregctl/src/tool_schema.rs", - "sha256": "28567d54120dd29d734ddcd3e10c4c6ee128eb7b2d195e4c77bd1cd6c9bc5cfd" + "sha256": "8197938479291503ba0975ae3c595b580a2f686bed6ae3cf4fe57cf01318c94d" }, "artifact": { "path": "products/breg/generated/tools/model-selection.v1alpha1.schema.json", @@ -4982,6 +4982,24 @@ "mediaType": "application/schema+json" } }, + { + "uri": "https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json", + "kind": "schema", + "status": "active", + "compatibilityLine": "v1alpha1", + "owner": "breg", + "title": "Registry Stack project development plan", + "description": "Base Registry Engine tool-file JSON Schemas at version v1alpha1.", + "source": { + "path": "crates/registry-bregctl/src/tool_schema.rs", + "sha256": "8197938479291503ba0975ae3c595b580a2f686bed6ae3cf4fe57cf01318c94d" + }, + "artifact": { + "path": "products/breg/generated/tools/project-dev.v1alpha1.schema.json", + "sha256": "30a0fcc71d24875309ae9cd96f41305add32ad69e89dcc9b4cbd1622478eeb43", + "mediaType": "application/schema+json" + } + }, { "uri": "https://id.registrystack.org/schemas/breg/project/project.v1alpha1.schema.json", "kind": "schema", @@ -5100,7 +5118,7 @@ "description": "Base Registry Engine tool-file JSON Schemas at version v1.", "source": { "path": "crates/registry-bregctl/src/tool_schema.rs", - "sha256": "28567d54120dd29d734ddcd3e10c4c6ee128eb7b2d195e4c77bd1cd6c9bc5cfd" + "sha256": "8197938479291503ba0975ae3c595b580a2f686bed6ae3cf4fe57cf01318c94d" }, "artifact": { "path": "products/breg/generated/tools/schema-test-credentials.v1.schema.json", @@ -5194,7 +5212,7 @@ }, "artifact": { "path": "products/casework/contracts/cli/report.schema.json", - "sha256": "442c29fe6db3bd93c2a35a5303e341d8aaa70c95d58dcd589fee9fff529fdff4", + "sha256": "9d701cf095137be5e1ce0203943626bcf7005597f3e5748527b510fabe180cb4", "mediaType": "application/schema+json" } }, diff --git a/products/platform/CHANGELOG.md b/products/platform/CHANGELOG.md index 45fddf7cac..ac4a9e0fc6 100644 --- a/products/platform/CHANGELOG.md +++ b/products/platform/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +- `rstack dev check|start|status|token|stop` runs a checkout's BReg and Casework + projects together against a PostgreSQL server you supply. See + [PROJECT-DEV.md](PROJECT-DEV.md). It is an alpha developer tool and may + change. + ## 0.46.0 - 2026-11-08 - `registry-platform-text`: `TextStep::StripNonspacingMarksIn` replaces diff --git a/products/platform/PROJECT-DEV.md b/products/platform/PROJECT-DEV.md new file mode 100644 index 0000000000..8859aed6ab --- /dev/null +++ b/products/platform/PROJECT-DEV.md @@ -0,0 +1,253 @@ +# Run a project for development + +Use `rstack dev` to start a checkout's BReg and Casework projects together. +Install matching BReg and Casework components and supply a dedicated development +PostgreSQL server over TLS. `rstack dev` does not start PostgreSQL and does not +use Docker: you supply the server. The products' own development modes are +unchanged by `rstack dev`. + +```sh +rstack install breg casework +# Set RSTACK_DEV_DATABASE_URL through your shell's secret provider. +rstack dev check ./licensing --json +rstack dev start ./licensing --ttl 30m --json +rstack dev status RUN_ID +rstack dev token RUN_ID --as applicant --json +rstack dev stop RUN_ID +``` + +Run `rstack dev check PROJECT [--deny-warnings] [--json]` to validate the plan +and resolve its fixture, grant, and binding references with the installed +product authoring tools. The check works with the database offline. With +`--json`, it writes a `BRegCtlReport` with `command` set to `dev-check`. +Use `--deny-warnings` to refuse any unaccepted warning. + +Start reports the product URLs and the journey result. +A failed journey exits 1 and includes the live run ID so you can inspect the +result and stop the run. Startup failures clean up or report the run ID if +cleanup is pending. The TTL starts at readiness and defaults to 30 minutes. +Supply one whole-number value from `1s` to `24h` with an `s`, `m`, or `h` suffix. + +The [licensing example](examples/licensing/dev.yaml) creates an application in +BReg, requests approval of the captured application in Casework, and checks the +reviewer's decision. Point `start` at the directory containing `dev.yaml`. +Product files remain byte-for-byte unchanged. Generated runtime settings, keys, +and packages live in the run's owner-only scratch directory. + +## Supply the PostgreSQL server + +The server must be PostgreSQL 17 or newer and must serve TLS. The runner always +verifies the server certificate and host name. It trusts the platform's roots, +or the CA bundle in PEM format named by `RSTACK_DEV_DATABASE_CA`. + +The administrator role named in the URL needs `LOGIN CREATEROLE CREATEDB`. +Superuser is not needed for a non-spatial run. The runner sets +`createrole_self_grant` on its own sessions, so you do not configure it. The +administrator also installs `btree_gist`, which PostgreSQL treats as a trusted +extension. + +The administrator URL always carries a password. Set it through +`RSTACK_DEV_DATABASE_URL`, because `--database-url` is visible to other local +users in the process list. The URL must be a `postgres://` or `postgresql://` +URL with one TCP host, a user, a password and one database name. Percent-encode +special characters in the password. The URL accepts these query parameters and +refuses all others: + +- `sslmode`: `require`, `verify-ca` or `verify-full`. The runner verifies the + certificate and host name whichever you choose. +- `connect_timeout` and `application_name`. +- `sslrootcert` is refused. Set `RSTACK_DEV_DATABASE_CA` instead. + +### Spatial projects + +A spatial BReg project needs PostGIS 3.5 or later installed on the server. The +administrator must be allowed to `CREATE EXTENSION postgis`, which normally +means a superuser or the managed service's equivalent. The run checks that the +server offers a supported PostGIS before it creates anything. + +### A local server + +This recipe starts a PostgreSQL 17 or newer server on `127.0.0.1` with a +private certificate authority, and creates a least-privilege administrator. It +uses the PostgreSQL and OpenSSL command-line tools. Keep `DEV` outside any +repository. + +```sh +DEV="$HOME/rstack-dev-postgres" +mkdir -p "$DEV" +initdb -D "$DEV/data" -U postgres --auth=scram-sha-256 --pwprompt -E UTF8 + +openssl req -new -x509 -newkey rsa:2048 -nodes -days 30 -subj /CN=rstack-dev-ca \ + -addext basicConstraints=critical,CA:TRUE -keyout "$DEV/ca.key" -out "$DEV/ca.pem" +openssl req -new -newkey rsa:2048 -nodes -subj /CN=localhost \ + -keyout "$DEV/data/server.key" -out "$DEV/server.csr" +chmod 600 "$DEV/data/server.key" +printf 'subjectAltName=DNS:localhost,IP:127.0.0.1\nextendedKeyUsage=serverAuth\n' \ + > "$DEV/server.ext" +openssl x509 -req -in "$DEV/server.csr" -CA "$DEV/ca.pem" -CAkey "$DEV/ca.key" \ + -CAcreateserial -days 30 -extfile "$DEV/server.ext" -out "$DEV/data/server.crt" + +printf "listen_addresses = '127.0.0.1'\nport = 5432\nssl = on\n" \ + >> "$DEV/data/postgresql.conf" +pg_ctl -D "$DEV/data" -l "$DEV/postgres.log" -w start + +CONNECTION="host=127.0.0.1 port=5432 user=postgres dbname=postgres sslmode=verify-full sslrootcert=$DEV/ca.pem" +psql "$CONNECTION" -c "CREATE ROLE devadmin LOGIN CREATEROLE CREATEDB" +psql "$CONNECTION" -c '\password devadmin' + +export RSTACK_DEV_DATABASE_CA="$DEV/ca.pem" +# Set RSTACK_DEV_DATABASE_URL to +# postgresql://devadmin:PASSWORD@127.0.0.1:5432/postgres?sslmode=verify-full +``` + +For a spatial project, install PostGIS for this server and create the +administrator as a superuser instead. + +## Write the development plan + +Use the [versioned plan schema](../breg/generated/tools/project-dev.v1alpha1.schema.json). +The plan file starts with the schema modeline that editors read, as the +[licensing example](examples/licensing/dev.yaml) does. +List the project's issuer, product directories and audiences in startup order, +and test people with synthetic subjects and exact product grants. A grant selects +the access profile, scopes, and additional claims. Each selected profile gets +the test person's subject in its principal claim. The products enforce their +ordinary permissions. If several people match a step in the BReg project's +`tests/journeys.yaml`, select its alias under +`schemaTestPeople..`. The run obtains the normal +schema-test receipt before packaging. + +Every BReg grant must name an access profile the BReg project declares, +including a profile that carries only proposal permissions and so belongs to no +entity. `rstack dev check` refuses a grant that does not, and so does `start`, +before it provisions anything. The error names the plan pointer, for example +`/people/NAME/grants/breg/profile`. A Casework grant must name an access profile +of the Casework project, reported at `/people/NAME/grants/casework/profile`. The run does not compare +scopes with the profile: a profile's required scopes are a per-entity rule, and +a mismatch appears as a failed journey step. + +For a native review handoff, `bindings.bregReviewAuthority` identifies the +Casework producer, and `bindings.caseworkContextProvider` identifies the BReg +context provider and action. `rstack dev check` refuses a binding that names a +review authority or action the BReg project does not declare, or a producer or +context provider the Casework project does not declare; `start` refuses it +before it provisions anything. A binding requests tokens for the other +product's audience as an OAuth resource, so that audience must be an absolute +URI, such as `urn:example:review`. It requests the scopes of its identity's +grant for that product, so those scopes must be OAuth scope-tokens (no quotes or +backslashes) that fit in 4096 bytes when joined by spaces; `check` reports a +violation at the grant's `scopes`. The run pairs the provider on a private copy +before packaging. A reviewer whose Casework context call forwards the bearer +to BReg declares both grants, one shared client, and `caseworkContext: true`. +Both grants declare the same scopes and additional claims. Casework reads the +token subject or the BReg principal claim. The token carries both audiences +and the authority explicitly shared by both grants. +See the [native review example](../breg/acceptance/reviewed-evidence-journey/dev.yaml). + +### Journey steps + +A journey runs 1 to 32 steps in order. Each step names the test person it runs +as (`as`), the product, the method, and a relative path that starts with `/v1/`. +A step sends one of two request bodies: + +- `body` is a JSON body. +- `raw` is a non-JSON body for methods other than GET: `contentType` and + `base64`. + +A step can also set the `If-Match`, `Idempotency-Key` and +`Registry-Source-Profile` headers. Use URI-safe characters for values in a path. + +Each step asserts the response with `expect`: + +- `status` is one accepted HTTP status from 200 to 599, or a list of up to 16. +- `json` maps JSON pointers to exact expected values. +- `checks` lists up to 32 assertions. Each tests one JSON pointer with exactly + one of `equals`, `length` or `contains`. +- `code` is the problem code an error response must carry. + +A step remembers values from its response under `capture`. Refer to a capture +as `{{capture-name}}` in a later step's path, request body, header, expected +value, or `find` value. A step cannot use its own captures. +Capture names are unique across the journey. A capture has one `type`: + +- `pointer` remembers the string or number at a JSON pointer in the body. +- `header` remembers a response header. `etag` is the supported header. +- `find` searches the array at a JSON pointer for the one item whose `where` + pointer equals a value, and remembers the item's `pointer` value. The step + fails if no item or more than one item matches. +- `value` remembers the value at a pointer. With `query`, the value must be a + URL, and the capture is that query parameter's value. + +Use a bounded GET `poll` for asynchronous delivery or result retrieval. Set +`maximumWaitMilliseconds` up to 60 seconds and choose a +`pollIntervalMilliseconds` from 50 to 5000. A journey can make at most 512 +requests and transfer at most 8 MiB of request and response bodies in total. + +`token` writes a short-lived bearer header file for each grant. Its JSON report +gives the person's `alias`, the `expires_at` time when the tokens expire +(five minutes after issue, or with the run if that is sooner), and a +`credentials` list whose entries carry the `product`, the selected `profile` +and the `header_file` path. +Pass the file to curl with `--header @HEADER_FILE`. With BReg, include the +selected profile as the `accessProfile` query parameter. With Casework, the +header file carries the profile. + +## Run isolation + +Each run listens on loopback, creates separate databases and migration and +runtime logins, and pins its own public signing key in the runtime files. The +local issuer signs under the project's issuer name with a key unique to the run. +Use synthetic test people and keep the issuer's private directory under your +user account. + +Stop and TTL expiry terminate the services and remove the databases, roles, +keys, and run files. A spatial BReg run also creates a `NOLOGIN` role whose name +starts with `breg_bbox_`; cleanup removes it with the others. + +The state directory holds one directory per run. It defaults to +`rstack-dev-UID` in the system temporary directory. Set `RSTACK_DEV_STATE_DIR` +to use another location. A run directory is renamed to `..removing` +before it is deleted, and `status` and `stop` ignore that name. If the deletion +is interrupted, delete that directory yourself. + +## When a run fails + +### Logs + +During a run, `/logs/supervisor.log` holds the supervisor's log and +`/logs/-.log` holds each product stage's log, for example +`breg-check.log`. `status` names the logs directory. When a start fails, the run +directory is removed but its logs are kept at `/failed//logs`, +and the error names that path. Delete kept logs yourself. + +### Recovery + +If PostgreSQL becomes unavailable during cleanup, the supervisor retries with +backoff for up to 30 minutes. It then writes `cleanup-pending.json` in the run +directory, and `status` reports `cleanup-pending` with the remaining databases, +roles and processes. If the supervisor is lost, `stop` and `status` report the +run directory, the process groups, and the exact database and role names. + +Release the listed resources by hand, in this order: + +1. Stop each listed process group, for example `kill -TERM -PGID`. +2. As the administrator, run `DROP DATABASE IF EXISTS "NAME" WITH (FORCE)` for + each listed database. +3. As the administrator, run `DROP ROLE IF EXISTS "NAME"` for each listed role, + including a `breg_bbox_*` role from a spatial run. +4. Remove the run directory. + +### Errors and limitations + +With `--json`, a failed `rstack dev` command prints one JSON error object with +`schema_version` (`registry.rstack.dev.v1`), `status` set to `error`, and +`error`. It adds `run_id` when the run needs cleanup and `recovery` with the +resources listed above. Failures that occur before the helper starts, such as a +missing component, print a `registry.rstack.error.v1` object with `status` set +to `refused` and an `error` object holding `class` and `message`. The reports of +`rstack dev check` use `BRegCtlReport` instead. + +`rstack dev` does not configure field encryption yet. `rstack dev check` and +`start` refuse a BReg project with encrypted fields at the product's +`/products/N/project` pointer, and the error names the encrypted +`ENTITY.FIELD`; `start` refuses it before it provisions anything. diff --git a/products/platform/config-conventions-exceptions.yaml b/products/platform/config-conventions-exceptions.yaml index df71be0a31..fd83d89688 100644 --- a/products/platform/config-conventions-exceptions.yaml +++ b/products/platform/config-conventions-exceptions.yaml @@ -71,6 +71,36 @@ exceptions: class: protocol-constant reason: RS384 is written as JSON Web Algorithms (RFC 7518) defines it resolution: Permanent. +- rule: CFG-NAME-2 + format: breg/project-dev + location: products/breg/generated/tools/project-dev.v1alpha1.schema.json#/$defs/Method/enum/DELETE + class: protocol-constant + reason: DELETE is written as HTTP methods (RFC 9110) defines it + resolution: Permanent. +- rule: CFG-NAME-2 + format: breg/project-dev + location: products/breg/generated/tools/project-dev.v1alpha1.schema.json#/$defs/Method/enum/GET + class: protocol-constant + reason: GET is written as HTTP methods (RFC 9110) defines it + resolution: Permanent. +- rule: CFG-NAME-2 + format: breg/project-dev + location: products/breg/generated/tools/project-dev.v1alpha1.schema.json#/$defs/Method/enum/PATCH + class: protocol-constant + reason: PATCH is written as HTTP PATCH (RFC 5789) defines it + resolution: Permanent. +- rule: CFG-NAME-2 + format: breg/project-dev + location: products/breg/generated/tools/project-dev.v1alpha1.schema.json#/$defs/Method/enum/POST + class: protocol-constant + reason: POST is written as HTTP methods (RFC 9110) defines it + resolution: Permanent. +- rule: CFG-NAME-2 + format: breg/project-dev + location: products/breg/generated/tools/project-dev.v1alpha1.schema.json#/$defs/Method/enum/PUT + class: protocol-constant + reason: PUT is written as HTTP methods (RFC 9110) defines it + resolution: Permanent. - rule: CFG-NAME-2 format: breg/queries-explanation location: products/breg/contracts/explain/QueriesExplanation.schema.json#/properties/operations/items/properties/filterable/items/properties/wireOperators/items/enum/eq null diff --git a/products/platform/config-formats.yaml b/products/platform/config-formats.yaml index 75a0f0a3d3..f64424772b 100644 --- a/products/platform/config-formats.yaml +++ b/products/platform/config-formats.yaml @@ -719,6 +719,65 @@ formats: boolean: none securityMembers: [] restrictingMembers: [] + - id: breg/project-dev + product: breg + title: Registry Stack project development plan + files: [dev.yaml] + syntax: yaml + audience: authored + stability: experimental + current: + apiVersion: id.registrystack.org/formats/breg/project-dev/v1alpha1 + kind: BRegProjectDev + checkedBy: + file: crates/registry-bregctl/src/project_dev/plan.rs + symbol: API_VERSION + target: + apiVersion: id.registrystack.org/formats/breg/project-dev/v1alpha1 + kind: BRegProjectDev + schema: + path: products/breg/generated/tools/project-dev.v1alpha1.schema.json + id: https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json + origin: generated + generator: cargo run -p registry-bregctl --features schema --example tool-schema -- --output products/breg/generated/tools + driftCheck: products/breg/scripts/check-tool-schemas.sh + reader: + crate: registry-bregctl + file: crates/registry-bregctl/src/project_dev/plan.rs + function: load + type: Plan + check: rstack dev check {project} + example: products/platform/conformance/projects/breg-project-dev/dev.yaml + conformance: + requiredText: /issuer + optionalText: none + integer: /journey/steps/0/expect/status + boolean: none + idList: /journey/steps + set: /people/operator/grants/breg/scopes + reference: /journey/steps/0/as + operand: none + relativePath: none + securityMembers: + - pointer: /issuer + whenOmitted: refused + - pointer: /products/*/audience + whenOmitted: refused + - pointer: /people/*/grants/breg/profile + whenOmitted: refused + - pointer: /people/*/grants/breg/scopes + whenOmitted: refused + - pointer: /people/*/grants/breg/claims + whenOmitted: closed + - pointer: /people/*/grants/casework/profile + whenOmitted: refused + - pointer: /people/*/grants/casework/scopes + whenOmitted: refused + - pointer: /people/*/grants/casework/claims + whenOmitted: closed + - pointer: /people/*/caseworkContext + whenOmitted: closed + restrictingMembers: [/issuer, /products/*/audience, /people/*/grants/breg/profile, /people/*/grants/casework/profile, /people/*/caseworkContext] - id: breg/example-scenarios product: breg title: Example scenarios diff --git a/products/platform/conformance/projects/breg-project-dev/dev.yaml b/products/platform/conformance/projects/breg-project-dev/dev.yaml new file mode 100644 index 0000000000..322bec7874 --- /dev/null +++ b/products/platform/conformance/projects/breg-project-dev/dev.yaml @@ -0,0 +1,25 @@ +# yaml-language-server: $schema=https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json +apiVersion: id.registrystack.org/formats/breg/project-dev/v1alpha1 +kind: BRegProjectDev +issuer: https://issuer.example.invalid +products: + - product: breg + project: registry + audience: urn:example:registry +people: + operator: + subject: operator + client: operator-client + grants: + breg: + profile: operator + scopes: [registry:read] +journey: + id: smoke + steps: + - id: read + product: breg + as: operator + method: GET + path: /v1/records/records + expect: {status: 200} diff --git a/products/platform/conformance/projects/breg-project-dev/registry/registry.yaml b/products/platform/conformance/projects/breg-project-dev/registry/registry.yaml new file mode 100644 index 0000000000..2175f1a432 --- /dev/null +++ b/products/platform/conformance/projects/breg-project-dev/registry/registry.yaml @@ -0,0 +1,66 @@ +# This deliberately small product is compiled by `rstack dev check`; it has no +# journeys fixture because the conformance plan has no explicit assignments. +apiVersion: id.registrystack.org/formats/breg/project/v1alpha1 +kind: BRegProject +project: + id: conformance-registry + canonicalBaseIri: https://registry.example.invalid + version: 0.1.0 + defaultLanguage: en + display: + label: Conformance registry + description: Minimal product for the development-plan conformance corpus. +package: + sourceRevision: conformance-registry-0.1.0 +datasets: + - id: records + display: {label: Records, description: Conformance records.} +reviewAuthorities: + - id: local-review + protocol: id.registrystack.org/protocols/review/v1 + display: {label: Local review, description: Review authority used by development-plan checks.} +recordTypes: + - id: record + route: records + primaryDataset: records + mutationMode: mutable + display: {label: Record, description: A conformance record.} + fields: + - {id: name, type: string, required: true, minimumLength: 1, maximumLength: 100, classification: internal} + - {id: owner, type: string, required: true, minimumLength: 1, maximumLength: 100, classification: internal} +actions: + - id: review-action + display: {label: Review action, description: Create one conformance record.} + execution: {mode: immediate} + inputs: + - {id: name, type: string, required: true, minimumLength: 1, maximumLength: 100, classification: internal} + - {id: owner, type: string, required: true, minimumLength: 1, maximumLength: 100, classification: internal} + plan: + type: fixed + effects: + - id: record + target: {recordType: record} + operation: create + set: + name: {fromInput: name} + owner: {fromInput: owner} +accessProfiles: + - id: operator + default: true + principalClaim: registry_principal + requiredScopes: unrestricted + permissions: + records: + - recordType: record + operations: [get, list] + readableFields: [name, owner] + rowBoundaries: + - {field: owner, claim: registry_principal, operator: equals} + actions: + - action: review-action + operations: [invoke] + targets: + - recordType: record + rowBoundaries: + - {field: owner, claim: registry_principal, operator: equals} + results: [record] diff --git a/products/platform/examples/licensing/casework/casework.yaml b/products/platform/examples/licensing/casework/casework.yaml new file mode 100644 index 0000000000..a3b3bccdef --- /dev/null +++ b/products/platform/examples/licensing/casework/casework.yaml @@ -0,0 +1,59 @@ +apiVersion: id.registrystack.org/formats/casework/project/v1alpha1 +kind: CaseworkProject +project: {id: licensing-review, version: "1"} +accessProfiles: + administrator: + principalClaim: registry_principal + requiredScopes: ["casework:admin"] + actsAs: administrator + reviewer: + principalClaim: sub + requiredScopes: ["casework:review"] + actsAs: staff + supervisor: + principalClaim: sub + requiredScopes: ["casework:supervise"] + actsAs: supervisor + intake: + principalClaim: producer_identity + requiredScopes: ["casework:intake"] + actsAs: requester +queues: + applications: + label: Applications awaiting review +reviewPolicies: + licensing-approval: + version: "1" + purpose: approval + context: {mode: submitted} + stages: + - id: review + queue: applications + decidingProfiles: [reviewer] + requiredApprovals: 1 + excludeInitiator: true + retention: + terminalRetentionDays: 30 + accountabilityRetentionDays: 90 + displaySchema: + type: object + additionalProperties: false + required: [reference, profession] + properties: + reference: {type: string, maxLength: 80} + profession: {type: string, maxLength: 80} + outcomes: + approved: + label: Approve application + settlement: approved + reasonRequired: false + publicReason: optional +trustedProducers: + licensing-intake: + profile: intake + issuer: https://identity.licensing.example.test + subject: intake + trustedInitiatorIssuer: https://identity.licensing.example.test + sourceNamespaces: [licensing] + reviewPolicies: [licensing-approval] + retention: {receiptRetentionDays: 7} diff --git a/products/platform/examples/licensing/dev.yaml b/products/platform/examples/licensing/dev.yaml new file mode 100644 index 0000000000..eee70505e6 --- /dev/null +++ b/products/platform/examples/licensing/dev.yaml @@ -0,0 +1,122 @@ +# yaml-language-server: $schema=https://id.registrystack.org/schemas/breg/project-dev/project-dev.v1alpha1.schema.json +apiVersion: id.registrystack.org/formats/breg/project-dev/v1alpha1 +kind: BRegProjectDev +issuer: https://identity.licensing.example.test +products: + - {product: breg, project: registry, audience: "urn:example:licensing:registry"} + - {product: casework, project: casework, audience: "urn:example:licensing:review"} +people: + applicant: + subject: applicant + client: applicant-portal + grants: + breg: + profile: applicant + scopes: ["licensing:apply"] + claims: {application_reference: application-001, registry_actor_kind: human} + intake: + subject: intake + client: licensing-intake + grants: + casework: + profile: intake + scopes: ["casework:intake"] + claims: {registry_actor_kind: service} + administrator: + subject: administrator + client: licensing-workbench + grants: + casework: + profile: administrator + scopes: ["casework:admin"] + claims: {registry_actor_kind: human} + reviewer: + subject: reviewer + client: licensing-workbench + grants: + casework: + profile: reviewer + scopes: ["casework:review"] + claims: {registry_actor_kind: human} +journey: + id: licensing-approval + steps: + - id: create-application + product: breg + as: applicant + method: POST + path: /v1/records/applications + headers: {Idempotency-Key: create-application} + body: {data: {reference: application-001, profession: Surveyor}} + expect: {status: 201} + capture: {application: {type: pointer, pointer: /data/recordIdentifier}} + - id: staff-review-queue + product: casework + as: administrator + method: POST + path: /v1/directory/bootstrap + headers: {If-Match: '"0"', Idempotency-Key: staff-review-queue} + body: + teamId: licensing-reviewers + staff: [{issuer: https://identity.licensing.example.test, subject: reviewer}] + supervisors: [] + queueId: applications + expect: {status: 200} + - id: request-approval + product: casework + as: intake + method: POST + path: /v1/review-requests + headers: {Idempotency-Key: request-approval} + body: + policy: {id: licensing-approval} + subject: + source: licensing + type: application + id: "{{application}}" + version: "1" + # SHA-256 of the synthetic submitted snapshot's canonical JSON. + digest: "sha256:847c1042e1013e5b80dc8bdb7c021b584be3a7b778a4f242e42777b07539b9e5" + requesterReference: application-001 + initiator: {issuer: https://identity.licensing.example.test, subject: applicant} + context: + mode: submitted + snapshot: {reference: application-001, profession: Surveyor} + expect: {status: 201} + capture: {request: {type: pointer, pointer: /requestId}} + - id: find-review-task + product: casework + as: reviewer + method: GET + path: /v1/review-tasks?queue=applications + expect: {status: 200, json: {/items/0/requestId: "{{request}}"}} + capture: + task: {type: pointer, pointer: /items/0/taskId} + task-revision: {type: pointer, pointer: /items/0/revision} + - id: claim-review + product: casework + as: reviewer + method: POST + path: "/v1/review-tasks/{{task}}/claim" + headers: {If-Match: '"{{task-revision}}"', Idempotency-Key: claim-review} + expect: {status: 200} + capture: {claim-revision: {type: pointer, pointer: /revision}} + - id: approve-application + product: casework + as: reviewer + method: POST + path: "/v1/review-tasks/{{task}}/decisions" + headers: {If-Match: '"{{claim-revision}}"', Idempotency-Key: approve-application} + body: {decision: {type: approve, outcome: approved}} + expect: {status: 204} + - id: read-approval + product: casework + as: intake + method: GET + path: "/v1/review-requests/{{request}}/result" + expect: + status: 200 + json: + /subject/id: "{{application}}" + /status: approved + /outcome: approved diff --git a/products/platform/examples/licensing/registry/registry.yaml b/products/platform/examples/licensing/registry/registry.yaml new file mode 100644 index 0000000000..f53edc63ca --- /dev/null +++ b/products/platform/examples/licensing/registry/registry.yaml @@ -0,0 +1,47 @@ +apiVersion: id.registrystack.org/formats/breg/project/v1alpha1 +kind: BRegProject +project: + id: licensing + canonicalBaseIri: https://licensing.example.test + version: 0.1.0 + defaultLanguage: en + display: {label: Licensing registry, description: Synthetic professional licence applications.} +package: + sourceRevision: licensing-example-0.1.0 +datasets: + - id: licensing + display: {label: Licensing, description: Applications awaiting an independent review.} +recordTypes: + - id: application + route: applications + primaryDataset: licensing + mutationMode: mutable + display: {label: Application, description: One licence application.} + fields: + - {id: reference, type: string, required: true, minimumLength: 1, maximumLength: 80, classification: internal} + - {id: profession, type: string, required: true, minimumLength: 1, maximumLength: 80, classification: internal} +accessProfiles: + - id: applicant + default: true + principalClaim: registry_principal + requiredScopes: ["licensing:apply"] + permissions: + records: + - recordType: application + operations: [create, get, list] + readableFields: [reference, profession] + writableFields: [reference, profession] + rowBoundaries: + - {field: reference, claim: application_reference, operator: equals} + - id: auditor + principalClaim: registry_principal + actorKind: human + requesterClients: [licensing-auditor] + requiredScopes: ["licensing:audit"] + permissions: + records: + - recordType: application + operations: [get] + readableFields: [reference] + rowBoundaries: + - {field: reference, claim: application_reference, operator: equals} diff --git a/products/platform/examples/licensing/registry/tests/journeys.yaml b/products/platform/examples/licensing/registry/tests/journeys.yaml new file mode 100644 index 0000000000..2b3b358f78 --- /dev/null +++ b/products/platform/examples/licensing/registry/tests/journeys.yaml @@ -0,0 +1,19 @@ +apiVersion: id.registrystack.org/formats/breg/journeys/v1 +kind: BRegJourneys +journeys: + - id: submit-application + steps: + - id: submit + recordType: application + accessProfile: applicant + claims: + principal: applicant + scopes: ["licensing:apply"] + directClaims: {application_reference: application-001} + request: + type: create + data: {reference: application-001, profession: Surveyor} + expect: + outcome: success + status: 201 + fields: {reference: application-001, profession: Surveyor} diff --git a/products/platform/scripts/run-config-conformance.py b/products/platform/scripts/run-config-conformance.py index 6ddddd13b4..f927a6c196 100644 --- a/products/platform/scripts/run-config-conformance.py +++ b/products/platform/scripts/run-config-conformance.py @@ -1934,6 +1934,9 @@ def prepare_problems(fmt: Format, staged: "Staged", env: dict[str, str], bin_dir def check_environment(bin_dir: Path) -> dict[str, str]: env = dict(os.environ) env["PATH"] = str(bin_dir) + os.pathsep + env.get("PATH", "") + bregctl = bin_dir / "bregctl" + if bregctl.is_file(): + env["RSTACK_BREGCTL_BIN"] = str(bregctl) env["NO_COLOR"] = "1" env[VARIABLE] = MARKER env.pop(UNSET_VARIABLE, None) diff --git a/products/platform/scripts/test-rstack-dev.py b/products/platform/scripts/test-rstack-dev.py new file mode 100644 index 0000000000..97dba72c73 --- /dev/null +++ b/products/platform/scripts/test-rstack-dev.py @@ -0,0 +1,917 @@ +#!/usr/bin/env python3 +"""Exercise rstack dev with real BReg, Casework and a disposable TLS PostgreSQL.""" + +from __future__ import annotations + +import argparse +import base64 +import concurrent.futures +import hashlib +import json +import os +import re +import secrets +import shutil +import signal +import socket +import subprocess +import tempfile +import time +from pathlib import Path +from urllib.error import HTTPError +from urllib.parse import unquote, urlsplit +from urllib.request import ProxyHandler, Request, build_opener + +STACK = Path(__file__).resolve().parents[3] +NATIVE_REVIEW = STACK / "products/breg/acceptance/reviewed-evidence-journey" +SPATIAL = STACK / "products/breg/acceptance/spatial-service-sites" +HTTP = build_opener(ProxyHandler({})) + + +def command(*args, env=None, data=None, cwd=None, timeout=120): + return subprocess.run( + args, + env=env, + input=data, + cwd=cwd, + capture_output=True, + timeout=timeout, + check=False, + ) + + +def checked(*args, **kwargs): + result = command(*args, **kwargs) + if result.returncode: + raise RuntimeError(f"{Path(args[0]).name} failed with exit {result.returncode}") + return result.stdout + + +def port(): + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + return listener.getsockname()[1] + + +def prepare_postgres(root, pg): + password = secrets.token_urlsafe(32) + cluster = root / "postgres" + checked( + pg / "initdb", + "-D", + cluster, + "-U", + "devadmin", + "--pwfile=/dev/stdin", + "--auth-local=trust", + "--auth-host=scram-sha-256", + "--no-locale", + "-E", + "UTF8", + data=f"{password}\n".encode(), + ) + ca = root / "ca.pem" + checked( + "openssl", + "req", + "-new", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-subj", + "/CN=rstack-dev-test", + "-days", + "1", + "-keyout", + root / "ca.key", + "-out", + ca, + "-addext", + "basicConstraints=critical,CA:TRUE", + ) + checked( + "openssl", + "req", + "-new", + "-newkey", + "rsa:2048", + "-nodes", + "-subj", + "/CN=127.0.0.1", + "-keyout", + cluster / "server.key", + "-out", + root / "server.csr", + ) + (cluster / "server.key").chmod(0o600) + extensions = root / "server.ext" + extensions.write_text("subjectAltName=IP:127.0.0.1\nextendedKeyUsage=serverAuth\n") + checked( + "openssl", + "x509", + "-req", + "-in", + root / "server.csr", + "-CA", + ca, + "-CAkey", + root / "ca.key", + "-CAcreateserial", + "-days", + "1", + "-extfile", + extensions, + "-out", + cluster / "server.crt", + ) + db_port = port() + with (cluster / "postgresql.conf").open("a") as config: + config.write( + f"\nlisten_addresses = '127.0.0.1'\nport = {db_port}\n" + "unix_socket_directories = ''\nssl = on\nlog_min_error_statement = panic\n" + ) + pg_env = { + **os.environ, + "PGHOST": "127.0.0.1", + "PGPORT": str(db_port), + "PGDATABASE": "postgres", + "PGUSER": "devadmin", + "PGPASSWORD": password, + "PGSSLMODE": "verify-full", + "PGSSLROOTCERT": str(ca), + } + url = f"postgresql://devadmin:{password}@127.0.0.1:{db_port}/postgres" + return cluster, ca, url, pg_env + + +def psql(pg, env, query): + return checked( + pg / "psql", "-XAtq", "-v", "ON_ERROR_STOP=1", env=env, data=query.encode() + ).decode() + + +def inventory(pg, env): + databases = psql( + pg, env, "SELECT datname FROM pg_database WHERE datname LIKE 'rstack%';" + ) + roles = psql( + pg, + env, + "SELECT rolname FROM pg_roles WHERE rolname LIKE 'rstack%' OR rolname LIKE 'breg_bbox_%';", + ) + return set(databases.splitlines()), set(roles.splitlines()) + + +def administrator_locks(pg, env): + locks = psql( + pg, + env, + """SELECT classid::text || ':' || objid::text || ':' || objsubid::text + FROM pg_locks + WHERE locktype = 'advisory' AND granted + AND database = (SELECT oid FROM pg_database WHERE datname = current_database());""", + ) + return set(locks.splitlines()) + + +def files(root): + return { + path.relative_to(root): hashlib.sha256(path.read_bytes()).digest() + for path in root.rglob("*") + if path.is_file() + } + + +def dev(bin_dir, env, cwd, *args, expected=0): + install_root = Path(env["RSTACK_DEV_STATE_DIR"]).with_name("rstack-install") + result = command( + bin_dir / "rstack", + "--install-root", + install_root, + "dev", + *args, + "--json", + env=env, + cwd=cwd, + timeout=390, + ) + try: + report = json.loads(result.stdout) + except ValueError: + raise RuntimeError( + f"rstack dev {args[0]} did not return JSON (exit {result.returncode})" + ) from None + if result.returncode != expected: + raise RuntimeError(f"rstack dev {args[0]} exited {result.returncode}: {report}") + if args[0] == "check": + assert report["kind"] == "BRegCtlReport" and report["command"] == "dev-check" + else: + assert report["schema_version"] == "registry.rstack.dev.v1" + return report + + +def credential(bin_dir, env, cwd, run, alias): + before = time.time() + report = dev(bin_dir, env, cwd, "token", run, "--as", alias) + # The report gives the expiry the tokens carry, at most five minutes away. + assert before < report["expires_at"] <= time.time() + 300, report + [entry] = report["credentials"] + header_file = Path(entry["header_file"]) + assert header_file.stat().st_mode & 0o077 == 0 + return dict(line.split(": ", 1) for line in header_file.read_text().splitlines()) + + +def get(url, headers): + try: + with HTTP.open(Request(url, headers=headers), timeout=5) as response: + return response.status + except HTTPError as error: + return error.code + + +def kid(headers): + jwt_header = headers["Authorization"].split()[1].split(".")[0] + return json.loads(base64.urlsafe_b64decode(jwt_header + "==="))["kid"] + + +def wait_for_stop(bin_dir, env, cwd, run): + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + if dev(bin_dir, env, cwd, "status", run)["status"] == "stopped": + return + time.sleep(0.2) + raise RuntimeError("the run did not finish its TTL cleanup") + + +def live_runs(state): + """Runs that may still be alive; `failed/` only holds kept logs and a + `.RUN_ID.removing` directory is a stopped run whose files are being deleted.""" + if not state.exists(): + return [] + return [ + path + for path in state.iterdir() + if path.name != "failed" and not path.name.startswith(".") + ] + + +def nothing_left_behind(state): + """Every stopped run's files are eventually deleted.""" + eventually( + lambda: ( + not state.exists() or all(path.name == "failed" for path in state.iterdir()) + ), + "a stopped run's files were not deleted", + ) + + +def kept_logs(state, error): + """The log directory a failed run's error names, after checking it.""" + marker = "logs kept at " + assert marker in error, error + logs = Path(error.rsplit(marker, 1)[1].strip()) + assert logs.name == "logs" and logs.parent.parent == state / "failed", logs + assert logs.is_dir() and logs.stat().st_mode & 0o777 == 0o700, logs + assert (logs / "supervisor.log").is_file(), logs + return logs + + +def eventually(check, message, timeout=10): + deadline = time.monotonic() + timeout + while not check(): + assert time.monotonic() < deadline, message + time.sleep(0.1) + + +def accepts_connections(url): + parts = urlsplit(url) + with socket.socket() as probe: + probe.settimeout(2) + return probe.connect_ex((parts.hostname, parts.port)) == 0 + + +def product_pids(bin_dir): + """Processes running this build's BReg or Casework (not their ctl tools).""" + pattern = f"{re.escape(str(bin_dir))}/(breg|casework)( |$)" + result = command("pgrep", "-f", pattern) + assert result.returncode in (0, 1), "pgrep failed" + return result.stdout.split() + + +def assert_nothing_listening(bin_dir, urls=()): + """No product URL accepts a connection and no product process remains.""" + for url in urls: + eventually( + lambda url=url: not accepts_connections(url), f"{url} is still listening" + ) + eventually(lambda: not product_pids(bin_dir), "a product process is still alive") + + +def verify_deployment_key(root, bin_dir, env, run_root, headers): + """Start the real verifier with a separate trust pin and the same issuer.""" + key = root / "deployment-issuer.key" + checked( + "openssl", "ecparam", "-name", "prime256v1", "-genkey", "-noout", "-out", key + ) + point = checked("openssl", "pkey", "-in", key, "-pubout", "-outform", "DER")[-65:] + assert point[0] == 4 + + def coordinate(value): + return base64.urlsafe_b64encode(value).decode().rstrip("=") + + jwks = run_root / "deployment-jwks" + jwks.touch(mode=0o600) + jwks.write_text( + json.dumps( + { + "keys": [ + { + "kty": "EC", + "alg": "ES256", + "crv": "P-256", + "kid": "deployment-key", + "x": coordinate(point[1:33]), + "y": coordinate(point[33:]), + } + ] + } + ) + ) + config = json.loads((run_root / "breg-runtime.yaml").read_text()) + config["authentication"]["oidc"]["jwksSource"]["documentRef"] = ( + "secret:file/deployment-jwks" + ) + config["audit"]["path"] = str(root / "deployment-audit" / "audit.jsonl") + config["listener"]["bind"] = f"127.0.0.1:{port()}" + runtime_file = root / "deployment-verifier.yaml" + runtime_file.write_text(json.dumps(config)) + process = subprocess.Popen( + [bin_dir / "breg", "--runtime-config", runtime_file], + env={ + key: value for key, value in env.items() if key != "RSTACK_DEV_DATABASE_URL" + }, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + origin = "http://" + config["listener"]["bind"] + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + assert process.poll() is None, "the independently pinned verifier exited" + try: + status = get(origin + "/ready", {}) + except OSError: + # The verifier may still be opening its listener. + status = None + if status == 200: + break + time.sleep(0.2) + else: + raise RuntimeError("the independently pinned verifier did not become ready") + assert ( + get(origin + "/v1/records/applications?accessProfile=applicant", headers) + == 401 + ) + finally: + process.terminate() + process.wait(timeout=15) + + +def create_owner(pg, pg_env, url): + """A least-privilege administrator: it can create roles and databases only.""" + password = secrets.token_urlsafe(32) + psql( + pg, + pg_env, + f"CREATE ROLE devowner LOGIN CREATEROLE CREATEDB PASSWORD '{password}';", + ) + parts = urlsplit(url) + owner_url = ( + f"postgresql://devowner:{password}@{parts.hostname}:{parts.port}/postgres" + ) + return owner_url, {**pg_env, "PGUSER": "devowner", "PGPASSWORD": password} + + +def exercise(root, bin_dir, pg, project, ca, url, pg_env, spatial): + checkout = root / "licensing" + shutil.copytree(project, checkout) + before = files(checkout) + state = root / "runs" + owner_url, owner_env = create_owner(pg, pg_env, url) + env = { + **os.environ, + "RSTACK_DEV_STATE_DIR": str(state), + "RSTACK_DEV_DATABASE_URL": owner_url, + "RSTACK_DEV_DATABASE_CA": str(ca), + "RSTACK_BREGCTL_BIN": str(bin_dir / "bregctl"), + "RSTACK_BREG_BIN": str(bin_dir / "breg"), + "RSTACK_CASEWORKCTL_BIN": str(bin_dir / "caseworkctl"), + "RSTACK_CASEWORK_BIN": str(bin_dir / "casework"), + } + try: + exercise_runs(root, bin_dir, pg, checkout, state, env, pg_env) + verify_lost_supervisor(bin_dir, pg, checkout, state, env, pg_env, owner_env) + verify_provisioning_deadline(bin_dir, pg, checkout, state, env, pg_env) + verify_spatial_refusal(root, bin_dir, pg, state, env, pg_env) + if spatial: + verify_spatial( + root, bin_dir, pg, {**env, "RSTACK_DEV_DATABASE_URL": url}, pg_env + ) + assert files(checkout) == before + nothing_left_behind(state) + finally: + for run_root in live_runs(state): + dev(bin_dir, env, checkout, "stop", run_root.name) + + +def exercise_runs(root, bin_dir, pg, checkout, state, env, pg_env): + baseline = inventory(pg, pg_env) + baseline_locks = administrator_locks(pg, pg_env) + with concurrent.futures.ThreadPoolExecutor(2) as pool: + runs = list( + pool.map(lambda _: dev(bin_dir, env, checkout, "start", "."), range(2)) + ) + a, b = runs + assert all(run["journey"]["result"] == "passed" for run in runs) + assert a["run_id"] != b["run_id"] + assert len(administrator_locks(pg, pg_env) - baseline_locks) == 2 + assert len({url for run in runs for url in run["products"].values()}) == 4 + databases, roles = inventory(pg, pg_env) + assert all(sum(run["run_id"] in item for item in databases) == 3 for run in runs) + assert all(sum(run["run_id"] in item for item in roles) == 6 for run in runs) + for product in ("breg", "casework"): + own_db = next( + name + for name in databases + if a["run_id"] in name and name.endswith(f"_{product}") + ) + other_db = next( + name + for name in databases + if b["run_id"] in name and name.endswith(f"_{product}") + ) + url = urlsplit((state / a["run_id"] / f"{product}-runtime-url").read_text()) + role_env = { + **pg_env, + "PGUSER": unquote(url.username), + "PGPASSWORD": unquote(url.password), + "PGDATABASE": own_db, + } + assert psql(pg, role_env, "SELECT current_database();").strip() == own_db + refused = command( + pg / "psql", + "-XAtq", + "-v", + "ON_ERROR_STOP=1", + env={**role_env, "PGDATABASE": other_db}, + data=b"SELECT 1;", + ) + assert refused.returncode != 0 + for product, alias, path in [ + ("breg", "applicant", "/v1/records/applications?accessProfile=applicant"), + ("casework", "intake", "/v1/review-policies/licensing-approval"), + ]: + first = credential(bin_dir, env, checkout, a["run_id"], alias) + second = credential(bin_dir, env, checkout, b["run_id"], alias) + assert kid(first) != kid(second) + assert get(a["products"][product] + path, first) == 200 + assert get(b["products"][product] + path, second) == 200 + assert get(a["products"][product] + path, second) == 401 + assert get(b["products"][product] + path, first) == 401 + verify_deployment_key( + root, + bin_dir, + env, + state / a["run_id"], + credential(bin_dir, env, checkout, a["run_id"], "applicant"), + ) + print( + "Concurrent licensing journeys passed with isolated product tokens, ports, databases and roles" + ) + + for run in runs: + assert dev(bin_dir, env, checkout, "stop", run["run_id"])["status"] == "stopped" + assert dev(bin_dir, env, checkout, "stop", run["run_id"])["status"] == "stopped" + assert not (state / run["run_id"]).exists() + assert inventory(pg, pg_env) == baseline + assert_nothing_listening( + bin_dir, [url for run in runs for url in run["products"].values()] + ) + + short = dev(bin_dir, env, checkout, "start", ".", "--ttl", "1s") + wait_for_stop(bin_dir, env, checkout, short["run_id"]) + assert inventory(pg, pg_env) == baseline + assert live_runs(state) == [] + assert_nothing_listening(bin_dir, short["products"].values()) + + failing = root / "failing-casework" + failing.write_text("#!/bin/sh\nprintf 'boom\\n' >&2\nexit 1\n") + failing.chmod(0o700) + failure_env = {**env, "RSTACK_CASEWORK_BIN": str(failing)} + report = dev(bin_dir, failure_env, checkout, "start", ".", expected=1) + assert "casework exited before readiness" in report["error"], report + assert "boom" in report["error"], report + logs = kept_logs(state, report["error"]) + assert any("boom" in log.read_text() for log in logs.glob("casework-*.log")), report + assert inventory(pg, pg_env) == baseline, report + assert live_runs(state) == [], report + assert_nothing_listening(bin_dir) + verify_native_handoff(root, bin_dir, env) + assert inventory(pg, pg_env) == baseline + assert administrator_locks(pg, pg_env) == baseline_locks + assert live_runs(state) == [] + print( + "Stop, TTL expiry and failed startup cleaned up; authored product files stayed byte-identical" + ) + + +def verify_provisioning_deadline(bin_dir, pg, checkout, state, env, pg_env): + baseline = inventory(pg, pg_env) + baseline_locks = administrator_locks(pg, pg_env) + lock = subprocess.Popen( + [pg / "psql", "-XAtq", "-v", "ON_ERROR_STOP=1"], + env={**pg_env, "PGAPPNAME": "rstack-dev-provisioning-lock"}, + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + text=True, + ) + lock_query = """ + SELECT EXISTS ( + SELECT 1 FROM pg_locks locks JOIN pg_stat_activity sessions USING (pid) + WHERE sessions.application_name = 'rstack-dev-provisioning-lock' + AND locks.relation = 'pg_catalog.pg_database'::regclass + AND locks.mode = 'ShareLock' AND locks.granted + ); + """ + try: + # A SHARE lock allows new connections and reads but blocks CREATE DATABASE. + # The idle deadline releases the lock even if the runner stalls. + lock.stdin.write( + "SET lock_timeout = '5s';\n" + "SET idle_in_transaction_session_timeout = '45s';\n" + "BEGIN;\n" + "LOCK TABLE pg_catalog.pg_database IN SHARE MODE;\n" + ) + lock.stdin.flush() + deadline = time.monotonic() + 10 + while psql(pg, pg_env, lock_query).strip() != "t": + assert lock.poll() is None, "the PostgreSQL lock fixture exited" + assert time.monotonic() < deadline, "the PostgreSQL lock was not acquired" + time.sleep(0.1) + + report = dev(bin_dir, env, checkout, "start", ".", expected=1) + error = report["error"] + assert "cannot create an isolated PostgreSQL database" in error, report + assert "statement deadline" in error, report + assert psql(pg, pg_env, lock_query).strip() == "t" + assert inventory(pg, pg_env) == baseline + assert administrator_locks(pg, pg_env) == baseline_locks + assert live_runs(state) == [] + if "logs kept at " in error: + kept_logs(state, error) + finally: + lock.stdin.close() + try: + lock.wait(timeout=5) + except subprocess.TimeoutExpired: + lock.kill() + lock.wait(timeout=5) + raise RuntimeError("the PostgreSQL lock fixture did not exit") from None + print("A blocked PostgreSQL operation timed out and released earlier resources") + + +def listener_pids(url): + result = command( + "lsof", "-nP", "-Fp", f"-iTCP:{urlsplit(url).port}", "-sTCP:LISTEN" + ) + return {line[1:] for line in result.stdout.decode().split() if line[0] == "p"} + + +def recover_lost_supervisor(pg, owner_env, recovery): + """Release a lost run by hand, from the facts `stop` reports. + + Order: stop each process group (TERM, then KILL), drop every database + (WITH (FORCE) ends leftover sessions), drop every role, then remove the + run directory. + """ + for process in recovery["processes"]: + group = process["pgid"] + os.killpg(group, signal.SIGTERM) + + def gone(group=group): + try: + os.killpg(group, 0) + except ProcessLookupError: + return True + return False + + try: + eventually(gone, "a process group ignored SIGTERM", timeout=10) + except AssertionError: + os.killpg(group, signal.SIGKILL) + eventually(gone, "a process group survived SIGKILL", timeout=10) + objects = recovery["objects"] + statements = [ + f'DROP DATABASE IF EXISTS "{item["database"]}" WITH (FORCE);' + for item in objects + ] + [f'DROP ROLE IF EXISTS "{role}";' for item in objects for role in item["roles"]] + psql(pg, owner_env, "\n".join(statements) + "\n") + shutil.rmtree(recovery["run_directory"]) + + +def verify_lost_supervisor(bin_dir, pg, checkout, state, env, pg_env, owner_env): + baseline = inventory(pg, pg_env) + baseline_locks = administrator_locks(pg, pg_env) + report = dev(bin_dir, env, checkout, "start", ".") + run = report["run_id"] + worker = command( + "pgrep", "-f", f"project-dev --worker {re.escape(str(state / run))}" + ) + [supervisor] = worker.stdout.split() + os.kill(int(supervisor), signal.SIGKILL) + status = dev(bin_dir, env, checkout, "status", run, expected=1) + stopped = dev(bin_dir, env, checkout, "stop", run, expected=1) + for lost in (status, stopped): + assert lost["recovery"]["kind"] == "lost-supervisor", lost + assert lost["run_id"] == run, lost + recovery = stopped["recovery"] + assert recovery["run_directory"] == str(state / run), recovery + processes = recovery["processes"] + assert sorted(p["product"] for p in processes) == ["breg", "casework"], recovery + for process in processes: + os.kill(process["pid"], 0) + assert os.getpgid(process["pid"]) == process["pgid"], recovery + listeners = listener_pids(report["products"][process["product"]]) + assert str(process["pid"]) in listeners, recovery + # The pattern behind assert_nothing_listening sees the live products. + assert len(product_pids(bin_dir)) == 2, recovery + databases, roles = inventory(pg, pg_env) + assert {item["database"] for item in recovery["objects"]} == ( + databases - baseline[0] + ), recovery + assert {role for item in recovery["objects"] for role in item["roles"]} == ( + roles - baseline[1] + ), recovery + recover_lost_supervisor(pg, owner_env, recovery) + assert_nothing_listening(bin_dir, report["products"].values()) + assert inventory(pg, pg_env) == baseline + eventually( + lambda: administrator_locks(pg, pg_env) == baseline_locks, + "the lost supervisor's administrator lock was not released", + ) + assert live_runs(state) == [] + assert dev(bin_dir, env, checkout, "status", run)["status"] == "stopped" + print("A lost supervisor was recovered by hand from the facts stop reported") + + +def verify_native_handoff(root, bin_dir, env): + checkout = root / "native-review" + checkout.mkdir() + for product in ("registry", "casework"): + shutil.copytree(NATIVE_REVIEW / product, checkout / product) + shutil.copyfile(NATIVE_REVIEW / "dev.yaml", checkout / "dev.yaml") + before = files(checkout) + report = dev(bin_dir, env, checkout, "start", ".") + try: + assert report["journey"]["id"] == "native-review-handoff" + assert report["journey"]["result"] == "passed" + assert files(checkout) == before + print( + "Native review handoff, reviewer context and separately authorized BReg apply passed" + ) + finally: + dev(bin_dir, env, checkout, "stop", report["run_id"]) + + +def spatial_project(root, name): + """Copy the spatial registry and write a plan that creates and reads a site.""" + checkout = root / name + registry = checkout / "registry" + shutil.copytree(SPATIAL, registry) + fixture = json.loads((registry / "tests/journeys.yaml").read_text()) + people = {} + for journey in fixture["journeys"]: + for step in journey["steps"]: + profile = step["accessProfile"] + claims = step["claims"] + people[profile] = { + "subject": claims["principal"], + "client": "spatial-smoke", + "grants": { + "breg": { + "profile": profile, + "scopes": claims["scopes"], + "claims": { + **claims.get("directClaims", {}), + "registry_actor_kind": claims.get("actorKind", "human"), + "registry_purpose": claims["purpose"], + }, + } + }, + } + with (registry / "fixtures/seed-service-sites.jsonl").open() as records: + site = json.loads(next(records))["data"] + longitude, latitude = site["location"]["coordinates"] + bbox = ( + f"{longitude - 0.01:.6f},{latitude - 0.01:.6f}," + f"{longitude + 0.01:.6f},{latitude + 0.01:.6f}" + ) + plan = { + "apiVersion": "id.registrystack.org/formats/breg/project-dev/v1alpha1", + "kind": "BRegProjectDev", + "issuer": "https://spatial-smoke.example.invalid", + "products": [{"product": "breg", "project": "registry", "audience": "breg"}], + "people": people, + "journey": { + "id": "spatial-smoke", + "steps": [ + { + "id": "create-site", + "product": "breg", + "as": "service-site-admin", + "method": "POST", + "path": "/v1/records/service-sites", + "headers": {"Idempotency-Key": "spatial-smoke"}, + "body": {"data": site}, + "expect": {"status": 201}, + }, + { + "id": "read-bbox", + "product": "breg", + "as": "map-reader", + "method": "GET", + "path": f"/v1/records/service-sites?bbox={bbox}", + "expect": { + "status": 200, + "checks": [{"pointer": "/items", "length": 1}], + }, + }, + ], + }, + } + (checkout / "dev.yaml").write_text(json.dumps(plan)) + return checkout, registry, bbox + + +def verify_spatial_refusal(root, bin_dir, pg, state, env, pg_env): + """A spatial start is refused cleanly when the administrator cannot offer PostGIS.""" + checkout, _, _ = spatial_project(root, "spatial-refusal") + baseline = inventory(pg, pg_env) + offered = psql( + pg, + pg_env, + "SELECT default_version FROM pg_available_extensions WHERE name = 'postgis';", + ).strip() + supported = bool(offered) and tuple(map(int, offered.split(".")[:2])) >= (3, 5) + report = dev(bin_dir, env, checkout, "start", ".", expected=1) + if supported: + assert "must be allowed to CREATE EXTENSION postgis" in report["error"], report + else: + assert "PostGIS 3.5 or later installed" in report["error"], report + assert inventory(pg, pg_env) == baseline, report + assert live_runs(state) == [], report + print("A spatial start without PostGIS rights was refused and left nothing behind") + + +def verify_spatial(root, bin_dir, pg, env, pg_env): + checkout, registry, bbox = spatial_project(root, "spatial") + before = files(registry) + baseline = inventory(pg, pg_env) + report = dev(bin_dir, env, checkout, "start", ".") + run = report["run_id"] + try: + assert report["journey"]["result"] == "passed" + origin = report["products"]["breg"] + collection = f"{origin}/v1/gis/collections/service-site.map-reader" + headers = credential(bin_dir, env, checkout, run, "map-reader") + with HTTP.open(Request(collection, headers=headers), timeout=5) as response: + metadata = json.load(response) + items = next( + link["href"] for link in metadata["links"] if link["rel"] == "items" + ) + assert items == f"{collection}/items" + request = Request(f"{items}?bbox={bbox}&f=json", headers=headers) + with HTTP.open(request, timeout=5) as response: + features = json.load(response) + assert features["type"] == "FeatureCollection" + assert len(features["features"]) == 1 + owners = set() + for label in ("breg_test", "breg"): + prefix = f"rstack_{run}_{label}" + db_env = {**pg_env, "PGDATABASE": prefix} + extension = psql( + pg, + db_env, + """SELECT n.nspname + FROM pg_extension e JOIN pg_namespace n ON n.oid = e.extnamespace + WHERE e.extname = 'postgis';""", + ).strip() + assert extension == "registry_spatial_ext" + owner = psql( + pg, + db_env, + f"""SELECT granted.rolname + FROM pg_auth_members membership + JOIN pg_roles grantee ON grantee.oid = membership.member + JOIN pg_roles granted ON granted.oid = membership.roleid + WHERE grantee.rolname = '{prefix}_migration' + AND granted.rolname LIKE 'breg_bbox_%' AND NOT granted.rolcanlogin + AND membership.set_option AND NOT membership.inherit_option + AND NOT membership.admin_option + AND NOT pg_has_role('{prefix}_runtime', granted.oid, 'MEMBER');""", + ).strip() + assert owner.startswith("breg_bbox_") and "\n" not in owner + owners.add(owner) + assert len(owners) == 2 + assert files(registry) == before + finally: + assert dev(bin_dir, env, checkout, "stop", run)["status"] == "stopped" + assert inventory(pg, pg_env) == baseline + print("Spatial BReg bbox journey passed with distinct, cleaned-up spatial roles") + + +def verify_offline_plan(root, bin_dir, project): + state = root / "offline-check-runs" + check_env = { + key: value + for key, value in os.environ.items() + if key + not in { + "RSTACK_DEV_DATABASE_URL", + "RSTACK_DEV_DATABASE_CA", + "RSTACK_BREG_BIN", + "RSTACK_CASEWORK_BIN", + } + } + check_env["RSTACK_BREGCTL_BIN"] = str(bin_dir / "bregctl") + check_env["RSTACK_CASEWORKCTL_BIN"] = str(bin_dir / "caseworkctl") + check_env["RSTACK_DEV_STATE_DIR"] = str(state) + report = dev(bin_dir, check_env, project, "check", ".") + assert report["ok"] and report["status"] == "complete" + assert report["diagnostics"] == [] + assert not state.exists() + + invalid = root / "offline-invalid" + shutil.copytree(project, invalid) + plan = invalid / "dev.yaml" + authored = plan.read_text() + profile = "profile: intake" + unknown = "missing-check-profile" + assert authored.count(profile) == 1 + plan.write_text(authored.replace(profile, f"profile: {unknown}")) + refused = dev(bin_dir, check_env, invalid, "check", ".", expected=1) + [diagnostic] = refused["diagnostics"] + assert refused["status"] == "domain-refusal" + assert diagnostic["path"] == "/people/intake/grants/casework/profile" + assert diagnostic["suggestedAction"] + assert unknown not in diagnostic["message"] + diagnostic["suggestedAction"] + assert not state.exists() + print("Offline checks resolve product profiles with only the authoring tools") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--bin-dir", type=Path, required=True) + parser.add_argument("--postgres-bin-dir", type=Path, required=True) + parser.add_argument("--scratch-parent", type=Path) + parser.add_argument("--spatial", action="store_true", help="also verify PostGIS") + args = parser.parse_args() + with tempfile.TemporaryDirectory( + prefix="rstack-dev-", dir=args.scratch_parent + ) as directory: + root = Path(directory) + bin_dir = args.bin_dir.resolve() + project = STACK / "products/platform/examples/licensing" + verify_offline_plan(root, bin_dir, project) + pg = args.postgres_bin_dir.resolve() + cluster, ca, url, pg_env = prepare_postgres(root, pg) + checked( + pg / "pg_ctl", "-D", cluster, "-l", root / "postgres.log", "-w", "start" + ) + try: + assert ( + psql( + pg, + pg_env, + "SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid();", + ).strip() + == "t" + ) + exercise( + root, + bin_dir, + pg, + project, + ca, + url, + pg_env, + args.spatial, + ) + finally: + checked(pg / "pg_ctl", "-D", cluster, "-m", "fast", "-w", "stop") + + +if __name__ == "__main__": + main() diff --git a/products/platform/scripts/test_run_config_conformance.py b/products/platform/scripts/test_run_config_conformance.py index 93d015c0fd..7003687e0f 100644 --- a/products/platform/scripts/test_run_config_conformance.py +++ b/products/platform/scripts/test_run_config_conformance.py @@ -1053,6 +1053,14 @@ def test_native_breg_editor_association_pins_kind_version_and_schema(self) -> No class PlanTest(unittest.TestCase): + def test_check_child_selects_the_built_breg_helper_for_source_rstack(self) -> None: + with tempfile.TemporaryDirectory() as directory: + binary = Path(directory) / "bregctl" + binary.write_text("fixture", encoding="utf-8") + with unittest.mock.patch.dict(runner.os.environ, {}, clear=True): + environment = runner.check_environment(Path(directory)) + self.assertEqual(environment["RSTACK_BREGCTL_BIN"], str(binary)) + def test_macos_check_child_restores_the_cargo_runtime_library_path(self) -> None: with ( unittest.mock.patch.object(runner.sys, "platform", "darwin"),