From ff8a39aad22534e3b6d6b709e99cca695d3fb267 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 09:42:30 +0700 Subject: [PATCH 01/64] feat(breg): add read-only external views over Adapter projections Baseline of the external record projections candidate: BReg serves read-only views over records an Adapter projects from an external source, with grants checked on every read, sealed page cursors and a retained projection store. Review fixes follow in separate commits. Signed-off-by: Jeremi Joslin --- Cargo.lock | 2 + crates/registry-adapter-core/src/checks.rs | 21 +- .../registry-adapter-core/src/diagnostics.rs | 4 + crates/registry-adapter-core/src/job.rs | 36 +- crates/registry-adapter-core/src/lib.rs | 2 + crates/registry-adapter-core/src/pack.rs | 25 +- .../registry-adapter-core/src/projection.rs | 211 ++ crates/registry-adapter-engine/src/egress.rs | 35 + .../registry-adapter-engine/src/evidence.rs | 79 +- crates/registry-adapter-engine/src/lib.rs | 1 + .../registry-adapter-engine/src/projection.rs | 525 ++++ crates/registry-adapter-engine/src/route.rs | 27 +- .../tests/projection_acquisition.rs | 507 ++++ crates/registry-breg/Cargo.toml | 5 + .../registry-breg/src/api/external_views.rs | 479 ++++ crates/registry-breg/src/api/mod.rs | 17 + crates/registry-breg/src/api/service.rs | 10 + crates/registry-breg/src/artifacts.rs | 24 + crates/registry-breg/src/compiler.rs | 9 + crates/registry-breg/src/contract.rs | 10 + crates/registry-breg/src/cursor.rs | 43 +- crates/registry-breg/src/external_source.rs | 884 +++++++ .../src/external_view_artifacts.rs | 266 ++ crates/registry-breg/src/external_views.rs | 621 +++++ crates/registry-breg/src/lib.rs | 4 + crates/registry-breg/src/metrics.rs | 5 +- crates/registry-breg/src/migration.rs | 1 + crates/registry-breg/src/model.rs | 11 + crates/registry-breg/src/package.rs | 26 + crates/registry-breg/src/postgres/catalog.rs | 61 +- .../postgres/external_label_acquisition.rs | 581 +++++ .../src/postgres/external_projection.rs | 2239 +++++++++++++++++ .../src/postgres/external_runtime.rs | 1765 +++++++++++++ .../src/postgres/external_view_query.rs | 1103 ++++++++ .../registry-breg/src/postgres/interlock.rs | 6 + crates/registry-breg/src/postgres/mod.rs | 4 + crates/registry-breg/src/postgres/schema.rs | 3 + crates/registry-breg/src/runtime_config.rs | 20 + crates/registry-breg/src/startup.rs | 92 +- crates/registry-breg/tests/external_views.rs | 170 ++ .../tests/external_views_example.rs | 84 + crates/registry-breg/tests/startup_http.rs | 16 +- crates/registry-bregctl/src/dev/config.rs | 22 +- crates/registry-bregctl/src/doctor.rs | 10 + crates/registry-bregctl/src/lib.rs | 150 +- crates/registry-bregctl/tests/tool_schema.rs | 4 +- crates/registry-cli-docs/catalog.json | 7 +- products/adapter/CONCEPT.md | 58 + products/adapter/README.md | 6 + .../examples/external-projections/README.md | 36 + .../examples/external-projections/pack.yaml | 118 + .../external-projections/schemas/holding.json | 11 + .../external-projections/schemas/person.json | 8 + .../scripts/projection.rhai | 42 + .../adapter/schemas/pack.v1alpha1.schema.json | 114 +- .../scripts/check_dependency_direction.py | 21 +- .../scripts/test_dependency_direction.py | 29 +- .../explain/ModelExplanation.schema.json | 26 + .../contracts/security-invariant-matrix.yaml | 4 + .../contracts/security-test-traceability.yaml | 4 + products/breg/examples/README.md | 4 + .../breg/examples/external-views/README.md | 169 ++ .../breg/examples/external-views/SCALE.md | 87 + .../examples/external-views/dev-clients.yaml | 45 + .../breg/examples/external-views/journey.py | 348 +++ .../breg/examples/external-views/recovery.py | 168 ++ .../examples/external-views/registry.yaml | 231 ++ .../examples/external-views/run_candidate.py | 415 +++ .../examples/external-views/scale_exercise.py | 475 ++++ .../external-views/synthetic_source.py | 400 +++ .../examples/external-views/test_scale.py | 93 + .../examples/external-views/test_source.py | 249 ++ .../external-views/tests/journeys.yaml | 20 + products/breg/external-projection-storage.md | 81 + .../authoring/registry-project.schema.json | 320 +++ .../generated/runtime/runtime.schema.json | 669 +++++ .../migration-descriptor.v1alpha1.schema.json | 6 +- products/breg/scripts/check-contracts.sh | 2 + .../breg/scripts/test_validate_product.py | 4 +- products/breg/scripts/validate_product.py | 2 +- scripts/external-projection-scope.sh | 35 + 81 files changed, 14450 insertions(+), 77 deletions(-) create mode 100644 crates/registry-adapter-core/src/projection.rs create mode 100644 crates/registry-adapter-engine/src/projection.rs create mode 100644 crates/registry-adapter-engine/tests/projection_acquisition.rs create mode 100644 crates/registry-breg/src/api/external_views.rs create mode 100644 crates/registry-breg/src/external_source.rs create mode 100644 crates/registry-breg/src/external_view_artifacts.rs create mode 100644 crates/registry-breg/src/external_views.rs create mode 100644 crates/registry-breg/src/postgres/external_label_acquisition.rs create mode 100644 crates/registry-breg/src/postgres/external_projection.rs create mode 100644 crates/registry-breg/src/postgres/external_runtime.rs create mode 100644 crates/registry-breg/src/postgres/external_view_query.rs create mode 100644 crates/registry-breg/tests/external_views.rs create mode 100644 crates/registry-breg/tests/external_views_example.rs create mode 100644 products/adapter/examples/external-projections/README.md create mode 100644 products/adapter/examples/external-projections/pack.yaml create mode 100644 products/adapter/examples/external-projections/schemas/holding.json create mode 100644 products/adapter/examples/external-projections/schemas/person.json create mode 100644 products/adapter/examples/external-projections/scripts/projection.rhai create mode 100644 products/breg/examples/external-views/README.md create mode 100644 products/breg/examples/external-views/SCALE.md create mode 100644 products/breg/examples/external-views/dev-clients.yaml create mode 100644 products/breg/examples/external-views/journey.py create mode 100644 products/breg/examples/external-views/recovery.py create mode 100644 products/breg/examples/external-views/registry.yaml create mode 100644 products/breg/examples/external-views/run_candidate.py create mode 100644 products/breg/examples/external-views/scale_exercise.py create mode 100644 products/breg/examples/external-views/synthetic_source.py create mode 100644 products/breg/examples/external-views/test_scale.py create mode 100644 products/breg/examples/external-views/test_source.py create mode 100644 products/breg/examples/external-views/tests/journeys.yaml create mode 100644 products/breg/external-projection-storage.md create mode 100755 scripts/external-projection-scope.sh diff --git a/Cargo.lock b/Cargo.lock index 59c106eabc..0b2f2c4d79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6115,6 +6115,8 @@ dependencies = [ "jsonwebtoken", "pg_query", "rcgen", + "registry-adapter-core", + "registry-adapter-engine", "registry-breg-client", "registry-evidence-authoring", "registry-evidence-client", diff --git a/crates/registry-adapter-core/src/checks.rs b/crates/registry-adapter-core/src/checks.rs index 56ae0fd8c1..6b89c6705e 100644 --- a/crates/registry-adapter-core/src/checks.rs +++ b/crates/registry-adapter-core/src/checks.rs @@ -1713,11 +1713,24 @@ fn source_route_checks(pack: &AdapterPack) -> Vec { .collect(); for (index, source) in pack.source_routes.iter().enumerate() { let prefix = format!("/sourceRoutes/{index}"); - let ingestion = source.evidence.is_none() - || source.scan.is_some() + let job_declared = source.scan.is_some() || source.next_recovery.is_some() || source.edits.is_some() || source.rescan_delay_seconds.is_some(); + let ingestion = (source.evidence.is_none() && source.projection.is_none()) || job_declared; + if source.projection.as_ref().is_some_and(|projection| { + source.evidence.is_some() + || job_declared + || source.read_support.is_some() + || !projection.identifiers_valid() + }) { + findings.push(finding( + CheckCode::PackProjectionInvalid, + &format!("{prefix}/projection"), + "the projection contract has incompatible route roles or identifiers", + "Use a separate projection route with valid authority and scheme identifiers.", + )); + } if ingestion && source.scan.is_none() { findings.push(job_finding( JobCode::SourceScanRequired, @@ -2857,11 +2870,11 @@ fn evidence_only_endpoint_ids(pack: &AdapterPack) -> HashSet { if source.evidence.is_some() { collect_source_endpoint_ids(&source.steps, &bodies, &mut evidence); } - let ingestion = source.evidence.is_none() - || source.scan.is_some() + let job_declared = source.scan.is_some() || source.next_recovery.is_some() || source.edits.is_some() || source.rescan_delay_seconds.is_some(); + let ingestion = (source.evidence.is_none() && source.projection.is_none()) || job_declared; if ingestion { collect_source_endpoint_ids(&source.steps, &bodies, &mut ordinary); } diff --git a/crates/registry-adapter-core/src/diagnostics.rs b/crates/registry-adapter-core/src/diagnostics.rs index 1c16066874..242c714a2f 100644 --- a/crates/registry-adapter-core/src/diagnostics.rs +++ b/crates/registry-adapter-core/src/diagnostics.rs @@ -53,6 +53,7 @@ pub enum CheckCode { PackInvalidHeaderName, PackMapParameterMisplaced, PackReadSupportInvalid, + PackProjectionInvalid, PackReservedHeaderName, PackScriptUncompilable, PackSqliteExtractInvalid, @@ -207,6 +208,7 @@ impl CheckCode { Self::PackGraphQlQueryRequired, Self::PackInvalidHeaderName, Self::PackMapParameterMisplaced, + Self::PackProjectionInvalid, Self::PackReadSupportInvalid, Self::PackReservedHeaderName, Self::PackScriptUncompilable, @@ -363,6 +365,7 @@ impl CheckCode { Self::PackInvalidHeaderName => "adapter.pack.invalid-header-name", Self::PackMapParameterMisplaced => "adapter.pack.map-parameter-misplaced", Self::PackReadSupportInvalid => "adapter.pack.read-support-invalid", + Self::PackProjectionInvalid => "adapter.pack.projection-invalid", Self::PackWriteSupportInvalid => "adapter.pack.write-support-invalid", Self::PackReservedHeaderName => "adapter.pack.reserved-header-name", Self::PackSqliteExtractInvalid => "adapter.pack.sqlite-extract-invalid", @@ -583,6 +586,7 @@ impl CheckCode { | Self::PackInvalidHeaderName | Self::PackMapParameterMisplaced | Self::PackReadSupportInvalid + | Self::PackProjectionInvalid | Self::PackWriteSupportInvalid | Self::PackReservedHeaderName | Self::PackScriptUncompilable diff --git a/crates/registry-adapter-core/src/job.rs b/crates/registry-adapter-core/src/job.rs index d4156e1ea0..f16f5e9903 100644 --- a/crates/registry-adapter-core/src/job.rs +++ b/crates/registry-adapter-core/src/job.rs @@ -1278,12 +1278,16 @@ fn is_default_csv_trim(value: &bool) -> bool { #[must_use] pub fn check_job(job: &AdapterJob, pack: &AdapterPack) -> Vec { let mut findings = Vec::new(); - if pack.source_routes.get(job.source.route.as_str()).is_none() { + if pack + .source_routes + .get(job.source.route.as_str()) + .is_none_or(|route| route.projection.is_some()) + { findings.push(job_finding( JobCode::RouteUnknown, "/source/route", - "the job names no source route its pinned pack declares", - "Name a sourceRoutes id of the pinned pack.", + "the job names no ingestion source route its pinned pack declares", + "Name an ingestion sourceRoutes id of the pinned pack.", )); } findings.extend(profile_consistency_checks(&job.profile)); @@ -2278,6 +2282,32 @@ mod tests { assert_eq!(findings[0].artifact.as_deref(), Some(ADAPTER_JOB_KIND)); } + #[test] + fn check_job_cannot_acquire_from_a_projection_route() { + let mut value = serde_json::to_value(pack()).expect("pack serializes"); + let source = value["sourceRoutes"][0] + .as_object_mut() + .expect("source route"); + for member in ["scan", "edits", "nextRecovery", "rescanDelaySeconds"] { + source.remove(member); + } + source.insert( + "projection".to_owned(), + serde_json::json!({ + "operation":"exact", "operationVersion":"1", "authority":"urn:example:registry", + "purpose":"licensing", "scope":"licensing-subjects", + "selectorScheme":"urn:example:id", "recordKeyScheme":"urn:example:id", + "retention":"retain", "absence":"protected", "localDisclosure":"host-profiles", + "fieldsSchema":{"path":"fields.json","digest":DIGEST}, + "maximumBatchKeys":1, "maximumPages":1, "maximumRows":1 + }), + ); + let projection_pack = serde_json::from_value(value).expect("projection pack parses"); + let findings = check_job(&job("kobo-submissions"), &projection_pack); + assert_eq!(findings[0].code, JobCode::RouteUnknown.code()); + assert_eq!(findings[0].artifact.as_deref(), Some(ADAPTER_JOB_KIND)); + } + /// Each profile bound is a typed member: a value outside its /// 1-to-maximum range fails to parse, so a job that declares one never /// loads. The edge of each range parses; one past it does not. diff --git a/crates/registry-adapter-core/src/lib.rs b/crates/registry-adapter-core/src/lib.rs index fbf9741b34..cd8c791454 100644 --- a/crates/registry-adapter-core/src/lib.rs +++ b/crates/registry-adapter-core/src/lib.rs @@ -45,6 +45,7 @@ mod preview_expectations; mod problem; mod profile; mod project; +mod projection; mod result_schema; mod route; mod runtime; @@ -66,6 +67,7 @@ pub use preview_expectations::*; pub use problem::*; pub use profile::*; pub use project::*; +pub use projection::*; pub use result_schema::*; pub use route::*; pub use runtime::*; diff --git a/crates/registry-adapter-core/src/pack.rs b/crates/registry-adapter-core/src/pack.rs index eeeca76600..452aa4aa3f 100644 --- a/crates/registry-adapter-core/src/pack.rs +++ b/crates/registry-adapter-core/src/pack.rs @@ -446,11 +446,10 @@ pub struct EvidenceRouteContract { pub acquisition_posture: EvidenceAcquisitionPosture, } -/// One source route of the governed-ingestion mode -/// (`products/adapter/CONCEPT.md` sections 17.1 and 17.2): a route -/// definition's steps plus the item-contract declaration. A source route -/// is an entry point of the job mode, never a subroutine: a facade -/// binding naming it, or a `direct:` send targeting it, refuses +/// One source entry point for an ingestion, Evidence or projection host. +/// The host's declaration supplies the operation contract alongside the +/// steps. A source route is never a subroutine: a facade binding naming it, +/// or a `direct:` send targeting it, refuses /// (`adapter.job.source-bound-refused`, /// `adapter.job.source-called-refused`). There is no `from`: the /// `from: file:` consumer form was rejected with the job's read step @@ -459,7 +458,7 @@ pub struct EvidenceRouteContract { #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct SourceRouteDecl { - /// The source route identifier the job definition names. + /// The identifier the embedding host names. pub id: LocalId, /// Whether the scan ends. Mandatory; the checks refuse its absence /// under `adapter.job.source-scan-required`. @@ -492,6 +491,9 @@ pub struct SourceRouteDecl { /// listener-free, read-only Evidence entry point. #[serde(default, skip_serializing_if = "Option::is_none")] pub evidence: Option, + /// Separately governed acquisition for a host that may retain a projection. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub projection: Option, /// The source-neutral read semantics this route explicitly supports. A /// facade host maps these native inputs and guarantees to its public /// protocol; it never infers them from a route id or source brand. @@ -567,9 +569,8 @@ pub struct AdapterPack { /// The internal `direct:` routes shared sub-flows call. #[serde(default)] pub routes: UniqueIdList, - /// The governed-ingestion source routes the job mode runs - /// (`products/adapter/CONCEPT.md` sections 17.1 and 17.2). Entry - /// points of the job mode; a facade route never reaches one. + /// The source routes of the ingestion, Evidence and projection hosts. + /// A facade route never reaches one. #[serde(default)] pub source_routes: UniqueIdList, /// The code maps routes convert values through. @@ -614,6 +615,9 @@ impl AdapterPack { ) .collect(); for source in &self.source_routes { + if let Some(projection) = &source.projection { + files.push(&projection.fields_schema); + } if let Some(evidence) = &source.evidence { files.push(&evidence.inputs_schema); files.push(&evidence.facts_schema); @@ -861,6 +865,7 @@ mod tests { edits: Some("ordered".to_string()), rescan_delay_seconds: None, evidence: None, + projection: None, steps: Vec::new(), }]) .expect("unique"), @@ -948,6 +953,7 @@ mod tests { edits: None, rescan_delay_seconds: None, evidence: None, + projection: None, steps: Vec::new(), }, SourceRouteDecl { @@ -958,6 +964,7 @@ mod tests { edits: None, rescan_delay_seconds: None, evidence: None, + projection: None, steps: Vec::new(), }, ]) diff --git a/crates/registry-adapter-core/src/projection.rs b/crates/registry-adapter-core/src/projection.rs new file mode 100644 index 0000000000..c00f1d4a99 --- /dev/null +++ b/crates/registry-adapter-core/src/projection.rs @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Normalized read operations for an independently governed projection host. + +use registry_platform_yaml::BoundedU64; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +use crate::ScriptReference; + +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ProjectionOperationKind { + Exact, + Batch, + ByParent, +} + +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ProjectionRetention { + LiveOnly, + Retain, +} + +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ProjectionAbsence { + Protected, + Authoritative, +} + +/// Whether the source authorizes disclosure under the host's narrower +/// profiles when acquisition uses a shared service identity. +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ProjectionLocalDisclosure { + AcquisitionOnly, + HostProfiles, +} + +/// A reviewed source operation's ceiling. The host still has to establish its +/// own current project grant, connection and eligible selectors before I/O. +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ProjectionRouteContract { + pub operation: ProjectionOperationKind, + pub operation_version: String, + pub authority: String, + pub purpose: String, + pub scope: String, + pub selector_scheme: String, + pub record_key_scheme: String, + pub retention: ProjectionRetention, + pub absence: ProjectionAbsence, + pub local_disclosure: ProjectionLocalDisclosure, + pub fields_schema: ScriptReference, + pub maximum_batch_keys: BoundedU64<1, 1000>, + pub maximum_pages: BoundedU64<1, 100>, + pub maximum_rows: BoundedU64<1, 10000>, +} + +impl ProjectionRouteContract { + #[must_use] + pub fn identifiers_valid(&self) -> bool { + [ + &self.operation_version, + &self.authority, + &self.purpose, + &self.scope, + &self.selector_scheme, + &self.record_key_scheme, + ] + .into_iter() + .all(|identifier| projection_token_valid(identifier)) + && (self.operation == ProjectionOperationKind::ByParent + || self.selector_scheme == self.record_key_scheme) + } +} + +/// Bounds a selector, row key, revision or cursor without normalizing it. +/// The binding reviewed with the source establishes the actual scheme. +#[must_use] +pub fn projection_token_valid(value: &str) -> bool { + !value.is_empty() + && value.len() <= 512 + && value.trim() == value + && !value.chars().any(char::is_control) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ProjectionMode { + Live, + Indexed, +} + +/// The complete root selection already authorized by the host. Collection +/// cursors are internal to the Adapter; callers cannot supply them. +#[derive(Clone, Eq, PartialEq, Serialize, Deserialize)] +#[serde( + tag = "type", + rename_all = "kebab-case", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum ProjectionRequest { + Exact { key: String }, + Batch { keys: Vec }, + ByParent { parent_key: String }, +} + +impl ProjectionRequest { + #[must_use] + pub fn kind(&self) -> ProjectionOperationKind { + match self { + Self::Exact { .. } => ProjectionOperationKind::Exact, + Self::Batch { .. } => ProjectionOperationKind::Batch, + Self::ByParent { .. } => ProjectionOperationKind::ByParent, + } + } +} + +#[derive(Clone, PartialEq, Serialize, Deserialize)] +#[serde( + tag = "outcome", + rename_all = "kebab-case", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum ProjectionRecord { + Present { + key: String, + fields: Map, + #[serde(default, skip_serializing_if = "Option::is_none")] + source_revision: Option, + }, + AuthoritativeAbsent { + key: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + source_revision: Option, + }, + Unavailable { + key: String, + }, +} + +impl ProjectionRecord { + #[must_use] + pub fn key(&self) -> &str { + match self { + Self::Present { key, .. } + | Self::AuthoritativeAbsent { key, .. } + | Self::Unavailable { key } => key, + } + } +} + +#[derive(Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ProjectionRow { + pub key: String, + pub parent_key: String, + pub fields: Map, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_revision: Option, +} + +/// Returned only after structural and scope checks. A ByParent result means +/// every page of one snapshot passed; interruption yields no collection. +#[derive(Clone, PartialEq, Serialize)] +#[serde( + tag = "type", + rename_all = "kebab-case", + rename_all_fields = "camelCase" +)] +pub enum ProjectionResult { + Exact { + record: ProjectionRecord, + }, + Batch { + records: Vec, + }, + ByParent { + parent_key: String, + snapshot: String, + rows: Vec, + }, +} + +// Do not make source values visible through diagnostics or tracing fields. +macro_rules! redact_projection_debug { + ($($name:ident),+) => {$( + impl std::fmt::Debug for $name { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(concat!(stringify!($name), "([REDACTED])")) + } + } + )+}; +} +redact_projection_debug!( + ProjectionRequest, + ProjectionRecord, + ProjectionRow, + ProjectionResult +); diff --git a/crates/registry-adapter-engine/src/egress.rs b/crates/registry-adapter-engine/src/egress.rs index 4d4f4ebaf8..90f46130a4 100644 --- a/crates/registry-adapter-engine/src/egress.rs +++ b/crates/registry-adapter-engine/src/egress.rs @@ -1349,6 +1349,18 @@ impl EndpointEgress { recovery: Option<&SourceRecoveryHeader>, traceparent: Option<&HeaderValue>, source_idempotency_key: Option<&HeaderValue>, + ) -> Result { + self.send_observed(send, recovery, traceparent, source_idempotency_key, None) + .await + } + + pub(crate) async fn send_observed( + &self, + send: OutboundSend<'_>, + recovery: Option<&SourceRecoveryHeader>, + traceparent: Option<&HeaderValue>, + source_idempotency_key: Option<&HeaderValue>, + projection: Option<&crate::projection::ProjectionHttpState>, ) -> Result { // A declared attempt above the ceiling the authoring check // bounds `attemptTimeoutMilliseconds` with is refused, never @@ -1406,6 +1418,7 @@ impl EndpointEgress { source_idempotency_key, }, &link, + projection, ); } let EndpointTransport::Live(live) = self.transport.as_ref() else { @@ -1465,6 +1478,18 @@ impl EndpointEgress { .map_err(destination_send_outcome)?; let status = reply.status().as_u16(); + if let Some(projection) = projection { + // Read the raw transport headers before the pack's allowlist or + // status handling can discard the source's retention restriction. + match reply + .selected_host_protocol_response_header_fields(std::iter::once("cache-control")) + { + Ok(fields) => { + projection.observe_cache_control(fields.iter().map(|(_, value)| value.as_str())) + } + Err(_) => projection.restrict_retention(), + } + } if (300..400).contains(&status) { // A redirect is never followed: the answer raises // `upstream.redirect` whatever the reviewed range says, and @@ -1797,6 +1822,7 @@ impl EndpointEgress { outbound: &OutboundHeaders, system_headers: ReplaySystemHeaders<'_>, link: &PinnedLink, + projection: Option<&crate::projection::ProjectionHttpState>, ) -> Result { let ReplaySystemHeaders { recovery, @@ -1872,6 +1898,15 @@ impl EndpointEgress { .map_err(EgressError::Refused)?; let status = u16::try_from(response.status.get()) .expect("a recorded status is bounded to an HTTP status"); + if let Some(projection) = projection { + projection.observe_cache_control( + response + .headers + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case("cache-control")) + .map(|(_, value)| value.as_str()), + ); + } if (300..400).contains(&status) { return Err(EgressError::Upstream(UpstreamFailure::new( ErrorClass::UpstreamRedirect, diff --git a/crates/registry-adapter-engine/src/evidence.rs b/crates/registry-adapter-engine/src/evidence.rs index a2dc891eba..7d985909f7 100644 --- a/crates/registry-adapter-engine/src/evidence.rs +++ b/crates/registry-adapter-engine/src/evidence.rs @@ -1358,7 +1358,7 @@ impl EvidenceProgram { &self, hooks: Arc, ) -> Result { - self.begin_acquisition_inner(hooks, None, chrono::Utc::now()) + self.begin_acquisition_inner(hooks, None, chrono::Utc::now(), None) .await } @@ -1370,7 +1370,7 @@ impl EvidenceProgram { hooks: Arc, attribution: Arc, ) -> Result { - self.begin_acquisition_inner(hooks, Some(attribution), chrono::Utc::now()) + self.begin_acquisition_inner(hooks, Some(attribution), chrono::Utc::now(), None) .await } @@ -1382,7 +1382,7 @@ impl EvidenceProgram { hooks: Arc, evaluation_instant: chrono::DateTime, ) -> Result { - self.begin_acquisition_inner(hooks, None, evaluation_instant) + self.begin_acquisition_inner(hooks, None, evaluation_instant, None) .await } @@ -1395,16 +1395,20 @@ impl EvidenceProgram { attribution: Arc, evaluation_instant: chrono::DateTime, ) -> Result { - self.begin_acquisition_inner(hooks, Some(attribution), evaluation_instant) + self.begin_acquisition_inner(hooks, Some(attribution), evaluation_instant, None) .await } - async fn begin_acquisition_inner( + pub(crate) async fn begin_acquisition_inner( &self, hooks: Arc, attribution: Option>, evaluation_instant: chrono::DateTime, + maximum_deadline: Option, ) -> Result { + if maximum_deadline.is_some_and(|deadline| tokio::time::Instant::now() >= deadline) { + return Err(EvidenceTerminalError::Budget.into()); + } let intent = EvidenceAcquisitionIntent { pack: self.pack.clone(), route: self.route.clone(), @@ -1419,6 +1423,7 @@ impl EvidenceProgram { let deadline = tokio::time::Instant::now() .checked_add(self.limits.deadline) .ok_or(EvidenceTerminalError::Budget)?; + let deadline = maximum_deadline.map_or(deadline, |maximum| deadline.min(maximum)); let mut endpoints = BTreeMap::new(); let mut host_protocols = BTreeMap::new(); for (id, plan) in &self.endpoint_plans { @@ -1553,6 +1558,22 @@ impl EvidenceProgram { acquisition: &EvidenceAcquisition, inputs: Value, ) -> Result { + let body = self.execute_raw(acquisition, inputs).await?; + let lookup: EvidenceLookup = + serde_json::from_value(body).map_err(|_| EvidenceExecutionError::LookupInvalid)?; + if let EvidenceLookup::Matched { facts } = &lookup { + if !self.facts_schema.validate(facts) { + return Err(EvidenceExecutionError::FactsInvalid); + } + } + Ok(lookup) + } + + pub(crate) async fn execute_raw( + &self, + acquisition: &EvidenceAcquisition, + inputs: Value, + ) -> Result { if acquisition.pack != self.pack || acquisition.route != self.route { return Err(EvidenceExecutionError::Terminal( EvidenceTerminalError::Authority, @@ -1590,14 +1611,7 @@ impl EvidenceProgram { return Err(EvidenceExecutionError::Route(error)); } }; - let lookup: EvidenceLookup = serde_json::from_value(exchange.message.body) - .map_err(|_| EvidenceExecutionError::LookupInvalid)?; - if let EvidenceLookup::Matched { facts } = &lookup { - if !self.facts_schema.validate(facts) { - return Err(EvidenceExecutionError::FactsInvalid); - } - } - Ok(lookup) + Ok(exchange.message.body) } #[must_use] @@ -1686,6 +1700,26 @@ pub fn compile_evidence_pack_with_protocols( &contract.facts_schema, EvidenceSchemaKind::Facts, )?; + compile_read_pack_with_schemas( + pack, + source, + pack_artifacts, + limits, + protocols, + input_schema, + facts_schema, + ) +} + +pub(crate) fn compile_read_pack_with_schemas( + pack: &AdapterPack, + source: &SourceRouteDecl, + pack_artifacts: &PackArtifactBytes, + limits: EvidenceLimits, + protocols: &HostProtocolRegistry, + input_schema: CheckedEvidenceSchema, + facts_schema: CheckedEvidenceSchema, +) -> Result { check_read_only_closure(pack, source)?; let worst_case = registry_adapter_core::evidence_route_worst_case(pack, &source.steps); if worst_case.uncountable { @@ -2005,7 +2039,7 @@ fn authentication_authority_matches( } } -fn load_schema( +pub(crate) fn load_schema( artifacts: &PackArtifactBytes, reference: ®istry_adapter_core::ScriptReference, kind: EvidenceSchemaKind, @@ -2145,7 +2179,7 @@ fn ensure_reachable_endpoints_bound( Ok(()) } -fn reachable_endpoint_ids( +pub(crate) fn reachable_endpoint_ids( pack: &AdapterPack, source: &SourceRouteDecl, ) -> Result, EvidenceCompileError> { @@ -2291,6 +2325,7 @@ pub struct EvidenceAcquisition { trusted_protocol_metadata: tokio::sync::Mutex>, trusted_protocol_attachment: tokio::sync::Mutex>, pub(crate) token_budget: tokio::sync::Mutex, + pub(crate) projection_http: Option, } struct EvidenceAcquisitionStart { @@ -2345,6 +2380,7 @@ impl EvidenceAcquisition { trusted_protocol_metadata: tokio::sync::Mutex::new(Vec::new()), trusted_protocol_attachment: tokio::sync::Mutex::new(None), token_budget: tokio::sync::Mutex::new(TokenBudget::new_exchange()), + projection_http: None, }) } @@ -2365,6 +2401,7 @@ impl EvidenceAcquisition { trusted_protocol_metadata: tokio::sync::Mutex::new(Vec::new()), trusted_protocol_attachment: tokio::sync::Mutex::new(None), token_budget: tokio::sync::Mutex::new(TokenBudget::new_exchange()), + projection_http: None, } } @@ -2447,7 +2484,10 @@ impl EvidenceAcquisition { } { let mut state = self.state.lock().await; - if state.data_operations >= self.limits.maximum_data_operations { + if state.data_operations >= self.limits.maximum_data_operations + || (self.projection_http.is_some() + && state.response_bytes >= self.limits.maximum_response_bytes) + { return Err(EvidenceTerminalError::Budget); } state.data_operations += 1; @@ -2456,6 +2496,13 @@ impl EvidenceAcquisition { Ok(()) } + pub(crate) async fn remaining_response_bytes(&self) -> usize { + let state = self.state.lock().await; + self.limits + .maximum_response_bytes + .saturating_sub(state.response_bytes) + } + pub(crate) async fn record_response_bytes( &self, bytes: usize, diff --git a/crates/registry-adapter-engine/src/lib.rs b/crates/registry-adapter-engine/src/lib.rs index 40c256b373..2124fda749 100644 --- a/crates/registry-adapter-engine/src/lib.rs +++ b/crates/registry-adapter-engine/src/lib.rs @@ -34,6 +34,7 @@ pub mod language; pub mod load; pub mod processor; pub mod project_transform; +pub mod projection; pub mod public_schema; pub mod rabbitmq_config; pub mod read_request; diff --git a/crates/registry-adapter-engine/src/projection.rs b/crates/registry-adapter-engine/src/projection.rs new file mode 100644 index 0000000000..f3d9a9a5c9 --- /dev/null +++ b/crates/registry-adapter-engine/src/projection.rs @@ -0,0 +1,525 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Listener-free acquisition for externally governed record projections. +//! The caller authorizes purpose, selectors and fields before entering this +//! seam and rechecks its grant before publishing. This module cannot store +//! records and never creates acquisition authority from a source response. + +use std::{ + collections::BTreeSet, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, + }, +}; + +use registry_adapter_core::{ + projection_token_valid, AdapterPack, ComponentKind, EvidenceAuthenticationPolicy, + EvidenceClientIdentityPolicy, EvidenceConnectionPolicy, ProjectionAbsence, ProjectionMode, + ProjectionRecord, ProjectionRequest, ProjectionResult, ProjectionRetention, + ProjectionRouteContract, ProjectionRow, +}; +use serde::Deserialize; +use serde_json::{json, Map, Value}; +use thiserror::Error; + +use crate::{ + artifacts::PackArtifactBytes, + evidence::{ + check_schema, compile_read_pack_with_schemas, load_schema, reachable_endpoint_ids, + CheckedEvidenceSchema, EvidenceAcquisition, EvidenceCompileError, EvidenceHostHooks, + EvidenceLimits, EvidencePackProgram, EvidenceProgram, EvidenceRuntimeBindings, + EvidenceSchemaKind, + }, + host_protocol::HostProtocolRegistry, + replay::ReplayTranscript, +}; + +#[derive(Debug, Error)] +pub enum ProjectionError { + #[error("the reviewed projection operation is missing or incompatible")] + Contract, + #[error("the projection route or its operator connection did not compile")] + Compile(#[from] EvidenceCompileError), + #[error("the indexed operation lacks explicit retention permission")] + RetentionRefused, + #[error("the source response does not permit retention")] + NoStore, + #[error("the request is outside the projection operation contract")] + InvalidRequest, + #[error("the source result does not satisfy the projection contract")] + InvalidResult, + #[error("the source could not supply a verified complete result")] + Unavailable, +} + +/// Trusted transport metadata; scripts never see or modify these flags. +#[derive(Default)] +pub(crate) struct ProjectionHttpState { + retention_restricted: AtomicBool, + failed: AtomicBool, + successful: AtomicU64, +} + +impl ProjectionHttpState { + pub(crate) fn restrict_retention(&self) { + self.retention_restricted.store(true, Ordering::Relaxed); + } + + pub(crate) fn observe_cache_control<'a>(&self, fields: impl Iterator) { + // Treat no-store with a nonstandard argument as restrictive too. + // Splitting quoted extension values conservatively can only withhold + // retention. A malformed/non-UTF-8 transport field also fails closed. + if fields.flat_map(|field| field.split(',')).any(|directive| { + directive + .split('=') + .next() + .is_some_and(|name| name.trim().eq_ignore_ascii_case("no-store")) + }) { + self.restrict_retention(); + } + } + + pub(crate) fn observe_response(&self, successful: bool) { + if successful { + self.successful.fetch_add(1, Ordering::Relaxed); + } else { + self.failed.store(true, Ordering::Relaxed); + } + } +} + +pub struct ProjectionPackProgram { + contract: ProjectionRouteContract, + program: EvidencePackProgram, +} + +/// Compile a route in the same closed read engine used by other embedding +/// hosts. The caller supplies the AdapterPack from the pinned pack document; +/// this compiler verifies the exact bytes of every consumed pack artifact. +pub fn compile_projection_pack( + pack: &AdapterPack, + route: &str, + artifacts: &PackArtifactBytes, + limits: EvidenceLimits, +) -> Result { + let source = pack + .source_routes + .get(route) + .ok_or(ProjectionError::Contract)?; + let contract = source + .projection + .as_ref() + .ok_or(ProjectionError::Contract)?; + if !contract.identifiers_valid() + || source.evidence.is_some() + || source.read_support.is_some() + || source.scan.is_some() + || source.next_recovery.is_some() + || source.edits.is_some() + || source.rescan_delay_seconds.is_some() + { + return Err(ProjectionError::Contract); + } + // This initial seam observes every raw data response. Permit only the + // ordinary HTTP path, so a different component or host protocol cannot + // evade that observer. + let endpoints = reachable_endpoint_ids(pack, source)?; + if endpoints.is_empty() + || endpoints.iter().any(|id| { + !pack.endpoints.get(id).is_some_and(|endpoint| { + endpoint.component == ComponentKind::Http && endpoint.host_protocol.is_none() + }) + }) + { + return Err(ProjectionError::Contract); + } + + let input_schema = check_schema( + EvidenceSchemaKind::Inputs, + json!({ + "type":"object", + "additionalProperties":false, + "required":["type","fields"], + "properties":{ + "type":{"enum":["exact","batch","by-parent"]}, + "fields":{"type":"array","items":{"type":"string"}}, + "key":{"type":"string"}, + "keys":{"type":"array","items":{"type":"string"}}, + "parentKey":{"type":"string"}, + "cursor":{"type":"string"} + } + }), + ) + .map_err(EvidenceCompileError::from)?; + let fields_schema = load_schema( + artifacts, + &contract.fields_schema, + EvidenceSchemaKind::Facts, + )?; + if !fields_schema + .as_value() + .get("properties") + .is_some_and(Value::is_object) + { + return Err(ProjectionError::Contract); + } + let program = compile_read_pack_with_schemas( + pack, + source, + artifacts, + limits, + &HostProtocolRegistry::default(), + input_schema, + fields_schema, + )?; + Ok(ProjectionPackProgram { + contract: contract.clone(), + program, + }) +} + +impl ProjectionPackProgram { + #[must_use] + pub fn contract(&self) -> &ProjectionRouteContract { + &self.contract + } + + #[must_use] + pub fn fields_schema(&self) -> &CheckedEvidenceSchema { + self.program.facts_schema() + } + + pub fn bind_live( + self, + runtime: &EvidenceRuntimeBindings, + connection_id: &str, + connection: &[EvidenceConnectionPolicy], + ) -> Result { + if connection.is_empty() + || connection.iter().any(|policy| { + matches!(policy.authentication, EvidenceAuthenticationPolicy::None {}) + && policy.client_identity != EvidenceClientIdentityPolicy::Required + }) + { + return Err(ProjectionError::Contract); + } + Ok(ProjectionProgram { + contract: self.contract, + program: self + .program + .bind_live(runtime, Some(connection_id), connection)?, + }) + } + + /// Execute the same reviewed route against a sealed synthetic transcript. + /// This path has no live connection, identity, or credentials. + pub fn bind_replay( + self, + transcript: Arc, + ) -> Result { + Ok(ProjectionProgram { + contract: self.contract, + program: self.program.bind_replay(transcript)?, + }) + } +} + +pub struct ProjectionProgram { + contract: ProjectionRouteContract, + program: EvidenceProgram, +} + +#[derive(Deserialize)] +#[serde( + tag = "type", + rename_all = "kebab-case", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +enum SourceResult { + Exact { + record: ProjectionRecord, + }, + Batch { + records: Vec, + }, + ByParent { + parent_key: String, + snapshot: String, + rows: Vec, + complete: bool, + next_cursor: Option, + }, + Unavailable {}, +} + +impl ProjectionProgram { + #[must_use] + pub fn contract(&self) -> &ProjectionRouteContract { + &self.contract + } + + #[must_use] + pub fn fields_schema(&self) -> &CheckedEvidenceSchema { + self.program.facts_schema() + } + + /// Run one bounded acquisition under the caller's current grant. All + /// pages share the same audited acquisition and operation/byte budgets. + pub async fn acquire( + &self, + hooks: Arc, + mode: ProjectionMode, + request: ProjectionRequest, + fields: &BTreeSet, + ) -> Result { + self.acquire_until(hooks, mode, request, fields, None).await + } + + /// Apply a stricter host grant deadline without resetting the operation + /// budget or cancelling a durable audit append in progress. + pub async fn acquire_until( + &self, + hooks: Arc, + mode: ProjectionMode, + request: ProjectionRequest, + fields: &BTreeSet, + maximum_deadline: Option, + ) -> Result { + if mode == ProjectionMode::Indexed && self.contract.retention != ProjectionRetention::Retain + { + return Err(ProjectionError::RetentionRefused); + } + self.check_request(mode, &request, fields)?; + let mut acquisition = self + .program + .begin_acquisition_inner(hooks, None, chrono::Utc::now(), maximum_deadline) + .await + .map_err(|_| ProjectionError::Unavailable)?; + acquisition.projection_http = Some(ProjectionHttpState::default()); + + match request { + ProjectionRequest::Exact { key } => { + match self + .execute( + &acquisition, + mode, + json!({"type":"exact","key":key,"fields":fields}), + ) + .await? + { + SourceResult::Exact { record } if record.key() == key => { + self.check_record(&record, fields)?; + Ok(ProjectionResult::Exact { record }) + } + _ => Err(ProjectionError::InvalidResult), + } + } + ProjectionRequest::Batch { keys } => { + match self + .execute( + &acquisition, + mode, + json!({"type":"batch","keys":keys,"fields":fields}), + ) + .await? + { + SourceResult::Batch { records } => { + let mut remaining: BTreeSet<&str> = + keys.iter().map(String::as_str).collect(); + for record in &records { + if !remaining.remove(record.key()) { + return Err(ProjectionError::InvalidResult); + } + self.check_record(record, fields)?; + } + if !remaining.is_empty() { + return Err(ProjectionError::InvalidResult); + } + Ok(ProjectionResult::Batch { records }) + } + _ => Err(ProjectionError::InvalidResult), + } + } + ProjectionRequest::ByParent { parent_key } => { + self.collect(&acquisition, mode, parent_key, fields).await + } + } + } + + fn check_request( + &self, + mode: ProjectionMode, + request: &ProjectionRequest, + fields: &BTreeSet, + ) -> Result<(), ProjectionError> { + let known_fields = self.fields_schema().as_value()["properties"] + .as_object() + .ok_or(ProjectionError::Contract)?; + if request.kind() != self.contract.operation + || fields.len() > 256 + || fields.iter().any(|field| !known_fields.contains_key(field)) + { + return Err(ProjectionError::InvalidRequest); + } + let valid = match request { + ProjectionRequest::Exact { key } => projection_token_valid(key), + ProjectionRequest::Batch { keys } => { + mode == ProjectionMode::Indexed + && !keys.is_empty() + && keys.len() as u64 <= self.contract.maximum_batch_keys.get() + && keys.iter().all(|key| projection_token_valid(key)) + && keys.iter().collect::>().len() == keys.len() + } + ProjectionRequest::ByParent { parent_key } => projection_token_valid(parent_key), + }; + if !valid { + return Err(ProjectionError::InvalidRequest); + } + Ok(()) + } + + async fn execute( + &self, + acquisition: &EvidenceAcquisition, + mode: ProjectionMode, + input: Value, + ) -> Result { + let http = acquisition + .projection_http + .as_ref() + .ok_or(ProjectionError::Contract)?; + let previous_successes = http.successful.load(Ordering::Relaxed); + let result = self.program.execute_raw(acquisition, input).await; + if http.failed.load(Ordering::Relaxed) + || http.successful.load(Ordering::Relaxed) == previous_successes + { + return Err(ProjectionError::Unavailable); + } + let value = result.map_err(|_| ProjectionError::Unavailable)?; + if mode == ProjectionMode::Indexed && http.retention_restricted.load(Ordering::Relaxed) { + return Err(ProjectionError::NoStore); + } + match serde_json::from_value(value) { + Ok(SourceResult::Unavailable {}) => Err(ProjectionError::Unavailable), + Ok(result) => Ok(result), + Err(_) => Err(ProjectionError::InvalidResult), + } + } + + fn check_record( + &self, + record: &ProjectionRecord, + fields: &BTreeSet, + ) -> Result<(), ProjectionError> { + match record { + ProjectionRecord::Present { + fields: returned, + source_revision, + .. + } => self.check_fields(returned, source_revision.as_deref(), fields), + ProjectionRecord::AuthoritativeAbsent { + source_revision, .. + } => { + if self.contract.absence != ProjectionAbsence::Authoritative { + return Err(ProjectionError::Unavailable); + } + check_revision(source_revision.as_deref()) + } + ProjectionRecord::Unavailable { .. } => Ok(()), + } + } + + fn check_fields( + &self, + returned: &Map, + revision: Option<&str>, + fields: &BTreeSet, + ) -> Result<(), ProjectionError> { + // A missing requested field is not an authoritative absence. A + // nullable field can be explicitly null under the reviewed schema. + if returned.len() != fields.len() + || returned.keys().any(|field| !fields.contains(field)) + || !self + .fields_schema() + .validate(&Value::Object(returned.clone())) + { + return Err(ProjectionError::InvalidResult); + } + check_revision(revision) + } + + async fn collect( + &self, + acquisition: &EvidenceAcquisition, + mode: ProjectionMode, + parent_key: String, + fields: &BTreeSet, + ) -> Result { + let mut cursor = None; + let mut seen_cursors = BTreeSet::new(); + let mut seen_rows = BTreeSet::new(); + let mut snapshot = None; + let mut rows = Vec::new(); + for _ in 0..self.contract.maximum_pages.get() { + let mut input = json!({"type":"by-parent","parentKey":parent_key,"fields":fields}); + if let Some(cursor) = cursor.take() { + input["cursor"] = Value::String(cursor); + } + let SourceResult::ByParent { + parent_key: returned_parent, + snapshot: current_snapshot, + rows: page, + complete, + next_cursor, + } = self.execute(acquisition, mode, input).await? + else { + return Err(ProjectionError::InvalidResult); + }; + if returned_parent != parent_key + || !projection_token_valid(¤t_snapshot) + || snapshot + .as_ref() + .is_some_and(|prior| prior != ¤t_snapshot) + || rows.len().saturating_add(page.len()) as u64 > self.contract.maximum_rows.get() + { + return Err(ProjectionError::InvalidResult); + } + for row in &page { + if row.parent_key != parent_key + || !projection_token_valid(&row.key) + || !seen_rows.insert(row.key.clone()) + { + return Err(ProjectionError::InvalidResult); + } + self.check_fields(&row.fields, row.source_revision.as_deref(), fields)?; + } + rows.extend(page); + if complete && next_cursor.is_none() { + return Ok(ProjectionResult::ByParent { + parent_key, + snapshot: current_snapshot, + rows, + }); + } + if complete { + return Err(ProjectionError::InvalidResult); + } + let next = next_cursor.ok_or(ProjectionError::Unavailable)?; + if !projection_token_valid(&next) || !seen_cursors.insert(next.clone()) { + return Err(ProjectionError::InvalidResult); + } + cursor = Some(next); + snapshot = Some(current_snapshot); + } + Err(ProjectionError::Unavailable) + } +} + +fn check_revision(revision: Option<&str>) -> Result<(), ProjectionError> { + if revision.is_some_and(|value| !projection_token_valid(value)) { + return Err(ProjectionError::InvalidResult); + } + Ok(()) +} diff --git a/crates/registry-adapter-engine/src/route.rs b/crates/registry-adapter-engine/src/route.rs index b868545b2c..b5302ce06d 100644 --- a/crates/registry-adapter-engine/src/route.rs +++ b/crates/registry-adapter-engine/src/route.rs @@ -3367,6 +3367,9 @@ impl Run<'_, '_> { { return Err(RouteFailure::of(ErrorClass::EgressRefused)); } + let projection_acquisition = self + .evidence + .filter(|acquisition| acquisition.projection_http.is_some()); let reply = if let Some(prepared) = prepared_protocol.take() { let crate::host_protocol::PreparedHostProtocolParts { body, @@ -3488,10 +3491,20 @@ impl Run<'_, '_> { ) } } else { + let maximum_response_bytes = if let Some(acquisition) = projection_acquisition { + endpoint + .maximum_response_bytes + .min(acquisition.remaining_response_bytes().await) + } else { + endpoint.maximum_response_bytes + }; + if maximum_response_bytes == 0 { + return Err(RouteFailure::of(ErrorClass::BudgetExhausted)); + } PhysicalSendReply::Ordinary( tokio::time::timeout_at( self.deadline, - endpoint.egress.send_with_system_headers( + endpoint.egress.send_observed( OutboundSend { link: &link, exchange_headers: &headers, @@ -3504,7 +3517,7 @@ impl Run<'_, '_> { }, authorization: credential.as_ref().map(|c| &c.header), attempt: endpoint.attempt.min(remaining), - maximum_response_bytes: endpoint.maximum_response_bytes, + maximum_response_bytes, ok_status_ranges: ok_range.clone(), throw_exception_on_failure: parameters .map(|parameters| parameters.throw_exception_on_failure) @@ -3513,6 +3526,8 @@ impl Run<'_, '_> { recovery_header.as_ref(), system_traceparent.as_ref(), source_idempotency_key.as_ref(), + self.evidence + .and_then(|acquisition| acquisition.projection_http.as_ref()), ), ) .await @@ -3520,6 +3535,14 @@ impl Run<'_, '_> { ) }; self.ensure_deadline()?; + if let Some(projection) = self + .evidence + .and_then(|acquisition| acquisition.projection_http.as_ref()) + { + let successful = matches!(&reply, PhysicalSendReply::Ordinary(Ok(reply)) + if (200..300).contains(&reply.status) && reply.status != 202); + projection.observe_response(successful); + } if let Some(acquisition) = self.evidence { let response_bytes = reply.response_bytes(); if response_bytes != 0 { diff --git a/crates/registry-adapter-engine/tests/projection_acquisition.rs b/crates/registry-adapter-engine/tests/projection_acquisition.rs new file mode 100644 index 0000000000..2c76a39d3a --- /dev/null +++ b/crates/registry-adapter-engine/tests/projection_acquisition.rs @@ -0,0 +1,507 @@ +// SPDX-License-Identifier: Apache-2.0 + +use registry_adapter_core::{ + AdapterPack, ProjectionMode, ProjectionRecord, ProjectionRequest, ProjectionResult, +}; +use registry_adapter_engine::{ + artifacts::PackArtifactBytes, + evidence::{EvidenceAcquisitionIntent, EvidenceHookFuture, EvidenceHostHooks, EvidenceLimits}, + projection::{compile_projection_pack, ProjectionError}, + replay::ReplayTranscript, +}; +use serde_json::{json, Value}; +use std::{collections::BTreeSet, sync::Arc}; + +struct Hooks; + +impl EvidenceHostHooks for Hooks { + fn before_acquisition<'a>(&'a self, _: EvidenceAcquisitionIntent) -> EvidenceHookFuture<'a> { + Box::pin(async { Ok(()) }) + } +} + +fn operation(retention: &str) -> (AdapterPack, PackArtifactBytes) { + let schema = serde_json::to_vec(&json!({ + "type":"object", + "additionalProperties":false, + "properties":{"familyName":{"type":"string"}} + })) + .unwrap(); + let digest = registry_platform_config::sha256_uri(&schema); + let steps = json!([{"to":{"uri":"http:source","parameters":{ + "httpMethod":"GET", "path":"/people" + }}}]); + let pack: AdapterPack = serde_json::from_value(json!({ + "id":"synthetic","version":"1","testedVersions":[], + "endpoints":[{"id":"source","component":"http","authentication":{"type":"none"},"attemptTimeoutMilliseconds":1000,"maximumResponseBytes":4096}], + "sourceRoutes":[{"id":"lookup","projection":{ + "operation":"exact","operationVersion":"1","authority":"population-register","purpose":"licensing", + "scope":"licensing-subjects","selectorScheme":"person-number","recordKeyScheme":"person-number", + "retention":retention,"absence":"authoritative","localDisclosure":"host-profiles", + "fieldsSchema":{"path":"schemas/person.json","digest":digest}, + "maximumBatchKeys":3,"maximumPages":2,"maximumRows":4 + }, "steps": steps}] + })) + .unwrap(); + ( + pack, + PackArtifactBytes::new([("schemas/person.json".to_owned(), schema)]), + ) +} + +fn transcript(replies: Vec) -> Arc { + Arc::new(ReplayTranscript::from_exchanges( + serde_json::from_value(json!([{"type":"endpoint","endpoint":"source"}])).unwrap(), + serde_json::from_value(json!(replies)).unwrap(), + )) +} + +fn exact_reply(record: Value, no_store: bool) -> Value { + let mut response = json!({"status":200,"body":{"type":"exact","record":record}}); + if no_store { + response["headers"] = json!({"Cache-Control":"private, NO-STORE"}); + } + json!({"request":{"endpoint":"source","method":"GET","path":"/people"},"response":response}) +} + +async fn exact( + mode: ProjectionMode, + retention: &str, + exchanges: Arc, +) -> Result { + let (pack, artifacts) = operation(retention); + let program = compile_projection_pack(&pack, "lookup", &artifacts, EvidenceLimits::default()) + .unwrap() + .bind_replay(exchanges) + .unwrap(); + program + .acquire( + Arc::new(Hooks), + mode, + ProjectionRequest::Exact { + key: "person-7".to_owned(), + }, + &BTreeSet::from(["familyName".to_owned()]), + ) + .await +} + +#[tokio::test] +async fn governed_retention_and_raw_response_restrictions_are_independent() { + let no_call = transcript(vec![]); + assert!(matches!( + exact(ProjectionMode::Indexed, "live-only", Arc::clone(&no_call)).await, + Err(ProjectionError::RetentionRefused) + )); + no_call.finish().unwrap(); + + let record = json!({"outcome":"present","key":"person-7","fields":{"familyName":"Lee"}}); + let protected = transcript(vec![exact_reply(record.clone(), true)]); + assert!(matches!( + exact(ProjectionMode::Indexed, "retain", Arc::clone(&protected)).await, + Err(ProjectionError::NoStore) + )); + protected.finish().unwrap(); + + let live = transcript(vec![exact_reply(record, true)]); + assert!(matches!( + exact(ProjectionMode::Live, "live-only", Arc::clone(&live)).await, + Ok(ProjectionResult::Exact { + record: ProjectionRecord::Present { .. } + }) + )); + live.finish().unwrap(); +} + +#[tokio::test] +async fn a_present_record_must_include_every_requested_field() { + let response = transcript(vec![exact_reply( + json!({"outcome": "present", "key": "person-7", "fields": {}}), + false, + )]); + assert!(matches!( + exact(ProjectionMode::Indexed, "retain", Arc::clone(&response)).await, + Err(ProjectionError::InvalidResult) + )); + response.finish().unwrap(); +} + +#[tokio::test] +async fn keyed_absence_is_distinct_from_ambiguous_or_wrong_identity() { + let absence = transcript(vec![exact_reply( + json!({"outcome":"authoritative-absent","key":"person-7"}), + false, + )]); + assert!(matches!( + exact(ProjectionMode::Indexed, "retain", Arc::clone(&absence)).await, + Ok(ProjectionResult::Exact { + record: ProjectionRecord::AuthoritativeAbsent { .. } + }) + )); + absence.finish().unwrap(); + + for result in [ + json!({"outcome":"present","key":"person-8","fields":{"familyName":"Wrong"}}), + json!({"outcome":"present","key":"person-7","fields":{"rawSecret":"canary"}}), + ] { + let exchange = transcript(vec![exact_reply(result, false)]); + assert!(matches!( + exact(ProjectionMode::Indexed, "retain", Arc::clone(&exchange)).await, + Err(ProjectionError::InvalidResult) + )); + exchange.finish().unwrap(); + } +} + +fn maintained_pack() -> (AdapterPack, PackArtifactBytes) { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../products/adapter/examples/external-projections"); + let pack: AdapterPack = registry_platform_yaml::Reader::new("pack.yaml") + .decode( + &std::fs::read(root.join("pack.yaml")).unwrap(), + ®istry_platform_yaml::Expect::one(®istry_platform_yaml::FormatSpec { + kind: registry_adapter_core::ADAPTER_PACK_KIND, + envelope: registry_platform_yaml::EnvelopeRule::ApiVersionKind { + api_versions: &[registry_platform_yaml::ApiVersion::current( + registry_adapter_core::ADAPTER_PACK_API_VERSION, + )], + retired_api_versions: &[], + }, + removed_keys: &[], + }), + ) + .unwrap() + .value; + assert_eq!(registry_adapter_core::check_pack(&pack), vec![]); + let artifacts = PackArtifactBytes::new( + [ + "schemas/person.json", + "schemas/holding.json", + "scripts/projection.rhai", + ] + .into_iter() + .map(|name| (name.to_owned(), std::fs::read(root.join(name)).unwrap())), + ); + (pack, artifacts) +} + +async fn acquire_fixture( + route: &str, + request: ProjectionRequest, + fields: &[&str], + records: Vec, + limits: EvidenceLimits, +) -> Result { + let (pack, artifacts) = maintained_pack(); + let replay = Arc::new(ReplayTranscript::from_exchanges( + serde_json::from_value(json!([{"type":"endpoint","endpoint":"synthetic-registry"}])) + .unwrap(), + serde_json::from_value(json!(records)).unwrap(), + )); + let program = compile_projection_pack(&pack, route, &artifacts, limits) + .unwrap() + .bind_replay(Arc::clone(&replay)) + .unwrap(); + let fields = fields.iter().map(|field| (*field).to_owned()).collect(); + let result = program + .acquire(Arc::new(Hooks), ProjectionMode::Indexed, request, &fields) + .await; + replay.finish().unwrap(); + result +} + +fn fixture_exchange(path: &str, request: Value, response: Value) -> Value { + json!({ + "request":{"endpoint":"synthetic-registry","method":"POST","path":path,"body":request}, + "response":response + }) +} + +#[tokio::test] +async fn native_batch_requires_exactly_one_outcome_for_each_requested_key() { + let requested = json!({"type":"batch","keys":["person-7","person-8"],"fields":["family-name"]}); + let found = json!({"subject":"person-7","state":"found","properties":{"family-name":"Lee"}}); + let absent = json!({"subject":"person-8","state":"confirmed-absent"}); + for (records, accepted) in [ + (vec![found.clone(), absent.clone()], true), + (vec![found.clone(), found.clone()], false), + (vec![found.clone()], false), + ( + vec![ + found, + absent, + json!({"subject":"outside","state":"ambiguous"}), + ], + false, + ), + ] { + let result = acquire_fixture( + "person-batch", + ProjectionRequest::Batch { + keys: vec!["person-7".to_owned(), "person-8".to_owned()], + }, + &["family-name"], + vec![fixture_exchange( + "/people/batch", + requested.clone(), + json!({"status":200,"body":{"records":records}}), + )], + EvidenceLimits::default(), + ) + .await; + if accepted { + let ProjectionResult::Batch { records } = result.unwrap() else { + panic!("expected typed batch result") + }; + assert!(matches!(records[0], ProjectionRecord::Present { .. })); + assert!(matches!( + records[1], + ProjectionRecord::AuthoritativeAbsent { .. } + )); + } else { + assert!(matches!(result, Err(ProjectionError::InvalidResult))); + } + } +} + +fn holdings_page(cursor: Option<&str>, status: u16, page: Value, no_store: bool) -> Value { + let mut request = + json!({"type":"by-parent","parentKey":"business-1","fields":["shareholder-type"]}); + if let Some(cursor) = cursor { + request["cursor"] = json!(cursor); + } + let mut response = json!({"status":status,"body":page}); + if no_store { + response["headers"] = json!({"cache-control":"no-store"}); + } + fixture_exchange("/holdings/by-business", request, response) +} + +async fn holdings( + exchanges: Vec, + limits: EvidenceLimits, +) -> Result { + acquire_fixture( + "holdings-by-business", + ProjectionRequest::ByParent { + parent_key: "business-1".to_owned(), + }, + &["shareholder-type"], + exchanges, + limits, + ) + .await +} + +#[tokio::test] +async fn a_collection_is_released_only_as_one_complete_parent_snapshot() { + let first = json!({ + "state":"snapshot","business":"business-1","snapshotId":"snapshot-a","finished":false,"next":"page-2", + "items":[{"holding":"holding-1","business":"business-1","properties":{"shareholder-type":"person"}}] + }); + let last = json!({ + "state":"snapshot","business":"business-1","snapshotId":"snapshot-a","finished":true, + "items":[{"holding":"holding-2","business":"business-1","properties":{"shareholder-type":"business"}}] + }); + let first_page = holdings_page(None, 200, first.clone(), false); + let last_page = holdings_page(Some("page-2"), 200, last.clone(), false); + let result = holdings( + vec![first_page.clone(), last_page.clone()], + EvidenceLimits::default(), + ) + .await + .unwrap(); + let ProjectionResult::ByParent { rows, .. } = result else { + panic!("expected complete typed snapshot") + }; + assert_eq!(rows.len(), 2); + assert_eq!(rows[1].fields["shareholder-type"], "business"); + + let empty = json!({"state":"snapshot","business":"business-1","snapshotId":"empty","finished":true,"items":[]}); + assert!(matches!( + holdings(vec![holdings_page(None, 200, empty, false)], EvidenceLimits::default()).await, + Ok(ProjectionResult::ByParent {rows, ..}) if rows.is_empty() + )); + + let interrupted = holdings_page(Some("page-2"), 403, json!({"state":"denied"}), false); + assert!(matches!( + holdings( + vec![first_page.clone(), interrupted], + EvidenceLimits::default() + ) + .await, + Err(ProjectionError::Unavailable) + )); + let blocked = holdings_page(None, 200, first, true); + assert!(matches!( + holdings(vec![blocked], EvidenceLimits::default()).await, + Err(ProjectionError::NoStore) + )); + + let mut wrong_snapshot = last.clone(); + wrong_snapshot["snapshotId"] = json!("snapshot-b"); + let mut wrong_parent = last.clone(); + wrong_parent["items"][0]["business"] = json!("business-2"); + let mut duplicate = last; + duplicate["items"][0]["holding"] = json!("holding-1"); + for malformed in [wrong_snapshot, wrong_parent, duplicate] { + assert!(matches!( + holdings( + vec![ + first_page.clone(), + holdings_page(Some("page-2"), 200, malformed, false) + ], + EvidenceLimits::default() + ) + .await, + Err(ProjectionError::InvalidResult) + )); + } + assert!(matches!( + holdings( + vec![first_page], + EvidenceLimits::try_new(1, 1000, 4096).unwrap() + ) + .await, + Err(ProjectionError::Unavailable) + )); +} + +#[cfg(unix)] +#[tokio::test] +async fn live_source_sees_the_service_identity_and_only_the_requested_fields() { + use registry_adapter_core::{AdapterEndpointBindings, EvidenceConnectionPolicy}; + use registry_adapter_engine::evidence::EvidenceRuntimeBindings; + use registry_platform_config::{SecretProvider, SecretResolver}; + use std::os::unix::fs::PermissionsExt; + use std::time::Duration; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + const SYNTHETIC_CREDENTIAL: &str = "fixture-projection-service-identity"; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let source = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut buffer = Vec::new(); + let mut chunk = [0; 512]; + let (header_length, content_length) = loop { + let count = socket.read(&mut chunk).await.unwrap(); + assert!(count > 0 && buffer.len() + count <= 8192); + buffer.extend_from_slice(&chunk[..count]); + if let Some(end) = buffer.windows(4).position(|bytes| bytes == b"\r\n\r\n") { + let header = std::str::from_utf8(&buffer[..end]).unwrap(); + assert!(header + .lines() + .next() + .unwrap() + .starts_with("POST /people/batch ")); + let headers: std::collections::BTreeMap<_, _> = header + .lines() + .skip(1) + .map(|line| line.split_once(':').unwrap()) + .map(|(name, value)| (name.to_ascii_lowercase(), value.trim())) + .collect(); + assert!(headers.get("x-registry-service-key") == Some(&SYNTHETIC_CREDENTIAL)); + break (end + 4, headers["content-length"].parse::().unwrap()); + } + }; + assert!(header_length + content_length <= 8192); + while buffer.len() < header_length + content_length { + let count = socket.read(&mut chunk).await.unwrap(); + assert!(count > 0 && buffer.len() + count <= 8192); + buffer.extend_from_slice(&chunk[..count]); + } + let body: Value = + serde_json::from_slice(&buffer[header_length..header_length + content_length]).unwrap(); + assert_eq!( + body, + json!({"type":"batch","keys":["person-7"],"fields":["family-name"]}) + ); + let response = br#"{"records":[{"subject":"person-7","state":"found","properties":{"family-name":"Lee"}}]}"#; + socket.write_all(format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nCache-Control: private\r\nCache-Control: NO-STORE\r\nConnection: close\r\n\r\n", + response.len() + ).as_bytes()).await.unwrap(); + socket.write_all(response).await.unwrap(); + }); + let root = tempfile::tempdir().unwrap(); + let secret = root.path().join("service-key"); + std::fs::write(&secret, SYNTHETIC_CREDENTIAL).unwrap(); + std::fs::set_permissions(&secret, std::fs::Permissions::from_mode(0o600)).unwrap(); + let runtime = EvidenceRuntimeBindings::new( + serde_json::from_value::(json!({ + "id":"synthetic-connection", + "endpoints":[{ + "endpoint":"synthetic-registry","baseUrl":origin, + "credential":{"type":"static-api-key","valueRef":"secret:file/service-key"} + }] + })) + .unwrap(), + SecretResolver::new([SecretProvider::File], root.path()).unwrap(), + ); + let policy: EvidenceConnectionPolicy = serde_json::from_value(json!({ + "endpoint":"synthetic-registry","baseUrl":origin,"clientIdentity":"forbidden", + "authentication":{"type":"static-api-key"} + })) + .unwrap(); + let (pack, artifacts) = maintained_pack(); + let program = + compile_projection_pack(&pack, "person-batch", &artifacts, EvidenceLimits::default()) + .unwrap() + .bind_live(&runtime, "synthetic-connection", &[policy]) + .unwrap(); + let result = program + .acquire( + Arc::new(Hooks), + ProjectionMode::Indexed, + ProjectionRequest::Batch { + keys: vec!["person-7".to_owned()], + }, + &BTreeSet::from(["family-name".to_owned()]), + ) + .await; + assert!( + matches!(result, Err(ProjectionError::NoStore)), + "{result:?}" + ); + tokio::time::timeout(Duration::from_secs(2), source) + .await + .unwrap() + .unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn an_audit_delay_cannot_start_source_io_after_the_grant_deadline() { + struct SlowAudit; + impl EvidenceHostHooks for SlowAudit { + fn before_acquisition<'a>( + &'a self, + _: EvidenceAcquisitionIntent, + ) -> EvidenceHookFuture<'a> { + Box::pin(async { + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + Ok(()) + }) + } + } + + let exchanges = transcript(vec![]); + let (pack, artifacts) = operation("retain"); + let program = compile_projection_pack(&pack, "lookup", &artifacts, EvidenceLimits::default()) + .unwrap() + .bind_replay(Arc::clone(&exchanges)) + .unwrap(); + let result = program + .acquire_until( + Arc::new(SlowAudit), + ProjectionMode::Indexed, + ProjectionRequest::Exact { + key: "person-7".to_owned(), + }, + &BTreeSet::from(["familyName".to_owned()]), + Some(tokio::time::Instant::now() + std::time::Duration::from_secs(1)), + ) + .await; + assert!(matches!(result, Err(ProjectionError::Unavailable))); + exchanges.finish().unwrap(); +} diff --git a/crates/registry-breg/Cargo.toml b/crates/registry-breg/Cargo.toml index 49d7689b43..bd6c82e762 100644 --- a/crates/registry-breg/Cargo.toml +++ b/crates/registry-breg/Cargo.toml @@ -268,6 +268,8 @@ runtime = [ "dep:ipnet", "dep:jsonwebtoken", "dep:reqwest", + "dep:registry-adapter-core", + "dep:registry-adapter-engine", "dep:registry-platform-audit", "dep:registry-platform-authcommon", "dep:registry-platform-buildinfo", @@ -295,6 +297,7 @@ postgres-test = ["runtime", "dep:tower", "registry-platform-httputil/test-suppor postgres-tls-test = ["runtime"] schema = [ "dep:schemars", + "registry-adapter-core?/schema", "registry-platform-audit?/schema", "registry-platform-config/schema", "registry-platform-hooks/schema", @@ -305,6 +308,8 @@ tooling = ["dep:tempfile", "dep:tower"] wasm = ["registry-platform-hooks/wasm", "registry-platform-script/wasm"] [dependencies] +registry-adapter-core = { workspace = true, optional = true } +registry-adapter-engine = { workspace = true, optional = true } chrono.workspace = true registry-evidence-client.workspace = true registry-evidence-verifier.workspace = true diff --git a/crates/registry-breg/src/api/external_views.rs b/crates/registry-breg/src/api/external_views.rs new file mode 100644 index 0000000000..3b1073543b --- /dev/null +++ b/crates/registry-breg/src/api/external_views.rs @@ -0,0 +1,479 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Explicit external views and machine-only refresh over an authorised local root. + +use super::*; +use crate::external_views::{ + ExternalCardinality, ExternalFieldType, ExternalLinkPermissionSource, ExternalProjectionMode, +}; +use crate::query::{ComparisonOp, FilterExpr, FilterPredicate, Literal, OrderDirection}; + +pub(crate) struct ExternalViewRequest { + pub(crate) context: AuthorizedRequestContext, + pub(crate) correlation: RequestCorrelation, + pub(crate) view: String, + pub(crate) link: String, + pub(crate) root: Option, + pub(crate) query: ExternalQueryMode, + pub(crate) deadline: tokio::time::Instant, +} + +pub(crate) enum ExternalQueryMode { + First(ExternalQuery), + Continue(String), +} + +#[derive(Clone, Default, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub(crate) struct ExternalQuery { + pub(crate) equality: Option, + pub(crate) order: Option, + pub(crate) page_size: u16, + pub(crate) include_count: bool, +} + +#[derive(Clone, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub(crate) struct ExternalEquality { + pub(crate) field: String, + pub(crate) value: Value, +} + +#[derive(Clone, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub(crate) struct ExternalOrder { + pub(crate) field: String, + pub(crate) descending: bool, +} + +#[derive(Clone, Copy)] +pub(crate) enum ExternalViewError { + Concealed, + InvalidCursor, + Incomplete, + Unavailable, +} + +#[derive(Clone, Copy, Eq, PartialEq)] +pub(crate) enum RefreshMode { + Projection, + Labels, +} + +pub(crate) trait ExternalViewService: Send + Sync { + fn read( + &self, + request: ExternalViewRequest, + ) -> ServiceFuture<'_, Result>; + fn refresh( + &self, + request: ExternalViewRequest, + mode: RefreshMode, + ) -> ServiceFuture<'_, Result>; +} + +pub(super) fn routes(registry: &CompiledRegistry) -> Router> { + let mut app = Router::new(); + if !registry.external_views().views.is_empty() { + app = app + .route("/v1/external-views/{view}/{link}/records", get(collection)) + .route( + "/v1/external-views/{view}/{link}/records/{root}", + get(detail), + ) + .route("/v1/external-views/{view}/{link}/refresh", post(refresh)) + .route( + "/v1/external-views/{view}/{link}/refresh-labels", + post(refresh_labels), + ); + } + app +} + +struct ReadTarget { + view: String, + link: String, + root: Option, + refresh: Option, +} + +async fn collection( + State(service): State>, + Authenticated(claims): Authenticated, + Extension(correlation): Extension, + deadline: Option>, + RawQuery(raw): RawQuery, + headers: HeaderMap, + Path((view, link)): Path<(String, String)>, +) -> Response { + handle( + &service, + &claims, + correlation, + deadline, + raw, + &headers, + ReadTarget { + view, + link, + root: None, + refresh: None, + }, + ) + .await +} + +async fn detail( + State(service): State>, + Authenticated(claims): Authenticated, + Extension(correlation): Extension, + deadline: Option>, + RawQuery(raw): RawQuery, + headers: HeaderMap, + Path((view, link, root)): Path<(String, String, String)>, +) -> Response { + handle( + &service, + &claims, + correlation, + deadline, + raw, + &headers, + ReadTarget { + view, + link, + root: Some(root), + refresh: None, + }, + ) + .await +} + +async fn refresh( + State(service): State>, + Authenticated(claims): Authenticated, + Extension(correlation): Extension, + deadline: Option>, + RawQuery(raw): RawQuery, + headers: HeaderMap, + Path((view, link)): Path<(String, String)>, +) -> Response { + handle( + &service, + &claims, + correlation, + deadline, + raw, + &headers, + ReadTarget { + view, + link, + root: None, + refresh: Some(RefreshMode::Projection), + }, + ) + .await +} + +async fn refresh_labels( + State(service): State>, + Authenticated(claims): Authenticated, + Extension(correlation): Extension, + deadline: Option>, + RawQuery(raw): RawQuery, + headers: HeaderMap, + Path((view, link)): Path<(String, String)>, +) -> Response { + handle( + &service, + &claims, + correlation, + deadline, + raw, + &headers, + ReadTarget { + view, + link, + root: None, + refresh: Some(RefreshMode::Labels), + }, + ) + .await +} + +async fn handle( + service: &HttpService, + claims: &VerifiedRequestClaims, + correlation: RequestCorrelation, + deadline: Option>, + raw: Option, + headers: &HeaderMap, + target: ReadTarget, +) -> Response { + let inventory = service.registry.external_views(); + let Some(view) = inventory.views.get(&target.view) else { + return concealed(); + }; + let Some(link) = view + .links + .iter() + .find(|link| link.id.as_str() == target.link) + else { + return concealed(); + }; + let Some(projection) = inventory.projections.get(link.projection.as_str()) else { + return unavailable(); + }; + let is_detail = target.root.is_some(); + let root_operation = if is_detail { + Operation::Get + } else { + Operation::List + }; + let Some(root_route) = service.registry.routes().routes.iter().find(|route| { + route.entity_id == view.record_type.as_str() + && route.operation == root_operation + && read_path_for_route(service, route).is_none() + }) else { + return concealed(); + }; + let mut root_options = QueryOptions::default(); + root_options.parsed.access_profile = admission_profile(raw.as_deref()); + let Some(authorised) = authorize_route(service, root_route, claims, &root_options) else { + return concealed(); + }; + let mut context = authorised.context; + if context + .bind_access_attribution(authorised.response_entity, headers) + .is_err() + { + return concealed(); + } + let profile = context.selected_profile(); + if context.purpose() != Some(projection.purpose.as_str()) { + return concealed(); + } + let grant = if let Some(mode) = target.refresh { + let Some(ids) = inventory.refresh.get(profile) else { + return concealed(); + }; + if projection.mode != ExternalProjectionMode::Indexed || !ids.contains(&projection.id) { + return concealed(); + } + if mode == RefreshMode::Labels + && (projection.cardinality != ExternalCardinality::Many + || !link + .label + .as_ref() + .is_some_and(|label| ids.contains(&label.projection))) + { + return concealed(); + } + None + } else { + let Some(grant) = inventory + .grants + .get(profile) + .and_then(|views| views.get(&target.view)) + .and_then(|grant| grant.links.iter().find(|grant| grant.link == link.id)) + else { + return concealed(); + }; + if !is_detail + && (projection.mode != ExternalProjectionMode::Indexed + || projection.cardinality != ExternalCardinality::One) + { + return concealed(); + } + Some(grant) + }; + let root = match target.root { + Some(root) => match Uuid::parse_str(&root) { + Ok(id) if id.hyphenated().to_string() == root => Some(id), + _ => return concealed(), + }, + None => None, + }; + let options = match QueryOptions::parse(raw.as_deref(), true) { + Ok(value) => value, + Err(error) => return error.problem(), + }; + if options.parsed.as_of.is_some() + || options.parsed.access_profile != root_options.parsed.access_profile + { + return invalid_query_at("query"); + } + let Some(record_type) = inventory.record_types.get(projection.record_type.as_str()) else { + return unavailable(); + }; + let allows_query = projection.mode == ExternalProjectionMode::Indexed + && (!is_detail || projection.cardinality == ExternalCardinality::Many); + let query = match plan_query( + options.parsed.mode, + grant, + &record_type.fields, + allows_query, + target.refresh.is_some(), + ) { + Ok(query) => query, + Err(parameter) => return invalid_query_at(parameter), + }; + let Some(backend) = &service.external_views else { + return unavailable(); + }; + let deadline = deadline + .map(|Extension(value)| value.0) + .unwrap_or_else(|| tokio::time::Instant::now() + std::time::Duration::from_secs(10)); + let request = ExternalViewRequest { + context, + correlation, + view: target.view, + link: target.link, + root, + query, + deadline, + }; + let result = if let Some(mode) = target.refresh { + backend.refresh(request, mode).await + } else { + backend.read(request).await + }; + let mut response = match result { + Ok(value) => Json(value).into_response(), + Err(ExternalViewError::Concealed) => concealed(), + Err(ExternalViewError::InvalidCursor) => cursor_invalid(), + Err(ExternalViewError::Incomplete) => ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({"status": "incomplete"})), + ) + .into_response(), + Err(ExternalViewError::Unavailable) => unavailable(), + }; + response + .headers_mut() + .insert(CACHE_CONTROL, HeaderValue::from_static("no-store")); + response +} + +fn admission_profile(raw: Option<&str>) -> Option { + raw.filter(|value| value.len() <= MAX_RAW_QUERY_BYTES)? + .split('&') + .filter_map(|pair| pair.split_once('=')) + .find_map(|(name, value)| { + (percent_decode(name).ok().as_deref() == Some("accessProfile")) + .then(|| percent_decode(value).ok()) + .flatten() + }) +} + +fn plan_query( + mode: strict_query::ParsedReadQueryMode, + grant: Option<&ExternalLinkPermissionSource>, + fields: &[crate::external_views::ExternalFieldSource], + allows_query: bool, + refresh: bool, +) -> Result { + let options = match mode { + strict_query::ParsedReadQueryMode::SkipToken { token } if allows_query || refresh => { + return Ok(ExternalQueryMode::Continue(token)); + } + strict_query::ParsedReadQueryMode::SkipToken { .. } => return Err("$skiptoken"), + strict_query::ParsedReadQueryMode::Query(options) => options, + }; + if options.select.is_some() { + return Err("$select"); + } + if options.bbox.is_some() { + return Err("bbox"); + } + if options.top.is_some() && !allows_query && !refresh { + return Err("$top"); + } + let page_size = u16::try_from(options.top.unwrap_or(50)) + .ok() + .filter(|size| (1..=100).contains(size)) + .ok_or("$top")?; + let equality = options + .filter + .map(|filter| { + if !allows_query || refresh { + return Err("$filter"); + } + let FilterExpr::Predicate(FilterPredicate::Compare { + field, + op: ComparisonOp::Eq, + literal, + }) = filter + else { + return Err("$filter"); + }; + let id = registry_platform_yaml::LocalId::new(field.as_str()).map_err(|_| "$filter")?; + if !grant.is_some_and(|grant| grant.filterable_fields.contains(&id)) { + return Err("$filter"); + } + let field_type = fields + .iter() + .find(|candidate| candidate.id == id) + .map(|field| field.value_type) + .ok_or("$filter")?; + let value = literal_value(field_type, literal).ok_or("$filter")?; + Ok(ExternalEquality { + field: id.as_str().to_owned(), + value, + }) + }) + .transpose()?; + let order = options + .orderby + .map(|order| { + if !allows_query || refresh { + return Err("$orderby"); + } + let id = registry_platform_yaml::LocalId::new(order.field.as_str()) + .map_err(|_| "$orderby")?; + if !grant.is_some_and(|grant| grant.sortable_fields.contains(&id)) { + return Err("$orderby"); + } + Ok(ExternalOrder { + field: id.as_str().to_owned(), + descending: order.direction == OrderDirection::Desc, + }) + }) + .transpose()?; + if options.count.is_some() + && (!allows_query || refresh || !grant.is_some_and(|grant| grant.allow_count)) + { + return Err("$count"); + } + Ok(ExternalQueryMode::First(ExternalQuery { + equality, + order, + page_size, + include_count: options.count.unwrap_or(false), + })) +} + +fn literal_value(field_type: ExternalFieldType, literal: Literal) -> Option { + match (field_type, literal) { + (_, Literal::Null) => Some(Value::Null), + (ExternalFieldType::String, Literal::String(value)) if value.len() <= 4096 => { + Some(Value::String(value)) + } + (ExternalFieldType::Integer, Literal::Integer(value)) => { + value.parse::().ok().map(Value::from) + } + (ExternalFieldType::Decimal, Literal::Decimal(value) | Literal::Integer(value)) => { + Some(Value::String(value)) + } + (ExternalFieldType::Boolean, Literal::Boolean(value)) => Some(Value::Bool(value)), + (ExternalFieldType::Date, Literal::String(value)) => { + chrono::NaiveDate::parse_from_str(&value, "%Y-%m-%d") + .ok() + .filter(|date| date.format("%Y-%m-%d").to_string() == value) + .map(|_| Value::String(value)) + } + (ExternalFieldType::Uuid, Literal::String(value)) => Uuid::parse_str(&value) + .ok() + .map(|id| Value::String(id.hyphenated().to_string())), + _ => None, + } +} diff --git a/crates/registry-breg/src/api/mod.rs b/crates/registry-breg/src/api/mod.rs index 852cd34c02..da15314208 100644 --- a/crates/registry-breg/src/api/mod.rs +++ b/crates/registry-breg/src/api/mod.rs @@ -7,6 +7,7 @@ mod access_log; mod actions; mod attachments; mod context; +pub(crate) mod external_views; mod gis; mod ingestion; mod metadata; @@ -229,6 +230,7 @@ fn route_set(service: Arc) -> Router { .merge(attachments::routes(&service)) .merge(gis::routes()) .merge(statistics::routes(&service)) + .merge(external_views::routes(&service.registry)) .merge(ingestion::routes(&service)) .route_layer(middleware::from_fn_with_state( Arc::clone(&service), @@ -516,6 +518,21 @@ async fn openapi( Some(&visible_proposals), ); statistics::append_openapi(&service, &claims, &options, &mut paths, &mut schemas); + if service.external_views.is_some() { + crate::external_view_artifacts::append( + &mut paths, + service.registry.external_views(), + |profile, entity, purpose, operation| { + visible.iter().any(|surface| { + surface.route.entity_id == entity + && surface.route.operation == operation + && surface.read_path.is_none() + && surface.context.selected_profile() == profile + && surface.context.purpose() == Some(purpose) + }) + }, + ); + } if service.review_completions.is_some() { crate::artifacts::append_review_completion_openapi(&mut paths, &mut schemas); } diff --git a/crates/registry-breg/src/api/service.rs b/crates/registry-breg/src/api/service.rs index 1ad06fa219..defa85799c 100644 --- a/crates/registry-breg/src/api/service.rs +++ b/crates/registry-breg/src/api/service.rs @@ -824,6 +824,7 @@ pub struct HttpService { pub(crate) cursors: Arc, pub(crate) mutations: Option>, pub(crate) statistics: Option>, + pub(crate) external_views: Option>, pub(crate) review_completions: Option>, pub(crate) readiness: Arc, pub(crate) public_origin: Option, @@ -866,6 +867,7 @@ impl HttpService { cursors, mutations: None, statistics: None, + external_views: None, review_completions: None, readiness, public_origin: None, @@ -873,6 +875,14 @@ impl HttpService { } } + pub(crate) fn with_external_views( + mut self, + service: Arc, + ) -> Self { + self.external_views = Some(service); + self + } + /// Install the package-fenced statistical dataset service. #[must_use] pub fn with_statistics( diff --git a/crates/registry-breg/src/artifacts.rs b/crates/registry-breg/src/artifacts.rs index 82b4b57a51..2e8f668f1c 100644 --- a/crates/registry-breg/src/artifacts.rs +++ b/crates/registry-breg/src/artifacts.rs @@ -96,6 +96,8 @@ pub(crate) struct EffectiveModel<'a> { pub entities: &'a BTreeMap, #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub statistical_datasets: &'a BTreeMap, + #[serde(skip_serializing_if = "crate::external_views::CompiledExternalViews::is_empty")] + pub external_views: &'a crate::external_views::CompiledExternalViews, pub physical_names: &'a PhysicalNameInventory, #[serde(skip_serializing_if = "CompiledActionInventory::is_empty")] pub action_inventory: &'a CompiledActionInventory, @@ -121,6 +123,7 @@ pub(crate) fn generate_artifacts( module_closure: &[CompiledModuleIdentity], entities: &BTreeMap, statistical_datasets: &BTreeMap, + external_views: &crate::external_views::CompiledExternalViews, physical_names: &PhysicalNameInventory, actions: &CompiledActionInventory, routes: &CompiledRouteInventory, @@ -146,6 +149,7 @@ pub(crate) fn generate_artifacts( module_closure, entities, statistical_datasets, + external_views, physical_names, action_inventory: actions, metadata_inventory: metadata, @@ -179,6 +183,13 @@ pub(crate) fn generate_artifacts( &action_contracts_schema(), )?; } + if !external_views.is_empty() { + insert_json( + &mut artifacts, + "compiled/external-views.json", + external_views, + )?; + } insert_json(&mut artifacts, "compiled/routes.json", routes)?; insert_json(&mut artifacts, "compiled/access.json", access)?; insert_json(&mut artifacts, "compiled/metadata-inventory.json", metadata)?; @@ -293,6 +304,7 @@ pub(crate) fn generate_artifacts( query, schemas: &schemas, statistical_datasets, + external_views, }); insert_json_value(&mut artifacts, "generated/openapi.json", &openapi)?; if let Some(projection) = manifest_projection { @@ -2015,6 +2027,7 @@ struct OpenApiDocumentInput<'a> { query: &'a CompiledQueryInventory, schemas: &'a BTreeMap, statistical_datasets: &'a BTreeMap, + external_views: &'a crate::external_views::CompiledExternalViews, } fn openapi_document(input: OpenApiDocumentInput<'_>) -> Value { @@ -2027,6 +2040,7 @@ fn openapi_document(input: OpenApiDocumentInput<'_>) -> Value { query, schemas, statistical_datasets, + external_views, } = input; let mut paths = Map::new(); let mut input_schemas = Map::new(); @@ -2136,6 +2150,16 @@ fn openapi_document(input: OpenApiDocumentInput<'_>) -> Value { ); } append_proposal_openapi(&mut paths, actions, None); + crate::external_view_artifacts::append( + &mut paths, + external_views, + |profile, entity, _, operation| { + entities + .get(entity) + .and_then(|entity| entity.access_profiles.get(profile)) + .is_some_and(|access| access.operations.contains(&operation)) + }, + ); let mut component_schemas: Map = schemas .iter() .map(|(id, schema)| (id.clone(), schema.clone())) diff --git a/crates/registry-breg/src/compiler.rs b/crates/registry-breg/src/compiler.rs index ff160faf6d..8040c4402b 100644 --- a/crates/registry-breg/src/compiler.rs +++ b/crates/registry-breg/src/compiler.rs @@ -371,6 +371,13 @@ pub fn compile_project_with_assets( "statistical release compilation", ) })?; + let external_views = crate::external_views::compile(project, &entities).map_err(|errors| { + stopped_compile_failure( + CompileFailure::from_errors(errors), + &findings, + "external view compilation", + ) + })?; crate::membership::compile(&mut entities); let recipients = crate::consent::compile(project, &sources, &mut entities); crate::event_record::compile(&sources, &mut entities); @@ -512,6 +519,7 @@ pub fn compile_project_with_assets( &module_closure, &entities, &statistical_datasets, + &external_views, &physical_names, &action_inventory, &route_inventory, @@ -551,6 +559,7 @@ pub fn compile_project_with_assets( module_closure, entities, statistical_datasets, + external_views, physical_names, action_inventory, route_inventory, diff --git a/crates/registry-breg/src/contract.rs b/crates/registry-breg/src/contract.rs index 036f45f9eb..18aa372343 100644 --- a/crates/registry-breg/src/contract.rs +++ b/crates/registry-breg/src/contract.rs @@ -108,6 +108,12 @@ pub struct RegistryProject { /// Declared count tables computed from governed Registry records. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub statistical_datasets: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub external_record_types: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub external_projections: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub external_views: Vec, /// Named organizations and frozen groups that consent may be given to. #[serde(default, skip_serializing_if = "Option::is_none")] pub recipients: Option, @@ -4755,6 +4761,10 @@ pub struct PermissionsSource { /// What the profile may do with statistical datasets: one permission for each dataset it names. #[serde(default)] pub datasets: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub external_views: Vec, + #[serde(default, skip_serializing_if = "UniqueSet::is_empty")] + pub refresh_external_projections: UniqueSet, } /// What a profile may do with one statistical dataset. diff --git a/crates/registry-breg/src/cursor.rs b/crates/registry-breg/src/cursor.rs index 498c94ebfb..8700f4fb21 100644 --- a/crates/registry-breg/src/cursor.rs +++ b/crates/registry-breg/src/cursor.rs @@ -26,6 +26,7 @@ const MAX_PAYLOAD_BYTES: usize = 8 * 1024; const MAX_TOKEN_BYTES: usize = ((1 + NONCE_BYTES + MAX_PAYLOAD_BYTES + TAG_BYTES) * 4).div_ceil(3); const MAX_AGE_SECONDS: u64 = 86_400; const CURSOR_AAD: &[u8] = b"breg-cursor-v3"; +const EXTERNAL_CURSOR_AAD: &[u8] = b"breg-external-cursor-v1"; const AEAD_LABEL: &[u8] = b"breg-cursor-aead-key-v3"; const BINDING_LABEL: &[u8] = b"breg-cursor-binding-key-v3"; @@ -58,11 +59,31 @@ impl CursorCodec { { return Err(CursorError::Malformed); } + self.seal(CURSOR_AAD, payload) + } + + pub(crate) fn seal_external(&self, payload: &T) -> Result { + self.seal(EXTERNAL_CURSOR_AAD, payload) + } + + pub(crate) fn open_external( + &self, + token: &str, + ) -> Result { + let plaintext = self.open(EXTERNAL_CURSOR_AAD, token)?; + serde_json::from_slice(&plaintext).map_err(|_| CursorError::Invalid) + } + + pub(crate) fn max_age(&self) -> Duration { + self.max_age + } + + fn seal(&self, aad: &[u8], payload: &T) -> Result { let plaintext = serde_json::to_vec(payload).map_err(|_| CursorError::Malformed)?; if plaintext.is_empty() || plaintext.len() > MAX_PAYLOAD_BYTES { return Err(CursorError::Malformed); } - let sealed = xchacha_seal::seal(&self.aead_key, CURSOR_AAD, &plaintext) + let sealed = xchacha_seal::seal(&self.aead_key, aad, &plaintext) .map_err(|_| CursorError::Configuration)?; let mut envelope = Vec::with_capacity(1 + sealed.len()); envelope.push(WIRE_VERSION); @@ -83,7 +104,7 @@ impl CursorCodec { Ok(payload) } - fn decode(&self, token: &str, now_unix_seconds: u64) -> Result { + fn open(&self, aad: &[u8], token: &str) -> Result>, CursorError> { if token.is_empty() || token.len() > MAX_TOKEN_BYTES { return Err(CursorError::Invalid); } @@ -96,11 +117,16 @@ impl CursorCodec { { return Err(CursorError::Invalid); } - let plaintext = xchacha_seal::open(&self.aead_key, CURSOR_AAD, &envelope[1..]) + let plaintext = xchacha_seal::open(&self.aead_key, aad, &envelope[1..]) .map_err(|_| CursorError::Invalid)?; if plaintext.is_empty() || plaintext.len() > MAX_PAYLOAD_BYTES { return Err(CursorError::Invalid); } + Ok(plaintext) + } + + fn decode(&self, token: &str, now_unix_seconds: u64) -> Result { + let plaintext = self.open(CURSOR_AAD, token)?; let payload: CursorPayload = serde_json::from_slice(&plaintext).map_err(|_| CursorError::Invalid)?; if payload.version != WIRE_VERSION { @@ -538,6 +564,17 @@ mod tests { use chacha20poly1305::aead::{Aead, Payload}; use chacha20poly1305::{KeyInit, XChaCha20Poly1305, XNonce}; + #[test] + fn external_cursor_is_bounded_and_has_a_separate_encryption_context() { + let codec = codec(); + let value = serde_json::json!({"sourceKey": "fixture-sensitive-id"}); + let token = codec.seal_external(&value).unwrap(); + assert!(!token.contains("fixture-sensitive-id")); + assert_eq!(codec.open_external::(&token).unwrap(), value); + assert!(codec.open(CURSOR_AAD, &token).is_err()); + assert!(codec.seal_external(&"x".repeat(MAX_PAYLOAD_BYTES)).is_err()); + } + fn codec() -> CursorCodec { CursorCodec::new(Zeroizing::new(vec![0x51; 32]), Duration::from_secs(60)) .expect("cursor codec accepts strong test secret") diff --git a/crates/registry-breg/src/external_source.rs b/crates/registry-breg/src/external_source.rs new file mode 100644 index 0000000000..1de1c93bb4 --- /dev/null +++ b/crates/registry-breg/src/external_source.rs @@ -0,0 +1,884 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Bind reviewed Adapter acquisitions to the active BReg projection contract. +//! The HTTP host proves each local selector and field grant before invoking +//! the service. This layer establishes the independent acquisition grant. + +use std::{ + collections::{BTreeMap, BTreeSet}, + fs::File, + io::Read, + path::{Component, Path}, + sync::Arc, +}; + +use chrono::{DateTime, Utc}; +use registry_adapter_core::{ + projection_token_valid, AdapterEndpointBindings, AdapterPack, EvidenceConnectionPolicy, + ProjectionLocalDisclosure, ProjectionMode, ProjectionOperationKind, ProjectionRecord, + ProjectionRequest, ProjectionResult, ProjectionRetention, ADAPTER_PACK_API_VERSION, + ADAPTER_PACK_KIND, +}; +use registry_adapter_engine::{ + artifacts::PackArtifactBytes, + evidence::{EvidenceHostHooks, EvidenceLimits, EvidenceRuntimeBindings}, + projection::{ + compile_projection_pack, ProjectionError, ProjectionPackProgram, ProjectionProgram, + }, +}; +use registry_platform_config::{sha256_uri, SecretResolver}; +use registry_platform_yaml::{ + lookalike_local_id_pairs, ApiVersion, BoundedU64, Digest, EnvelopeRule, Expect, FormatSpec, + Identified, LocalId, Reader, UniqueIdList, UniqueList, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Map, Value}; +use thiserror::Error; + +use crate::{ + external_views::{ + ExternalCardinality, ExternalFieldType, ExternalProjectionMode, ExternalProjectionSource, + ExternalRecordTypeSource, + }, + model::CompiledRegistry, +}; + +const MAXIMUM_BINDINGS: usize = 128; +const MAXIMUM_PACK_BYTES: usize = 1_048_576; +const MAXIMUM_CLOSURE_FILES: usize = 512; +const MAXIMUM_CLOSURE_FILE_BYTES: usize = 2 * 1_048_576; +const MAXIMUM_CLOSURE_BYTES: usize = 16 * 1_048_576; +const MAXIMUM_RESPONSE_BYTES: u64 = 16 * 1_048_576; +const MAXIMUM_CA_BYTES: usize = + registry_platform_httputil::destination::MAX_DESTINATION_CA_BUNDLE_BYTES; +const MAXIMUM_RETENTION_SECONDS: u64 = 31_536_000; + +/// Operator binding for one already declared projection. The reviewed pack +/// provides the acquisition ceiling; runtime configuration can narrow it. +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ExternalSourceBinding { + pub id: LocalId, + pub pack_file: String, + pub pack_digest: Digest, + pub connection: AdapterEndpointBindings, + pub endpoint_policies: UniqueIdList, + pub allowed_fields: UniqueList, + /// The reviewed source grant cutoff, as an RFC 3339 timestamp. + pub grant_valid_until: String, + /// An optional further restriction on locally authorized selectors. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub allowed_selectors: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retention: Option, + pub acquisition: ExternalSourceAcquisition, +} + +impl Identified for ExternalSourceBinding { + fn id(&self) -> &str { + self.id.as_str() + } +} + +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Copy, Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ExternalSourceRetention { + pub maximum_retention_seconds: BoundedU64<1, MAXIMUM_RETENTION_SECONDS>, + pub refresh_interval_seconds: BoundedU64<1, MAXIMUM_RETENTION_SECONDS>, +} + +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Copy, Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ExternalSourceAcquisition { + pub maximum_data_operations: BoundedU64<1, 1000>, + pub timeout_milliseconds: BoundedU64<1, 60000>, + pub maximum_response_bytes: BoundedU64<1, MAXIMUM_RESPONSE_BYTES>, +} + +#[derive(Debug, Error)] +pub enum ExternalSourceError { + #[error("the external source binding is invalid")] + Configuration, + #[error("the external source pack or its pinned artifact could not be verified")] + Artifact, + #[error("the external source operation does not match the compiled projection")] + Contract, + #[error("no external source is bound to this projection")] + Unbound, + #[error("the external source grant does not permit this acquisition")] + Grant, + #[error("the source result is incompatible with the approved BReg field types")] + Result, + #[error("the external source acquisition was refused: {0}")] + Acquisition(#[from] ProjectionError), +} + +#[derive(Default)] +pub struct ExternalSourceService { + bindings: BTreeMap, +} + +pub struct BoundExternalSource { + program: ProjectionProgram, + fields: BTreeSet, + allowed_selectors: Option>, + mode: ExternalProjectionMode, + grant_valid_until: DateTime, + binding_fingerprint: String, + grant_fingerprint: String, + cache_ttl_seconds: Option, + retention: Option, + field_types: BTreeMap, +} + +impl BoundExternalSource { + pub fn binding_fingerprint(&self) -> &str { + &self.binding_fingerprint + } + + pub fn grant_fingerprint(&self) -> &str { + &self.grant_fingerprint + } + + pub fn grant_valid_until(&self) -> DateTime { + self.grant_valid_until + } + + pub fn mode(&self) -> ExternalProjectionMode { + self.mode + } + + pub fn effective_cache_ttl_seconds(&self) -> Option { + self.cache_ttl_seconds + } + + pub fn maximum_retention_seconds(&self) -> Option { + self.retention + .map(|retention| retention.maximum_retention_seconds.get() as u32) + } + + pub fn refresh_interval_seconds(&self) -> Option { + self.retention + .map(|retention| retention.refresh_interval_seconds.get() as u32) + } + + pub fn maximum_batch_keys(&self) -> usize { + self.program.contract().maximum_batch_keys.get() as usize + } + + pub fn allowed_fields(&self) -> &BTreeSet { + &self.fields + } +} + +impl ExternalSourceService { + /// Capture the pack and public TLS material without source calls or + /// resolving service credentials. Missing bindings never get a fallback. + pub fn load( + compiled: &CompiledRegistry, + bindings: &[ExternalSourceBinding], + resolver: Arc, + ) -> Result { + validate_bindings(bindings)?; + let catalog = compiled.external_views(); + let mut result = Self::default(); + let mut connections = BTreeMap::new(); + for binding in bindings { + let id = binding.id.as_str(); + let projection = catalog + .projections + .get(id) + .ok_or(ExternalSourceError::Contract)?; + let record = catalog + .record_types + .get(projection.record_type.as_str()) + .ok_or(ExternalSourceError::Contract)?; + let (pack, artifacts) = load_pack(Path::new(&binding.pack_file), &binding.pack_digest)?; + let limits = EvidenceLimits::try_new( + binding.acquisition.maximum_data_operations.get(), + binding.acquisition.timeout_milliseconds.get(), + binding.acquisition.maximum_response_bytes.get() as usize, + ) + .map_err(|_| ExternalSourceError::Configuration)?; + let program = + compile_projection_pack(&pack, projection.operation.as_str(), &artifacts, limits) + .map_err(|_| ExternalSourceError::Contract)?; + check_pack_contract(projection, record, &program)?; + let (fields, field_types) = project_fields(projection, record, binding)?; + let cache_ttl_seconds = effective_ttl(projection, binding.retention)?; + let grant_valid_until = parse_grant_cutoff(&binding.grant_valid_until)?; + let allowed_selectors = binding + .allowed_selectors + .as_ref() + .map(|list| list.iter().cloned().collect::>()); + + let ca_bundles = capture_ca_bundles(&binding.connection)?; + let ca_digests: BTreeMap<_, _> = ca_bundles + .iter() + .map(|(name, bytes)| (name, sha256_uri(bytes))) + .collect(); + let connection_fingerprint = fingerprint(&json!({ + "connection":binding.connection, + "endpointPolicies":binding.endpoint_policies, + "caDigests":ca_digests + }))?; + let connection_id = binding.connection.id.as_str(); + let connection = match connections.entry(connection_id) { + std::collections::btree_map::Entry::Vacant(entry) => { + &entry + .insert(( + connection_fingerprint.clone(), + EvidenceRuntimeBindings::from_shared_http( + binding.connection.clone(), + Arc::clone(&resolver), + ca_bundles, + ), + )) + .1 + } + std::collections::btree_map::Entry::Occupied(entry) => { + let (previous, runtime) = entry.into_mut(); + if previous.as_str() != connection_fingerprint.as_str() { + return Err(ExternalSourceError::Configuration); + } + runtime + } + }; + let program = program + .bind_live(connection, connection_id, &binding.endpoint_policies) + .map_err(|_| ExternalSourceError::Contract)?; + let binding_fingerprint = fingerprint(&json!({ + "packDigest":binding.pack_digest, + "operation":projection.operation, + "connectionFingerprint":connection_fingerprint, + "acquisition":binding.acquisition + }))?; + let grant_fingerprint = fingerprint(&json!({ + "projection":projection, + "externalType":record, + "sourceGrant":program.contract(), + "allowedFields":fields, + "allowedSelectors":allowed_selectors, + "grantValidUntil":grant_valid_until.to_rfc3339(), + "retention":binding.retention + }))?; + result.bindings.insert( + id.to_owned(), + BoundExternalSource { + program, + fields, + allowed_selectors, + mode: projection.mode, + grant_valid_until, + binding_fingerprint, + grant_fingerprint, + cache_ttl_seconds, + retention: binding.retention, + field_types, + }, + ); + } + Ok(result) + } + + pub fn binding(&self, projection: &str) -> Option<&BoundExternalSource> { + self.bindings.get(projection) + } + + /// Acquire only the preauthorized local keys and fields. A grant that + /// expires during I/O never produces an accepted result. + pub async fn acquire( + &self, + projection: &str, + hooks: Arc, + request: ProjectionRequest, + fields: &BTreeSet, + clock: impl Fn() -> DateTime + Send + Sync, + ) -> Result { + let bound = self + .binding(projection) + .ok_or(ExternalSourceError::Unbound)?; + let now = clock(); + if now >= bound.grant_valid_until + || fields.is_empty() + || !fields.is_subset(&bound.fields) + || !selectors_allowed(&request, bound.allowed_selectors.as_ref()) + { + return Err(ExternalSourceError::Grant); + } + let mode = match bound.mode { + ExternalProjectionMode::Live => ProjectionMode::Live, + ExternalProjectionMode::Indexed => ProjectionMode::Indexed, + }; + let remaining = bound + .grant_valid_until + .signed_duration_since(now) + .to_std() + .map_err(|_| ExternalSourceError::Grant)?; + let grant_deadline = tokio::time::Instant::now() + .checked_add(remaining) + .ok_or(ExternalSourceError::Grant)?; + let result = bound + .program + .acquire_until(hooks, mode, request, fields, Some(grant_deadline)) + .await?; + check_result(&result, &bound.field_types)?; + if clock() >= bound.grant_valid_until { + return Err(ExternalSourceError::Grant); + } + Ok(result) + } +} + +pub(crate) fn validate_bindings( + bindings: &[ExternalSourceBinding], +) -> Result<(), ExternalSourceError> { + let mut ids = BTreeSet::new(); + if bindings.len() > MAXIMUM_BINDINGS + || !bindings + .iter() + .all(|binding| ids.insert(binding.id.as_str())) + || !lookalike_local_id_pairs(bindings.iter().map(Identified::id)).is_empty() + { + return Err(ExternalSourceError::Configuration); + } + for binding in bindings { + parse_grant_cutoff(&binding.grant_valid_until)?; + if !valid_absolute_file(&binding.pack_file) + || binding.allowed_fields.is_empty() + || binding.allowed_fields.len() > 256 + || binding.endpoint_policies.is_empty() + || binding.allowed_selectors.as_ref().is_some_and(|selectors| { + selectors.len() > 10_000 || selectors.iter().any(|key| !projection_token_valid(key)) + }) + || binding.retention.is_some_and(|retention| { + retention.refresh_interval_seconds.get() + >= retention.maximum_retention_seconds.get() + }) + { + return Err(ExternalSourceError::Configuration); + } + } + Ok(()) +} + +fn parse_grant_cutoff(value: &str) -> Result, ExternalSourceError> { + if value.len() > 64 { + return Err(ExternalSourceError::Configuration); + } + DateTime::parse_from_rfc3339(value) + .map(|cutoff| cutoff.to_utc()) + .map_err(|_| ExternalSourceError::Configuration) +} + +fn check_pack_contract( + projection: &ExternalProjectionSource, + record: &ExternalRecordTypeSource, + program: &ProjectionPackProgram, +) -> Result<(), ExternalSourceError> { + let contract = program.contract(); + let expected_operation = match (projection.mode, projection.cardinality) { + (_, ExternalCardinality::Many) => ProjectionOperationKind::ByParent, + (ExternalProjectionMode::Live, ExternalCardinality::One) => ProjectionOperationKind::Exact, + (ExternalProjectionMode::Indexed, ExternalCardinality::One) => { + ProjectionOperationKind::Batch + } + }; + if contract.local_disclosure != ProjectionLocalDisclosure::HostProfiles + || contract.operation != expected_operation + || contract.operation_version != projection.operation_version.as_str() + || contract.authority != record.authority.as_str() + || contract.purpose != projection.purpose.as_str() + || contract.scope != projection.scope.as_str() + || contract.selector_scheme != projection.selector_scheme.as_str() + || contract.record_key_scheme != record.identifier_scheme.as_str() + || (projection.mode == ExternalProjectionMode::Indexed + && contract.retention != ProjectionRetention::Retain) + { + return Err(ExternalSourceError::Contract); + } + for id in projection.fields.iter() { + let field = record + .fields + .iter() + .find(|field| field.id == *id) + .ok_or(ExternalSourceError::Contract)?; + let schema = program.fields_schema().as_value()["properties"] + .get(id.as_str()) + .ok_or(ExternalSourceError::Contract)?; + if !compatible_field(field.value_type, field.nullable, schema) { + return Err(ExternalSourceError::Contract); + } + } + Ok(()) +} + +fn compatible_field(value_type: ExternalFieldType, nullable: bool, schema: &Value) -> bool { + let (expected, format) = match value_type { + ExternalFieldType::String => ("string", None), + ExternalFieldType::Integer => ("integer", None), + ExternalFieldType::Boolean => ("boolean", None), + ExternalFieldType::Decimal => ("number", None), + ExternalFieldType::Date => ("string", Some("date")), + ExternalFieldType::Uuid => ("string", Some("uuid")), + }; + if format.is_some_and(|format| schema["format"].as_str() != Some(format)) { + return false; + } + if let Some(declared) = schema["type"].as_str() { + return declared == expected; + } + if let Some(variants) = schema["type"].as_array() { + return variants + .iter() + .any(|variant| variant.as_str() == Some(expected)) + && variants.iter().all(|variant| { + variant.as_str() == Some(expected) || nullable && variant.as_str() == Some("null") + }); + } + schema["enum"].as_array().is_some_and(|values| { + !values.is_empty() + && values + .iter() + .all(|value| typed_value(value_type, nullable, value)) + }) +} + +type FieldTypes = BTreeMap; + +fn project_fields( + projection: &ExternalProjectionSource, + record: &ExternalRecordTypeSource, + binding: &ExternalSourceBinding, +) -> Result<(BTreeSet, FieldTypes), ExternalSourceError> { + let approved: BTreeSet<_> = binding.allowed_fields.iter().map(LocalId::as_str).collect(); + let mut fields = BTreeSet::new(); + let mut types = BTreeMap::new(); + for id in projection.fields.iter() { + if !approved.contains(id.as_str()) { + return Err(ExternalSourceError::Grant); + } + let field = record + .fields + .iter() + .find(|field| field.id == *id) + .ok_or(ExternalSourceError::Contract)?; + fields.insert(id.as_str().to_owned()); + types.insert(id.as_str().to_owned(), (field.value_type, field.nullable)); + } + Ok((fields, types)) +} + +fn effective_ttl( + projection: &ExternalProjectionSource, + retention: Option, +) -> Result, ExternalSourceError> { + match (projection.mode, projection.cache_ttl_seconds, retention) { + (ExternalProjectionMode::Live, None, None) => Ok(None), + (ExternalProjectionMode::Indexed, Some(ttl), Some(retention)) => { + let effective = ttl.min(retention.maximum_retention_seconds.get() as u32); + if u64::from(effective) <= retention.refresh_interval_seconds.get() { + return Err(ExternalSourceError::Configuration); + } + Ok(Some(effective)) + } + _ => Err(ExternalSourceError::Configuration), + } +} + +fn selectors_allowed(request: &ProjectionRequest, allowed: Option<&BTreeSet>) -> bool { + let Some(allowed) = allowed else { + return true; + }; + match request { + ProjectionRequest::Exact { key } => allowed.contains(key), + ProjectionRequest::Batch { keys } => keys.iter().all(|key| allowed.contains(key)), + ProjectionRequest::ByParent { parent_key } => allowed.contains(parent_key), + } +} + +fn check_result(result: &ProjectionResult, types: &FieldTypes) -> Result<(), ExternalSourceError> { + let check = |fields: &Map| { + fields.iter().all(|(name, value)| { + types + .get(name) + .is_some_and(|&(value_type, nullable)| typed_value(value_type, nullable, value)) + }) + }; + let record_valid = |record: &ProjectionRecord| match record { + ProjectionRecord::Present { fields, .. } => check(fields), + _ => true, + }; + let valid = match result { + ProjectionResult::Exact { record } => record_valid(record), + ProjectionResult::Batch { records } => records.iter().all(record_valid), + ProjectionResult::ByParent { rows, .. } => rows.iter().all(|row| check(&row.fields)), + }; + if valid { + Ok(()) + } else { + Err(ExternalSourceError::Result) + } +} + +fn typed_value(value_type: ExternalFieldType, nullable: bool, value: &Value) -> bool { + if value.is_null() { + return nullable; + } + match value_type { + ExternalFieldType::String => value.is_string(), + ExternalFieldType::Integer => value.as_i64().is_some(), + ExternalFieldType::Boolean => value.is_boolean(), + ExternalFieldType::Decimal => value.is_number(), + ExternalFieldType::Date => value.as_str().is_some_and(|raw| { + chrono::NaiveDate::parse_from_str(raw, "%Y-%m-%d") + .is_ok_and(|date| date.to_string() == raw) + }), + ExternalFieldType::Uuid => value.as_str().is_some_and(|raw| { + uuid::Uuid::parse_str(raw).is_ok_and(|id| id.hyphenated().to_string() == raw) + }), + } +} + +fn fingerprint(value: &Value) -> Result { + serde_json::to_vec(value) + .map(|bytes| sha256_uri(&bytes)) + .map_err(|_| ExternalSourceError::Configuration) +} + +fn load_pack( + path: &Path, + expected: &Digest, +) -> Result<(AdapterPack, PackArtifactBytes), ExternalSourceError> { + let parent = path.parent().ok_or(ExternalSourceError::Artifact)?; + let file_name = path.file_name().ok_or(ExternalSourceError::Artifact)?; + let root = open_directory(parent)?; + let bytes = read_relative(&root, Path::new(file_name), MAXIMUM_PACK_BYTES)?; + if sha256_uri(&bytes) != expected.as_str() { + return Err(ExternalSourceError::Artifact); + } + let format = FormatSpec { + kind: ADAPTER_PACK_KIND, + envelope: EnvelopeRule::ApiVersionKind { + api_versions: &[ApiVersion::current(ADAPTER_PACK_API_VERSION)], + retired_api_versions: &[], + }, + removed_keys: &[], + }; + let pack: AdapterPack = Reader::new("pack.yaml") + .decode(&bytes, &Expect::one(&format)) + .map_err(|_| ExternalSourceError::Artifact)? + .value; + if !registry_adapter_core::check_pack(&pack).is_empty() { + return Err(ExternalSourceError::Contract); + } + let files = pack.pinned_files(); + if files.len() > MAXIMUM_CLOSURE_FILES { + return Err(ExternalSourceError::Artifact); + } + let mut retained = BTreeMap::)>::new(); + let mut total = 0; + for file in files { + if !valid_relative_file(&file.path) { + return Err(ExternalSourceError::Artifact); + } + if let Some((previous, _)) = retained.get(file.path.as_str()) { + if previous != file.digest.as_str() { + return Err(ExternalSourceError::Artifact); + } + continue; + } + let data = read_relative(&root, Path::new(&file.path), MAXIMUM_CLOSURE_FILE_BYTES)?; + let digest = sha256_uri(&data); + total += data.len(); + if digest != file.digest.as_str() || total > MAXIMUM_CLOSURE_BYTES { + return Err(ExternalSourceError::Artifact); + } + retained.insert(file.path.to_string(), (digest, data)); + } + let artifacts = + PackArtifactBytes::new(retained.into_iter().map(|(path, (_, data))| (path, data))); + Ok((pack, artifacts)) +} + +fn capture_ca_bundles( + connection: &AdapterEndpointBindings, +) -> Result>, ExternalSourceError> { + let mut bundles = BTreeMap::new(); + if let Some(tls) = &connection.outbound_tls { + if tls.trust_profiles.len() > 32 { + return Err(ExternalSourceError::Configuration); + } + for (id, profile) in &tls.trust_profiles { + if !valid_absolute_file(&profile.ca_bundle_file) { + return Err(ExternalSourceError::Configuration); + } + let path = Path::new(&profile.ca_bundle_file); + let parent = path.parent().ok_or(ExternalSourceError::Artifact)?; + let name = path.file_name().ok_or(ExternalSourceError::Artifact)?; + let root = open_directory(parent)?; + bundles.insert( + id.clone(), + read_relative(&root, Path::new(name), MAXIMUM_CA_BYTES)?, + ); + } + } + Ok(bundles) +} + +fn valid_absolute_file(value: &str) -> bool { + value.strip_prefix('/').is_some_and(valid_relative_file) +} + +fn valid_relative_file(value: &str) -> bool { + !value.is_empty() + && value.len() <= 512 + && !value.contains('\\') + && !value.chars().any(char::is_control) + && value + .split('/') + .all(|part| !part.is_empty() && part != "." && part != "..") +} + +#[cfg(unix)] +fn open_directory(path: &Path) -> Result { + use rustix::fs::{open, Mode, OFlags}; + + open( + path, + OFlags::RDONLY | OFlags::CLOEXEC | OFlags::DIRECTORY | OFlags::NONBLOCK, + Mode::empty(), + ) + .map(File::from) + .map_err(|_| ExternalSourceError::Artifact) +} + +#[cfg(not(unix))] +fn open_directory(_: &Path) -> Result { + Err(ExternalSourceError::Artifact) +} + +#[cfg(unix)] +fn read_relative(root: &File, path: &Path, maximum: usize) -> Result, ExternalSourceError> { + use rustix::fs::{openat, Mode, OFlags}; + + let mut dir = root + .try_clone() + .map_err(|_| ExternalSourceError::Artifact)?; + let mut parts = path.components().peekable(); + while let Some(part) = parts.next() { + let Component::Normal(name) = part else { + return Err(ExternalSourceError::Artifact); + }; + let leaf = parts.peek().is_none(); + let mut flags = OFlags::RDONLY | OFlags::CLOEXEC | OFlags::NOFOLLOW | OFlags::NONBLOCK; + if !leaf { + flags |= OFlags::DIRECTORY; + } + let opened = openat(&dir, name, flags, Mode::empty()) + .map(File::from) + .map_err(|_| ExternalSourceError::Artifact)?; + if !leaf { + dir = opened; + continue; + } + let metadata = opened + .metadata() + .map_err(|_| ExternalSourceError::Artifact)?; + if !metadata.is_file() || metadata.len() == 0 || metadata.len() > maximum as u64 { + return Err(ExternalSourceError::Artifact); + } + let mut data = Vec::with_capacity(metadata.len() as usize); + opened + .take(maximum as u64 + 1) + .read_to_end(&mut data) + .map_err(|_| ExternalSourceError::Artifact)?; + if data.len() as u64 != metadata.len() || data.len() > maximum { + return Err(ExternalSourceError::Artifact); + } + return Ok(data); + } + Err(ExternalSourceError::Artifact) +} + +#[cfg(not(unix))] +fn read_relative(_: &File, _: &Path, _: usize) -> Result, ExternalSourceError> { + Err(ExternalSourceError::Artifact) +} + +#[cfg(test)] +mod tests { + use super::*; + use registry_adapter_engine::evidence::{EvidenceAcquisitionIntent, EvidenceHookFuture}; + use registry_platform_yaml::ExternalId; + + struct Hooks; + impl EvidenceHostHooks for Hooks { + fn before_acquisition<'a>( + &'a self, + _: EvidenceAcquisitionIntent, + ) -> EvidenceHookFuture<'a> { + Box::pin(async { Ok(()) }) + } + } + + fn project_contracts() -> (ExternalProjectionSource, ExternalRecordTypeSource) { + let projection = serde_json::from_value(json!({ + "id":"holder","recordType":"person","operation":"person-exact","operationVersion":"1", + "purpose":"licensing","scope":"licensing-holders","selectorScheme":"urn:example:person-number", + "mode":"live","cardinality":"one","fields":["family-name"] + })).unwrap(); + let record = serde_json::from_value(json!({ + "id":"person","authority":"urn:example:population-register","identifierScheme":"urn:example:person-number", + "fields":[{"id":"family-name","type":"string","classification":"public"}] + })).unwrap(); + (projection, record) + } + + fn example() -> (AdapterPack, PackArtifactBytes) { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../products/adapter/examples/external-projections"); + let bytes = std::fs::read(root.join("pack.yaml")).unwrap(); + let digest = Digest::new(sha256_uri(&bytes)).unwrap(); + load_pack(&root.join("pack.yaml"), &digest).unwrap() + } + + #[test] + fn project_operation_cannot_change_source_scope_scheme_type_or_retention() { + let (pack, artifacts) = example(); + let program = + compile_projection_pack(&pack, "person-exact", &artifacts, EvidenceLimits::default()) + .unwrap(); + let (projection, record) = project_contracts(); + assert!(check_pack_contract(&projection, &record, &program).is_ok()); + + let mut restricted_pack = serde_json::to_value(&pack).unwrap(); + restricted_pack["sourceRoutes"][0]["projection"]["localDisclosure"] = + json!("acquisition-only"); + let restricted = compile_projection_pack( + &serde_json::from_value(restricted_pack).unwrap(), + "person-exact", + &artifacts, + EvidenceLimits::default(), + ) + .unwrap(); + assert!(matches!( + check_pack_contract(&projection, &record, &restricted), + Err(ExternalSourceError::Contract) + )); + + let mut wrong_scope = projection.clone(); + wrong_scope.scope = ExternalId::new("different-scope").unwrap(); + assert!(matches!( + check_pack_contract(&wrong_scope, &record, &program), + Err(ExternalSourceError::Contract) + )); + + let mut wrong_scheme = record.clone(); + wrong_scheme.identifier_scheme = ExternalId::new("urn:example:unrelated").unwrap(); + assert!(matches!( + check_pack_contract(&projection, &wrong_scheme, &program), + Err(ExternalSourceError::Contract) + )); + + let mut wrong_type = record.clone(); + wrong_type.fields[0].value_type = ExternalFieldType::Integer; + assert!(matches!( + check_pack_contract(&projection, &wrong_type, &program), + Err(ExternalSourceError::Contract) + )); + + let mut indexed = projection; + indexed.mode = ExternalProjectionMode::Indexed; + assert!(matches!( + check_pack_contract(&indexed, &record, &program), + Err(ExternalSourceError::Contract) + )); + } + + #[test] + fn exact_pack_pin_and_every_referenced_file_are_checked() { + let root = tempfile::tempdir().unwrap(); + let pack = root.path().join("pack.yaml"); + std::fs::write(&pack, b"synthetic mismatch").unwrap(); + assert!(matches!( + load_pack(&pack, &Digest::new(sha256_uri(b"expected bytes")).unwrap()), + Err(ExternalSourceError::Artifact) + )); + + let source = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../products/adapter/examples/external-projections/pack.yaml"); + let bytes = std::fs::read(source).unwrap(); + std::fs::write(&pack, &bytes).unwrap(); + assert!(matches!( + load_pack(&pack, &Digest::new(sha256_uri(&bytes)).unwrap()), + Err(ExternalSourceError::Artifact) + )); + } + + #[tokio::test] + async fn a_result_finished_at_grant_cutoff_is_not_accepted() { + let (pack, artifacts) = example(); + let transcript = Arc::new(registry_adapter_engine::replay::ReplayTranscript::from_exchanges( + serde_json::from_value(json!([{"type":"endpoint","endpoint":"synthetic-registry"}])).unwrap(), + serde_json::from_value(json!([{ + "request":{"endpoint":"synthetic-registry","method":"POST","path":"/people/exact","body":{ + "type":"exact","key":"person-7","fields":["family-name"] + }}, + "response":{"status":200,"body":{"subject":"person-7","state":"found","properties":{"family-name":"Lee"}}} + }])).unwrap(), + )); + let program = + compile_projection_pack(&pack, "person-exact", &artifacts, EvidenceLimits::default()) + .unwrap() + .bind_replay(Arc::clone(&transcript)) + .unwrap(); + let cutoff = DateTime::parse_from_rfc3339("2030-01-02T00:00:00Z") + .unwrap() + .to_utc(); + let fields = BTreeSet::from(["family-name".to_owned()]); + let service = ExternalSourceService { + bindings: BTreeMap::from([( + "holder".to_owned(), + BoundExternalSource { + program, + fields: fields.clone(), + allowed_selectors: None, + mode: ExternalProjectionMode::Live, + grant_valid_until: cutoff, + binding_fingerprint: "binding".to_owned(), + grant_fingerprint: "grant".to_owned(), + cache_ttl_seconds: None, + retention: None, + field_types: BTreeMap::from([( + "family-name".to_owned(), + (ExternalFieldType::String, false), + )]), + }, + )]), + }; + let samples = std::sync::atomic::AtomicUsize::new(0); + let result = service + .acquire( + "holder", + Arc::new(Hooks), + ProjectionRequest::Exact { + key: "person-7".to_owned(), + }, + &fields, + || { + if samples.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 { + cutoff - chrono::Duration::seconds(1) + } else { + cutoff + } + }, + ) + .await; + assert!(matches!(result, Err(ExternalSourceError::Grant))); + transcript.finish().unwrap(); + } +} diff --git a/crates/registry-breg/src/external_view_artifacts.rs b/crates/registry-breg/src/external_view_artifacts.rs new file mode 100644 index 0000000000..85766e32a0 --- /dev/null +++ b/crates/registry-breg/src/external_view_artifacts.rs @@ -0,0 +1,266 @@ +// SPDX-License-Identifier: Apache-2.0 +//! OpenAPI for the external view surface, using the same named link grants. + +use std::collections::BTreeSet; + +use serde_json::{json, Map, Value}; + +use crate::contract::Operation; +use crate::external_views::{ + CompiledExternalViews, ExternalCardinality, ExternalFieldType, ExternalLinkPermissionSource, + ExternalProjectionMode, ExternalRecordTypeSource, +}; + +pub(crate) fn append( + paths: &mut Map, + inventory: &CompiledExternalViews, + admitted: impl Fn(&str, &str, &str, Operation) -> bool, +) { + for view in inventory.views.values() { + for link in &view.links { + let projection = &inventory.projections[link.projection.as_str()]; + let external = &inventory.record_types[projection.record_type.as_str()]; + let base = format!("/v1/external-views/{}/{}", view.id, link.id); + let grants = |operation: Operation| { + inventory + .grants + .iter() + .filter_map(|(profile, views)| { + if !admitted( + profile, + view.record_type.as_str(), + projection.purpose.as_str(), + operation, + ) { + return None; + } + views + .get(view.id.as_str())? + .links + .iter() + .find(|grant| grant.link == link.id) + }) + .collect::>() + }; + let label = link.label.as_ref().map(|label| { + &inventory.record_types[inventory.projections[label.projection.as_str()] + .record_type + .as_str()] + }); + let detail = grants(Operation::Get); + let is_indexed = projection.mode == ExternalProjectionMode::Indexed; + let is_many = projection.cardinality == ExternalCardinality::Many; + if !detail.is_empty() { + paths.insert( + format!("{base}/records/{{root}}"), + json!({"get": read_operation( + view.id.as_str(), link.id.as_str(), true, is_indexed && is_many, + is_many, external, label, &detail + )}), + ); + } + if is_indexed && !is_many { + let collection = grants(Operation::List); + if !collection.is_empty() { + paths.insert( + format!("{base}/records"), + json!({"get": read_operation( + view.id.as_str(), link.id.as_str(), false, true, + false, external, None, &collection + )}), + ); + } + } + let refresh_admitted = |labels: bool| { + is_indexed + && inventory.refresh.iter().any(|(profile, projections)| { + projections.contains(&projection.id) + && (!labels + || (is_many + && link.label.as_ref().is_some_and(|label| { + projections.contains(&label.projection) + }))) + && admitted( + profile, + view.record_type.as_str(), + projection.purpose.as_str(), + Operation::List, + ) + }) + }; + for (suffix, summary, labels) in [ + ( + "refresh", + "Refresh a bounded page of authorised local roots", + false, + ), + ( + "refresh-labels", + "Refresh labels selected from fresh authorised related rows", + true, + ), + ] { + if refresh_admitted(labels) { + paths.insert( + format!("{base}/{suffix}"), + refresh_operation( + format!("external-{}-{}-{suffix}", view.id, link.id), + summary, + ), + ); + } + } + } + } +} + +fn refresh_operation(operation_id: String, summary: &str) -> Value { + json!({"post": { + "operationId": operation_id, + "summary": summary, + "security": [{"bearerAuth": []}], + "parameters": [profile_parameter(), size_parameter(), cursor_parameter()], + "responses": { + "200": {"description": "Value-free refresh report", "content": {"application/json": {"schema": { + "type": "object", "additionalProperties": false, + "required": ["processed", "published", "reused", "unavailable"], + "properties": {"processed": count_schema(), "published": count_schema(), + "reused": count_schema(), "unavailable": count_schema(), "nextCursor": {"type":"string"}} + }}}}, + "400": {"description": "Invalid page request"}, + "404": {"description": "Refresh is unavailable to the caller"}, + "503": {"description": "The current source or projection is unavailable"} + } + }}) +} + +#[allow(clippy::too_many_arguments)] +fn read_operation( + view: &str, + link: &str, + detail: bool, + pageable: bool, + many: bool, + external: &ExternalRecordTypeSource, + label: Option<&ExternalRecordTypeSource>, + grants: &[&ExternalLinkPermissionSource], +) -> Value { + let mut parameters = vec![profile_parameter()]; + if detail { + parameters.push(json!({"name":"root","in":"path","required":true,"schema":{"type":"string","format":"uuid"}})); + } + let allow_count = pageable && grants.iter().any(|grant| grant.allow_count); + if pageable { + parameters.extend([size_parameter(), cursor_parameter()]); + if grants + .iter() + .any(|grant| !grant.filterable_fields.is_empty()) + { + parameters.push(json!({"name":"$filter","in":"query","description":"Exact equality on a granted field","schema":{"type":"string"}})); + } + if grants.iter().any(|grant| !grant.sortable_fields.is_empty()) { + parameters.push(json!({"name":"$orderby","in":"query","description":"One granted field; missing values sort last","schema":{"type":"string"}})); + } + if allow_count { + parameters.push(json!({"name":"$count","in":"query","schema":{"type":"boolean"}})); + } + } + let readable = grants + .iter() + .flat_map(|grant| grant.readable_fields.iter().map(|id| id.as_str())) + .collect::>(); + let mut row = json!({ + "rootId": {"type":"string","format":"uuid"}, + "availability": availability_schema(), + "fields": fields(external, &readable) + }); + if many { + row["rowId"] = json!({"type":"string","description":"Opaque stable row handle within this link and root"}); + } + let label_fields = grants + .iter() + .flat_map(|grant| grant.label_fields.iter().map(|id| id.as_str())) + .collect::>(); + if let Some(label) = label.filter(|_| !label_fields.is_empty()) { + row["label"] = json!({ + "type":"object","additionalProperties":false,"required":["authority","availability","fields"], + "properties":{"authority":{"const":label.authority},"availability":availability_schema(),"fields":fields(label,&label_fields)} + }); + } + let mut properties = json!({ + "status": {"enum":["available","unavailable"]}, + "authority": {"const":external.authority}, + "records": {"type":"array","items":{"type":"object","additionalProperties":false, + "required":["rootId","availability","fields"],"properties":row}} + }); + if pageable { + properties["nextCursor"] = json!({"type":"string"}); + } + if allow_count { + properties["count"] = count_schema(); + } + let suffix = if detail { "detail" } else { "list" }; + json!({ + "operationId":format!("external-{view}-{link}-{suffix}"), + "summary":"Read a named view of external records", + "security":[{"bearerAuth":[]}], + "parameters":parameters, + "responses":{ + "200":{"description":"The external fields available under the selected local grant","content":{"application/json":{"schema":{ + "type":"object","additionalProperties":false,"required":["status","authority","records"],"properties":properties + }}}}, + "400":{"description":"Invalid query or continuation"}, + "404":{"description":"The view or root is unavailable to the caller"}, + "503":{"description":"The external result is unavailable or indexed coverage is incomplete"} + } + }) +} + +fn fields(external: &ExternalRecordTypeSource, readable: &BTreeSet<&str>) -> Value { + let properties = external + .fields + .iter() + .filter(|field| readable.contains(field.id.as_str())) + .map(|field| { + let value = match field.value_type { + ExternalFieldType::String => json!({"type":"string"}), + ExternalFieldType::Integer => json!({"type":"integer"}), + ExternalFieldType::Boolean => json!({"type":"boolean"}), + ExternalFieldType::Decimal => { + json!({"type":"string","pattern":"^-?[0-9]+(\\.[0-9]+)?$"}) + } + ExternalFieldType::Date => json!({"type":"string","format":"date"}), + ExternalFieldType::Uuid => json!({"type":"string","format":"uuid"}), + }; + ( + field.id.to_string(), + if field.nullable { + json!({"anyOf":[value,{"type":"null"}]}) + } else { + value + }, + ) + }) + .collect::>(); + json!({"type":"object","additionalProperties":false,"properties":properties}) +} + +fn availability_schema() -> Value { + json!({"enum":["available","absent","unavailable"]}) +} + +fn profile_parameter() -> Value { + json!({"name":"accessProfile","in":"query","schema":{"type":"string"}}) +} + +fn size_parameter() -> Value { + json!({"name":"$top","in":"query","schema":{"type":"integer","minimum":1,"maximum":100}}) +} + +fn cursor_parameter() -> Value { + json!({"name":"$skiptoken","in":"query","description":"Use alone with the same access profile","schema":{"type":"string"}}) +} + +fn count_schema() -> Value { + json!({"type":"integer","minimum":0}) +} diff --git a/crates/registry-breg/src/external_views.rs b/crates/registry-breg/src/external_views.rs new file mode 100644 index 0000000000..03de33d2b1 --- /dev/null +++ b/crates/registry-breg/src/external_views.rs @@ -0,0 +1,621 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Governed configuration for read views over external source projections. + +use std::collections::{BTreeMap, BTreeSet}; + +use registry_platform_yaml::{ExternalId, LocalId}; +use serde::{Deserialize, Serialize}; + +use crate::contract::{ + ActorKindSource, Classification, DisplaySource, FieldTypeSource, Operation, RegistryProject, + UniqueSet, +}; +use crate::diagnostics::Diagnostic; +use crate::model::CompiledEntity; + +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct ExternalRecordTypeSource { + pub id: LocalId, + pub authority: ExternalId, + pub identifier_scheme: ExternalId, + #[serde(default)] + pub display: DisplaySource, + pub fields: Vec, +} + +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct ExternalFieldSource { + pub id: LocalId, + #[serde(rename = "type")] + pub value_type: ExternalFieldType, + pub classification: Classification, + #[serde(default)] + pub nullable: bool, +} + +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ExternalFieldType { + String, + Integer, + Boolean, + Decimal, + Date, + Uuid, +} + +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ExternalProjectionMode { + Live, + Indexed, +} + +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ExternalCardinality { + One, + Many, +} + +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct ExternalProjectionSource { + pub id: LocalId, + pub record_type: LocalId, + pub operation: LocalId, + pub operation_version: ExternalId, + pub purpose: ExternalId, + pub scope: ExternalId, + pub selector_scheme: ExternalId, + pub mode: ExternalProjectionMode, + pub cardinality: ExternalCardinality, + pub fields: UniqueSet, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_ttl_seconds: Option, +} + +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct ExternalViewSource { + pub id: LocalId, + pub record_type: LocalId, + pub links: Vec, +} + +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct ExternalLinkSource { + pub id: LocalId, + pub projection: LocalId, + pub from_field: LocalId, + pub identifier_scheme: ExternalId, + pub fields: UniqueSet, + /// An optional to-one label on a row of an indexed related collection. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub label: Option, +} + +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct ExternalLabelSource { + pub projection: LocalId, + pub from_field: LocalId, + pub identifier_scheme: ExternalId, + pub fields: UniqueSet, +} + +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct ExternalViewPermissionSource { + pub view: LocalId, + pub links: Vec, +} + +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct ExternalLinkPermissionSource { + pub link: LocalId, + pub readable_fields: UniqueSet, + #[serde(default, skip_serializing_if = "UniqueSet::is_empty")] + pub filterable_fields: UniqueSet, + #[serde(default, skip_serializing_if = "UniqueSet::is_empty")] + pub sortable_fields: UniqueSet, + #[serde(default)] + pub allow_count: bool, + /// A separate grant for label fields reached through the related row. + #[serde(default, skip_serializing_if = "UniqueSet::is_empty")] + pub label_fields: UniqueSet, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct CompiledExternalViews { + pub record_types: BTreeMap, + pub projections: BTreeMap, + pub views: BTreeMap, + /// A named local access profile grants a particular view and link. + pub grants: BTreeMap>, + pub refresh: BTreeMap>, +} + +impl CompiledExternalViews { + pub fn is_empty(&self) -> bool { + self.record_types.is_empty() + && self.projections.is_empty() + && self.views.is_empty() + && self.grants.is_empty() + && self.refresh.is_empty() + } +} + +pub(crate) fn compile( + project: &RegistryProject, + entities: &BTreeMap, +) -> Result> { + let mut diagnostics = Vec::new(); + let record_types = index( + &project.external_record_types, + |source| &source.id, + "/externalRecordTypes", + &mut diagnostics, + ); + let projections = index( + &project.external_projections, + |source| &source.id, + "/externalProjections", + &mut diagnostics, + ); + let views = index( + &project.external_views, + |source| &source.id, + "/externalViews", + &mut diagnostics, + ); + + for (index, record_type) in project.external_record_types.iter().enumerate() { + let path = format!("/externalRecordTypes/{index}"); + validate_uri( + &record_type.authority, + &format!("{path}/authority"), + &mut diagnostics, + ); + validate_uri( + &record_type.identifier_scheme, + &format!("{path}/identifierScheme"), + &mut diagnostics, + ); + let mut ids = BTreeSet::new(); + if record_type.fields.is_empty() + || !record_type.fields.iter().all(|field| ids.insert(&field.id)) + { + error( + &mut diagnostics, + "field-set", + format!("{path}/fields"), + "Declare a nonempty set of external field identifiers without duplicates.", + ); + } + } + + for (index, projection) in project.external_projections.iter().enumerate() { + let path = format!("/externalProjections/{index}"); + validate_uri( + &projection.selector_scheme, + &format!("{path}/selectorScheme"), + &mut diagnostics, + ); + let Some(record_type) = record_types.get(projection.record_type.as_str()) else { + error( + &mut diagnostics, + "unknown-type", + format!("{path}/recordType"), + "Select a declared external record type.", + ); + continue; + }; + let field_ids = record_type + .fields + .iter() + .map(|field| &field.id) + .collect::>(); + if projection.fields.is_empty() + || !projection + .fields + .iter() + .all(|field| field_ids.contains(field)) + { + error( + &mut diagnostics, + "field-set", + format!("{path}/fields"), + "Select at least one field declared by the external record type.", + ); + } + if projection.cardinality == ExternalCardinality::One + && projection.selector_scheme != record_type.identifier_scheme + { + error( + &mut diagnostics, + "identifier-scheme-mismatch", + format!("{path}/selectorScheme"), + "An exact operation selects the external record by its declared identifier scheme.", + ); + } + match (projection.mode, projection.cache_ttl_seconds) { + (ExternalProjectionMode::Live, Some(_)) => error( + &mut diagnostics, + "live-cache", + format!("{path}/cacheTtlSeconds"), + "Remove the cache lifetime for a live projection.", + ), + (ExternalProjectionMode::Indexed, None | Some(0)) => error( + &mut diagnostics, + "indexed-cache", + format!("{path}/cacheTtlSeconds"), + "Set a positive cache lifetime for an indexed projection.", + ), + _ => {} + } + } + + let mut local_schemes = BTreeMap::new(); + for (index, view) in project.external_views.iter().enumerate() { + let path = format!("/externalViews/{index}"); + let Some(root) = entities.get(view.record_type.as_str()) else { + error( + &mut diagnostics, + "unknown-root", + format!("{path}/recordType"), + "Select a local record type to root the view.", + ); + continue; + }; + if view.links.is_empty() { + error( + &mut diagnostics, + "empty-links", + format!("{path}/links"), + "Declare at least one external link on the view.", + ); + } + let mut link_ids = BTreeSet::new(); + for (link_index, link) in view.links.iter().enumerate() { + let path = format!("{path}/links/{link_index}"); + if !link_ids.insert(&link.id) { + error( + &mut diagnostics, + "duplicate-id", + format!("{path}/id"), + "Give each link on the view a distinct identifier.", + ); + } + let Some(projection) = projections.get(link.projection.as_str()) else { + error( + &mut diagnostics, + "unknown-projection", + format!("{path}/projection"), + "Select a declared external projection.", + ); + continue; + }; + if link.identifier_scheme != projection.selector_scheme + || local_schemes + .insert( + (&view.record_type, &link.from_field), + &link.identifier_scheme, + ) + .is_some_and(|scheme| scheme != &link.identifier_scheme) + { + error(&mut diagnostics, "identifier-scheme-mismatch", format!("{path}/identifierScheme"), + "Use one consistent local identifier scheme that matches the approved source selector."); + } + if !root.stored_fields.iter().any(|field| { + field.logical.id == link.from_field.as_str() + && field.logical.encryption.is_none() + && matches!( + field.logical.field_type, + FieldTypeSource::Uuid + | FieldTypeSource::String { + max_length: 1..=512, + .. + } + ) + }) { + error(&mut diagnostics, "invalid-root-key", format!("{path}/fromField"), + "Select a stored UUID or bounded string identifier supported by direct binding."); + } + if link.fields.is_empty() || !link.fields.is_subset(&projection.fields) { + error( + &mut diagnostics, + "field-set", + format!("{path}/fields"), + "Select at least one field from the approved projection.", + ); + } + if let Some(label) = &link.label { + validate_label( + label, + projection, + &record_types, + &projections, + &path, + &mut diagnostics, + ); + } + } + } + + let mut grants = BTreeMap::new(); + let mut refresh = BTreeMap::new(); + for (profile_index, profile) in project.access_profiles.iter().enumerate() { + let path = format!("/accessProfiles/{profile_index}/permissions"); + let mut view_grants = BTreeMap::new(); + for (grant_index, permission) in profile.permissions.external_views.iter().enumerate() { + let path = format!("{path}/externalViews/{grant_index}"); + let Some(view) = views.get(permission.view.as_str()) else { + error( + &mut diagnostics, + "unknown-view", + format!("{path}/view"), + "Select a declared external view.", + ); + continue; + }; + let root = profile + .permissions + .entities + .iter() + .find(|grant| grant.entity == view.record_type.as_str()); + if !root.is_some_and(|root| root.operations.contains(&Operation::Get)) { + error( + &mut diagnostics, + "root-read-missing", + format!("{path}/view"), + "Grant get on the local root record type in this same profile.", + ); + } + if view_grants + .insert(permission.view.as_str().to_owned(), permission.clone()) + .is_some() + { + error( + &mut diagnostics, + "duplicate-view-grant", + format!("{path}/view"), + "Grant each external view once per access profile.", + ); + } + let mut link_grants = BTreeSet::new(); + for (link_index, grant) in permission.links.iter().enumerate() { + let path = format!("{path}/links/{link_index}"); + if !link_grants.insert(&grant.link) { + error( + &mut diagnostics, + "duplicate-link-grant", + format!("{path}/link"), + "Grant each link once per view permission.", + ); + } + let Some(link) = view.links.iter().find(|link| link.id == grant.link) else { + error( + &mut diagnostics, + "unknown-link", + format!("{path}/link"), + "Grant a link declared on the selected view.", + ); + continue; + }; + if grant.readable_fields.is_empty() + || !grant.readable_fields.is_subset(&link.fields) + || (!grant.label_fields.is_empty() + && !link + .label + .as_ref() + .is_some_and(|label| grant.label_fields.is_subset(&label.fields))) + { + error(&mut diagnostics, "invalid-read-grant", path.clone(), + "Grant only declared link fields, with an explicit separate grant for labels."); + } + if !grant.filterable_fields.is_subset(&grant.readable_fields) + || !grant.sortable_fields.is_subset(&grant.readable_fields) + { + error( + &mut diagnostics, + "processing-wider-than-read", + path.clone(), + "Grant reading a field before permitting it for filtering or sorting.", + ); + } + let uses_query = !grant.filterable_fields.is_empty() + || !grant.sortable_fields.is_empty() + || grant.allow_count; + if let Some(projection) = projections.get(link.projection.as_str()) { + if !profile + .required_purposes + .contains(projection.purpose.as_str()) + { + error( + &mut diagnostics, + "purpose-not-granted", + path.clone(), + "Name the projection purpose in this profile's required purposes.", + ); + } + if uses_query && projection.mode != ExternalProjectionMode::Indexed { + error( + &mut diagnostics, + "live-query", + path.clone(), + "Grant filtering, sorting, or counts only for an indexed projection.", + ); + } + } + if uses_query + && !root.is_some_and(|root| root.operations.contains(&Operation::List)) + || grant.allow_count && !root.is_some_and(|root| root.allow_count) + { + error( + &mut diagnostics, + "root-query-missing", + path, + "Grant list and, for a count, allowCount on the local record type.", + ); + } + } + } + if !view_grants.is_empty() { + grants.insert(profile.id.clone(), view_grants); + } + if !profile.permissions.refresh_external_projections.is_empty() { + let path = format!("{path}/refreshExternalProjections"); + let scoped_service = profile.actor_kind == Some(ActorKindSource::Service) + && !profile.requester_clients.is_empty() + && !profile.required_scopes.is_empty() + && !profile.required_purposes.is_empty(); + if !scoped_service { + error(&mut diagnostics, "refresh-service-profile", path.clone(), + "Grant refresh only to a service profile with explicit clients, scopes, and purposes."); + } + for id in &profile.permissions.refresh_external_projections { + let may_refresh = projections.get(id.as_str()).is_some_and(|projection| { + projection.mode == ExternalProjectionMode::Indexed + && profile + .required_purposes + .contains(projection.purpose.as_str()) + && views.values().any(|view| { + view.links.iter().any(|link| { + link.projection == *id + || link.label.as_ref().is_some_and(|label| { + label.projection == *id + && profile + .permissions + .refresh_external_projections + .contains(&link.projection) + }) + }) && profile.permissions.entities.iter().any(|root| { + root.entity == view.record_type.as_str() + && root.operations.contains(&Operation::List) + }) + }) + }); + if !may_refresh { + error(&mut diagnostics, "invalid-refresh-grant", path.clone(), + "Grant refresh under the projection purpose and a permitted root list. Refreshing labels also requires a refresh grant for the related collection."); + } + } + refresh.insert( + profile.id.clone(), + profile.permissions.refresh_external_projections.clone(), + ); + } + } + if !diagnostics.is_empty() { + return Err(diagnostics); + } + Ok(CompiledExternalViews { + record_types, + projections, + views, + grants, + refresh, + }) +} + +fn validate_label( + label: &ExternalLabelSource, + parent: &ExternalProjectionSource, + record_types: &BTreeMap, + projections: &BTreeMap, + path: &str, + diagnostics: &mut Vec, +) { + let valid = parent.cardinality == ExternalCardinality::Many + && parent.mode == ExternalProjectionMode::Indexed + && parent.fields.contains(&label.from_field) + && record_types + .get(parent.record_type.as_str()) + .is_some_and(|record_type| { + record_type.fields.iter().any(|field| { + field.id == label.from_field + && matches!( + field.value_type, + ExternalFieldType::String | ExternalFieldType::Uuid + ) + }) + }) + && projections + .get(label.projection.as_str()) + .is_some_and(|projection| { + projection.mode == ExternalProjectionMode::Indexed + && projection.cardinality == ExternalCardinality::One + && projection.purpose == parent.purpose + && projection.selector_scheme == label.identifier_scheme + && !label.fields.is_empty() + && label.fields.is_subset(&projection.fields) + }); + if !valid { + error(diagnostics, "invalid-label", format!("{path}/label"), + "Link an indexed related row to an indexed exact label with compatible identifiers, fields, and purpose."); + } +} + +fn validate_uri(value: &ExternalId, path: &str, diagnostics: &mut Vec) { + let value = value.as_str(); + if value.chars().any(char::is_whitespace) + || !url::Url::parse(value).is_ok_and(|uri| { + uri.username().is_empty() && uri.password().is_none() && !uri.path().is_empty() + }) + { + error( + diagnostics, + "invalid-identifier-uri", + path, + "Use an absolute identifier URI without embedded credentials.", + ); + } +} + +fn index( + items: &[T], + id: impl Fn(&T) -> &LocalId, + path: &str, + diagnostics: &mut Vec, +) -> BTreeMap { + let mut by_id = BTreeMap::new(); + for (index, item) in items.iter().enumerate() { + if by_id + .insert(id(item).as_str().to_owned(), item.clone()) + .is_some() + { + error( + diagnostics, + "duplicate-id", + format!("{path}/{index}/id"), + "Give each declaration in this group a distinct identifier.", + ); + } + } + by_id +} + +fn error(diagnostics: &mut Vec, code: &str, path: impl Into, message: &str) { + diagnostics.push(Diagnostic::error( + &format!("breg.external-view.{code}"), + path, + message, + )); +} diff --git a/crates/registry-breg/src/lib.rs b/crates/registry-breg/src/lib.rs index 10255258eb..c3e0894a68 100644 --- a/crates/registry-breg/src/lib.rs +++ b/crates/registry-breg/src/lib.rs @@ -71,6 +71,10 @@ pub mod evidence_source; #[cfg(feature = "tooling")] pub mod example_references; #[cfg(feature = "runtime")] +pub mod external_source; +pub(crate) mod external_view_artifacts; +pub mod external_views; +#[cfg(feature = "runtime")] pub mod field_encryption; #[cfg(feature = "runtime")] pub mod field_encryption_backfill; diff --git a/crates/registry-breg/src/metrics.rs b/crates/registry-breg/src/metrics.rs index 1b04d0882d..889d56d895 100644 --- a/crates/registry-breg/src/metrics.rs +++ b/crates/registry-breg/src/metrics.rs @@ -127,15 +127,17 @@ pub enum ProgressWorker { AttachmentVerification, Review, SubjectAccessLogRetention, + ExternalProjectionRetention, } impl ProgressWorker { - pub const ALL: [Self; 5] = [ + pub const ALL: [Self; 6] = [ Self::Hook, Self::RecordTimer, Self::AttachmentVerification, Self::Review, Self::SubjectAccessLogRetention, + Self::ExternalProjectionRetention, ]; #[must_use] @@ -146,6 +148,7 @@ impl ProgressWorker { Self::AttachmentVerification => "attachment-verification", Self::Review => "review", Self::SubjectAccessLogRetention => "subject-access-log-retention", + Self::ExternalProjectionRetention => "external-projection-retention", } } } diff --git a/crates/registry-breg/src/migration.rs b/crates/registry-breg/src/migration.rs index fd5290163b..5f39c8958e 100644 --- a/crates/registry-breg/src/migration.rs +++ b/crates/registry-breg/src/migration.rs @@ -2622,6 +2622,7 @@ mod tests { registry_revision: "ignored-descriptor-revision".to_owned(), entities: BTreeMap::new(), statistical_datasets: BTreeMap::new(), + external_views: Default::default(), physical_names: PhysicalNameInventory { entities: BTreeMap::new(), }, diff --git a/crates/registry-breg/src/model.rs b/crates/registry-breg/src/model.rs index 22947bf3c4..1e620e193c 100644 --- a/crates/registry-breg/src/model.rs +++ b/crates/registry-breg/src/model.rs @@ -1882,6 +1882,11 @@ pub struct CompiledRegistry { entities: BTreeMap, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] statistical_datasets: BTreeMap, + #[serde( + default, + skip_serializing_if = "crate::external_views::CompiledExternalViews::is_empty" + )] + external_views: crate::external_views::CompiledExternalViews, physical_names: PhysicalNameInventory, action_inventory: CompiledActionInventory, route_inventory: CompiledRouteInventory, @@ -1914,6 +1919,7 @@ impl CompiledRegistry { module_closure: Vec, entities: BTreeMap, statistical_datasets: BTreeMap, + external_views: crate::external_views::CompiledExternalViews, physical_names: PhysicalNameInventory, action_inventory: CompiledActionInventory, route_inventory: CompiledRouteInventory, @@ -1942,6 +1948,7 @@ impl CompiledRegistry { module_closure, entities, statistical_datasets, + external_views, physical_names, action_inventory, route_inventory, @@ -2069,6 +2076,10 @@ impl CompiledRegistry { }) } + pub fn external_views(&self) -> &crate::external_views::CompiledExternalViews { + &self.external_views + } + pub fn statistical_datasets(&self) -> &BTreeMap { &self.statistical_datasets } diff --git a/crates/registry-breg/src/package.rs b/crates/registry-breg/src/package.rs index 4da46a3b86..7bc2588725 100644 --- a/crates/registry-breg/src/package.rs +++ b/crates/registry-breg/src/package.rs @@ -264,6 +264,11 @@ pub struct CompiledRegistryMigrationBaseline { pub entities: BTreeMap, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub statistical_datasets: BTreeMap, + #[serde( + default, + skip_serializing_if = "crate::external_views::CompiledExternalViews::is_empty" + )] + pub external_views: crate::external_views::CompiledExternalViews, pub physical_names: PhysicalNameInventory, pub routes: CompiledRouteInventory, pub access: CompiledAccessInventory, @@ -293,6 +298,7 @@ impl CompiledRegistryMigrationBaseline { registry_revision: compiled.revision().to_owned(), entities: compiled.entities().clone(), statistical_datasets: compiled.statistical_datasets().clone(), + external_views: compiled.external_views().clone(), physical_names: compiled.physical_names().clone(), routes: compiled.routes().clone(), access: compiled.access().clone(), @@ -401,6 +407,7 @@ pub enum CompiledRegistryChangeCode { StatisticalDatasetAdded, StatisticalDatasetRemoved, StatisticalDatasetChanged, + ExternalViewsChanged, } #[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] @@ -432,6 +439,7 @@ pub enum CompiledRegistryChangeTargetKind { Action, Recipient, StatisticalDataset, + ExternalViews, } /// What package loading needs from the deployment. A package carries no @@ -1007,6 +1015,14 @@ pub fn compiled_registry_change_set_from_baseline( compare_registry_identity(previous, &candidate_baseline, &mut changes); compare_entities(previous, &candidate_baseline, &mut changes); compare_statistical_datasets(previous, &candidate_baseline, &mut changes); + if previous.external_views != candidate_baseline.external_views { + push_change( + &mut changes, + CompiledRegistryChangeClass::AccessOrDisclosureChange, + CompiledRegistryChangeCode::ExternalViewsChanged, + target(CompiledRegistryChangeTargetKind::ExternalViews, None, None), + ); + } compare_routes(previous, &candidate_baseline, &mut changes); compare_query_inventory(previous, &candidate_baseline, &mut changes); compare_actions(previous, &candidate_baseline, &mut changes); @@ -4402,6 +4418,7 @@ struct PredecessorGovernedModel { model_revision: String, entities: BTreeMap, statistical_datasets: BTreeMap, + external_views: crate::external_views::CompiledExternalViews, physical_names: PhysicalNameInventory, routes: CompiledRouteInventory, access: CompiledAccessInventory, @@ -4425,6 +4442,7 @@ impl PredecessorGovernedModel { registry_revision: self.model_revision.clone(), entities: self.entities.clone(), statistical_datasets: self.statistical_datasets.clone(), + external_views: self.external_views.clone(), physical_names: self.physical_names.clone(), routes: self.routes.clone(), access: self.access.clone(), @@ -4479,6 +4497,13 @@ fn package_predecessor_governed_model( .transpose() .map_err(|_| PackageError::Derivation)? .unwrap_or_default(); + let external_views = value + .get("externalViews") + .cloned() + .map(serde_json::from_value) + .transpose() + .map_err(|_| PackageError::Derivation)? + .unwrap_or_default(); let effective_physical_names: PhysicalNameInventory = serde_json::from_value( value .get("physicalNames") @@ -4545,6 +4570,7 @@ fn package_predecessor_governed_model( model_revision: entry.sha256.clone(), entities, statistical_datasets, + external_views, physical_names, routes, access, diff --git a/crates/registry-breg/src/postgres/catalog.rs b/crates/registry-breg/src/postgres/catalog.rs index bc6fd5b79c..e832a27637 100644 --- a/crates/registry-breg/src/postgres/catalog.rs +++ b/crates/registry-breg/src/postgres/catalog.rs @@ -238,6 +238,13 @@ pub async fn catalog_drift_objects( if !registry.actions().actions.is_empty() { expected_indexes.insert(IMMEDIATE_ACTION_RESULT_TARGET_INDEX.to_owned()); } + if super::external_projection::enabled(registry) { + expected_indexes.extend( + super::external_projection::INDEXES + .iter() + .map(|name| (*name).to_owned()), + ); + } let index_rows = client .query( "SELECT n.nspname || '.' || i.relname @@ -569,6 +576,21 @@ impl ExpectedManagedCatalog { ); } catalog.add_statistical_release_store(); + if super::external_projection::enabled(registry) { + for table in super::external_projection::BACKUP_EXCLUDED_TABLES { + catalog.table( + table, + ["SELECT"], + std::iter::empty::<&str>(), + Some((false, false)), + ); + } + for (identity, _, _, _, runtime_execute) in + super::external_projection::MANAGED_FUNCTIONS + { + catalog.function(identity, runtime_execute.then_some("EXECUTE"), None); + } + } for (table, privileges) in crate::import_authority::IMPORT_AUTHORITY_TABLES { let name = format!("registry_internal.{table}"); catalog.table( @@ -1385,7 +1407,7 @@ pub(crate) async fn verify_managed_catalog( ) -> Result<()> { verify_managed_owners_for_catalog(client, migration_role, runtime_role, expected_catalog) .await?; - verify_closed_ambient_catalog(client).await?; + verify_closed_ambient_catalog(client, expected_catalog).await?; verify_exact_acl( client, runtime_role, @@ -1405,7 +1427,28 @@ pub(crate) async fn verify_managed_catalog( Ok(()) } -async fn verify_closed_ambient_catalog(client: &impl GenericClient) -> Result<()> { +async fn verify_closed_ambient_catalog( + client: &impl GenericClient, + expected_catalog: &ExpectedManagedCatalog, +) -> Result<()> { + let mut projection_identities = Vec::::new(); + let mut projection_returns = Vec::::new(); + let mut projection_sets = Vec::::new(); + let mut projection_definers = Vec::::new(); + for (identity, result, returns_set, security_definer, _) in + super::external_projection::MANAGED_FUNCTIONS + { + if expected_catalog + .objects + .iter() + .any(|object| object.kind == ManagedObjectKind::Function && object.name == *identity) + { + projection_identities.push((*identity).to_owned()); + projection_returns.push((*result).to_owned()); + projection_sets.push(*returns_set); + projection_definers.push(*security_definer); + } + } let row = client .query_one( "SELECT @@ -1485,6 +1528,16 @@ async fn verify_closed_ambient_catalog(client: &impl GenericClient) -> Result<() AND p.provolatile = 'v' AND p.prolang = (SELECT oid FROM pg_catalog.pg_language WHERE lanname = 'sql') AND p.proconfig = ARRAY['search_path=pg_catalog, registry_internal']) + OR (n.nspname = 'registry_internal' + AND p.prokind = 'f' AND p.provolatile = 'v' + AND NOT p.proisstrict AND NOT p.proleakproof + AND p.prolang = (SELECT oid FROM pg_catalog.pg_language WHERE lanname = 'plpgsql') + AND p.proconfig = ARRAY['search_path=pg_catalog, registry_internal'] + AND EXISTS (SELECT 1 FROM unnest($2::text[],$3::text[],$4::boolean[],$5::boolean[]) + AS allowed(identity,result_type,returns_set,security_definer) + WHERE allowed.identity=n.nspname || '.' || p.proname || '(' || pg_catalog.pg_get_function_identity_arguments(p.oid) || ')' + AND p.prorettype=allowed.result_type::regtype + AND p.proretset=allowed.returns_set AND p.prosecdef=allowed.security_definer)) OR (n.nspname = 'registry_internal' AND p.proname = 'reserve_generated_identifier' AND pg_catalog.pg_get_function_identity_arguments(p.oid) = 'text, text, jsonb, text' @@ -1509,7 +1562,7 @@ async fn verify_closed_ambient_catalog(client: &impl GenericClient) -> Result<() ) OR EXISTS ( SELECT 1 FROM pg_catalog.pg_publication WHERE puballtables )", - &[&MANAGED_SCHEMAS], + &[&MANAGED_SCHEMAS, &projection_identities, &projection_returns, &projection_sets, &projection_definers], ) .await?; if (0..5).any(|index| row.get::<_, bool>(index)) { @@ -1984,7 +2037,7 @@ pub async fn managed_schema_fingerprint( let migration_role = current_role(client).await?; verify_managed_owners_for_catalog(client, &migration_role, runtime_role, expected_catalog) .await?; - verify_closed_ambient_catalog(client).await?; + verify_closed_ambient_catalog(client, expected_catalog).await?; verify_exact_acl( client, runtime_role, diff --git a/crates/registry-breg/src/postgres/external_label_acquisition.rs b/crates/registry-breg/src/postgres/external_label_acquisition.rs new file mode 100644 index 0000000000..b93040b245 --- /dev/null +++ b/crates/registry-breg/src/postgres/external_label_acquisition.rs @@ -0,0 +1,581 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Internal label selectors reached through authorized, fresh related rows. + +use std::fmt; + +use serde::Deserialize; +use uuid::Uuid; + +use crate::contract::{ActorKindSource, Operation}; +use crate::external_views::{ExternalCardinality, ExternalFieldType, ExternalProjectionMode}; +use crate::model::CompiledRegistry; + +use super::external_projection::{ProjectionGeneration, ProjectionScope}; +use super::external_view_query::ExternalQueryError; +use super::read::ReadRelations; +use super::GuardedTransaction; + +type Result = std::result::Result; + +/// Internal position, carried only in an encrypted, generation-bound cursor. +#[derive(Clone)] +pub(crate) struct LabelAfter { + pub(crate) root_id: Uuid, + pub(crate) member_key: String, +} + +impl fmt::Debug for LabelAfter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("LabelAfter()") + } +} + +#[derive(Deserialize)] +pub(crate) struct LabelSelector { + pub(crate) root_id: Uuid, + pub(crate) member_key: String, + pub(crate) selector: String, + parent_expires_at: i64, +} + +impl fmt::Debug for LabelSelector { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("LabelSelector()") + } +} + +pub(crate) struct LabelSelectorPage { + pub(crate) rows: Vec, + pub(crate) continuation: Option, + pub(crate) minimum_parent_expires_at: Option, +} + +pub(crate) struct LabelAcquisition { + page_sql: String, + publication_sql: String, +} + +impl LabelAcquisition { + /// The caller admits the service identity and current source grants, then + /// installs the root claim context in its guarded transaction. This seam + /// rechecks the compiled grants for both hops without disclosing selectors. + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + registry: &CompiledRegistry, + profile_id: &str, + purpose: &str, + view_id: &str, + link_id: &str, + parent_scope: &ProjectionScope, + parent_generation: ProjectionGeneration, + ) -> Result { + let inventory = registry.external_views(); + let view = inventory + .views + .get(view_id) + .ok_or(ExternalQueryError::NotPermitted)?; + let link = view + .links + .iter() + .find(|link| link.id.as_str() == link_id) + .ok_or(ExternalQueryError::NotPermitted)?; + let label = link + .label + .as_ref() + .ok_or(ExternalQueryError::NotPermitted)?; + let parent = inventory + .projections + .get(link.projection.as_str()) + .ok_or(ExternalQueryError::NotPermitted)?; + let target = inventory + .projections + .get(label.projection.as_str()) + .ok_or(ExternalQueryError::NotPermitted)?; + let entity = registry + .entities() + .get(view.record_type.as_str()) + .ok_or(ExternalQueryError::NotPermitted)?; + let profile = entity + .access_profiles + .get(profile_id) + .ok_or(ExternalQueryError::NotPermitted)?; + let refresh = inventory + .refresh + .get(profile_id) + .ok_or(ExternalQueryError::NotPermitted)?; + let direct_selector = inventory + .record_types + .get(parent.record_type.as_str()) + .is_some_and(|record| { + record.fields.iter().any(|field| { + field.id == label.from_field + && matches!( + field.value_type, + ExternalFieldType::String | ExternalFieldType::Uuid + ) + }) + }); + if profile.actor_kind != Some(ActorKindSource::Service) + || profile.requester_clients.is_empty() + || profile.required_scopes.is_empty() + || !profile.required_purposes.contains(purpose) + || !profile.operations.contains(&Operation::List) + || !refresh.contains(&parent.id) + || !refresh.contains(&target.id) + || parent.mode != ExternalProjectionMode::Indexed + || target.mode != ExternalProjectionMode::Indexed + || parent.cardinality != ExternalCardinality::Many + || target.cardinality != ExternalCardinality::One + || parent.purpose.as_str() != purpose + || target.purpose.as_str() != purpose + || target.selector_scheme != label.identifier_scheme + || !parent.fields.contains(&label.from_field) + || !direct_selector + || parent_scope.projection_id != parent.id.as_str() + || parent_scope.epoch != parent_generation.epoch + || parent_generation.generation < 0 + { + return Err(ExternalQueryError::NotPermitted); + } + let key_field = link.from_field.as_str(); + let relations = ReadRelations::statistics(entity, &[key_field.to_owned()]) + .map_err(|_| ExternalQueryError::Unavailable)?; + let key = relations + .field_expression(entity, key_field) + .map_err(|_| ExternalQueryError::Unavailable)?; + if key.encrypted { + return Err(ExternalQueryError::NotPermitted); + } + let root_sql=format!("SELECT {} AS root_id,({})::text AS subject_key FROM {} WHERE current_setting('registry.access_profile',true)={} AND current_setting('registry.purpose',true)={}",relations.id_expression,key.sql,relations.from_sql,literal(profile_id),literal(purpose)); + let state_sql=format!("SELECT projection_id,epoch FROM registry_internal.registry_external_projection_scopes WHERE projection_id={} AND epoch={}::uuid AND binding_digest={} AND grant_digest={} AND generation={} AND active",literal(&parent_scope.projection_id),literal(&parent_scope.epoch.to_string()),literal(&parent_scope.binding_digest),literal(&parent_scope.grant_digest),parent_generation.generation); + Ok(Self::statements( + &root_sql, + &state_sql, + label.from_field.as_str(), + )) + } + + fn statements(root_sql: &str, state_sql: &str, selector_field: &str) -> Self { + // These replacements contain only compiled identifiers, profile names, + // and scope fingerprints. Member positions and selectors are bound. + let eligible = ELIGIBLE_SQL + .replace("__ROOT_SQL__", root_sql) + .replace("__SELECTOR_FIELD__", &literal(selector_field)); + let page_sql = PAGE_SQL + .replace("__STATE_SQL__", state_sql) + .replace("__ELIGIBLE_SQL__", &eligible); + let publication_sql = PUBLICATION_SQL + .replace("__STATE_SQL__", state_sql) + .replace("__ELIGIBLE_SQL__", &eligible); + Self { + page_sql, + publication_sql, + } + } + + /// Use with publish_for_roots: $6 is the label selector, $13 the admitted + /// root IDs. Current parent generation and member association are checked + /// in that same statement, without interpolating source keys into SQL. + pub(crate) fn publication_eligibility(&self) -> &str { + &self.publication_sql + } + + pub(crate) async fn page( + &self, + transaction: &GuardedTransaction<'_>, + after: Option<&LabelAfter>, + limit: u16, + ) -> Result { + if !(1..=1000).contains(&limit) + || after.is_some_and(|position| { + position.member_key.is_empty() || position.member_key.len() > 1024 + }) + { + return Err(ExternalQueryError::InvalidQuery); + } + let root = after.map(|position| position.root_id); + let member = after.map(|position| position.member_key.as_str()); + let bound = i64::from(limit) + 1; + let result = transaction + .transaction() + .query_one(&self.page_sql, &[&root, &member, &bound]) + .await + .map_err(|_| ExternalQueryError::Unavailable)?; + if !result.get::<_, bool>(0) { + return Err(ExternalQueryError::GenerationChanged); + } + let rows: Vec = + serde_json::from_value(result.get(1)).map_err(|_| ExternalQueryError::Unavailable)?; + Ok(bounded_page(rows, limit)) + } +} + +fn bounded_page(mut rows: Vec, limit: u16) -> LabelSelectorPage { + let has_more = rows.len() > usize::from(limit); + rows.truncate(usize::from(limit)); + let continuation = if has_more { + rows.last().map(|row| LabelAfter { + root_id: row.root_id, + member_key: row.member_key.clone(), + }) + } else { + None + }; + let minimum_parent_expires_at = rows.iter().map(|row| row.parent_expires_at).min(); + LabelSelectorPage { + rows, + continuation, + minimum_parent_expires_at, + } +} + +fn literal(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} + +const ELIGIBLE_SQL: &str = r#" +SELECT roots.root_id,members.source_key AS member_key,members.fields->>__SELECTOR_FIELD__ AS selector,snapshot.expires_at AS parent_expires_at +FROM (__ROOT_SQL__) AS roots +JOIN parent_state AS scope ON true +JOIN registry_internal.registry_external_projection_snapshots AS snapshot + ON snapshot.projection_id=scope.projection_id AND snapshot.epoch=scope.epoch + AND snapshot.subject_key=roots.subject_key AND snapshot.kind='complete-collection' + AND snapshot.expires_at>floor(extract(epoch FROM clock_timestamp()))::bigint +JOIN registry_internal.registry_external_projection_values AS members + ON members.projection_id=snapshot.projection_id AND members.epoch=snapshot.epoch AND members.subject_key=snapshot.subject_key +WHERE jsonb_typeof(members.fields->__SELECTOR_FIELD__)='string' + AND octet_length(members.fields->>__SELECTOR_FIELD__) BETWEEN 1 AND 1024 + AND (members.fields->>__SELECTOR_FIELD__) !~ '[[:cntrl:]]' +"#; + +const PAGE_SQL: &str = r#" +WITH parent_state AS MATERIALIZED (__STATE_SQL__), selected AS ( + __ELIGIBLE_SQL__ +), page AS MATERIALIZED ( + SELECT root_id,member_key,selector,parent_expires_at FROM selected + WHERE $1::uuid IS NULL OR root_id>$1::uuid OR (root_id=$1::uuid AND member_key COLLATE "C">$2::text COLLATE "C") + ORDER BY root_id,member_key COLLATE "C" LIMIT $3::bigint +) +SELECT EXISTS(SELECT 1 FROM parent_state),COALESCE((SELECT jsonb_agg(jsonb_build_object( + 'root_id',root_id,'member_key',member_key,'selector',selector,'parent_expires_at',parent_expires_at) ORDER BY root_id,member_key COLLATE "C") FROM page),'[]'::jsonb) +"#; + +const PUBLICATION_SQL: &str = r#" +WITH parent_state AS MATERIALIZED (__STATE_SQL__) +SELECT 1 FROM (__ELIGIBLE_SQL__) AS selected +WHERE selected.root_id=ANY($13::uuid[]) AND selected.selector=$6::text +"#; + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + pub(super) fn registry(grants: &[&str]) -> CompiledRegistry { + let project = json!({ + "apiVersion":crate::compiler::AUTHORING_API_VERSION,"kind":"BRegProject", + "project":{"id":"label-acquisition","version":"1","defaultLanguage":"en","canonicalBaseIri":"https://fixture.example.test"}, + "datasets":[{"id":"records"}], + "recordTypes":[{"id":"local-record","primaryDataset":"records","route":"records","mutationMode":"mutable","classification":"internal","fields":[ + {"id":"subject-key","type":"string","maximumLength":64,"classification":"internal"}, + {"id":"tenant","type":"string","maximumLength":32,"classification":"internal"}]}], + "externalRecordTypes":[ + {"id":"member","authority":"urn:test:source","identifierScheme":"urn:test:member","fields":[{"id":"holder","type":"string","classification":"restricted","nullable":true}]}, + {"id":"label-record","authority":"urn:test:source","identifierScheme":"urn:test:person","fields":[{"id":"name","type":"string","classification":"internal"}]}], + "externalProjections":[ + {"id":"parent","recordType":"member","operation":"members","operationVersion":"1","purpose":"consultation","scope":"eligible","selectorScheme":"urn:test:keys","mode":"indexed","cardinality":"many","cacheTtlSeconds":60,"fields":["holder"]}, + {"id":"label","recordType":"label-record","operation":"batch","operationVersion":"1","purpose":"consultation","scope":"eligible","selectorScheme":"urn:test:person","mode":"indexed","cardinality":"one","cacheTtlSeconds":60,"fields":["name"]}], + "externalViews":[{"id":"view","recordType":"local-record","links":[{"id":"members","projection":"parent","fromField":"subject-key","identifierScheme":"urn:test:keys","fields":["holder"],"label":{"projection":"label","fromField":"holder","identifierScheme":"urn:test:person","fields":["name"]}}]}], + "accessProfiles":[{"id":"refresh","principalClaim":"registry_principal","actorKind":"service","requesterClients":["worker"],"requiredScopes":["registry.refresh"],"requiredPurposes":["consultation"],"permissions":{ + "records":[{"recordType":"local-record","operations":["list"],"rowBoundaries":[{"field":"tenant","claim":"tenant","operator":"equals"}]}], + "refreshExternalProjections":["parent","label"]}}] + }); + let project = + crate::contract::parse_project_json(&serde_json::to_vec(&project).unwrap()).unwrap(); + let compiled = crate::compiler::compile_project( + &project, + &[], + crate::compiler::CompileProfile::Authoring, + ) + .unwrap(); + // Exercise the SQL seam's independent grant check on a bounded copy. + let mut value = serde_json::to_value(compiled).unwrap(); + value["externalViews"]["refresh"]["refresh"] = json!(grants); + serde_json::from_value(value).unwrap() + } + + #[test] + fn separate_machine_grants_and_same_purpose_are_required() { + let scope = + ProjectionScope::new("parent", Uuid::new_v4(), "binding", "grant", 60, 120).unwrap(); + let generation = ProjectionGeneration { + epoch: scope.epoch, + generation: 1, + }; + for grants in [vec!["parent"], vec!["label"]] { + assert!(LabelAcquisition::new( + ®istry(&grants), + "refresh", + "consultation", + "view", + "members", + &scope, + generation + ) + .is_err()); + } + let registry = registry(&["parent", "label"]); + assert!(LabelAcquisition::new( + ®istry, + "refresh", + "different", + "view", + "members", + &scope, + generation + ) + .is_err()); + let plan = LabelAcquisition::new( + ®istry, + "refresh", + "consultation", + "view", + "members", + &scope, + generation, + ) + .unwrap(); + assert!(plan.publication_eligibility().contains("$6::text")); + assert!(plan.publication_eligibility().contains("$13::uuid[]")); + assert!(plan + .publication_eligibility() + .contains("complete-collection")); + assert!(plan.publication_eligibility().contains("clock_timestamp()")); + assert!(plan.page_sql.contains("$2::text")); + assert!(plan.page_sql.contains("COLLATE \"C\"")); + } + + #[test] + fn acquisition_deadline_excludes_the_lookahead_row() { + let rows = [200, 300, 100] + .into_iter() + .enumerate() + .map(|(index, expiry)| LabelSelector { + root_id: Uuid::nil(), + member_key: index.to_string(), + selector: "internal".to_owned(), + parent_expires_at: expiry, + }) + .collect(); + let page = bounded_page(rows, 2); + assert_eq!(page.minimum_parent_expires_at, Some(200)); + assert_eq!(page.continuation.unwrap().member_key, "1"); + assert_eq!(bounded_page(vec![], 2).minimum_parent_expires_at, None); + } + + #[test] + fn cursor_and_selector_debug_never_disclose_source_values() { + let position = LabelAfter { + root_id: Uuid::nil(), + member_key: "PRIVATE_MEMBER_CANARY".to_owned(), + }; + let selector = LabelSelector { + root_id: Uuid::nil(), + member_key: "PRIVATE_MEMBER_CANARY".to_owned(), + selector: "PRIVATE_SELECTOR_CANARY".to_owned(), + parent_expires_at: 1000, + }; + assert!(!format!("{position:?} {selector:?}").contains("CANARY")); + } +} + +#[cfg(test)] +#[cfg(feature = "postgres-test")] +mod postgres_tests { + use super::super::external_projection::{configure_scope, publication_statement, INSTALL_SQL}; + use super::*; + use serde_json::{json, Value}; + use tokio_postgres::NoTls; + + #[allow(dead_code)] + mod harness { + use crate as registry_breg; + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/support/postgres_harness.rs" + )); + } + + #[tokio::test] + async fn indirect_label_uses_root_rls_fresh_complete_parent_and_current_membership() { + let database = harness::TestDatabase::create(2).await; + let (migration, migration_task) = database.connect_migration().await; + let registry = super::tests::registry(&["parent", "label"]); + super::super::install_compiled_schema(&migration, ®istry, &database.runtime_role) + .await + .unwrap(); + migration + .batch_execute(&INSTALL_SQL.replace( + "floor(extract(epoch FROM clock_timestamp()))::bigint", + "1000", + )) + .await + .unwrap(); + let parent = + ProjectionScope::new("parent", Uuid::new_v4(), "binding", "grant", 60, 120).unwrap(); + let label = ProjectionScope::new( + "label", + Uuid::new_v4(), + "label-binding", + "label-grant", + 60, + 120, + ) + .unwrap(); + configure_scope(&migration, &parent).await.unwrap(); + configure_scope(&migration, &label).await.unwrap(); + let entity = ®istry.entities()["local-record"]; + let key = &entity + .stored_fields + .iter() + .find(|field| field.logical.id == "subject-key") + .unwrap() + .physical_name; + let tenant = &entity + .stored_fields + .iter() + .find(|field| field.logical.id == "tenant") + .unwrap() + .physical_name; + database.admin.batch_execute(&format!("INSERT INTO registry_data.\"{}\" (record_id,active_package_revision,\"{key}\",\"{tenant}\") VALUES ('00000000-0000-4000-8000-000000000001','fixture','allowed-parent','allowed'),('00000000-0000-4000-8000-000000000002','fixture','denied-parent','other'),('00000000-0000-4000-8000-000000000003','fixture','expired-parent','allowed')",entity.physical_table)).await.unwrap(); + for subject in ["allowed-parent", "denied-parent", "expired-parent"] { + let ticket: i64 = migration + .query_one( + "SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,$5)", + &[ + &parent.projection_id, + &parent.epoch, + &parent.binding_digest, + &parent.grant_digest, + &subject, + ], + ) + .await + .unwrap() + .get(0); + let rows = json!([{"source_key":"a","fields":{"holder":"indirect-person"}},{"source_key":"b","fields":{"holder":null}},{"source_key":"c","fields":{}},{"source_key":"d","fields":{"holder":"second-person"}}]); + let answer:String=migration.query_one("SELECT registry_internal.publish_external_projection($1,$2,$3,$4,$5,$6,'members',990,1050,1110,'complete-collection',$7)",&[&parent.projection_id,&parent.epoch,&parent.binding_digest,&parent.grant_digest,&ticket,&subject,&rows]).await.unwrap().get(0); + assert_eq!(answer, "published"); + } + migration.execute("UPDATE registry_internal.registry_external_projection_snapshots SET expires_at=999 WHERE projection_id='parent' AND subject_key='expired-parent'",&[]).await.unwrap(); + let (runtime, connection) = database + .runtime_config + .postgres() + .connect(NoTls) + .await + .unwrap(); + let runtime_task = tokio::spawn(async move { + connection.await.unwrap(); + }); + runtime.batch_execute("SET registry.access_profile='refresh'; SET registry.principal='worker'; SET registry.purpose='consultation'; SET registry.row_boundaries='[{\"field\":\"tenant\",\"operator\":\"equals\",\"values\":[\"allowed\"]}]';").await.unwrap(); + let plan = LabelAcquisition::new( + ®istry, + "refresh", + "consultation", + "view", + "members", + &parent, + ProjectionGeneration { + epoch: parent.epoch, + generation: 3, + }, + ) + .unwrap(); + let page_sql = plan.page_sql.replace( + "floor(extract(epoch FROM clock_timestamp()))::bigint", + "1000", + ); + let page = runtime + .query_one(&page_sql, &[&None::, &None::, &2_i64]) + .await + .unwrap(); + assert!(page.get::<_, bool>(0)); + let rows: Value = page.get(1); + assert_eq!(rows.as_array().unwrap().len(), 2); + assert_eq!(rows[0]["selector"], "indirect-person"); + assert_eq!(rows[1]["member_key"], "d"); + let allowed = Uuid::parse_str("00000000-0000-4000-8000-000000000001").unwrap(); + let next = runtime + .query_one(&page_sql, &[&Some(allowed), &Some("a"), &2_i64]) + .await + .unwrap(); + assert_eq!(next.get::<_, Value>(1).as_array().unwrap().len(), 1); + let sql = publication_statement(plan.publication_eligibility()).replace( + "floor(extract(epoch FROM clock_timestamp()))::bigint", + "1000", + ); + let ticket:i64=runtime.query_one("SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,'indirect-person')",&[&label.projection_id,&label.epoch,&label.binding_digest,&label.grant_digest]).await.unwrap().get(0); + let rows = json!([{"source_key":"indirect-person","fields":{"name":"retained"}}]); + let roots = vec![allowed]; + let outcome: String = runtime + .query_one( + &sql, + &[ + &label.projection_id, + &label.epoch, + &label.binding_digest, + &label.grant_digest, + &ticket, + &"indirect-person", + &"label", + &999_i64, + &1059_i64, + &1119_i64, + &"one", + &rows, + &roots, + ], + ) + .await + .unwrap() + .get(0); + assert_eq!(outcome, "published"); + migration.execute("UPDATE registry_internal.registry_external_projection_values SET fields='{}' WHERE projection_id='parent' AND subject_key='allowed-parent' AND source_key='a'",&[]).await.unwrap(); + let outcome: String = runtime + .query_one( + &sql, + &[ + &label.projection_id, + &label.epoch, + &label.binding_digest, + &label.grant_digest, + &ticket, + &"indirect-person", + &"label", + &999_i64, + &1059_i64, + &1119_i64, + &"one", + &rows, + &roots, + ], + ) + .await + .unwrap() + .get(0); + assert_eq!(outcome, "ineligible"); + assert_eq!(runtime.query_one("SELECT fields->>'name' FROM registry_internal.registry_external_projection_values WHERE projection_id='label'",&[]).await.unwrap().get::<_,String>(0),"retained"); + migration.execute("UPDATE registry_internal.registry_external_projection_scopes SET generation=generation+1 WHERE projection_id='parent'",&[]).await.unwrap(); + let page = runtime + .query_one(&page_sql, &[&None::, &None::, &2_i64]) + .await + .unwrap(); + assert!(!page.get::<_, bool>(0)); + assert_eq!(page.get::<_, Value>(1), json!([])); + runtime_task.abort(); + migration_task.abort(); + drop(runtime); + drop(migration); + database.cleanup().await; + } +} diff --git a/crates/registry-breg/src/postgres/external_projection.rs b/crates/registry-breg/src/postgres/external_projection.rs new file mode 100644 index 0000000000..695ccf844f --- /dev/null +++ b/crates/registry-breg/src/postgres/external_projection.rs @@ -0,0 +1,2239 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Guarded, expiring external projections, separate from Record custody. + +use std::collections::BTreeSet; +use std::fmt; + +use serde::Serialize; +use serde_json::Value; +use tokio_postgres::GenericClient; +use uuid::Uuid; + +use super::{ExpectedRegistryIdentity, GuardedTransaction, RuntimeRevoke, SqlIdentifier}; + +pub(crate) const BIND_ONCE_FUNCTION: &str = "registry_internal.bind_external_projection_once(text,uuid,text,text,bigint,bigint,text,text,uuid)"; +pub(crate) const MAINTENANCE_FUNCTION:&str="registry_internal.maintain_external_projection_scope(text,text,uuid,text,uuid,text,text,bigint,bigint)"; +/// Exact catalog identities include PostgreSQL argument names and spacing. +/// ACL/install signatures above remain type-only SQL call signatures. +/// Entries are (identity, result type, returns set, security definer, runtime execute). +pub(crate) const MANAGED_FUNCTIONS: &[(&str,&str,bool,bool,bool)] = &[ + ("registry_internal.allocate_external_projection_refresh(p_projection text, p_epoch uuid, p_binding text, p_grant text, p_subject text)","bigint",false,true,true), + ("registry_internal.publish_external_projection(p_projection text, p_epoch uuid, p_binding text, p_grant text, p_sequence bigint, p_subject text, p_snapshot text, p_observed bigint, p_expires bigint, p_purge bigint, p_kind text, p_rows jsonb)","text",false,true,true), + ("registry_internal.fail_external_projection_refresh(p_projection text, p_epoch uuid, p_sequence bigint, p_subject text, p_reason text)","boolean",false,true,true), + ("registry_internal.withdraw_external_projection(p_projection text, p_epoch uuid, p_binding text, p_grant text)","boolean",false,true,true), + ("registry_internal.purge_external_projections(p_limit integer)","record",true,true,true), + ("registry_internal.lock_external_projection_generation(p_projection text, p_epoch uuid, p_binding text, p_grant text)","bigint",false,true,true), + ("registry_internal.bind_external_projection_once(p_projection text, p_epoch uuid, p_binding text, p_grant text, p_ttl bigint, p_retention bigint, p_contract text, p_package text, p_activation uuid)","uuid",false,true,true), + ("registry_internal.maintain_external_projection_scope(p_action text, p_projection text, p_epoch uuid, p_package text, p_activation uuid, p_binding text, p_grant text, p_ttl bigint, p_retention bigint)","jsonb",false,false,false), +]; + +pub(crate) const INDEXES: &[&str] = &[ + "registry_internal.registry_external_projection_expiry_idx", + "registry_internal.registry_external_projection_refresh_expiry_idx", + "registry_internal.registry_external_projection_refresh_subject_idx", + "registry_internal.registry_external_projection_allocated_idx", +]; + +pub(crate) fn enabled(registry: &crate::model::CompiledRegistry) -> bool { + registry + .external_views() + .projections + .values() + .any(|projection| projection.mode == crate::external_views::ExternalProjectionMode::Indexed) +} + +/// Scope identity includes the reviewed external inventory, so changing a hop, +/// field, grant or eligible acquisition path invalidates retained associations. +pub(crate) fn scope_contract_digest( + registry: &crate::model::CompiledRegistry, + projection_id: &str, +) -> Result { + let projection = registry + .external_views() + .projections + .get(projection_id) + .ok_or(ProjectionError::InvalidScope)?; + let value = serde_json::to_value((registry.external_views(), projection)) + .map_err(|_| ProjectionError::InvalidScope)?; + let canonical = registry_platform_canonical_json::canonicalize_json(&value) + .map_err(|_| ProjectionError::InvalidScope)?; + Ok(registry_platform_config::sha256_uri(&canonical)) +} + +/// Migration-only reconciliation under the existing exclusive package guard. +/// Featureless registries install no store. Removing the feature removes only +/// its rebuildable cache and fixed runtime authority in the same transaction. +pub(crate) async fn reconcile( + migration: &impl GenericClient, + registry: &crate::model::CompiledRegistry, + runtime_role: &SqlIdentifier, +) -> Result<(), ProjectionError> { + if !enabled(registry) { + let exists: bool=migration.query_one("SELECT to_regclass('registry_internal.registry_external_projection_scopes') IS NOT NULL",&[]).await.map_err(|_|ProjectionError::Unavailable)?.get(0); + if exists { + for function in RUNTIME_FUNCTIONS + .iter() + .copied() + .chain([BIND_ONCE_FUNCTION, MAINTENANCE_FUNCTION]) + { + migration + .batch_execute(&format!("DROP FUNCTION IF EXISTS {function};")) + .await + .map_err(|_| ProjectionError::Unavailable)?; + } + for table in [VALUES_TABLE, SNAPSHOTS_TABLE, REFRESHES_TABLE, SCOPES_TABLE] { + migration + .batch_execute(&format!("DROP TABLE {table};")) + .await + .map_err(|_| ProjectionError::Unavailable)?; + } + } + return Ok(()); + } + install(migration, runtime_role).await?; + let mut allowlist = serde_json::Map::new(); + for (id, projection) in ®istry.external_views().projections { + if projection.mode != crate::external_views::ExternalProjectionMode::Indexed { + continue; + } + let ttl = projection + .cache_ttl_seconds + .ok_or(ProjectionError::InvalidScope)?; + allowlist.insert( + id.clone(), + serde_json::json!({"contract":scope_contract_digest(registry,id)?,"ttl":ttl}), + ); + } + let allowed = Value::Object(allowlist); + // Source facts never enter this generated function. Its fixed allowlist is + // part of the managed catalog fingerprint and survives a data-only restore. + let declaration = format!("'{}'::jsonb", allowed.to_string().replace('\'', "''")); + let sql = BIND_ONCE_SQL.replace("__COMPILED_EXTERNAL_ALLOWLIST__", &declaration); + migration + .batch_execute(&sql) + .await + .map_err(|_| ProjectionError::Unavailable)?; + let lock_key = super::RegistryLockKey::derive(registry.registry_id()) + .map_err(|_| ProjectionError::InvalidScope)? + .get(); + let maintenance = MAINTENANCE_SQL + .replace("__COMPILED_EXTERNAL_ALLOWLIST__", &declaration) + .replace("__COMPILED_REGISTRY_LOCK__", &lock_key.to_string()); + migration + .batch_execute(&maintenance) + .await + .map_err(|_| ProjectionError::Unavailable)?; + let revoke = RuntimeRevoke::detect(migration, runtime_role) + .await + .map_err(|_| ProjectionError::Unavailable)? + .with_public(); + migration.batch_execute(&format!("REVOKE ALL ON FUNCTION {BIND_ONCE_FUNCTION} FROM {revoke}; GRANT EXECUTE ON FUNCTION {BIND_ONCE_FUNCTION} TO {};",runtime_role.quoted())).await.map_err(|_|ProjectionError::Unavailable)?; + migration + .batch_execute(&format!( + "REVOKE ALL ON FUNCTION {MAINTENANCE_FUNCTION} FROM {revoke};" + )) + .await + .map_err(|_| ProjectionError::Unavailable)?; + let ids = allowed + .as_object() + .ok_or(ProjectionError::InvalidScope)? + .keys() + .cloned() + .collect::>(); + for table in [SNAPSHOTS_TABLE, REFRESHES_TABLE, SCOPES_TABLE] { + migration + .execute( + &format!("DELETE FROM {table} WHERE NOT (projection_id=ANY($1::text[]))"), + &[&ids], + ) + .await + .map_err(|_| ProjectionError::Unavailable)?; + } + for (id, entry) in allowed.as_object().ok_or(ProjectionError::InvalidScope)? { + let contract = entry["contract"] + .as_str() + .ok_or(ProjectionError::InvalidScope)?; + migration + .execute( + "UPDATE registry_internal.registry_external_projection_scopes + SET contract_digest=$2,epoch='00000000-0000-0000-0000-000000000000'::uuid, + binding_digest='',grant_digest='',active=false,generation=generation+1 + WHERE projection_id=$1 AND contract_digest<>$2", + &[&id, &contract], + ) + .await + .map_err(|_| ProjectionError::Unavailable)?; + } + Ok(()) +} + +/// Initialize only a migration-allowlisted empty slot under the exact active +/// package. Same bindings reuse their stored epoch; withdrawal or mismatched +/// bindings require explicit migration authority and cannot reactivate here. +pub(crate) async fn bind_initial_scope( + transaction: &GuardedTransaction<'_>, + registry: &crate::model::CompiledRegistry, + expected: &ExpectedRegistryIdentity, + requested: &ProjectionScope, +) -> Result { + let contract = scope_contract_digest(registry, &requested.projection_id)?; + let activation = + Uuid::parse_str(&expected.activation_id).map_err(|_| ProjectionError::InvalidScope)?; + let row = transaction + .transaction() + .query_one( + "SELECT registry_internal.bind_external_projection_once($1,$2,$3,$4,$5,$6,$7,$8,$9)", + &[ + &requested.projection_id, + &requested.epoch, + &requested.binding_digest, + &requested.grant_digest, + &requested.cache_ttl_seconds, + &requested.retention_seconds, + &contract, + &expected.package_digest, + &activation, + ], + ) + .await + .map_err(|_| ProjectionError::Unavailable)?; + let epoch = row + .get::<_, Option>(0) + .ok_or(ProjectionError::ScopeChanged)?; + Ok(ProjectionScope { + epoch, + ..requested.clone() + }) +} + +const MAX_ROWS: usize = 10_000; +const MAX_BYTES: usize = 2 * 1024 * 1024; +const MAX_KEY_BYTES: usize = 1024; + +/// These tables are rebuildable, and must all be excluded from backups. +/// Restore provisioning installs empty tables before a current grant can +/// reacquire data. Restoring an old scope or refresh receipt is unsupported. +pub(crate) const BACKUP_EXCLUDED_TABLES: &[&str] = &[ + "registry_internal.registry_external_projection_scopes", + "registry_internal.registry_external_projection_refreshes", + "registry_internal.registry_external_projection_snapshots", + "registry_internal.registry_external_projection_values", +]; + +pub(crate) const SCOPES_TABLE: &str = BACKUP_EXCLUDED_TABLES[0]; +pub(crate) const REFRESHES_TABLE: &str = BACKUP_EXCLUDED_TABLES[1]; +pub(crate) const SNAPSHOTS_TABLE: &str = BACKUP_EXCLUDED_TABLES[2]; +pub(crate) const VALUES_TABLE: &str = BACKUP_EXCLUDED_TABLES[3]; + +pub(crate) const RUNTIME_FUNCTIONS: &[&str] = &[ + "registry_internal.allocate_external_projection_refresh(text,uuid,text,text,text)", + "registry_internal.publish_external_projection(text,uuid,text,text,bigint,text,text,bigint,bigint,bigint,text,jsonb)", + "registry_internal.fail_external_projection_refresh(text,uuid,bigint,text,text)", + "registry_internal.withdraw_external_projection(text,uuid,text,text)", + "registry_internal.purge_external_projections(integer)", + "registry_internal.lock_external_projection_generation(text,uuid,text,text)", +]; + +/// Errors are deliberately closed and value-free, including database failures. +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +pub(crate) enum ProjectionError { + #[error("indexed external projections require separate migration and runtime database roles")] + RoleSeparationRequired, + #[error("external projection scope is invalid")] + InvalidScope, + #[error("external projection snapshot is invalid")] + InvalidSnapshot, + #[error("external projection retention is not permitted")] + RetentionRefused, + #[error("external projection scope is no longer active")] + ScopeChanged, + #[error("external projection refresh is stale or unavailable")] + StaleRefresh, + #[error("external projection store is unavailable")] + Unavailable, +} + +pub(crate) fn kernel_error(error: ProjectionError) -> super::PostgresKernelError { + match error { + ProjectionError::RoleSeparationRequired => super::PostgresKernelError::Configuration( + "indexed external projections require separate migration and runtime database roles", + ), + _ => super::PostgresKernelError::Connection, + } +} + +/// Current deployment authority. An epoch is never reused after withdrawal or +/// a changed source grant, binding, acquisition scope, or retention policy. +#[derive(Clone)] +pub(crate) struct ProjectionScope { + pub(crate) projection_id: String, + pub(crate) epoch: Uuid, + pub(crate) binding_digest: String, + pub(crate) grant_digest: String, + pub(crate) cache_ttl_seconds: i64, + /// Maximum retention from snapshot time, not extra time after expiry. + pub(crate) retention_seconds: i64, +} + +impl fmt::Debug for ProjectionScope { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("ProjectionScope()") + } +} + +impl ProjectionScope { + pub(crate) fn new( + projection_id: &str, + epoch: Uuid, + binding_digest: &str, + grant_digest: &str, + cache_ttl_seconds: i64, + retention_seconds: i64, + ) -> Result { + if !valid_key(projection_id) + || !valid_key(binding_digest) + || !valid_key(grant_digest) + || cache_ttl_seconds < 1 + || retention_seconds < cache_ttl_seconds + || retention_seconds > 31_536_000 + { + return Err(ProjectionError::InvalidScope); + } + Ok(Self { + projection_id: projection_id.to_owned(), + epoch, + binding_digest: binding_digest.to_owned(), + grant_digest: grant_digest.to_owned(), + cache_ttl_seconds, + retention_seconds, + }) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum RetentionPermission { + // Retain an explicit refusal variant even though the source adapter also rejects it. + #[allow(dead_code)] + NoStore, + /// Source-approved absolute last retention time, as Unix seconds. + PermittedUntil(i64), +} + +#[derive(Clone, Serialize)] +pub(crate) struct ProjectionRow { + source_key: String, + fields: Value, +} + +impl fmt::Debug for ProjectionRow { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("ProjectionRow()") + } +} + +impl ProjectionRow { + /// Call only after the source contract validates the approved field names, + /// field types, classification ceiling and requested-key binding. + pub(crate) fn new(source_key: &str, fields: Value) -> Result { + if !valid_key(source_key) || fields.as_object().is_none_or(|fields| fields.len() > 256) { + return Err(ProjectionError::InvalidSnapshot); + } + Ok(Self { + source_key: source_key.to_owned(), + fields, + }) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum SnapshotKind { + One, + Absent, + CompleteCollection, +} + +impl SnapshotKind { + fn as_str(self) -> &'static str { + match self { + Self::One => "one", + Self::Absent => "absent", + Self::CompleteCollection => "complete-collection", + } + } +} + +/// A source-contract-validated complete snapshot. There is deliberately no +/// constructor for denied, ambiguous, failed, or partial source outcomes. +pub(crate) struct AcceptedSnapshot { + subject_key: String, + snapshot_id: String, + observed_at: i64, + source_expires_at: i64, + retain_until: i64, + kind: SnapshotKind, + rows: Vec, +} + +impl fmt::Debug for AcceptedSnapshot { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("AcceptedSnapshot()") + } +} + +impl AcceptedSnapshot { + pub(crate) fn one( + subject_key: &str, + snapshot_id: &str, + observed_at: i64, + source_expires_at: i64, + retention: RetentionPermission, + row: ProjectionRow, + ) -> Result { + if row.source_key != subject_key { + return Err(ProjectionError::InvalidSnapshot); + } + Self::accepted( + subject_key, + snapshot_id, + observed_at, + source_expires_at, + retention, + SnapshotKind::One, + vec![row], + ) + } + + /// Only an authoritative, disclosure-permitted absence reaches this + /// constructor. A source's protected-existence response never does. + pub(crate) fn authorized_absence( + subject_key: &str, + snapshot_id: &str, + observed_at: i64, + source_expires_at: i64, + retention: RetentionPermission, + ) -> Result { + Self::accepted( + subject_key, + snapshot_id, + observed_at, + source_expires_at, + retention, + SnapshotKind::Absent, + vec![], + ) + } + + /// The caller has verified every bounded page and the final completeness + /// marker for this exact parent. Empty membership is an accepted snapshot. + pub(crate) fn complete_collection( + subject_key: &str, + snapshot_id: &str, + observed_at: i64, + source_expires_at: i64, + retention: RetentionPermission, + rows: Vec, + ) -> Result { + Self::accepted( + subject_key, + snapshot_id, + observed_at, + source_expires_at, + retention, + SnapshotKind::CompleteCollection, + rows, + ) + } + + #[allow(clippy::too_many_arguments)] + fn accepted( + subject_key: &str, + snapshot_id: &str, + observed_at: i64, + source_expires_at: i64, + retention: RetentionPermission, + kind: SnapshotKind, + mut rows: Vec, + ) -> Result { + let RetentionPermission::PermittedUntil(retain_until) = retention else { + return Err(ProjectionError::RetentionRefused); + }; + if retain_until < source_expires_at { + return Err(ProjectionError::RetentionRefused); + } + if !valid_key(subject_key) + || !valid_key(snapshot_id) + || observed_at < 0 + || source_expires_at <= observed_at + || rows.len() > MAX_ROWS + || rows + .iter() + .map(|row| &row.source_key) + .collect::>() + .len() + != rows.len() + { + return Err(ProjectionError::InvalidSnapshot); + } + rows.sort_by(|a, b| a.source_key.cmp(&b.source_key)); + if serde_json::to_vec(&rows) + .map_err(|_| ProjectionError::InvalidSnapshot)? + .len() + > MAX_BYTES + { + return Err(ProjectionError::InvalidSnapshot); + } + Ok(Self { + subject_key: subject_key.to_owned(), + snapshot_id: snapshot_id.to_owned(), + observed_at, + source_expires_at, + retain_until, + kind, + rows, + }) + } + + fn bounds(&self, scope: &ProjectionScope) -> Result<(i64, i64), ProjectionError> { + let expires_at = self + .observed_at + .checked_add(scope.cache_ttl_seconds) + .ok_or(ProjectionError::InvalidSnapshot)? + .min(self.source_expires_at); + let purge_at = self + .observed_at + .checked_add(scope.retention_seconds) + .ok_or(ProjectionError::InvalidSnapshot)? + .min(self.retain_until); + Ok((expires_at, purge_at)) + } +} + +pub(crate) struct RefreshTicket { + scope: ProjectionScope, + subject_key: String, + sequence: i64, +} + +impl fmt::Debug for RefreshTicket { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("RefreshTicket()") + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum Publication { + Ineligible, + Published, + Replay, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct ProjectionGeneration { + pub(crate) epoch: Uuid, + pub(crate) generation: i64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum RefreshFailure { + Abandoned, + Unavailable, + Denied, + Incomplete, + InvalidBinding, + RetentionRefused, +} + +impl RefreshFailure { + fn as_str(self) -> &'static str { + match self { + Self::Abandoned => "abandoned", + Self::Unavailable => "unavailable", + Self::Denied => "denied", + Self::Incomplete => "incomplete", + Self::InvalidBinding => "invalid-binding", + Self::RetentionRefused => "retention-refused", + } + } +} + +fn valid_key(value: &str) -> bool { + !value.is_empty() && value.len() <= MAX_KEY_BYTES && !value.chars().any(char::is_control) +} + +/// Called only by the verified migration role. The runtime has SELECT and +/// fixed-function EXECUTE privileges, never direct projection-table DML. +pub(crate) async fn install( + migration: &impl GenericClient, + runtime_role: &SqlIdentifier, +) -> Result<(), ProjectionError> { + let migration_role: String = migration + .query_one("SELECT current_user", &[]) + .await + .map_err(|_| ProjectionError::Unavailable)? + .get(0); + if migration_role == runtime_role.as_str() { + return Err(ProjectionError::RoleSeparationRequired); + } + migration + .batch_execute(INSTALL_SQL) + .await + .map_err(|_| ProjectionError::Unavailable)?; + migration + .batch_execute(&format!( + "GRANT USAGE ON SCHEMA registry_internal TO {};", + runtime_role.quoted() + )) + .await + .map_err(|_| ProjectionError::Unavailable)?; + let revoke = RuntimeRevoke::detect(migration, runtime_role) + .await + .map_err(|_| ProjectionError::Unavailable)? + .with_public(); + for table in BACKUP_EXCLUDED_TABLES { + migration + .batch_execute(&format!( + "REVOKE ALL ON TABLE {table} FROM {revoke}; GRANT SELECT ON TABLE {table} TO {};", + runtime_role.quoted() + )) + .await + .map_err(|_| ProjectionError::Unavailable)?; + } + for function in RUNTIME_FUNCTIONS { + migration.batch_execute(&format!("REVOKE ALL ON FUNCTION {function} FROM {revoke}; GRANT EXECUTE ON FUNCTION {function} TO {};", runtime_role.quoted())).await.map_err(|_| ProjectionError::Unavailable)?; + } + Ok(()) +} + +/// Apply only under the migration/package guard. A withdrawn epoch cannot be +/// reactivated; recovery requires a fresh epoch and authorized acquisition. +#[cfg(all(test, feature = "postgres-test"))] +pub(crate) async fn configure_scope( + migration: &impl GenericClient, + scope: &ProjectionScope, +) -> Result<(), ProjectionError> { + let changed = migration.execute( + "INSERT INTO registry_internal.registry_external_projection_scopes + (projection_id,epoch,binding_digest,grant_digest,cache_ttl_seconds,retention_seconds) + VALUES ($1,$2,$3,$4,$5,$6) + ON CONFLICT (projection_id) DO UPDATE SET + epoch=EXCLUDED.epoch,binding_digest=EXCLUDED.binding_digest,grant_digest=EXCLUDED.grant_digest, + cache_ttl_seconds=EXCLUDED.cache_ttl_seconds,retention_seconds=EXCLUDED.retention_seconds, + active=true,generation=registry_internal.registry_external_projection_scopes.generation + + CASE WHEN registry_internal.registry_external_projection_scopes.epoch=EXCLUDED.epoch THEN 0 ELSE 1 END + WHERE registry_internal.registry_external_projection_scopes.epoch <> EXCLUDED.epoch + OR (registry_internal.registry_external_projection_scopes.active + AND registry_internal.registry_external_projection_scopes.binding_digest=EXCLUDED.binding_digest + AND registry_internal.registry_external_projection_scopes.grant_digest=EXCLUDED.grant_digest + AND registry_internal.registry_external_projection_scopes.cache_ttl_seconds=EXCLUDED.cache_ttl_seconds + AND registry_internal.registry_external_projection_scopes.retention_seconds=EXCLUDED.retention_seconds)", + &[&scope.projection_id,&scope.epoch,&scope.binding_digest,&scope.grant_digest,&scope.cache_ttl_seconds,&scope.retention_seconds], + ).await.map_err(|_| ProjectionError::Unavailable)?; + if changed != 1 { + return Err(ProjectionError::ScopeChanged); + } + Ok(()) +} + +pub(crate) async fn allocate_refresh( + transaction: &GuardedTransaction<'_>, + scope: &ProjectionScope, + subject_key: &str, +) -> Result { + allocate_inner(transaction.transaction(), scope, subject_key).await +} + +/// Read the operator-provisioned binding after the ordinary exact-package +/// guard. A runtime cannot create or rotate scope authority through this API. +pub(crate) async fn load_active_scope( + transaction: &GuardedTransaction<'_>, + projection_id: &str, + binding_digest: &str, + grant_digest: &str, +) -> Result { + let row = transaction + .transaction() + .query_opt( + "SELECT epoch,cache_ttl_seconds,retention_seconds + FROM registry_internal.registry_external_projection_scopes + WHERE projection_id=$1 AND binding_digest=$2 AND grant_digest=$3 AND active", + &[&projection_id, &binding_digest, &grant_digest], + ) + .await + .map_err(|_| ProjectionError::Unavailable)? + .ok_or(ProjectionError::ScopeChanged)?; + ProjectionScope::new( + projection_id, + row.get(0), + binding_digest, + grant_digest, + row.get(1), + row.get(2), + ) +} + +/// Capture once at the beginning of a sweep, then bind this sequence into its +/// continuation. A separate manual sweep gets a fresh acquisition baseline. +pub(crate) async fn next_refresh_sequence( + transaction: &GuardedTransaction<'_>, + scope: &ProjectionScope, +) -> Result { + let row = transaction + .transaction() + .query_opt( + "SELECT next_sequence+1 FROM registry_internal.registry_external_projection_scopes + WHERE projection_id=$1 AND epoch=$2 AND binding_digest=$3 AND grant_digest=$4 AND active", + &[ + &scope.projection_id, + &scope.epoch, + &scope.binding_digest, + &scope.grant_digest, + ], + ) + .await + .map_err(|_| ProjectionError::Unavailable)? + .ok_or(ProjectionError::ScopeChanged)?; + Ok(row.get(0)) +} + +/// One active-scope-bound lookup for a bounded root page. Source keys are +/// parameters and never enter SQL diagnostics; duplicates are returned once. +pub(crate) async fn refresh_due_keys( + transaction: &GuardedTransaction<'_>, + scope: &ProjectionScope, + subject_keys: &[String], + minimum_sequence: i64, + now: i64, +) -> Result, ProjectionError> { + if subject_keys.len() > 1000 + || subject_keys.iter().any(|key| !valid_key(key)) + || now < 0 + || minimum_sequence < 1 + { + return Err(ProjectionError::InvalidScope); + } + let row = transaction + .transaction() + .query_one( + REFRESH_DUE_KEYS_SQL, + &[ + &scope.projection_id, + &scope.epoch, + &scope.binding_digest, + &scope.grant_digest, + &subject_keys, + &now, + &minimum_sequence, + ], + ) + .await + .map_err(|_| ProjectionError::Unavailable)?; + if !row.get::<_, bool>(0) { + return Err(ProjectionError::ScopeChanged); + } + Ok(row.get::<_, Vec>(1).into_iter().collect()) +} + +const REFRESH_DUE_KEYS_SQL: &str = r#" +WITH active_scope AS MATERIALIZED ( + SELECT projection_id,epoch FROM registry_internal.registry_external_projection_scopes + WHERE projection_id=$1 AND epoch=$2 AND binding_digest=$3 AND grant_digest=$4 AND active +), requested AS (SELECT DISTINCT unnest($5::text[]) AS subject_key), due AS ( + SELECT requested.subject_key FROM requested CROSS JOIN active_scope AS scope + LEFT JOIN registry_internal.registry_external_projection_snapshots AS snapshot + ON snapshot.projection_id=scope.projection_id AND snapshot.epoch=scope.epoch AND snapshot.subject_key=requested.subject_key + WHERE snapshot.subject_key IS NULL OR snapshot.expires_at<=$6 OR snapshot.refresh_sequence<$7 +) +SELECT EXISTS(SELECT 1 FROM active_scope),COALESCE((SELECT array_agg(subject_key ORDER BY subject_key COLLATE "C") FROM due),ARRAY[]::text[]) +"#; + +async fn allocate_inner( + client: &impl GenericClient, + scope: &ProjectionScope, + subject_key: &str, +) -> Result { + if !valid_key(subject_key) { + return Err(ProjectionError::InvalidSnapshot); + } + let row = client + .query_one( + "SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,$5)", + &[ + &scope.projection_id, + &scope.epoch, + &scope.binding_digest, + &scope.grant_digest, + &subject_key, + ], + ) + .await + .map_err(|_| ProjectionError::Unavailable)?; + let sequence: Option = row.get(0); + Ok(RefreshTicket { + scope: scope.clone(), + subject_key: subject_key.to_owned(), + sequence: sequence.ok_or(ProjectionError::StaleRefresh)?, + }) +} + +/// Evaluate only a trusted compiled root-read query in the publication statement. +/// Its parameters are $13::uuid[] for local roots and $6::text for the source key. +/// Ineligibility does not invalidate values shared by other authorized roots. +pub(crate) async fn publish_for_roots( + transaction: &GuardedTransaction<'_>, + ticket: &RefreshTicket, + snapshot: &AcceptedSnapshot, + eligibility_sql: &str, + root_ids: &[Uuid], +) -> Result { + if ticket.subject_key != snapshot.subject_key { + return Err(ProjectionError::InvalidSnapshot); + } + let scope = &ticket.scope; + let (expires_at, purge_at) = snapshot.bounds(scope)?; + let rows = + serde_json::to_value(&snapshot.rows).map_err(|_| ProjectionError::InvalidSnapshot)?; + let kind = snapshot.kind.as_str(); + let sql = publication_statement(eligibility_sql); + let result = transaction + .transaction() + .query_one( + &sql, + &[ + &scope.projection_id, + &scope.epoch, + &scope.binding_digest, + &scope.grant_digest, + &ticket.sequence, + &ticket.subject_key, + &snapshot.snapshot_id, + &snapshot.observed_at, + &expires_at, + &purge_at, + &kind, + &rows, + &root_ids, + ], + ) + .await + .map_err(|_| ProjectionError::Unavailable)?; + match result.get::<_, String>(0).as_str() { + "ineligible" => Ok(Publication::Ineligible), + "published" => Ok(Publication::Published), + "replay" => Ok(Publication::Replay), + "scope-changed" => Err(ProjectionError::ScopeChanged), + _ => Err(ProjectionError::StaleRefresh), + } +} + +pub(super) fn publication_statement(eligibility_sql: &str) -> String { + format!("SELECT CASE WHEN EXISTS ({eligibility_sql}) THEN registry_internal.publish_external_projection($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12) ELSE 'ineligible' END") +} + +#[cfg(all(test, feature = "postgres-test"))] +async fn publish_inner( + client: &impl GenericClient, + ticket: &RefreshTicket, + snapshot: &AcceptedSnapshot, +) -> Result { + if ticket.subject_key != snapshot.subject_key { + return Err(ProjectionError::InvalidSnapshot); + } + let scope = &ticket.scope; + let (expires_at, purge_at) = snapshot.bounds(scope)?; + let rows = + serde_json::to_value(&snapshot.rows).map_err(|_| ProjectionError::InvalidSnapshot)?; + let kind = snapshot.kind.as_str(); + let row = client.query_one("SELECT registry_internal.publish_external_projection($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)", &[&scope.projection_id,&scope.epoch,&scope.binding_digest,&scope.grant_digest,&ticket.sequence,&ticket.subject_key,&snapshot.snapshot_id,&snapshot.observed_at,&expires_at,&purge_at,&kind,&rows]).await.map_err(|_| ProjectionError::Unavailable)?; + match row.get::<_, String>(0).as_str() { + "published" => Ok(Publication::Published), + "replay" => Ok(Publication::Replay), + "scope-changed" => Err(ProjectionError::ScopeChanged), + _ => Err(ProjectionError::StaleRefresh), + } +} + +/// The scope row lock fences publication and withdrawal until the caller has +/// checked coverage, selected its page, rechecked expiry and audited release. +/// Query headers and values by this epoch AND current expiry in one statement. +pub(crate) async fn lock_generation( + transaction: &GuardedTransaction<'_>, + scope: &ProjectionScope, +) -> Result { + let row = transaction + .transaction() + .query_one( + "SELECT registry_internal.lock_external_projection_generation($1,$2,$3,$4)", + &[ + &scope.projection_id, + &scope.epoch, + &scope.binding_digest, + &scope.grant_digest, + ], + ) + .await + .map_err(|_| ProjectionError::Unavailable)?; + let generation = row + .get::<_, Option>(0) + .ok_or(ProjectionError::ScopeChanged)?; + Ok(ProjectionGeneration { + epoch: scope.epoch, + generation, + }) +} + +#[cfg(all(test, feature = "postgres-test"))] +pub(crate) async fn withdraw_scope( + transaction: &GuardedTransaction<'_>, + scope: &ProjectionScope, +) -> Result<(), ProjectionError> { + let row = transaction + .transaction() + .query_one( + "SELECT registry_internal.withdraw_external_projection($1,$2,$3,$4)", + &[ + &scope.projection_id, + &scope.epoch, + &scope.binding_digest, + &scope.grant_digest, + ], + ) + .await + .map_err(|_| ProjectionError::Unavailable)?; + if !row.get::<_, bool>(0) { + return Err(ProjectionError::ScopeChanged); + } + Ok(()) +} + +/// Exact-key failures invalidate coverage. Generic collection interruption +/// preserves its prior complete snapshot; denial and no-store invalidate both. +pub(crate) async fn fail_refresh( + transaction: &GuardedTransaction<'_>, + ticket: &RefreshTicket, + failure: RefreshFailure, +) -> Result<(), ProjectionError> { + let row = transaction + .transaction() + .query_one( + "SELECT registry_internal.fail_external_projection_refresh($1,$2,$3,$4,$5)", + &[ + &ticket.scope.projection_id, + &ticket.scope.epoch, + &ticket.sequence, + &ticket.subject_key, + &failure.as_str(), + ], + ) + .await + .map_err(|_| ProjectionError::Unavailable)?; + if !row.get::<_, bool>(0) { + return Err(ProjectionError::StaleRefresh); + } + Ok(()) +} + +/// Each lane removes at most `limit` rows, with values cascading only from the +/// selected bounded snapshot headers. No expired value is made readable. +pub(crate) async fn purge( + transaction: &GuardedTransaction<'_>, + limit: u32, +) -> Result<(u64, u64), ProjectionError> { + if !(1..=1000).contains(&limit) { + return Err(ProjectionError::InvalidScope); + } + let limit = i32::try_from(limit).map_err(|_| ProjectionError::InvalidScope)?; + let row = transaction + .transaction() + .query_one( + "SELECT * FROM registry_internal.purge_external_projections($1)", + &[&limit], + ) + .await + .map_err(|_| ProjectionError::Unavailable)?; + Ok(( + u64::try_from(row.get::<_, i64>(0)).map_err(|_| ProjectionError::Unavailable)?, + u64::try_from(row.get::<_, i64>(1)).map_err(|_| ProjectionError::Unavailable)?, + )) +} + +pub(crate) const INSTALL_SQL: &str = r#" +CREATE TABLE IF NOT EXISTS registry_internal.registry_external_projection_scopes ( + projection_id text PRIMARY KEY CHECK (octet_length(projection_id) BETWEEN 1 AND 1024), + contract_digest text NOT NULL DEFAULT '', + epoch uuid NOT NULL, + binding_digest text NOT NULL, + grant_digest text NOT NULL, + cache_ttl_seconds bigint NOT NULL CHECK (cache_ttl_seconds BETWEEN 1 AND 31536000), + retention_seconds bigint NOT NULL CHECK (retention_seconds BETWEEN cache_ttl_seconds AND 31536000), + active boolean NOT NULL DEFAULT true, + next_sequence bigint NOT NULL DEFAULT 0 CHECK (next_sequence >= 0), + generation bigint NOT NULL DEFAULT 0 CHECK (generation >= 0) +); +CREATE TABLE IF NOT EXISTS registry_internal.registry_external_projection_refreshes ( + projection_id text NOT NULL REFERENCES registry_internal.registry_external_projection_scopes(projection_id), + sequence bigint NOT NULL CHECK (sequence > 0), + epoch uuid NOT NULL, + subject_key text NOT NULL CHECK (octet_length(subject_key) BETWEEN 1 AND 1024), + started_at bigint NOT NULL, + lease_expires_at bigint NOT NULL, + state text NOT NULL CHECK (state IN ('allocated','published','unavailable','denied','incomplete','invalid-binding','retention-refused')), + PRIMARY KEY (projection_id,sequence) +); +CREATE TABLE IF NOT EXISTS registry_internal.registry_external_projection_snapshots ( + projection_id text NOT NULL REFERENCES registry_internal.registry_external_projection_scopes(projection_id), + epoch uuid NOT NULL, + subject_key text NOT NULL CHECK (octet_length(subject_key) BETWEEN 1 AND 1024), + snapshot_id text NOT NULL CHECK (octet_length(snapshot_id) BETWEEN 1 AND 1024), + observed_at bigint NOT NULL CHECK (observed_at >= 0), + expires_at bigint NOT NULL CHECK (expires_at > observed_at), + purge_at bigint NOT NULL CHECK (purge_at >= expires_at), + refresh_sequence bigint NOT NULL CHECK (refresh_sequence > 0), + kind text NOT NULL CHECK (kind IN ('one','absent','complete-collection')), + PRIMARY KEY (projection_id,epoch,subject_key) +); +CREATE TABLE IF NOT EXISTS registry_internal.registry_external_projection_values ( + projection_id text NOT NULL, + epoch uuid NOT NULL, + subject_key text NOT NULL, + source_key text NOT NULL CHECK (octet_length(source_key) BETWEEN 1 AND 1024), + fields jsonb NOT NULL CHECK (jsonb_typeof(fields)='object'), + PRIMARY KEY (projection_id,epoch,subject_key,source_key), + FOREIGN KEY (projection_id,epoch,subject_key) + REFERENCES registry_internal.registry_external_projection_snapshots(projection_id,epoch,subject_key) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS registry_external_projection_expiry_idx + ON registry_internal.registry_external_projection_snapshots(expires_at,projection_id,epoch,subject_key); +CREATE INDEX IF NOT EXISTS registry_external_projection_refresh_expiry_idx + ON registry_internal.registry_external_projection_refreshes(lease_expires_at,projection_id,sequence); +CREATE INDEX IF NOT EXISTS registry_external_projection_refresh_subject_idx + ON registry_internal.registry_external_projection_refreshes(projection_id,epoch,subject_key,sequence) + INCLUDE (state,lease_expires_at); +CREATE INDEX IF NOT EXISTS registry_external_projection_allocated_idx + ON registry_internal.registry_external_projection_refreshes(projection_id,lease_expires_at) + WHERE state='allocated'; + +CREATE OR REPLACE FUNCTION registry_internal.allocate_external_projection_refresh( + p_projection text,p_epoch uuid,p_binding text,p_grant text,p_subject text +) RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +SET search_path = pg_catalog, registry_internal AS $body$ +DECLARE scope registry_internal.registry_external_projection_scopes; projection_now bigint; allocated bigint; +BEGIN + projection_now := floor(extract(epoch FROM clock_timestamp()))::bigint; + SELECT * INTO scope FROM registry_internal.registry_external_projection_scopes + WHERE projection_id=p_projection FOR UPDATE; + IF NOT FOUND OR p_epoch IS NULL OR p_binding IS NULL OR p_grant IS NULL OR NOT scope.active OR scope.epoch<>p_epoch + OR scope.binding_digest<>p_binding OR scope.grant_digest<>p_grant + OR p_subject IS NULL OR octet_length(p_subject) NOT BETWEEN 1 AND 1024 THEN RETURN NULL; END IF; + IF (SELECT count(*) FROM registry_internal.registry_external_projection_refreshes + WHERE projection_id=p_projection AND state='allocated' AND lease_expires_at>projection_now) >= 1024 THEN RETURN NULL; END IF; + UPDATE registry_internal.registry_external_projection_scopes SET next_sequence=next_sequence+1 + WHERE projection_id=p_projection RETURNING next_sequence INTO allocated; + INSERT INTO registry_internal.registry_external_projection_refreshes + (projection_id,sequence,epoch,subject_key,started_at,lease_expires_at,state) + VALUES (p_projection,allocated,p_epoch,p_subject,projection_now,projection_now+120,'allocated'); + RETURN allocated; +END $body$; + +CREATE OR REPLACE FUNCTION registry_internal.publish_external_projection( + p_projection text,p_epoch uuid,p_binding text,p_grant text,p_sequence bigint,p_subject text, + p_snapshot text,p_observed bigint,p_expires bigint,p_purge bigint,p_kind text,p_rows jsonb +) RETURNS text LANGUAGE plpgsql SECURITY DEFINER +SET search_path = pg_catalog, registry_internal AS $body$ +DECLARE scope registry_internal.registry_external_projection_scopes; + attempt registry_internal.registry_external_projection_refreshes; + previous registry_internal.registry_external_projection_snapshots; + projection_now bigint; previous_rows jsonb; has_previous boolean; +BEGIN + projection_now := floor(extract(epoch FROM clock_timestamp()))::bigint; + SELECT * INTO scope FROM registry_internal.registry_external_projection_scopes + WHERE projection_id=p_projection FOR UPDATE; + IF NOT FOUND OR p_epoch IS NULL OR p_binding IS NULL OR p_grant IS NULL OR NOT scope.active OR scope.epoch<>p_epoch + OR scope.binding_digest<>p_binding OR scope.grant_digest<>p_grant THEN RETURN 'scope-changed'; END IF; + SELECT * INTO attempt FROM registry_internal.registry_external_projection_refreshes + WHERE projection_id=p_projection AND sequence=p_sequence AND epoch=p_epoch AND subject_key=p_subject FOR UPDATE; + IF NOT FOUND OR attempt.state NOT IN ('allocated','published') OR attempt.lease_expires_at<=projection_now THEN RETURN 'stale'; END IF; + -- Final receipts fence older results even when denial or no-store removed + -- the snapshot. Scope locking serializes this check with finalization. + IF EXISTS (SELECT 1 FROM registry_internal.registry_external_projection_refreshes + WHERE projection_id=p_projection AND epoch=p_epoch AND subject_key=p_subject + AND sequence>p_sequence AND state<>'allocated') THEN RETURN 'stale'; END IF; + IF p_snapshot IS NULL OR octet_length(p_snapshot) NOT BETWEEN 1 AND 1024 + OR p_observed IS NULL OR p_observed<0 OR p_observed>projection_now + OR p_expires IS NULL OR p_expires<=projection_now OR p_expires<=p_observed + OR p_expires-p_observed>scope.cache_ttl_seconds + OR p_purge IS NULL OR p_purgescope.retention_seconds + OR p_kind IS NULL OR p_kind NOT IN ('one','absent','complete-collection') + OR p_rows IS NULL OR jsonb_typeof(p_rows)<>'array' + OR octet_length(p_rows::text)>2097152 THEN RETURN 'invalid'; END IF; + IF jsonb_array_length(p_rows)>10000 + OR (p_kind='one' AND jsonb_array_length(p_rows)<>1) + OR (p_kind='absent' AND jsonb_array_length(p_rows)<>0) THEN RETURN 'invalid'; END IF; + IF EXISTS (SELECT 1 FROM jsonb_array_elements(p_rows) AS r + WHERE jsonb_typeof(r)<>'object' OR r - 'source_key' - 'fields' <> '{}'::jsonb + OR jsonb_typeof(r->'source_key') IS DISTINCT FROM 'string' + OR octet_length(r->>'source_key') NOT BETWEEN 1 AND 1024 + OR jsonb_typeof(r->'fields') IS DISTINCT FROM 'object' + OR (p_kind='one' AND r->>'source_key'<>p_subject)) + OR (SELECT count(*)<>count(DISTINCT r->>'source_key') FROM jsonb_array_elements(p_rows) AS r) + THEN RETURN 'invalid'; END IF; + SELECT * INTO previous FROM registry_internal.registry_external_projection_snapshots + WHERE projection_id=p_projection AND epoch=p_epoch AND subject_key=p_subject FOR UPDATE; + has_previous := FOUND; + IF has_previous AND previous.snapshot_id=p_snapshot THEN + SELECT COALESCE(jsonb_agg(jsonb_build_object('source_key',source_key,'fields',fields) ORDER BY source_key COLLATE "C"),'[]'::jsonb) + INTO previous_rows FROM registry_internal.registry_external_projection_values + WHERE projection_id=p_projection AND epoch=p_epoch AND subject_key=p_subject; + IF previous.kind<>p_kind OR previous.observed_at<>p_observed OR previous_rows<>p_rows THEN RETURN 'invalid'; END IF; + IF previous.expires_at<=projection_now THEN RETURN 'stale'; END IF; + UPDATE registry_internal.registry_external_projection_snapshots + SET refresh_sequence=greatest(refresh_sequence,p_sequence) + WHERE projection_id=p_projection AND epoch=p_epoch AND subject_key=p_subject; + UPDATE registry_internal.registry_external_projection_refreshes SET state='published' + WHERE projection_id=p_projection AND sequence=p_sequence; + RETURN 'replay'; + END IF; + IF attempt.state<>'allocated' OR (has_previous AND + (previous.refresh_sequence>=p_sequence OR previous.observed_at>p_observed)) THEN RETURN 'stale'; END IF; + INSERT INTO registry_internal.registry_external_projection_snapshots + (projection_id,epoch,subject_key,snapshot_id,observed_at,expires_at,purge_at,refresh_sequence,kind) + VALUES (p_projection,p_epoch,p_subject,p_snapshot,p_observed,p_expires,p_purge,p_sequence,p_kind) + ON CONFLICT (projection_id,epoch,subject_key) DO UPDATE SET + snapshot_id=EXCLUDED.snapshot_id,observed_at=EXCLUDED.observed_at, + expires_at=EXCLUDED.expires_at,purge_at=EXCLUDED.purge_at, + refresh_sequence=EXCLUDED.refresh_sequence,kind=EXCLUDED.kind; + DELETE FROM registry_internal.registry_external_projection_values + WHERE projection_id=p_projection AND epoch=p_epoch AND subject_key=p_subject; + INSERT INTO registry_internal.registry_external_projection_values + (projection_id,epoch,subject_key,source_key,fields) + SELECT p_projection,p_epoch,p_subject,r->>'source_key',r->'fields' FROM jsonb_array_elements(p_rows) AS r; + UPDATE registry_internal.registry_external_projection_refreshes SET state='published' + WHERE projection_id=p_projection AND sequence=p_sequence; + UPDATE registry_internal.registry_external_projection_scopes SET generation=generation+1 WHERE projection_id=p_projection; + RETURN 'published'; +END $body$; + +CREATE OR REPLACE FUNCTION registry_internal.fail_external_projection_refresh( + p_projection text,p_epoch uuid,p_sequence bigint,p_subject text,p_reason text +) RETURNS boolean LANGUAGE plpgsql SECURITY DEFINER +SET search_path = pg_catalog, registry_internal AS $body$ +DECLARE current_epoch uuid; allocated boolean; removed bigint; +BEGIN + IF p_reason IS NULL OR p_reason NOT IN ('abandoned','unavailable','denied','incomplete','invalid-binding','retention-refused') THEN RETURN false; END IF; + SELECT epoch INTO current_epoch FROM registry_internal.registry_external_projection_scopes + WHERE projection_id=p_projection AND active FOR UPDATE; + IF NOT FOUND OR p_epoch IS NULL OR current_epoch<>p_epoch THEN RETURN false; END IF; + IF p_reason='abandoned' THEN + DELETE FROM registry_internal.registry_external_projection_refreshes + WHERE projection_id=p_projection AND epoch=p_epoch AND sequence=p_sequence AND subject_key=p_subject AND state='allocated'; + RETURN true; + END IF; + SELECT state='allocated' INTO allocated FROM registry_internal.registry_external_projection_refreshes + WHERE projection_id=p_projection AND epoch=p_epoch AND sequence=p_sequence AND subject_key=p_subject + AND lease_expires_at>floor(extract(epoch FROM clock_timestamp()))::bigint FOR UPDATE; + IF NOT FOUND OR NOT allocated THEN RETURN false; END IF; + IF p_reason IN ('retention-refused','denied','unavailable','incomplete','invalid-binding') THEN + DELETE FROM registry_internal.registry_external_projection_snapshots + WHERE projection_id=p_projection AND epoch=p_epoch AND subject_key=p_subject AND refresh_sequence<=p_sequence + AND (p_reason IN ('retention-refused','denied') OR kind IN ('one','absent')); + GET DIAGNOSTICS removed=ROW_COUNT; + IF removed>0 THEN + UPDATE registry_internal.registry_external_projection_scopes SET generation=generation+1 WHERE projection_id=p_projection; + END IF; + END IF; + UPDATE registry_internal.registry_external_projection_refreshes SET state=p_reason + WHERE projection_id=p_projection AND epoch=p_epoch AND sequence=p_sequence AND subject_key=p_subject AND state='allocated'; + RETURN FOUND; +END $body$; + +CREATE OR REPLACE FUNCTION registry_internal.withdraw_external_projection( + p_projection text,p_epoch uuid,p_binding text,p_grant text +) RETURNS boolean LANGUAGE plpgsql SECURITY DEFINER +SET search_path = pg_catalog, registry_internal AS $body$ +BEGIN + UPDATE registry_internal.registry_external_projection_scopes SET active=false,generation=generation+1 + WHERE projection_id=p_projection AND epoch=p_epoch AND binding_digest=p_binding AND grant_digest=p_grant AND active; + RETURN FOUND; +END $body$; + +CREATE OR REPLACE FUNCTION registry_internal.purge_external_projections(p_limit integer) +RETURNS TABLE (snapshots_pruned bigint,refreshes_pruned bigint) LANGUAGE plpgsql SECURITY DEFINER +SET search_path = pg_catalog, registry_internal AS $body$ +DECLARE projection_now bigint; +BEGIN + IF p_limit IS NULL OR p_limit NOT BETWEEN 1 AND 1000 THEN + RAISE EXCEPTION USING ERRCODE='22023', MESSAGE='external projection purge limit is invalid'; + END IF; + projection_now := floor(extract(epoch FROM clock_timestamp()))::bigint; + WITH expired AS MATERIALIZED ( + SELECT o.projection_id,o.epoch,o.subject_key + FROM registry_internal.registry_external_projection_snapshots AS o + JOIN registry_internal.registry_external_projection_scopes AS s USING (projection_id) + WHERE o.expires_at<=projection_now OR o.epoch<>s.epoch OR NOT s.active + ORDER BY o.expires_at,o.projection_id,o.epoch,o.subject_key LIMIT p_limit FOR UPDATE OF o SKIP LOCKED + ), removed AS ( + DELETE FROM registry_internal.registry_external_projection_snapshots AS o USING expired AS e + WHERE o.projection_id=e.projection_id AND o.epoch=e.epoch AND o.subject_key=e.subject_key RETURNING 1 + ) SELECT count(*) INTO snapshots_pruned FROM removed; + WITH expired AS MATERIALIZED ( + SELECT r.projection_id,r.sequence + FROM registry_internal.registry_external_projection_refreshes AS r + JOIN registry_internal.registry_external_projection_scopes AS s USING (projection_id) + WHERE r.epoch<>s.epoch OR NOT s.active OR (r.lease_expires_at<=projection_now + AND (r.state='allocated' OR NOT EXISTS ( + SELECT 1 FROM registry_internal.registry_external_projection_refreshes AS earlier + WHERE earlier.projection_id=r.projection_id AND earlier.epoch=r.epoch + AND earlier.subject_key=r.subject_key AND earlier.sequenceprojection_now))) + ORDER BY r.lease_expires_at,r.projection_id,r.sequence LIMIT p_limit FOR UPDATE OF r SKIP LOCKED + ), removed AS ( + DELETE FROM registry_internal.registry_external_projection_refreshes AS r USING expired AS e + WHERE r.projection_id=e.projection_id AND r.sequence=e.sequence RETURNING 1 + ) SELECT count(*) INTO refreshes_pruned FROM removed; + RETURN NEXT; +END $body$; + +CREATE OR REPLACE FUNCTION registry_internal.lock_external_projection_generation( + p_projection text,p_epoch uuid,p_binding text,p_grant text +) RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER +SET search_path = pg_catalog, registry_internal AS $body$ +DECLARE result bigint; +BEGIN + SELECT generation INTO result FROM registry_internal.registry_external_projection_scopes + WHERE projection_id=p_projection AND epoch=p_epoch AND binding_digest=p_binding AND grant_digest=p_grant AND active + FOR SHARE; + RETURN result; +END $body$; +"#; + +const MAINTENANCE_SQL: &str = r#" +CREATE OR REPLACE FUNCTION registry_internal.maintain_external_projection_scope( + p_action text,p_projection text,p_epoch uuid,p_package text,p_activation uuid, + p_binding text,p_grant text,p_ttl bigint,p_retention bigint +) RETURNS jsonb LANGUAGE plpgsql SECURITY INVOKER +SET search_path = pg_catalog, registry_internal AS $body$ +DECLARE allowed jsonb := __COMPILED_EXTERNAL_ALLOWLIST__; + expected jsonb; scope registry_internal.registry_external_projection_scopes; + package_digest text; activation_id uuid; current_status text; counters jsonb := '{}'::jsonb; +BEGIN + IF p_action IS NULL OR p_action NOT IN ('inspect','withdraw','rotate') THEN + RAISE EXCEPTION USING ERRCODE='22023', MESSAGE='external projection maintenance action is invalid'; + END IF; + IF p_action='inspect' THEN + PERFORM pg_advisory_xact_lock_shared(__COMPILED_REGISTRY_LOCK__::bigint); + ELSE + PERFORM pg_advisory_xact_lock(__COMPILED_REGISTRY_LOCK__::bigint); + END IF; + expected := allowed -> p_projection; + IF expected IS NULL THEN + RAISE EXCEPTION USING ERRCODE='22023', MESSAGE='external projection is not installed'; + END IF; + SELECT active_package_digest,active_activation_id,maintenance_status + INTO package_digest,activation_id,current_status FROM registry_internal.registry_state WHERE singleton; + IF NOT FOUND OR current_status<>'ready' THEN + RAISE EXCEPTION USING ERRCODE='55000', MESSAGE='external projection package is not ready'; + END IF; + SELECT * INTO scope FROM registry_internal.registry_external_projection_scopes + WHERE projection_id=p_projection FOR UPDATE; + IF p_action<>'inspect' THEN + IF p_package IS DISTINCT FROM package_digest OR p_activation IS DISTINCT FROM activation_id + OR p_epoch IS DISTINCT FROM scope.epoch THEN + RAISE EXCEPTION USING ERRCODE='55000', MESSAGE='external projection maintenance identity changed'; + END IF; + IF p_action='withdraw' THEN + IF scope.projection_id IS NULL THEN + RAISE EXCEPTION USING ERRCODE='55000', MESSAGE='external projection scope is not bound'; + END IF; + UPDATE registry_internal.registry_external_projection_scopes SET active=false,generation=generation+1 + WHERE projection_id=p_projection RETURNING * INTO scope; + ELSE + IF p_binding IS NULL OR octet_length(p_binding) NOT BETWEEN 1 AND 1024 + OR p_grant IS NULL OR octet_length(p_grant) NOT BETWEEN 1 AND 1024 + OR p_ttl IS NULL OR p_ttl NOT BETWEEN 1 AND (expected->>'ttl')::bigint + OR p_retention IS NULL OR p_retention NOT BETWEEN p_ttl AND 31536000 THEN + RAISE EXCEPTION USING ERRCODE='22023', MESSAGE='external projection maintenance binding is invalid'; + END IF; + INSERT INTO registry_internal.registry_external_projection_scopes AS current + (projection_id,contract_digest,epoch,binding_digest,grant_digest,cache_ttl_seconds,retention_seconds) + VALUES (p_projection,expected->>'contract',gen_random_uuid(),p_binding,p_grant,p_ttl,p_retention) + ON CONFLICT (projection_id) DO UPDATE SET contract_digest=EXCLUDED.contract_digest, + epoch=EXCLUDED.epoch,binding_digest=EXCLUDED.binding_digest,grant_digest=EXCLUDED.grant_digest, + cache_ttl_seconds=EXCLUDED.cache_ttl_seconds,retention_seconds=EXCLUDED.retention_seconds, + active=true,generation=current.generation+1 + RETURNING * INTO scope; + END IF; + DELETE FROM registry_internal.registry_external_projection_snapshots WHERE projection_id=p_projection; + DELETE FROM registry_internal.registry_external_projection_refreshes WHERE projection_id=p_projection; + END IF; + IF p_action='inspect' THEN + -- These are physical cache/receipt counts, never caller-visible root + -- readiness. Fixed state names and integers disclose no source values. + WITH outcomes(state) AS (VALUES ('allocated'),('published'),('unavailable'),('denied'),('incomplete'),('invalid-binding'),('retention-refused')), + receipts AS (SELECT state,count(*) AS total FROM registry_internal.registry_external_projection_refreshes + WHERE projection_id=p_projection GROUP BY state), + snapshots AS (SELECT + count(*) FILTER (WHERE epoch=scope.epoch AND scope.active AND expires_at>floor(extract(epoch FROM statement_timestamp()))::bigint) AS fresh, + count(*) FILTER (WHERE expires_at<=floor(extract(epoch FROM statement_timestamp()))::bigint) AS expired, + count(*) FILTER (WHERE (epoch<>scope.epoch OR NOT scope.active) AND expires_at>floor(extract(epoch FROM statement_timestamp()))::bigint) AS invalidated + FROM registry_internal.registry_external_projection_snapshots WHERE projection_id=p_projection) + SELECT jsonb_build_object('refreshCounts',(SELECT jsonb_object_agg(outcomes.state,COALESCE(receipts.total,0)) FROM outcomes LEFT JOIN receipts USING (state)), + 'snapshotCounts',jsonb_build_object('fresh',fresh,'expired',expired,'invalidated',invalidated)) INTO counters FROM snapshots; + END IF; + RETURN jsonb_build_object('projection',p_projection,'epoch',scope.epoch,'active',COALESCE(scope.active,false), + 'bindingDigest',scope.binding_digest,'grantDigest',scope.grant_digest,'cacheTtlSeconds',scope.cache_ttl_seconds, + 'retentionSeconds',scope.retention_seconds,'packageDigest',package_digest,'activationId',activation_id) || counters; +END $body$; +"#; + +const BIND_ONCE_SQL: &str = r#" +CREATE OR REPLACE FUNCTION registry_internal.bind_external_projection_once( + p_projection text,p_epoch uuid,p_binding text,p_grant text,p_ttl bigint,p_retention bigint, + p_contract text,p_package text,p_activation uuid +) RETURNS uuid LANGUAGE plpgsql SECURITY DEFINER +SET search_path = pg_catalog, registry_internal AS $body$ +DECLARE allowed jsonb := __COMPILED_EXTERNAL_ALLOWLIST__; + expected jsonb; scope registry_internal.registry_external_projection_scopes; +BEGIN + expected := allowed -> p_projection; + IF expected IS NULL OR p_contract IS NULL OR expected->>'contract'<>p_contract + OR p_epoch IS NULL OR p_epoch='00000000-0000-0000-0000-000000000000'::uuid + OR p_binding IS NULL OR octet_length(p_binding) NOT BETWEEN 1 AND 1024 + OR p_grant IS NULL OR octet_length(p_grant) NOT BETWEEN 1 AND 1024 + OR p_ttl IS NULL OR p_ttl NOT BETWEEN 1 AND (expected->>'ttl')::bigint + OR p_retention IS NULL OR p_retention NOT BETWEEN p_ttl AND 31536000 + OR p_package IS NULL OR p_activation IS NULL THEN RETURN NULL; END IF; + IF NOT EXISTS (SELECT 1 FROM registry_internal.registry_state WHERE singleton + AND active_package_digest=p_package AND active_activation_id=p_activation AND maintenance_status='ready') THEN RETURN NULL; END IF; + INSERT INTO registry_internal.registry_external_projection_scopes + (projection_id,contract_digest,epoch,binding_digest,grant_digest,cache_ttl_seconds,retention_seconds) + VALUES (p_projection,p_contract,p_epoch,p_binding,p_grant,p_ttl,p_retention) + ON CONFLICT (projection_id) DO NOTHING; + SELECT * INTO scope FROM registry_internal.registry_external_projection_scopes WHERE projection_id=p_projection FOR UPDATE; + IF scope.active AND scope.contract_digest=p_contract AND scope.binding_digest=p_binding AND scope.grant_digest=p_grant + AND scope.cache_ttl_seconds=p_ttl AND scope.retention_seconds=p_retention THEN RETURN scope.epoch; END IF; + IF NOT scope.active AND scope.epoch='00000000-0000-0000-0000-000000000000'::uuid + AND scope.binding_digest='' AND scope.grant_digest='' AND scope.contract_digest=p_contract THEN + UPDATE registry_internal.registry_external_projection_scopes + SET epoch=p_epoch,binding_digest=p_binding,grant_digest=p_grant,cache_ttl_seconds=p_ttl,retention_seconds=p_retention,active=true,generation=generation+1 + WHERE projection_id=p_projection; + RETURN p_epoch; + END IF; + RETURN NULL; +END $body$; +"#; + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn scope() -> ProjectionScope { + ProjectionScope::new( + "people", + Uuid::nil(), + "sha256:binding", + "sha256:grant", + 60, + 120, + ) + .unwrap() + } + + fn row(key: &str) -> ProjectionRow { + ProjectionRow::new(key, json!({"label":"PRIVATE_SOURCE_CANARY"})).unwrap() + } + + #[test] + fn no_store_and_unbounded_retention_are_refused() { + assert_eq!( + AcceptedSnapshot::one( + "key", + "snapshot", + 100, + 160, + RetentionPermission::NoStore, + row("key") + ) + .unwrap_err(), + ProjectionError::RetentionRefused + ); + assert!(ProjectionScope::new("people", Uuid::nil(), "binding", "grant", 60, 59).is_err()); + let snapshot = AcceptedSnapshot::one( + "key", + "snapshot", + 100, + 160, + RetentionPermission::PermittedUntil(150), + row("key"), + ); + assert_eq!(snapshot.unwrap_err(), ProjectionError::RetentionRefused); + } + + #[test] + fn exact_identity_and_complete_collection_membership_are_bounded() { + assert_eq!( + AcceptedSnapshot::one( + "key", + "snapshot", + 100, + 160, + RetentionPermission::PermittedUntil(220), + row("other") + ) + .unwrap_err(), + ProjectionError::InvalidSnapshot + ); + assert!(AcceptedSnapshot::complete_collection( + "parent", + "snapshot", + 100, + 160, + RetentionPermission::PermittedUntil(220), + vec![row("same"), row("same")] + ) + .is_err()); + let empty = AcceptedSnapshot::complete_collection( + "parent", + "snapshot", + 100, + 160, + RetentionPermission::PermittedUntil(220), + vec![], + ) + .unwrap(); + assert_eq!(empty.kind, SnapshotKind::CompleteCollection); + let absent = AcceptedSnapshot::authorized_absence( + "key", + "snapshot", + 100, + 160, + RetentionPermission::PermittedUntil(220), + ) + .unwrap(); + assert_eq!(absent.kind, SnapshotKind::Absent); + } + + #[test] + fn original_snapshot_bounds_expiry_and_purge() { + let accepted = AcceptedSnapshot::one( + "key", + "snapshot", + 100, + 300, + RetentionPermission::PermittedUntil(400), + row("key"), + ) + .unwrap(); + assert_eq!(accepted.bounds(&scope()).unwrap(), (160, 220)); + let accepted = AcceptedSnapshot::one( + "key", + "snapshot", + 100, + 150, + RetentionPermission::PermittedUntil(180), + row("key"), + ) + .unwrap(); + assert_eq!(accepted.bounds(&scope()).unwrap(), (150, 180)); + } + + #[test] + fn source_values_and_keys_are_redacted_in_debug() { + let accepted = AcceptedSnapshot::one( + "PRIVATE_KEY_CANARY", + "PRIVATE_OBSERVATION_CANARY", + 100, + 160, + RetentionPermission::PermittedUntil(220), + row("PRIVATE_KEY_CANARY"), + ) + .unwrap(); + for debug in [ + format!("{accepted:?}"), + format!("{:?}", row("PRIVATE_KEY_CANARY")), + ] { + assert!(!debug.contains("PRIVATE_")); + } + assert_eq!( + ProjectionError::InvalidSnapshot.to_string(), + "external projection snapshot is invalid" + ); + } + + #[test] + fn backup_policy_excludes_every_projection_table() { + assert_eq!(BACKUP_EXCLUDED_TABLES.len(), 4); + for table in BACKUP_EXCLUDED_TABLES { + assert!(table.starts_with("registry_internal.registry_external_projection_")); + assert!(INSTALL_SQL.contains(table)); + } + } +} + +#[cfg(test)] +#[cfg(feature = "postgres-test")] +mod postgres_tests { + use super::*; + use serde_json::json; + use tokio_postgres::{Client, NoTls}; + + #[allow(dead_code)] + mod harness { + use crate as registry_breg; + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/support/postgres_harness.rs" + )); + } + + fn scope(epoch: Uuid) -> ProjectionScope { + ProjectionScope::new("projection", epoch, "binding", "grant", 60, 120).unwrap() + } + + fn record(key: &str, value: &str) -> ProjectionRow { + ProjectionRow::new(key, json!({"label":value})).unwrap() + } + + fn snapshot(id: &str, at: i64, value: &str) -> AcceptedSnapshot { + AcceptedSnapshot::one( + "subject", + id, + at, + at + 60, + RetentionPermission::PermittedUntil(at + 120), + record("subject", value), + ) + .unwrap() + } + + async fn clock(migration: &Client, now: i64) { + let sql = INSTALL_SQL.replace( + "floor(extract(epoch FROM clock_timestamp()))::bigint", + &now.to_string(), + ); + migration + .batch_execute(&sql) + .await + .expect("fixture clock installs"); + } + + #[tokio::test] + async fn package_catalog_and_bind_once_keep_operator_authority_separate() { + use super::super::{ExpectedManagedCatalog, RegistryLockKey, RegistryStateTestIdentity}; + let database = harness::TestDatabase::create(2).await; + let (migration, migration_task) = database.connect_migration().await; + let registry = super::super::external_view_query::tests::registry(); + super::super::install_compiled_schema(&migration, ®istry, &database.runtime_role) + .await + .unwrap(); + let expected = super::super::initialize_compiled_registry_state_for_test( + &migration, + &database.runtime_role, + ®istry, + RegistryStateTestIdentity { + package_id: "projection-package", + database_id: "projection-database", + label: "projection-catalog", + }, + ) + .await + .unwrap(); + let catalog = ExpectedManagedCatalog::compiled(®istry); + for (identity, result, returns_set, security_definer, _) in MANAGED_FUNCTIONS { + let row=migration.query_one("SELECT p.prorettype::regtype::text,p.proretset,p.prosecdef,r.rolname FROM pg_proc p JOIN pg_namespace n ON n.oid=p.pronamespace JOIN pg_roles r ON r.oid=p.proowner WHERE n.nspname||'.'||p.proname||'('||pg_get_function_identity_arguments(p.oid)||')'=$1",&[identity]).await.unwrap(); + assert_eq!(row.get::<_, String>(0), *result); + assert_eq!(row.get::<_, bool>(1), *returns_set); + assert_eq!(row.get::<_, bool>(2), *security_definer); + assert_eq!(row.get::<_, String>(3), database.migration_role.as_str()); + } + super::super::verify_catalog_identity_for_catalog( + &migration, + &expected, + &catalog, + &database.migration_role, + &database.runtime_role, + ) + .await + .unwrap(); + let pool = database.runtime_config.build_pool().unwrap(); + let mut runtime = pool.get().await.unwrap(); + let transaction = super::super::begin_registry_transaction( + &mut runtime, + RegistryLockKey::derive(registry.registry_id()).unwrap(), + std::time::Duration::from_secs(2), + &expected, + ) + .await + .unwrap(); + let mut requested = scope(Uuid::new_v4()); + requested.cache_ttl_seconds = 30; + let bound = bind_initial_scope(&transaction, ®istry, &expected, &requested) + .await + .unwrap(); + let restarted = bind_initial_scope(&transaction, ®istry, &expected, &requested) + .await + .unwrap(); + assert_eq!(bound.epoch, restarted.epoch); + let mut substituted = scope(Uuid::new_v4()); + substituted.binding_digest = "another-binding".to_owned(); + assert_eq!( + bind_initial_scope(&transaction, ®istry, &expected, &substituted) + .await + .unwrap_err(), + ProjectionError::ScopeChanged + ); + assert!( + load_active_scope(&transaction, "projection", "binding", "grant") + .await + .is_ok() + ); + let baseline = next_refresh_sequence(&transaction, &bound).await.unwrap(); + let now: i64 = transaction + .transaction() + .query_one( + "SELECT floor(extract(epoch FROM clock_timestamp()))::bigint", + &[], + ) + .await + .unwrap() + .get(0); + assert!( + refresh_due_keys(&transaction, &bound, &["subject".to_owned()], baseline, now) + .await + .unwrap() + .contains("subject") + ); + let ticket = allocate_refresh(&transaction, &bound, "subject") + .await + .unwrap(); + let accepted = AcceptedSnapshot::one( + "subject", + "sweep", + now, + now + 60, + RetentionPermission::PermittedUntil(now + 120), + record("subject", "fresh"), + ) + .unwrap(); + let eligibility = "SELECT 1 WHERE $6::text='subject' AND cardinality($13::uuid[])>0"; + assert_eq!( + publish_for_roots(&transaction, &ticket, &accepted, eligibility, &[]) + .await + .unwrap(), + Publication::Ineligible + ); + assert!( + refresh_due_keys(&transaction, &bound, &["subject".to_owned()], baseline, now) + .await + .unwrap() + .contains("subject") + ); + assert_eq!( + publish_for_roots( + &transaction, + &ticket, + &accepted, + eligibility, + &[Uuid::new_v4()] + ) + .await + .unwrap(), + Publication::Published + ); + assert!( + !refresh_due_keys(&transaction, &bound, &["subject".to_owned()], baseline, now) + .await + .unwrap() + .contains("subject") + ); + assert_eq!( + refresh_due_keys( + &transaction, + &bound, + &[ + "subject".to_owned(), + "missing".to_owned(), + "missing".to_owned() + ], + baseline, + now + ) + .await + .unwrap(), + BTreeSet::from(["missing".to_owned()]) + ); + assert!(refresh_due_keys(&transaction, &bound, &[], baseline, now) + .await + .unwrap() + .is_empty()); + assert_eq!( + refresh_due_keys( + &transaction, + &bound, + &["subject".to_owned()], + baseline, + now + 30 + ) + .await + .unwrap(), + BTreeSet::from(["subject".to_owned()]) + ); + assert_eq!( + refresh_due_keys( + &transaction, + &bound, + &vec!["subject".to_owned(); 1001], + baseline, + now + ) + .await + .unwrap_err(), + ProjectionError::InvalidScope + ); + assert_eq!( + refresh_due_keys(&transaction, &bound, &[String::new()], baseline, now) + .await + .unwrap_err(), + ProjectionError::InvalidScope + ); + let next_sweep = next_refresh_sequence(&transaction, &bound).await.unwrap(); + assert!(refresh_due_keys( + &transaction, + &bound, + &["subject".to_owned()], + next_sweep, + now + ) + .await + .unwrap() + .contains("subject")); + withdraw_scope(&transaction, &bound).await.unwrap(); + assert_eq!( + refresh_due_keys(&transaction, &bound, &[], baseline, now) + .await + .unwrap_err(), + ProjectionError::ScopeChanged + ); + assert_eq!( + bind_initial_scope(&transaction, ®istry, &expected, &requested) + .await + .unwrap_err(), + ProjectionError::ScopeChanged + ); + transaction.commit().await.unwrap(); + drop(runtime); + // Reapplying an unchanged compiled inventory cannot undo withdrawal. + reconcile(&migration, ®istry, &database.runtime_role) + .await + .unwrap(); + assert!(!migration.query_one("SELECT active FROM registry_internal.registry_external_projection_scopes WHERE projection_id='projection'",&[]).await.unwrap().get::<_,bool>(0)); + migration + .batch_execute(&format!( + "GRANT UPDATE ON registry_internal.registry_external_projection_values TO {}", + database.runtime_role.quoted() + )) + .await + .unwrap(); + assert!(super::super::verify_catalog_identity_for_catalog( + &migration, + &expected, + &catalog, + &database.migration_role, + &database.runtime_role + ) + .await + .is_err()); + migration + .batch_execute(&format!( + "REVOKE UPDATE ON registry_internal.registry_external_projection_values FROM {}", + database.runtime_role.quoted() + )) + .await + .unwrap(); + super::super::verify_catalog_identity_for_catalog( + &migration, + &expected, + &catalog, + &database.migration_role, + &database.runtime_role, + ) + .await + .unwrap(); + let maintenance="SELECT registry_internal.maintain_external_projection_scope($1,$2,$3,$4,$5,$6,$7,$8,$9)"; + let activation = Uuid::parse_str(&expected.activation_id).unwrap(); + let package = expected.package_digest.as_str(); + let runtime = pool.get().await.unwrap(); + assert!(runtime + .query_one( + maintenance, + &[ + &"inspect", + &"projection", + &Some(bound.epoch), + &package, + &activation, + &"renewed-binding", + &"renewed-grant", + &20_i64, + &120_i64 + ] + ) + .await + .is_err()); + drop(runtime); + let rotated: Value = migration + .query_one( + maintenance, + &[ + &"rotate", + &"projection", + &Some(bound.epoch), + &package, + &activation, + &"renewed-binding", + &"renewed-grant", + &20_i64, + &120_i64, + ], + ) + .await + .unwrap() + .get(0); + let new_epoch = Uuid::parse_str(rotated["epoch"].as_str().unwrap()).unwrap(); + assert_ne!(bound.epoch, new_epoch); + assert_eq!(rotated["active"], true); + let inspected: Value = migration + .query_one( + maintenance, + &[ + &"inspect", + &"projection", + &Some(new_epoch), + &package, + &activation, + &"", + &"", + &20_i64, + &120_i64, + ], + ) + .await + .unwrap() + .get(0); + assert_eq!( + inspected["snapshotCounts"], + json!({"fresh":0,"expired":0,"invalidated":0}) + ); + assert_eq!(inspected["refreshCounts"].as_object().unwrap().len(), 7); + assert!(inspected["refreshCounts"] + .as_object() + .unwrap() + .values() + .all(|count| count == 0)); + assert!(migration + .query_one( + maintenance, + &[ + &"withdraw", + &"projection", + &Some(bound.epoch), + &package, + &activation, + &"", + &"", + &20_i64, + &120_i64 + ] + ) + .await + .is_err()); + let withdrawn: Value = migration + .query_one( + maintenance, + &[ + &"withdraw", + &"projection", + &Some(new_epoch), + &package, + &activation, + &"", + &"", + &20_i64, + &120_i64, + ], + ) + .await + .unwrap() + .get(0); + assert_eq!(withdrawn["active"], false); + let without = super::super::external_view_query::tests::registry_with_external(false); + reconcile(&migration, &without, &database.runtime_role) + .await + .unwrap(); + assert!(!migration.query_one("SELECT to_regclass('registry_internal.registry_external_projection_scopes') IS NOT NULL",&[]).await.unwrap().get::<_,bool>(0)); + reconcile(&migration, &without, &database.runtime_role) + .await + .unwrap(); + drop(pool); + migration_task.abort(); + drop(migration); + database.cleanup().await; + } + + #[tokio::test] + async fn projection_publication_fences_scope_sequence_replay_and_atomic_membership() { + let database = harness::TestDatabase::create(2).await; + let (migration, migration_task) = database.connect_migration().await; + assert_eq!( + install(&migration, &database.migration_role) + .await + .unwrap_err(), + ProjectionError::RoleSeparationRequired + ); + install(&migration, &database.runtime_role).await.unwrap(); + clock(&migration, 1000).await; + let scope = scope(Uuid::new_v4()); + configure_scope(&migration, &scope).await.unwrap(); + let (runtime, connection) = database + .runtime_config + .postgres() + .connect(NoTls) + .await + .unwrap(); + let runtime_task = tokio::spawn(async move { + connection.await.expect("fixture runtime connection"); + }); + + // Direct DML is unavailable even though the runtime can execute the + // bounded publication function and select its retained projection. + assert!(runtime + .execute( + "UPDATE registry_internal.registry_external_projection_scopes SET active=false", + &[] + ) + .await + .is_err()); + assert!(runtime + .execute( + "DELETE FROM registry_internal.registry_external_projection_values", + &[] + ) + .await + .is_err()); + let older = allocate_inner(&runtime, &scope, "subject").await.unwrap(); + let newer = allocate_inner(&runtime, &scope, "subject").await.unwrap(); + let fresh = snapshot("newer", 999, "fresh"); + assert_eq!( + publish_inner(&runtime, &newer, &fresh).await.unwrap(), + Publication::Published + ); + assert_eq!( + publish_inner(&runtime, &older, &snapshot("older", 998, "stale")) + .await + .unwrap_err(), + ProjectionError::StaleRefresh + ); + let expiry_before: i64 = runtime.query_one("SELECT expires_at FROM registry_internal.registry_external_projection_snapshots WHERE subject_key='subject'", &[]).await.unwrap().get(0); + assert_eq!( + publish_inner(&runtime, &newer, &fresh).await.unwrap(), + Publication::Replay + ); + let expiry_after: i64 = runtime.query_one("SELECT expires_at FROM registry_internal.registry_external_projection_snapshots WHERE subject_key='subject'", &[]).await.unwrap().get(0); + assert_eq!(expiry_before, expiry_after); + let substitution = snapshot("newer", 999, "substitution"); + assert!(publish_inner(&runtime, &newer, &substitution) + .await + .is_err()); + assert!(runtime.query_one("SELECT registry_internal.fail_external_projection_refresh($1,$2,$3,$4,'retention-refused')",&[&scope.projection_id,&scope.epoch,&older.sequence,&older.subject_key]).await.unwrap().get::<_,bool>(0)); + assert_eq!(runtime.query_one("SELECT count(*) FROM registry_internal.registry_external_projection_snapshots WHERE subject_key='subject'",&[]).await.unwrap().get::<_,i64>(0),1); + let failed = allocate_inner(&runtime, &scope, "subject").await.unwrap(); + assert!(runtime.query_one("SELECT registry_internal.fail_external_projection_refresh($1,$2,$3,$4,'unavailable')",&[&scope.projection_id,&scope.epoch,&failed.sequence,&failed.subject_key]).await.unwrap().get::<_,bool>(0)); + assert_eq!(runtime.query_one("SELECT count(*) FROM registry_internal.registry_external_projection_snapshots WHERE subject_key='subject'",&[]).await.unwrap().get::<_,i64>(0),0); + let restore = allocate_inner(&runtime, &scope, "subject").await.unwrap(); + publish_inner(&runtime, &restore, &fresh).await.unwrap(); + let no_store = allocate_inner(&runtime, &scope, "subject").await.unwrap(); + assert!(runtime.query_one("SELECT registry_internal.fail_external_projection_refresh($1,$2,$3,$4,'retention-refused')",&[&scope.projection_id,&scope.epoch,&no_store.sequence,&no_store.subject_key]).await.unwrap().get::<_,bool>(0)); + assert_eq!(runtime.query_one("SELECT count(*) FROM registry_internal.registry_external_projection_snapshots WHERE subject_key='subject'",&[]).await.unwrap().get::<_,i64>(0),0); + + let first = allocate_inner(&runtime, &scope, "parent").await.unwrap(); + let collection = AcceptedSnapshot::complete_collection( + "parent", + "members-1", + 998, + 1058, + RetentionPermission::PermittedUntil(1118), + vec![record("a", "first"), record("b", "second")], + ) + .unwrap(); + publish_inner(&runtime, &first, &collection).await.unwrap(); + let count = || async { + runtime.query_one("SELECT count(*) FROM registry_internal.registry_external_projection_values WHERE subject_key='parent'", &[]).await.unwrap().get::<_,i64>(0) + }; + assert_eq!(count().await, 2); + let interrupted = allocate_inner(&runtime, &scope, "parent").await.unwrap(); + runtime.query_one("SELECT registry_internal.fail_external_projection_refresh($1,$2,$3,$4,'incomplete')", &[&scope.projection_id,&scope.epoch,&interrupted.sequence,&interrupted.subject_key]).await.unwrap(); + assert_eq!(count().await, 2); + let next = allocate_inner(&runtime, &scope, "parent").await.unwrap(); + let empty = AcceptedSnapshot::complete_collection( + "parent", + "members-2", + 999, + 1059, + RetentionPermission::PermittedUntil(1119), + vec![], + ) + .unwrap(); + publish_inner(&runtime, &next, &empty).await.unwrap(); + assert_eq!(count().await, 0); + assert_eq!(runtime.query_one("SELECT kind FROM registry_internal.registry_external_projection_snapshots WHERE subject_key='parent'", &[]).await.unwrap().get::<_,String>(0), "complete-collection"); + + let pending = allocate_inner(&runtime, &scope, "subject").await.unwrap(); + runtime + .query_one( + "SELECT registry_internal.withdraw_external_projection($1,$2,$3,$4)", + &[ + &scope.projection_id, + &scope.epoch, + &scope.binding_digest, + &scope.grant_digest, + ], + ) + .await + .unwrap(); + assert_eq!( + publish_inner( + &runtime, + &pending, + &snapshot("after-withdrawal", 1000, "forbidden") + ) + .await + .unwrap_err(), + ProjectionError::ScopeChanged + ); + assert!(configure_scope(&migration, &scope).await.is_err()); + let replacement = super::postgres_tests::scope(Uuid::new_v4()); + configure_scope(&migration, &replacement).await.unwrap(); + assert_eq!( + publish_inner(&runtime, &pending, &fresh).await.unwrap_err(), + ProjectionError::ScopeChanged + ); + + runtime_task.abort(); + migration_task.abort(); + drop(runtime); + drop(migration); + database.cleanup().await; + } + + #[tokio::test] + async fn allocation_abandonment_preserves_snapshots_without_a_terminal_fence() { + let database = harness::TestDatabase::create(2).await; + let (migration, migration_task) = database.connect_migration().await; + install(&migration, &database.runtime_role).await.unwrap(); + clock(&migration, 1000).await; + let scope = scope(Uuid::new_v4()); + configure_scope(&migration, &scope).await.unwrap(); + let initial = allocate_inner(&migration, &scope, "subject").await.unwrap(); + publish_inner(&migration, &initial, &snapshot("initial", 999, "retained")) + .await + .unwrap(); + let older = allocate_inner(&migration, &scope, "subject").await.unwrap(); + let abandoned = allocate_inner(&migration, &scope, "subject").await.unwrap(); + let fail = "SELECT registry_internal.fail_external_projection_refresh($1,$2,$3,$4,$5)"; + assert!(migration + .query_one( + fail, + &[ + &scope.projection_id, + &scope.epoch, + &abandoned.sequence, + &"other", + &RefreshFailure::Abandoned.as_str() + ] + ) + .await + .unwrap() + .get::<_, bool>(0)); + assert!(migration + .query_one( + fail, + &[ + &scope.projection_id, + &scope.epoch, + &abandoned.sequence, + &abandoned.subject_key, + &RefreshFailure::Abandoned.as_str() + ] + ) + .await + .unwrap() + .get::<_, bool>(0)); + assert_eq!(migration.query_one("SELECT count(*) FROM registry_internal.registry_external_projection_refreshes WHERE projection_id=$1 AND sequence=$2",&[&scope.projection_id,&abandoned.sequence]).await.unwrap().get::<_,i64>(0),0); + assert_eq!(migration.query_one("SELECT snapshot_id,expires_at FROM registry_internal.registry_external_projection_snapshots WHERE projection_id=$1 AND subject_key='subject'",&[&scope.projection_id]).await.unwrap().get::<_,String>(0),"initial"); + publish_inner(&migration, &older, &snapshot("older", 999, "accepted")) + .await + .unwrap(); + assert!(migration + .query_one( + fail, + &[ + &scope.projection_id, + &scope.epoch, + &older.sequence, + &older.subject_key, + &RefreshFailure::Abandoned.as_str() + ] + ) + .await + .unwrap() + .get::<_, bool>(0)); + assert_eq!(migration.query_one("SELECT lease_expires_at-started_at FROM registry_internal.registry_external_projection_refreshes WHERE projection_id=$1 AND sequence=$2",&[&scope.projection_id,&older.sequence]).await.unwrap().get::<_,i64>(0),120); + migration_task.abort(); + drop(migration); + database.cleanup().await; + } + + #[tokio::test] + async fn terminal_receipts_fence_older_results_after_snapshot_removal() { + let database = harness::TestDatabase::create(2).await; + let (migration, migration_task) = database.connect_migration().await; + install(&migration, &database.runtime_role).await.unwrap(); + clock(&migration, 1000).await; + let scope = scope(Uuid::new_v4()); + configure_scope(&migration, &scope).await.unwrap(); + for reason in [ + "denied", + "retention-refused", + "unavailable", + "incomplete", + "invalid-binding", + ] { + let initial = allocate_inner(&migration, &scope, "subject").await.unwrap(); + let accepted = snapshot(reason, 999, "retained"); + publish_inner(&migration, &initial, &accepted) + .await + .unwrap(); + let older = allocate_inner(&migration, &scope, "subject").await.unwrap(); + let newer = allocate_inner(&migration, &scope, "subject").await.unwrap(); + assert!(migration + .query_one( + "SELECT registry_internal.fail_external_projection_refresh($1,$2,$3,$4,$5)", + &[ + &scope.projection_id, + &scope.epoch, + &newer.sequence, + &newer.subject_key, + &reason + ] + ) + .await + .unwrap() + .get::<_, bool>(0)); + assert!(migration.query_one("SELECT registry_internal.fail_external_projection_refresh($1,$2,$3,$4,'abandoned')",&[&scope.projection_id,&scope.epoch,&newer.sequence,&newer.subject_key]).await.unwrap().get::<_,bool>(0)); + assert_eq!( + publish_inner(&migration, &older, &snapshot("older", 999, "forbidden")) + .await + .unwrap_err(), + ProjectionError::StaleRefresh + ); + assert_eq!( + publish_inner(&migration, &initial, &accepted) + .await + .unwrap_err(), + ProjectionError::StaleRefresh + ); + // Even an expired barrier must survive while an earlier attempt + // remains live, independently of wall-clock lease ordering. + migration.execute("UPDATE registry_internal.registry_external_projection_refreshes SET lease_expires_at=999 WHERE projection_id=$1 AND sequence=$2",&[&scope.projection_id,&newer.sequence]).await.unwrap(); + migration + .query_one( + "SELECT * FROM registry_internal.purge_external_projections(1000)", + &[], + ) + .await + .unwrap(); + assert_eq!( + publish_inner(&migration, &older, &accepted) + .await + .unwrap_err(), + ProjectionError::StaleRefresh + ); + let later = allocate_inner(&migration, &scope, "subject").await.unwrap(); + publish_inner(&migration, &later, &accepted).await.unwrap(); + migration.query_one("SELECT registry_internal.fail_external_projection_refresh($1,$2,$3,$4,'retention-refused')",&[&scope.projection_id,&scope.epoch,&older.sequence,&older.subject_key]).await.unwrap(); + assert_eq!(migration.query_one("SELECT refresh_sequence FROM registry_internal.registry_external_projection_snapshots WHERE projection_id=$1 AND subject_key='subject'",&[&scope.projection_id]).await.unwrap().get::<_,i64>(0),later.sequence); + } + migration_task.abort(); + drop(migration); + database.cleanup().await; + } + + #[tokio::test] + async fn projection_expiry_purge_and_excluded_backup_restore_are_closed() { + let database = harness::TestDatabase::create(2).await; + let (migration, migration_task) = database.connect_migration().await; + assert_eq!( + install(&migration, &database.migration_role) + .await + .unwrap_err(), + ProjectionError::RoleSeparationRequired + ); + install(&migration, &database.runtime_role).await.unwrap(); + clock(&migration, 1000).await; + let scope = scope(Uuid::new_v4()); + configure_scope(&migration, &scope).await.unwrap(); + let ticket = allocate_inner(&migration, &scope, "subject").await.unwrap(); + let observed = snapshot("original", 990, "retained"); + publish_inner(&migration, &ticket, &observed).await.unwrap(); + clock(&migration, 1050).await; + let visible: i64 = migration.query_one("SELECT count(*) FROM registry_internal.registry_external_projection_snapshots WHERE expires_at>1050", &[]).await.unwrap().get(0); + assert_eq!(visible, 0); + assert!(publish_inner(&migration, &ticket, &observed).await.is_err()); + // Purge at freshness expiry leaves retention headroom. + let pruned = migration + .query_one( + "SELECT * FROM registry_internal.purge_external_projections(100)", + &[], + ) + .await + .unwrap(); + assert_eq!(pruned.get::<_, i64>(0), 1); + assert_eq!( + migration + .query_one( + "SELECT count(*) FROM registry_internal.registry_external_projection_values", + &[] + ) + .await + .unwrap() + .get::<_, i64>(0), + 0 + ); + + // A projection-excluding backup restores the table definitions only. + // Model that exact boundary using a second empty disposable database, + // not by restoring expired authority from the first database. + let restored = harness::TestDatabase::create(1).await; + let (restore_migration, restore_task) = restored.connect_migration().await; + install(&restore_migration, &restored.runtime_role) + .await + .unwrap(); + for table in BACKUP_EXCLUDED_TABLES { + let count = restore_migration + .query_one(&format!("SELECT count(*) FROM {table}"), &[]) + .await + .unwrap() + .get::<_, i64>(0); + assert_eq!(count, 0); + } + assert!(allocate_inner(&restore_migration, &scope, "subject") + .await + .is_err()); + restore_task.abort(); + migration_task.abort(); + drop(restore_migration); + drop(migration); + restored.cleanup().await; + database.cleanup().await; + } +} diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs new file mode 100644 index 0000000000..9b520cf000 --- /dev/null +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -0,0 +1,1765 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Governed HTTP reads and bounded refresh of external views. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; +use std::time::Duration; + +use chrono::Utc; +use registry_adapter_core::{ProjectionRecord, ProjectionRequest, ProjectionResult}; +use registry_adapter_engine::evidence::{ + EvidenceAcquisitionIntent, EvidenceHookError, EvidenceHookFuture, EvidenceHostHooks, + EvidenceSendIntent, +}; +use registry_adapter_engine::projection::ProjectionError as AdapterProjectionError; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use tokio::time::{timeout_at, Instant}; +use uuid::Uuid; + +use crate::api::external_views::{ + ExternalQuery, ExternalQueryMode, ExternalViewError, ExternalViewRequest, ExternalViewService, + RefreshMode, +}; +use crate::api::ServiceFuture; +use crate::audit::{ + begin_pre_io_audit, profile_is_keyed, read_terminal_entry, PreIoAudit, PreIoAuditKind, + ReadTerminalAudit, RegistryAudit, TerminalAudit, TerminalAuditOutcome, +}; +use crate::cursor::CursorCodec; +use crate::external_source::{BoundExternalSource, ExternalSourceError, ExternalSourceService}; +use crate::external_views::{ + ExternalCardinality, ExternalLinkPermissionSource, ExternalLinkSource, ExternalProjectionMode, + ExternalProjectionSource, ExternalRecordTypeSource, +}; +use crate::model::{CompiledEntity, CompiledRegistry, HttpMethod}; + +use super::cancellation::QueryCancellationGuard; +use super::external_label_acquisition::{LabelAcquisition, LabelAfter}; +use super::external_projection::{ + self as store, AcceptedSnapshot, ProjectionGeneration, ProjectionRow, ProjectionScope, + RefreshFailure, RefreshTicket, RetentionPermission, +}; +use super::external_view_query::{ + self as indexed, ExternalAvailability, IndexedViewQuery, LabelScope, PageAfter, SortDirection, +}; +use super::read::{strict_claim_context, ReadRelations}; +use super::{ + begin_record_transaction, begin_registry_transaction, ClaimContext, ExpectedRegistryIdentity, + GuardedTransaction, RegistryLockKey, RuntimePool, +}; + +type Result = std::result::Result; + +pub(crate) struct PostgresExternalViewService { + pool: RuntimePool, + registry: Arc, + expected: ExpectedRegistryIdentity, + lock_key: RegistryLockKey, + lock_timeout: Duration, + audit: RegistryAudit, + cursors: Arc, + sources: ExternalSourceService, +} + +struct Plan<'a> { + root: &'a CompiledEntity, + link: &'a ExternalLinkSource, + projection: &'a ExternalProjectionSource, + external: &'a ExternalRecordTypeSource, + grant: Option<&'a ExternalLinkPermissionSource>, + source: &'a BoundExternalSource, + label_parent: Option>, +} + +struct LabelParent<'a> { + projection: &'a ExternalProjectionSource, + source: &'a BoundExternalSource, +} + +impl Plan<'_> { + fn grant_valid_until(&self) -> chrono::DateTime { + let cutoff = self.source.grant_valid_until(); + self.label_parent.as_ref().map_or(cutoff, |parent| { + cutoff.min(parent.source.grant_valid_until()) + }) + } + + fn read_grant(&self) -> Result<&ExternalLinkPermissionSource> { + self.grant.ok_or(ExternalViewError::Concealed) + } +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Continuation { + binding: String, + expires_at: i64, + query: ExternalQuery, + position: Position, +} + +#[derive(Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +enum Position { + Read { + root_id: Uuid, + source_key: String, + sort_value: Option, + }, + Refresh { + after: RefreshAfter, + minimum_sequence: i64, + }, +} + +#[derive(Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +enum RefreshAfter { + Root { root_id: Uuid }, + Label { root_id: Uuid, member_key: String }, +} + +impl From for Position { + fn from(value: PageAfter) -> Self { + Self::Read { + root_id: value.root_id, + source_key: value.source_key, + sort_value: value.sort_value, + } + } +} + +struct HeldResponse { + body: Value, + expires_at: i64, +} + +impl HeldResponse { + fn release(self) -> Result { + if Utc::now().timestamp() >= self.expires_at { + return Err(ExternalViewError::Unavailable); + } + Ok(self.body) + } +} + +impl PostgresExternalViewService { + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + pool: RuntimePool, + registry: Arc, + expected: ExpectedRegistryIdentity, + lock_key: RegistryLockKey, + lock_timeout: Duration, + audit: RegistryAudit, + cursors: Arc, + sources: ExternalSourceService, + ) -> Self { + Self { + pool, + registry, + expected, + lock_key, + lock_timeout, + audit, + cursors, + sources, + } + } + + pub(crate) async fn bind_configured_scopes(&self) -> Result<()> { + let configured = self + .registry + .external_views() + .projections + .values() + .filter_map(|projection| { + let bound = self.sources.binding(projection.id.as_str())?; + Some((projection, bound, bound.effective_cache_ttl_seconds()?)) + }) + .collect::>(); + if configured.is_empty() { + return Ok(()); + } + let mut session = self.session().await?; + let transaction = begin_registry_transaction( + session.client(), + self.lock_key, + self.lock_timeout, + &self.expected, + ) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + transaction + .set_statement_budget(Duration::from_secs(10)) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + for (projection, bound, ttl) in configured { + let retention = bound + .maximum_retention_seconds() + .ok_or(ExternalViewError::Unavailable)?; + let scope = ProjectionScope::new( + projection.id.as_str(), + Uuid::new_v4(), + bound.binding_fingerprint(), + bound.grant_fingerprint(), + i64::from(ttl), + i64::from(retention), + ) + .map_err(|_| ExternalViewError::Unavailable)?; + store::bind_initial_scope(&transaction, &self.registry, &self.expected, &scope) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + } + transaction + .commit() + .await + .map_err(|_| ExternalViewError::Unavailable)?; + session.disarm(); + Ok(()) + } + + pub(crate) async fn run_retention( + self: Arc, + last_success: Arc, + mut shutdown: tokio::sync::watch::Receiver, + ) { + let mut interval = tokio::time::interval(Duration::from_secs(5)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + tokio::select! { + change = shutdown.changed() => { + if change.is_err() || *shutdown.borrow() { return; } + } + _ = interval.tick() => { + match timeout_at(Instant::now() + Duration::from_secs(5), self.purge_tick()).await { + Ok(Ok(())) => last_success.record(), + _ => tracing::error!("external projection retention tick failed"), + } + } + } + } + } + + async fn purge_tick(&self) -> Result<()> { + const BATCH: u32 = 1000; + let mut session = self.session().await?; + for _ in 0..8 { + let transaction = begin_registry_transaction( + session.client(), + self.lock_key, + self.lock_timeout, + &self.expected, + ) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + transaction + .set_statement_budget(Duration::from_secs(2)) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + let (snapshots, receipts) = store::purge(&transaction, BATCH) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + transaction + .commit() + .await + .map_err(|_| ExternalViewError::Unavailable)?; + if snapshots < u64::from(BATCH) && receipts < u64::from(BATCH) { + break; + } + } + session.disarm(); + Ok(()) + } + + fn plan<'a>( + &'a self, + request: &ExternalViewRequest, + refresh: Option, + ) -> Result> { + let inventory = self.registry.external_views(); + let profile = request.context.selected_profile(); + let view = inventory + .views + .get(&request.view) + .ok_or(ExternalViewError::Concealed)?; + let link = view + .links + .iter() + .find(|link| link.id.as_str() == request.link) + .ok_or(ExternalViewError::Concealed)?; + let projection = inventory + .projections + .get(link.projection.as_str()) + .ok_or(ExternalViewError::Unavailable)?; + if request.context.purpose() != Some(projection.purpose.as_str()) { + return Err(ExternalViewError::Concealed); + } + let mut label_parent = None; + let mut projection = projection; + let grant = if let Some(mode) = refresh { + let ids = inventory + .refresh + .get(profile) + .ok_or(ExternalViewError::Concealed)?; + if request.context.requester_client().is_none() + || projection.mode != ExternalProjectionMode::Indexed + || !ids.contains(&projection.id) + { + return Err(ExternalViewError::Concealed); + } + if mode == RefreshMode::Labels { + let link_label = link.label.as_ref().ok_or(ExternalViewError::Concealed)?; + let label = inventory + .projections + .get(link_label.projection.as_str()) + .ok_or(ExternalViewError::Concealed)?; + if projection.cardinality != ExternalCardinality::Many + || label.mode != ExternalProjectionMode::Indexed + || label.cardinality != ExternalCardinality::One + || label.purpose != projection.purpose + || !ids.contains(&label.id) + { + return Err(ExternalViewError::Concealed); + } + label_parent = Some(LabelParent { + projection, + source: self + .sources + .binding(projection.id.as_str()) + .ok_or(ExternalViewError::Unavailable)?, + }); + projection = label; + } + None + } else { + Some( + inventory + .grants + .get(profile) + .and_then(|views| views.get(&request.view)) + .and_then(|view| view.links.iter().find(|grant| grant.link == link.id)) + .ok_or(ExternalViewError::Concealed)?, + ) + }; + let root = self + .registry + .entities() + .get(view.record_type.as_str()) + .ok_or(ExternalViewError::Unavailable)?; + let external = inventory + .record_types + .get(projection.record_type.as_str()) + .ok_or(ExternalViewError::Unavailable)?; + let source = self + .sources + .binding(projection.id.as_str()) + .ok_or(ExternalViewError::Unavailable)?; + Ok(Plan { + root, + link, + projection, + external, + grant, + source, + label_parent, + }) + } + + async fn run( + &self, + request: ExternalViewRequest, + refresh: Option, + ) -> Result { + if !profile_is_keyed(self.audit.profile()) { + return Err(ExternalViewError::Unavailable); + } + let plan = self.plan(&request, refresh)?; + let claims = strict_claim_context(&self.registry, &request.context, &plan.root.id) + .map_err(|_| ExternalViewError::Unavailable)?; + let operation = match refresh { + Some(RefreshMode::Labels) => "external-projection-label-refresh", + Some(RefreshMode::Projection) => "external-projection-refresh", + None if plan.projection.mode == ExternalProjectionMode::Live => { + "external-view-live-read" + } + None => "external-view-indexed-read", + }; + let method = if refresh.is_some() { + HttpMethod::Post + } else { + HttpMethod::Get + }; + let target = request.root.map(|root| root.to_string()); + // The durable attempt precedes both protected SQL and any source I/O. + let attempt = begin_pre_io_audit( + &self.audit, + &self.expected, + &claims, + PreIoAudit { + kind: PreIoAuditKind::Attempt, + method, + operation_id: operation, + target_record: target.as_deref(), + refusal_reason: None, + correlation: &request.correlation, + }, + ) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + let result = if refresh.is_some() { + self.refresh_page(&request, &plan, &claims).await + } else if plan.projection.mode == ExternalProjectionMode::Live { + self.read_live(&request, &plan, &claims, operation).await + } else { + self.read_indexed(&request, &plan, &claims, operation).await + }; + if !attempt.is_answered() { + self.terminal(&request, &plan, operation, method, result.is_ok()) + .await?; + } + result?.release() + } + + async fn transaction<'a>( + &self, + session: &'a mut QueryCancellationGuard, + claims: &ClaimContext, + deadline: Instant, + ) -> Result> { + let transaction = begin_record_transaction( + session.client(), + self.lock_key, + self.lock_timeout, + &self.expected, + claims, + ) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + let budget = deadline + .saturating_duration_since(Instant::now()) + .min(Duration::from_secs(30)); + if budget < Duration::from_millis(1) { + return Err(ExternalViewError::Unavailable); + } + transaction + .set_statement_budget(budget) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + Ok(transaction) + } + + async fn session(&self) -> Result { + let client = self + .pool + .get() + .await + .map_err(|_| ExternalViewError::Unavailable)?; + Ok(QueryCancellationGuard::new(self.pool.clone(), client)) + } + + async fn read_live( + &self, + request: &ExternalViewRequest, + plan: &Plan<'_>, + claims: &ClaimContext, + operation: &str, + ) -> Result { + let root = request.root.ok_or(ExternalViewError::Concealed)?; + if !matches!(&request.query, ExternalQueryMode::First(_)) { + return Err(ExternalViewError::InvalidCursor); + } + let mut session = self.session().await?; + let transaction = self + .transaction(&mut session, claims, request.deadline) + .await?; + let key = RootKeys::new(plan)? + .for_root(&transaction, root) + .await? + .ok_or(ExternalViewError::Concealed)?; + transaction + .commit() + .await + .map_err(|_| ExternalViewError::Unavailable)?; + session.disarm(); + drop(session); + + let fields = plan + .read_grant()? + .readable_fields + .iter() + .map(|field| field.as_str().to_owned()) + .collect(); + let result = if let Some(key) = &key { + let source_request = match plan.projection.cardinality { + ExternalCardinality::One => ProjectionRequest::Exact { key: key.clone() }, + ExternalCardinality::Many => ProjectionRequest::ByParent { + parent_key: key.clone(), + }, + }; + self.acquire(request, plan, source_request, &fields, None, None) + .await + .ok() + } else { + None + }; + let mut session = self.session().await?; + let transaction = self + .transaction(&mut session, claims, request.deadline) + .await?; + // Re-evaluate RLS and the association after the network round trip. + let current = RootKeys::new(plan)? + .for_root(&transaction, root) + .await? + .ok_or(ExternalViewError::Concealed)?; + let (status, records) = if current == key { + self.live_records(request, plan, root, result)? + } else { + ("unavailable", Vec::new()) + }; + self.log_roots(&transaction, request, plan, &[root.to_string()], operation) + .await?; + self.terminal(request, plan, operation, HttpMethod::Get, true) + .await?; + transaction + .commit() + .await + .map_err(|_| ExternalViewError::Unavailable)?; + session.disarm(); + Ok(HeldResponse { + body: view_response(plan, status, records, None, None), + expires_at: if status == "unavailable" { + i64::MAX + } else { + plan.source.grant_valid_until().timestamp() + }, + }) + } + + async fn read_indexed( + &self, + request: &ExternalViewRequest, + plan: &Plan<'_>, + claims: &ClaimContext, + operation: &str, + ) -> Result { + let mut session = self.session().await?; + let transaction = self + .transaction(&mut session, claims, request.deadline) + .await?; + let (scope, generation) = match self + .lock_scope(&transaction, plan.source, plan.projection.id.as_str()) + .await + { + Ok(scope) => scope, + Err(_) + if request.root.is_some() + && matches!(&request.query, + ExternalQueryMode::First(query) if query.equality.is_none() + && query.order.is_none() && !query.include_count) => + { + let root = request.root.ok_or(ExternalViewError::Concealed)?; + RootKeys::new(plan)? + .for_root(&transaction, root) + .await? + .ok_or(ExternalViewError::Concealed)?; + self.log_roots(&transaction, request, plan, &[root.to_string()], operation) + .await?; + self.terminal(request, plan, operation, HttpMethod::Get, true) + .await?; + transaction + .commit() + .await + .map_err(|_| ExternalViewError::Unavailable)?; + session.disarm(); + return Ok(HeldResponse { + body: view_response(plan, "unavailable", Vec::new(), None, None), + expires_at: i64::MAX, + }); + } + Err(_) if matches!(&request.query, ExternalQueryMode::Continue(_)) => { + return Err(ExternalViewError::InvalidCursor); + } + Err(_) => return Err(ExternalViewError::Incomplete), + }; + let grant = plan.read_grant()?; + let label = if grant.label_fields.is_empty() { + None + } else { + let label = plan + .link + .label + .as_ref() + .ok_or(ExternalViewError::Unavailable)?; + match self.sources.binding(label.projection.as_str()) { + Some(source) => self + .lock_scope(&transaction, source, label.projection.as_str()) + .await + .ok() + .map(|(scope, generation)| { + (scope, generation, source.grant_valid_until().timestamp()) + }), + None => None, + } + }; + let binding = self.cursor_binding( + request, + &scope, + Some(generation), + label.as_ref().map(|(_, generation, _)| *generation), + )?; + let (query, position) = self.page_request(&request.query, &binding)?; + let after = match position { + Some(Position::Read { + root_id, + source_key, + sort_value, + }) => Some(PageAfter { + root_id, + source_key, + sort_value, + }), + Some(Position::Refresh { .. }) => return Err(ExternalViewError::InvalidCursor), + None => None, + }; + let mut expires_at = plan.source.grant_valid_until().timestamp(); + if let Some((_, _, label_expiry)) = &label { + expires_at = expires_at.min(*label_expiry); + } + let page = indexed::query( + &transaction, + &self.registry, + IndexedViewQuery { + profile_id: request.context.selected_profile(), + view_id: &request.view, + link_id: &request.link, + scope: &scope, + generation, + label_scope: label.as_ref().map(|(scope, generation, _)| LabelScope { + scope, + generation: *generation, + }), + now: Utc::now().timestamp(), + root_id: request.root, + fields: grant + .readable_fields + .iter() + .map(|field| field.as_str().to_owned()) + .collect(), + filter: query.equality.as_ref().map(|filter| indexed::ExactFilter { + field: filter.field.clone(), + value: filter.value.clone(), + }), + order: query.order.as_ref().map(|order| indexed::ExternalOrder { + field: order.field.clone(), + direction: if order.descending { + SortDirection::Descending + } else { + SortDirection::Ascending + }, + }), + include_count: query.include_count, + page_size: query.page_size, + after, + }, + ) + .await + .map_err(|error| match error { + indexed::ExternalQueryError::GenerationChanged => ExternalViewError::InvalidCursor, + indexed::ExternalQueryError::NotPermitted => ExternalViewError::Concealed, + _ => ExternalViewError::Unavailable, + })?; + if request.root.is_some() && !page.root_found { + return Err(ExternalViewError::Concealed); + } + if !page.complete + && (request.root.is_none() + || query.equality.is_some() + || query.order.is_some() + || query.include_count + || matches!(&request.query, ExternalQueryMode::Continue(_))) + { + return Err(ExternalViewError::Incomplete); + } + if let Some(cutoff) = page.minimum_expires_at { + expires_at = expires_at.min(cutoff); + } + let next = page + .continuation + .map(|position| self.next_cursor(binding, query, position.into(), expires_at)) + .transpose()?; + let root_ids = request + .root + .into_iter() + .chain(page.rows.iter().map(|row| row.root_id)) + .map(|id| id.to_string()) + .collect::>() + .into_iter() + .collect::>(); + let status = if page.complete { + "available" + } else { + "unavailable" + }; + let records = page.rows.into_iter().map(|row| { + let mut value = response_row(row.root_id, availability(row.availability), row.fields); + if plan.projection.cardinality == ExternalCardinality::Many { + value["rowId"] = json!(self.row_id(request, plan, row.root_id, &row.source_key)?); + } + if let Some(label) = row.label { + let authority = plan.link.label.as_ref() + .and_then(|label| self.registry.external_views().projections.get(label.projection.as_str())) + .and_then(|projection| self.registry.external_views().record_types.get(projection.record_type.as_str())) + .map(|record_type| record_type.authority.as_str()) + .ok_or(ExternalViewError::Unavailable)?; + value["label"] = json!({"authority": authority, "availability": availability(label.availability), "fields": label.fields}); + } + Ok(value) + }) + .collect::>>()?; + self.log_roots(&transaction, request, plan, &root_ids, operation) + .await?; + self.terminal(request, plan, operation, HttpMethod::Get, true) + .await?; + transaction + .commit() + .await + .map_err(|_| ExternalViewError::Unavailable)?; + session.disarm(); + Ok(HeldResponse { + body: view_response(plan, status, records, page.total_count, next), + expires_at, + }) + } + + async fn lock_scope( + &self, + transaction: &GuardedTransaction<'_>, + source: &BoundExternalSource, + projection: &str, + ) -> Result<(ProjectionScope, ProjectionGeneration)> { + let scope = active_scope(transaction, source, projection).await?; + let generation = store::lock_generation(transaction, &scope) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + Ok((scope, generation)) + } + + async fn log_roots( + &self, + transaction: &GuardedTransaction<'_>, + request: &ExternalViewRequest, + plan: &Plan<'_>, + roots: &[String], + operation: &str, + ) -> Result<()> { + crate::subject_access_log::record_reads( + transaction.transaction(), + plan.root, + &plan.root.id, + &request.context, + roots, + operation, + &self.expected, + &request.correlation, + &self.audit, + ) + .await + .map_err(|_| ExternalViewError::Unavailable) + } + + async fn terminal( + &self, + request: &ExternalViewRequest, + plan: &Plan<'_>, + operation: &str, + method: HttpMethod, + permitted: bool, + ) -> Result<()> { + let entry = read_terminal_entry( + self.audit.profile(), + ReadTerminalAudit { + terminal: TerminalAudit { + grant: request.context.grant_audit().cloned(), + outcome: if permitted { + TerminalAuditOutcome::Returned + } else { + TerminalAuditOutcome::Refused + }, + method, + operation_id: operation.to_owned(), + entity_id: Some(plan.root.id.clone()), + action_id: None, + package_revision: self.expected.activation_id.clone(), + selected_access_profile: request.context.selected_profile().to_owned(), + purpose_present: request.context.purpose().is_some(), + principal_reference: request.context.principal().map(str::to_owned), + record_reference: request.root.map(|root| root.to_string()), + record_revision: None, + result_count: None, + field_set_reference: None, + correlation: request.correlation.clone(), + }, + query_reference: None, + row_boundary_reference: None, + }, + ) + .map_err(|_| ExternalViewError::Unavailable)?; + self.audit + .append(entry) + .await + .map_err(|_| ExternalViewError::Unavailable) + } + + async fn acquire( + &self, + request: &ExternalViewRequest, + plan: &Plan<'_>, + source_request: ProjectionRequest, + fields: &BTreeSet, + supporting_until: Option, + refresh_authority: Option>, + ) -> std::result::Result { + // Constructed only after the request's durable attempt and a + // guarded root lookup. Source credentials come solely from the + // pinned operator connection. + let mut grant_until = plan.grant_valid_until(); + if let Some(expiry) = supporting_until { + let cutoff = + chrono::DateTime::from_timestamp(expiry, 0).ok_or(ExternalSourceError::Grant)?; + grant_until = grant_until.min(cutoff); + } + let hooks = Arc::new(AcquisitionGate { + deadline: request.deadline, + grant_until, + route: plan.projection.operation.as_str().to_owned(), + refresh_authority, + }); + self.sources + .acquire( + plan.projection.id.as_str(), + hooks, + source_request, + fields, + Utc::now, + ) + .await + } + + fn row_id( + &self, + request: &ExternalViewRequest, + plan: &Plan<'_>, + root: Uuid, + source_key: &str, + ) -> Result { + self.cursors + .binding_digest( + b"breg-external-view-row-v1", + &json!([ + self.registry.registry_id(), + request.view, + request.link, + root, + plan.source.binding_fingerprint(), + source_key + ]), + ) + .map_err(|_| ExternalViewError::Unavailable) + } + + fn cursor_binding( + &self, + request: &ExternalViewRequest, + scope: &ProjectionScope, + generation: Option, + label: Option, + ) -> Result { + let context = &request.context; + let boundaries = context + .row_boundaries() + .iter() + .map(|boundary| { + json!([ + boundary.field(), + format!("{:?}", boundary.operator()), + boundary.values() + ]) + }) + .collect::>(); + self.cursors + .binding_digest( + b"breg-external-view-v1", + &json!({ + "activation": self.expected.activation_id, + "view": request.view, + "link": request.link, + "root": request.root, + "profile": context.selected_profile(), + "principal": context.principal(), + "requesterClient": context.requester_client(), + "purpose": context.purpose(), + "rowBoundaries": boundaries, + "recipients": context.recipients(), + "projection": scope.projection_id, + "epoch": scope.epoch, + "binding": scope.binding_digest, + "grant": scope.grant_digest, + "generation": generation.map(|value| value.generation), + "label": label.map(|value| (value.epoch, value.generation)), + }), + ) + .map_err(|_| ExternalViewError::Unavailable) + } + + fn page_request( + &self, + mode: &ExternalQueryMode, + binding: &str, + ) -> Result<(ExternalQuery, Option)> { + match mode { + ExternalQueryMode::First(query) => Ok((query.clone(), None)), + ExternalQueryMode::Continue(token) => { + let payload: Continuation = self + .cursors + .open_external(token) + .map_err(|_| ExternalViewError::InvalidCursor)?; + if payload.binding != binding || Utc::now().timestamp() >= payload.expires_at { + return Err(ExternalViewError::InvalidCursor); + } + Ok((payload.query, Some(payload.position))) + } + } + } + + fn next_cursor( + &self, + binding: String, + query: ExternalQuery, + position: Position, + expires_at: i64, + ) -> Result { + let expires_at = expires_at.min( + Utc::now() + .timestamp() + .saturating_add(self.cursors.max_age().as_secs() as i64), + ); + self.cursors + .seal_external(&Continuation { + binding, + query, + position, + expires_at, + }) + .map_err(|_| ExternalViewError::Unavailable) + } +} + +impl ExternalViewService for PostgresExternalViewService { + fn read(&self, request: ExternalViewRequest) -> ServiceFuture<'_, Result> { + Box::pin(async move { + timeout_at(request.deadline, self.run(request, None)) + .await + .map_err(|_| ExternalViewError::Unavailable)? + }) + } + + fn refresh( + &self, + request: ExternalViewRequest, + mode: RefreshMode, + ) -> ServiceFuture<'_, Result> { + Box::pin(async move { + timeout_at(request.deadline, self.run(request, Some(mode))) + .await + .map_err(|_| ExternalViewError::Unavailable)? + }) + } +} + +struct RefreshAuthority { + pool: RuntimePool, + expected: ExpectedRegistryIdentity, + lock_key: RegistryLockKey, + lock_timeout: Duration, + scope: ProjectionScope, + parent: Option<(ProjectionScope, ProjectionGeneration)>, +} + +impl RefreshAuthority { + async fn check(&self, deadline: Instant) -> std::result::Result<(), EvidenceHookError> { + let client = self + .pool + .get() + .await + .map_err(|_| EvidenceHookError::Authority)?; + let mut session = QueryCancellationGuard::new(self.pool.clone(), client); + let transaction = begin_registry_transaction( + session.client(), + self.lock_key, + self.lock_timeout, + &self.expected, + ) + .await + .map_err(|_| EvidenceHookError::Authority)?; + let budget = deadline + .saturating_duration_since(Instant::now()) + .min(Duration::from_secs(2)); + if budget < Duration::from_millis(1) { + return Err(EvidenceHookError::Authority); + } + transaction + .set_statement_budget(budget) + .await + .map_err(|_| EvidenceHookError::Authority)?; + let current = store::load_active_scope( + &transaction, + &self.scope.projection_id, + &self.scope.binding_digest, + &self.scope.grant_digest, + ) + .await + .map_err(|_| EvidenceHookError::Authority)?; + if current.epoch != self.scope.epoch { + return Err(EvidenceHookError::Authority); + } + if let Some((parent, generation)) = &self.parent { + let current = store::lock_generation(&transaction, parent) + .await + .map_err(|_| EvidenceHookError::Authority)?; + if ¤t != generation { + return Err(EvidenceHookError::Authority); + } + } + transaction + .commit() + .await + .map_err(|_| EvidenceHookError::Authority)?; + session.disarm(); + Ok(()) + } +} + +struct AcquisitionGate { + deadline: Instant, + grant_until: chrono::DateTime, + route: String, + refresh_authority: Option>, +} + +impl AcquisitionGate { + fn within_bounds(&self) -> std::result::Result<(), EvidenceHookError> { + if Instant::now() >= self.deadline || Utc::now() >= self.grant_until { + return Err(EvidenceHookError::Authority); + } + Ok(()) + } + + async fn admitted(&self) -> std::result::Result<(), EvidenceHookError> { + self.within_bounds()?; + if let Some(authority) = &self.refresh_authority { + authority.check(self.deadline).await?; + } + self.within_bounds() + } +} + +impl EvidenceHostHooks for AcquisitionGate { + fn before_acquisition<'a>( + &'a self, + intent: EvidenceAcquisitionIntent, + ) -> EvidenceHookFuture<'a> { + Box::pin(async move { + if intent.route != self.route { + return Err(EvidenceHookError::Authority); + } + self.admitted().await + }) + } + + fn before_credentials<'a>(&'a self, _intent: EvidenceSendIntent) -> EvidenceHookFuture<'a> { + Box::pin(async { self.admitted().await }) + } +} + +struct RootKeys { + select: String, + publication_eligibility: String, + id: String, +} + +impl RootKeys { + fn new(plan: &Plan<'_>) -> Result { + let key_field = plan.link.from_field.as_str(); + let relations = ReadRelations::statistics(plan.root, &[key_field.to_owned()]) + .map_err(|_| ExternalViewError::Unavailable)?; + let key = relations + .field_expression(plan.root, key_field) + .map_err(|_| ExternalViewError::Unavailable)?; + if key.encrypted { + return Err(ExternalViewError::Unavailable); + } + Ok(Self { + select: format!( + "SELECT {}, ({})::text FROM {}", + relations.id_expression, key.sql, relations.from_sql + ), + publication_eligibility: format!( + "SELECT 1 FROM {} WHERE {} = ANY($13::uuid[]) AND ({})::text = $6::text", + relations.from_sql, relations.id_expression, key.sql, + ), + id: relations.id_expression, + }) + } + + async fn for_root( + &self, + transaction: &GuardedTransaction<'_>, + root: Uuid, + ) -> Result>> { + let row = transaction + .transaction() + .query_opt(&format!("{} WHERE {} = $1", self.select, self.id), &[&root]) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + Ok(row.map(|row| row.get(1))) + } + + async fn page( + &self, + transaction: &GuardedTransaction<'_>, + after: Option, + limit: u16, + ) -> Result)>> { + // The row policy can make PostgreSQL underestimate the eligible + // population. Prefer the existing primary-key order for this + // bounded refresh transaction to avoid sorting the scope per page. + transaction + .transaction() + .batch_execute("SET LOCAL enable_sort = off") + .await + .map_err(|_| ExternalViewError::Unavailable)?; + let rows = transaction + .transaction() + .query( + &format!( + "{} WHERE ($1::uuid IS NULL OR {} > $1) ORDER BY {} LIMIT $2", + self.select, self.id, self.id + ), + &[&after, &i64::from(limit)], + ) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + Ok(rows + .into_iter() + .map(|row| (row.get(0), row.get(1))) + .collect()) + } +} + +async fn active_scope( + transaction: &GuardedTransaction<'_>, + source: &BoundExternalSource, + projection: &str, +) -> Result { + if Utc::now() >= source.grant_valid_until() { + return Err(ExternalViewError::Unavailable); + } + let scope = store::load_active_scope( + transaction, + projection, + source.binding_fingerprint(), + source.grant_fingerprint(), + ) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + if Some(scope.cache_ttl_seconds) != source.effective_cache_ttl_seconds().map(i64::from) + || Some(scope.retention_seconds) != source.maximum_retention_seconds().map(i64::from) + { + return Err(ExternalViewError::Unavailable); + } + Ok(scope) +} + +impl PostgresExternalViewService { + fn live_records( + &self, + request: &ExternalViewRequest, + plan: &Plan<'_>, + root: Uuid, + result: Option, + ) -> Result<(&'static str, Vec)> { + let rows = match result { + Some(ProjectionResult::Exact { + record: ProjectionRecord::Present { fields, .. }, + }) => ( + "available", + vec![response_row(root, "available", Value::Object(fields))], + ), + Some(ProjectionResult::Exact { + record: ProjectionRecord::AuthoritativeAbsent { .. }, + }) => ("available", vec![response_row(root, "absent", json!({}))]), + Some(ProjectionResult::ByParent { rows, .. }) => ( + "available", + rows.into_iter() + .map(|row| { + let mut value = response_row(root, "available", Value::Object(row.fields)); + value["rowId"] = json!(self.row_id(request, plan, root, &row.key)?); + Ok(value) + }) + .collect::>>()?, + ), + _ => ("unavailable", Vec::new()), + }; + Ok(rows) + } +} + +fn availability(value: ExternalAvailability) -> &'static str { + match value { + ExternalAvailability::Available => "available", + ExternalAvailability::Absent => "absent", + ExternalAvailability::Unavailable => "unavailable", + } +} + +fn response_row(root: Uuid, availability: &str, fields: Value) -> Value { + json!({"rootId": root, "availability": availability, "fields": fields}) +} + +fn view_response( + plan: &Plan<'_>, + status: &str, + records: Vec, + count: Option, + next: Option, +) -> Value { + let mut body = json!({ + "status": status, + "authority": plan.external.authority, + "records": records, + }); + if let Some(count) = count { + body["count"] = json!(count); + } + if let Some(next) = next { + body["nextCursor"] = json!(next); + } + body +} + +struct RefreshRows { + rows: Vec<(Uuid, Option)>, + next: Option, + eligibility: String, + supporting_until: Option, +} + +struct RefreshWork { + key: String, + roots: BTreeSet, + ticket: RefreshTicket, +} + +enum RefreshResult { + Accepted(AcceptedSnapshot), + Failed(RefreshFailure), +} + +impl PostgresExternalViewService { + async fn refresh_rows( + &self, + transaction: &GuardedTransaction<'_>, + request: &ExternalViewRequest, + plan: &Plan<'_>, + parent: Option<&(ProjectionScope, ProjectionGeneration)>, + after: Option, + page_size: u16, + ) -> Result { + if let Some((parent_scope, parent_generation)) = parent { + let after = match after { + Some(RefreshAfter::Label { + root_id, + member_key, + }) => Some(LabelAfter { + root_id, + member_key, + }), + Some(RefreshAfter::Root { .. }) => return Err(ExternalViewError::InvalidCursor), + None => None, + }; + let selectors = LabelAcquisition::new( + &self.registry, + request.context.selected_profile(), + request + .context + .purpose() + .ok_or(ExternalViewError::Concealed)?, + &request.view, + &request.link, + parent_scope, + *parent_generation, + ) + .map_err(|_| ExternalViewError::Concealed)?; + let page = selectors + .page(transaction, after.as_ref(), page_size) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + return Ok(RefreshRows { + rows: page + .rows + .into_iter() + .map(|row| (row.root_id, Some(row.selector))) + .collect(), + next: page.continuation.map(|after| RefreshAfter::Label { + root_id: after.root_id, + member_key: after.member_key, + }), + eligibility: selectors.publication_eligibility().to_owned(), + supporting_until: page.minimum_parent_expires_at, + }); + } + let after = match after { + Some(RefreshAfter::Root { root_id }) => Some(root_id), + Some(RefreshAfter::Label { .. }) => return Err(ExternalViewError::InvalidCursor), + None => None, + }; + // By-parent acquisitions use a smaller request budget. + let size = if plan.projection.cardinality == ExternalCardinality::Many { + page_size.min(10) + } else { + page_size + }; + let keys = RootKeys::new(plan)?; + let mut rows = keys.page(transaction, after, size + 1).await?; + let has_more = rows.len() > usize::from(size); + rows.truncate(usize::from(size)); + let next = rows + .last() + .filter(|_| has_more) + .map(|(root_id, _)| RefreshAfter::Root { root_id: *root_id }); + Ok(RefreshRows { + rows, + next, + eligibility: keys.publication_eligibility, + supporting_until: None, + }) + } + + async fn refresh_page( + &self, + request: &ExternalViewRequest, + plan: &Plan<'_>, + claims: &ClaimContext, + ) -> Result { + let mut session = self.session().await?; + let transaction = self + .transaction(&mut session, claims, request.deadline) + .await?; + let scope = active_scope(&transaction, plan.source, plan.projection.id.as_str()).await?; + let parent = if let Some(parent) = &plan.label_parent { + Some( + self.lock_scope(&transaction, parent.source, parent.projection.id.as_str()) + .await?, + ) + } else { + None + }; + let binding = self.cursor_binding( + request, + &scope, + None, + parent.as_ref().map(|(_, generation)| *generation), + )?; + let (query, position) = self.page_request(&request.query, &binding)?; + let (after, minimum_sequence) = match position { + Some(Position::Refresh { + after, + minimum_sequence, + }) => (Some(after), minimum_sequence), + Some(Position::Read { .. }) => return Err(ExternalViewError::InvalidCursor), + None => ( + None, + store::next_refresh_sequence(&transaction, &scope) + .await + .map_err(|_| ExternalViewError::Unavailable)?, + ), + }; + let page = self + .refresh_rows( + &transaction, + request, + plan, + parent.as_ref(), + after, + query.page_size, + ) + .await?; + let mut by_key = BTreeMap::>::new(); + let mut unavailable = 0usize; + for (root, key) in &page.rows { + if let Some(key) = key.as_ref().filter(|value| !value.is_empty()) { + by_key.entry(key.clone()).or_default().insert(*root); + } else { + unavailable += 1; + } + } + let keys = by_key.keys().cloned().collect::>(); + let due = store::refresh_due_keys( + &transaction, + &scope, + &keys, + minimum_sequence, + Utc::now().timestamp(), + ) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + let reused = keys.len() - due.len(); + let mut work = Vec::with_capacity(due.len()); + for (key, roots) in by_key { + if !due.contains(&key) { + continue; + } + let ticket = store::allocate_refresh(&transaction, &scope, &key) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + work.push(RefreshWork { key, roots, ticket }); + } + transaction + .commit() + .await + .map_err(|_| ExternalViewError::Unavailable)?; + session.disarm(); + drop(session); + + let mut published = 0usize; + if !work.is_empty() { + let authority = Arc::new(RefreshAuthority { + pool: self.pool.clone(), + expected: self.expected.clone(), + lock_key: self.lock_key, + lock_timeout: self.lock_timeout, + scope: scope.clone(), + parent: parent.clone(), + }); + let outcomes = self + .fetch_refresh(request, plan, &work, page.supporting_until, authority) + .await; + let mut session = self.session().await?; + let transaction = self + .transaction(&mut session, claims, request.deadline) + .await?; + let current = + active_scope(&transaction, plan.source, plan.projection.id.as_str()).await?; + if current.epoch != scope.epoch { + return Err(ExternalViewError::Unavailable); + } + if let (Some(parent), Some((_, selected_generation))) = (&plan.label_parent, &parent) { + let parent_is_current = self + .lock_scope(&transaction, parent.source, parent.projection.id.as_str()) + .await + .is_ok_and(|(_, generation)| &generation == selected_generation); + if !parent_is_current + || page + .supporting_until + .is_some_and(|cutoff| Utc::now().timestamp() >= cutoff) + { + for work in &work { + store::fail_refresh(&transaction, &work.ticket, RefreshFailure::Abandoned) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + } + transaction + .commit() + .await + .map_err(|_| ExternalViewError::Unavailable)?; + session.disarm(); + return Err(ExternalViewError::Unavailable); + } + } + for (work, result) in work.into_iter().zip(outcomes) { + let accepted = match result { + RefreshResult::Accepted(snapshot) => { + let root_ids = work.roots.into_iter().collect::>(); + // The RLS recheck and publication share a statement + // snapshot. A detached association does not erase a + // shared fact still used by another root. + match store::publish_for_roots( + &transaction, + &work.ticket, + &snapshot, + &page.eligibility, + &root_ids, + ) + .await + { + Ok(store::Publication::Published | store::Publication::Replay) => true, + Ok(store::Publication::Ineligible) + | Err(store::ProjectionError::StaleRefresh) => { + store::fail_refresh( + &transaction, + &work.ticket, + RefreshFailure::Abandoned, + ) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + false + } + Err(_) => return Err(ExternalViewError::Unavailable), + } + } + RefreshResult::Failed(failure) => { + store::fail_refresh(&transaction, &work.ticket, failure) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + false + } + }; + if accepted { + published += 1; + } else { + unavailable += 1; + } + } + transaction + .commit() + .await + .map_err(|_| ExternalViewError::Unavailable)?; + session.disarm(); + } + let expires_at = plan.grant_valid_until().timestamp(); + let mut body = json!({"processed": page.rows.len(), "published": published, "reused": reused, "unavailable": unavailable}); + if let Some(after) = page.next { + body["nextCursor"] = json!(self.next_cursor( + binding, + query, + Position::Refresh { + after, + minimum_sequence + }, + expires_at + )?); + } + Ok(HeldResponse { body, expires_at }) + } + + async fn fetch_refresh( + &self, + request: &ExternalViewRequest, + plan: &Plan<'_>, + work: &[RefreshWork], + supporting_until: Option, + authority: Arc, + ) -> Vec { + let approved_fields = plan + .projection + .fields + .iter() + .map(|field| field.as_str().to_owned()) + .collect(); + let expires_at = plan.source.grant_valid_until().timestamp(); + let mut results = Vec::with_capacity(work.len()); + match plan.projection.cardinality { + ExternalCardinality::One => { + for chunk in work.chunks(plan.source.maximum_batch_keys().max(1)) { + let observed_at = Utc::now().timestamp(); + let acquired = self + .acquire( + request, + plan, + ProjectionRequest::Batch { + keys: chunk.iter().map(|item| item.key.clone()).collect(), + }, + &approved_fields, + supporting_until, + Some(Arc::clone(&authority)), + ) + .await; + match acquired { + Ok(ProjectionResult::Batch { records }) => { + let mut records = records + .into_iter() + .map(|record| (record.key().to_owned(), record)) + .collect::>(); + results.extend(chunk.iter().map(|item| { + records.remove(&item.key).map_or( + RefreshResult::Failed(RefreshFailure::Incomplete), + |record| { + accepted_record( + &self.cursors, + plan.source.binding_fingerprint(), + record, + observed_at, + expires_at, + ) + }, + ) + })); + } + result => { + let failure = acquisition_failure(result.err()); + results.extend(chunk.iter().map(|_| RefreshResult::Failed(failure))); + } + } + } + } + ExternalCardinality::Many => { + for item in work { + let observed_at = Utc::now().timestamp(); + let acquired = self + .acquire( + request, + plan, + ProjectionRequest::ByParent { + parent_key: item.key.clone(), + }, + &approved_fields, + supporting_until, + Some(Arc::clone(&authority)), + ) + .await; + let result = match acquired { + Ok(ProjectionResult::ByParent { + parent_key, + snapshot, + rows, + }) if parent_key == item.key => { + let rows = rows + .into_iter() + .map(|row| ProjectionRow::new(&row.key, Value::Object(row.fields))) + .collect::, _>>(); + rows.and_then(|rows| { + let snapshot_id = snapshot_identity( + &self.cursors, + plan.source.binding_fingerprint(), + Some(&snapshot), + )?; + AcceptedSnapshot::complete_collection( + &item.key, + &snapshot_id, + observed_at, + expires_at, + RetentionPermission::PermittedUntil(expires_at), + rows, + ) + }) + .map_or( + RefreshResult::Failed(RefreshFailure::Incomplete), + RefreshResult::Accepted, + ) + } + result => RefreshResult::Failed(acquisition_failure(result.err())), + }; + results.push(result); + } + } + } + results + } +} + +fn accepted_record( + cursors: &CursorCodec, + binding: &str, + record: ProjectionRecord, + observed_at: i64, + expires_at: i64, +) -> RefreshResult { + let source_revision = match &record { + ProjectionRecord::Present { + source_revision, .. + } + | ProjectionRecord::AuthoritativeAbsent { + source_revision, .. + } => source_revision.as_deref(), + ProjectionRecord::Unavailable { .. } => None, + }; + let Ok(id) = snapshot_identity(cursors, binding, source_revision) else { + return RefreshResult::Failed(RefreshFailure::Incomplete); + }; + let retention = RetentionPermission::PermittedUntil(expires_at); + let snapshot = match record { + ProjectionRecord::Present { key, fields, .. } => { + ProjectionRow::new(&key, Value::Object(fields)).and_then(|row| { + AcceptedSnapshot::one(&key, &id, observed_at, expires_at, retention, row) + }) + } + ProjectionRecord::AuthoritativeAbsent { key, .. } => { + AcceptedSnapshot::authorized_absence(&key, &id, observed_at, expires_at, retention) + } + ProjectionRecord::Unavailable { .. } => { + return RefreshResult::Failed(RefreshFailure::Unavailable) + } + }; + snapshot.map_or( + RefreshResult::Failed(RefreshFailure::Incomplete), + RefreshResult::Accepted, + ) +} + +fn acquisition_failure(error: Option) -> RefreshFailure { + match error { + Some(ExternalSourceError::Grant) => RefreshFailure::Denied, + Some(ExternalSourceError::Acquisition( + AdapterProjectionError::NoStore | AdapterProjectionError::RetentionRefused, + )) => RefreshFailure::RetentionRefused, + Some( + ExternalSourceError::Unbound + | ExternalSourceError::Configuration + | ExternalSourceError::Artifact + | ExternalSourceError::Contract, + ) => RefreshFailure::InvalidBinding, + Some(ExternalSourceError::Result) => RefreshFailure::Incomplete, + _ => RefreshFailure::Unavailable, + } +} + +fn snapshot_identity( + cursors: &CursorCodec, + binding: &str, + source_revision: Option<&str>, +) -> std::result::Result { + // Every fresh acquisition has its own receipt. Preserve a scoped, + // keyed reference to an approved source revision when one was supplied. + let capture = Uuid::new_v4(); + let Some(source_revision) = source_revision else { + return Ok(capture.to_string()); + }; + let reference = cursors + .binding_digest( + b"breg-external-source-revision-v1", + &json!([binding, source_revision]), + ) + .map_err(|_| store::ProjectionError::InvalidSnapshot)?; + Ok(format!("{capture}:{reference}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_new_capture_preserves_a_scoped_source_revision_reference() { + let codec = CursorCodec::new( + zeroize::Zeroizing::new(vec![42; 32]), + Duration::from_secs(120), + ) + .unwrap(); + let revision = "synthetic-source-revision"; + let first = snapshot_identity(&codec, "binding-a", Some(revision)).unwrap(); + let second = snapshot_identity(&codec, "binding-a", Some(revision)).unwrap(); + let third = snapshot_identity(&codec, "binding-b", Some(revision)).unwrap(); + assert_ne!(first, second); + assert!(!first.contains(revision)); + assert_eq!( + first.split_once(':').unwrap().1, + second.split_once(':').unwrap().1 + ); + assert_ne!( + first.split_once(':').unwrap().1, + third.split_once(':').unwrap().1 + ); + assert!(Uuid::parse_str(&snapshot_identity(&codec, "binding-a", None).unwrap()).is_ok()); + } +} diff --git a/crates/registry-breg/src/postgres/external_view_query.rs b/crates/registry-breg/src/postgres/external_view_query.rs new file mode 100644 index 0000000000..ff03b48bb9 --- /dev/null +++ b/crates/registry-breg/src/postgres/external_view_query.rs @@ -0,0 +1,1103 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Indexed external views over the selected profile's ordinary root RLS. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; + +use serde::Deserialize; +use serde_json::Value; +use tokio_postgres::types::Type; +use uuid::Uuid; + +use crate::contract::Operation; +use crate::external_views::{ExternalCardinality, ExternalFieldType, ExternalProjectionMode}; +use crate::model::CompiledRegistry; + +use super::external_projection::{ProjectionGeneration, ProjectionScope}; +use super::read::ReadRelations; +use super::GuardedTransaction; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +pub(crate) enum ExternalQueryError { + #[error("external view is unavailable")] + Unavailable, + #[error("external view query is not permitted")] + NotPermitted, + #[error("external view query is invalid")] + InvalidQuery, + #[error("external view generation changed; restart the query")] + GenerationChanged, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum SortDirection { + Ascending, + Descending, +} + +impl SortDirection { + fn sql(self) -> &'static str { + match self { + Self::Ascending => "ASC", + Self::Descending => "DESC", + } + } + fn comparison(self) -> &'static str { + match self { + Self::Ascending => ">", + Self::Descending => "<", + } + } +} + +pub(crate) struct ExactFilter { + pub(crate) field: String, + pub(crate) value: Value, +} + +pub(crate) struct ExternalOrder { + pub(crate) field: String, + pub(crate) direction: SortDirection, +} + +/// Internal cursor material. The source key and sort value must only leave +/// BReg inside its encrypted, caller/query/generation-bound cursor. +pub(crate) struct PageAfter { + pub(crate) root_id: Uuid, + pub(crate) source_key: String, + pub(crate) sort_value: Option, +} + +impl fmt::Debug for PageAfter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("PageAfter()") + } +} + +pub(crate) struct LabelScope<'a> { + pub(crate) scope: &'a ProjectionScope, + pub(crate) generation: ProjectionGeneration, +} + +pub(crate) struct IndexedViewQuery<'a> { + pub(crate) profile_id: &'a str, + pub(crate) view_id: &'a str, + pub(crate) link_id: &'a str, + pub(crate) scope: &'a ProjectionScope, + /// Obtained with `lock_generation` in this same transaction. + pub(crate) generation: ProjectionGeneration, + pub(crate) label_scope: Option>, + pub(crate) now: i64, + pub(crate) root_id: Option, + pub(crate) fields: BTreeSet, + pub(crate) filter: Option, + pub(crate) order: Option, + pub(crate) include_count: bool, + pub(crate) page_size: u16, + pub(crate) after: Option, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum ExternalAvailability { + Available, + Absent, + Unavailable, +} + +#[derive(Deserialize)] +pub(crate) struct ExternalLabel { + pub(crate) availability: ExternalAvailability, + pub(crate) fields: Value, +} + +#[derive(Deserialize)] +pub(crate) struct ExternalViewRow { + pub(crate) root_id: Uuid, + /// Internal stable row identity, not an automatically disclosed field. + pub(crate) source_key: String, + pub(crate) availability: ExternalAvailability, + pub(crate) fields: Value, + pub(crate) label: Option, + sort_value: Option, +} + +impl fmt::Debug for ExternalViewRow { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("ExternalViewRow()") + } +} + +pub(crate) struct IndexedViewPage { + pub(crate) complete: bool, + /// A requested local root outside the selected profile remains concealed. + pub(crate) root_found: bool, + pub(crate) rows: Vec, + pub(crate) total_count: Option, + pub(crate) continuation: Option, + /// Recheck this bound after terminal audit and before releasing values. + pub(crate) minimum_expires_at: Option, +} + +/// Root authority is installed by `begin_record_transaction`. This function +/// rechecks all named view and query grants, and derives relations solely from +/// the compiled root inventory. It never accepts raw SQL or a source key. +pub(crate) async fn query( + transaction: &GuardedTransaction<'_>, + registry: &CompiledRegistry, + request: IndexedViewQuery<'_>, +) -> Result { + let statement = compile(registry, &request)?; + let parameters = statement + .parameters + .iter() + .map(|value| { + ( + value as &(dyn tokio_postgres::types::ToSql + Sync), + Type::TEXT, + ) + }) + .collect::>(); + let row = transaction + .transaction() + .query_typed_one(&statement.sql, ¶meters) + .await + .map_err(|_| ExternalQueryError::Unavailable)?; + if !row.get::<_, bool>(0) { + return Err(ExternalQueryError::GenerationChanged); + } + let complete: bool = row.get(1); + let root_found: bool = row.get(2); + let minimum_expires_at = row.get(3); + let count: Option = row.get(4); + let mut rows: Vec = + serde_json::from_value(row.get(5)).map_err(|_| ExternalQueryError::Unavailable)?; + let has_more = rows.len() > usize::from(request.page_size); + rows.truncate(usize::from(request.page_size)); + let continuation = if has_more { + rows.last().map(|row| PageAfter { + root_id: row.root_id, + source_key: row.source_key.clone(), + sort_value: row.sort_value.clone(), + }) + } else { + None + }; + Ok(IndexedViewPage { + complete, + root_found, + rows, + total_count: count + .map(u64::try_from) + .transpose() + .map_err(|_| ExternalQueryError::Unavailable)?, + continuation, + minimum_expires_at, + }) +} + +struct QueryStatement { + sql: String, + parameters: Vec>, +} + +fn compile( + registry: &CompiledRegistry, + request: &IndexedViewQuery<'_>, +) -> Result { + if !(1..=1000).contains(&request.page_size) || request.now < 0 { + return Err(ExternalQueryError::InvalidQuery); + } + let inventory = registry.external_views(); + let view = inventory + .views + .get(request.view_id) + .ok_or(ExternalQueryError::NotPermitted)?; + let link = view + .links + .iter() + .find(|link| link.id.as_str() == request.link_id) + .ok_or(ExternalQueryError::NotPermitted)?; + let grant = inventory + .grants + .get(request.profile_id) + .and_then(|grants| grants.get(request.view_id)) + .and_then(|grant| { + grant + .links + .iter() + .find(|grant| grant.link.as_str() == request.link_id) + }) + .ok_or(ExternalQueryError::NotPermitted)?; + let projection = inventory + .projections + .get(link.projection.as_str()) + .ok_or(ExternalQueryError::NotPermitted)?; + let external_type = inventory + .record_types + .get(projection.record_type.as_str()) + .ok_or(ExternalQueryError::NotPermitted)?; + let entity = registry + .entities() + .get(view.record_type.as_str()) + .ok_or(ExternalQueryError::NotPermitted)?; + let profile = entity + .access_profiles + .get(request.profile_id) + .ok_or(ExternalQueryError::NotPermitted)?; + let operation = if request.root_id.is_some() { + Operation::Get + } else { + Operation::List + }; + if !profile.operations.contains(&operation) + || projection.mode != ExternalProjectionMode::Indexed + || request.scope.projection_id != projection.id.as_str() + || request.generation.epoch != request.scope.epoch + || (projection.cardinality == ExternalCardinality::Many && request.root_id.is_none()) + || request.fields.iter().any(|field| { + !grant + .readable_fields + .iter() + .any(|allowed| allowed.as_str() == field) + }) + || (request.include_count && !grant.allow_count) + { + return Err(ExternalQueryError::NotPermitted); + } + let field_types = external_type + .fields + .iter() + .map(|field| (field.id.to_string(), field.value_type)) + .collect::>(); + if let Some(filter) = &request.filter { + if !grant + .filterable_fields + .iter() + .any(|field| field.as_str() == filter.field) + || !grant + .readable_fields + .iter() + .any(|field| field.as_str() == filter.field) + { + return Err(ExternalQueryError::NotPermitted); + } + } + if let Some(order) = &request.order { + if !grant + .sortable_fields + .iter() + .any(|field| field.as_str() == order.field) + || !grant + .readable_fields + .iter() + .any(|field| field.as_str() == order.field) + { + return Err(ExternalQueryError::NotPermitted); + } + } + let key_field = link.from_field.as_str(); + if key_field != "id" + && !entity + .stored_fields + .iter() + .any(|field| field.logical.id == key_field) + { + return Err(ExternalQueryError::InvalidQuery); + } + let relations = ReadRelations::statistics(entity, &[key_field.to_owned()]) + .map_err(|_| ExternalQueryError::Unavailable)?; + let key = relations + .field_expression(entity, key_field) + .map_err(|_| ExternalQueryError::Unavailable)?; + if key.encrypted { + return Err(ExternalQueryError::NotPermitted); + } + let mut parameters = vec![]; + let profile_param = parameter(&mut parameters, Some(request.profile_id.to_owned())); + let root_param = parameter(&mut parameters, request.root_id.map(|id| id.to_string())); + let now = parameter(&mut parameters, Some(request.now.to_string())); + let scope = scope_predicate("s", request.scope, request.generation, &mut parameters); + let root_sql = format!("SELECT {} AS root_id,({})::text AS subject_key FROM {} WHERE current_setting('registry.access_profile',true)={profile_param} AND ({root_param}::uuid IS NULL OR {}={root_param}::uuid)",relations.id_expression,key.sql,relations.from_sql,relations.id_expression); + let many = projection.cardinality == ExternalCardinality::Many; + let expected_kind = if many { + "o.kind='complete-collection'" + } else { + "o.kind IN ('one','absent')" + }; + let selected = project_fields("data", &request.fields); + let filter = if let Some(filter) = &request.filter { + let kind = *field_types + .get(&filter.field) + .ok_or(ExternalQueryError::InvalidQuery)?; + let expression = field_expression("data", &filter.field, kind); + match scalar(&filter.value, kind)? { + None => format!("{expression} IS NULL"), + Some(value) => format!( + "{expression}={}::{}", + parameter(&mut parameters, Some(value)), + sql_type(kind) + ), + } + } else { + "true".to_owned() + }; + let order_type = request + .order + .as_ref() + .map(|order| { + field_types + .get(&order.field) + .copied() + .ok_or(ExternalQueryError::InvalidQuery) + }) + .transpose()?; + let sort_expression = match (&request.order, order_type) { + (Some(order), Some(kind)) => field_expression("data", &order.field, kind), + _ => "NULL::text".to_owned(), + }; + let direction = request + .order + .as_ref() + .map_or(SortDirection::Ascending, |order| order.direction); + let order_sql = if request.order.is_some() { + format!( + "sort_key {} NULLS LAST,root_id ASC,source_key COLLATE \"C\" ASC", + direction.sql() + ) + } else { + "root_id ASC,source_key COLLATE \"C\" ASC".to_owned() + }; + let after = continuation_sql( + request + .after + .as_ref() + .map(|after| (after, order_type.map(|kind| ("sort_key", kind, direction)))), + &mut parameters, + )?; + let require_complete = + request.filter.is_some() || request.order.is_some() || request.include_count || many; + let limit = parameter( + &mut parameters, + Some((u32::from(request.page_size) + 1).to_string()), + ); + let count = if request.include_count { + "CASE WHEN coverage.complete THEN (SELECT count(*) FROM filtered) ELSE NULL END" + } else { + "NULL::bigint" + }; + + let (label_state, label_join, label_value, label_valid, label_expiry) = if let Some( + label_scope, + ) = + &request.label_scope + { + let label = link + .label + .as_ref() + .ok_or(ExternalQueryError::NotPermitted)?; + if grant.label_fields.is_empty() + || label_scope.scope.projection_id != label.projection.as_str() + || label_scope.generation.epoch != label_scope.scope.epoch + { + return Err(ExternalQueryError::NotPermitted); + } + let label_projection = inventory + .projections + .get(label.projection.as_str()) + .ok_or(ExternalQueryError::NotPermitted)?; + if label_projection.mode != ExternalProjectionMode::Indexed + || label_projection.cardinality != ExternalCardinality::One + { + return Err(ExternalQueryError::NotPermitted); + } + let predicate = scope_predicate( + "s", + label_scope.scope, + label_scope.generation, + &mut parameters, + ); + let label_fields = grant.label_fields.iter().map(ToString::to_string).collect(); + let from = literal(label.from_field.as_str()); + ( + format!(",label_state AS MATERIALIZED (SELECT * FROM registry_internal.registry_external_projection_scopes AS s WHERE {predicate})"), + format!("LEFT JOIN label_state AS ls ON true LEFT JOIN registry_internal.registry_external_projection_snapshots AS lo ON lo.projection_id=ls.projection_id AND lo.epoch=ls.epoch AND lo.subject_key=matched.data->>{from} AND lo.expires_at>{now}::bigint AND lo.kind IN ('one','absent') LEFT JOIN registry_internal.registry_external_projection_values AS lv ON lv.projection_id=lo.projection_id AND lv.epoch=lo.epoch AND lv.subject_key=lo.subject_key AND lv.source_key=lo.subject_key"), + format!("CASE WHEN matched.data->>{from} IS NULL THEN NULL::jsonb ELSE jsonb_build_object('availability',CASE WHEN lo.kind='one' THEN 'available' WHEN lo.kind='absent' THEN 'absent' ELSE 'unavailable' END,'fields',{}) END",project_fields("lv.fields",&label_fields)), + String::new(), + "lo.expires_at".to_owned(), + ) + } else { + let label_value = match &link.label { + Some(label) if !grant.label_fields.is_empty() => { + let from = literal(label.from_field.as_str()); + let fields = grant.label_fields.iter().map(ToString::to_string).collect(); + format!("CASE WHEN matched.data->>{from} IS NULL THEN NULL::jsonb ELSE jsonb_build_object('availability','unavailable','fields',{}) END",project_fields("NULL::jsonb",&fields)) + } + _ => "NULL::jsonb".to_owned(), + }; + ( + String::new(), + String::new(), + label_value, + String::new(), + "NULL::bigint".to_owned(), + ) + }; + let membership = if many { + "JOIN registry_internal.registry_external_projection_values AS v ON v.projection_id=c.projection_id AND v.epoch=c.epoch AND v.subject_key=c.subject_key WHERE c.covered" + } else { + "LEFT JOIN registry_internal.registry_external_projection_values AS v ON v.projection_id=c.projection_id AND v.epoch=c.epoch AND v.subject_key=c.subject_key AND v.source_key=c.subject_key AND c.covered" + }; + let sql = format!( + r#"WITH projection_state AS MATERIALIZED ( + SELECT * FROM registry_internal.registry_external_projection_scopes AS s WHERE {scope} + ){label_state}, authorized_roots AS MATERIALIZED ({root_sql}), + candidates AS MATERIALIZED ( + SELECT r.root_id,r.subject_key,s.projection_id,s.epoch,o.kind,o.expires_at, + COALESCE(o.expires_at>{now}::bigint AND {expected_kind},false) AS covered + FROM authorized_roots AS r LEFT JOIN projection_state AS s ON true + LEFT JOIN registry_internal.registry_external_projection_snapshots AS o + ON o.projection_id=s.projection_id AND o.epoch=s.epoch AND o.subject_key=r.subject_key + ), coverage AS MATERIALIZED ( + SELECT COALESCE(bool_and(covered),true) AND EXISTS (SELECT 1 FROM projection_state) AS complete, + count(*)>0 AS root_found,min(expires_at) FILTER (WHERE covered) AS expires_at FROM candidates + ), matched AS MATERIALIZED ( + SELECT c.root_id,COALESCE(v.source_key,'') AS source_key, + CASE WHEN c.covered AND c.kind='absent' THEN 'absent' WHEN c.covered THEN 'available' ELSE 'unavailable' END AS availability, + CASE WHEN c.covered THEN v.fields ELSE NULL::jsonb END AS data + FROM candidates AS c {membership} + ), filtered AS MATERIALIZED ( + SELECT *,{sort_expression} AS sort_key FROM matched WHERE {filter} + ), page AS MATERIALIZED ( + SELECT *,row_number() OVER (ORDER BY {order_sql}) AS ordinal + FROM filtered WHERE {after} AND (NOT {require_complete} OR (SELECT complete FROM coverage)) + ORDER BY {order_sql} LIMIT {limit}::bigint + ), displayed AS ( + SELECT matched.ordinal,{label_expiry} AS label_expires_at, + jsonb_build_object('root_id',matched.root_id,'source_key',matched.source_key, + 'availability',matched.availability,'fields',{selected},'label',{label_value},'sort_value',matched.sort_key::text) AS payload + FROM page AS matched {label_join} + ) SELECT EXISTS (SELECT 1 FROM projection_state){label_valid},coverage.complete,coverage.root_found, + LEAST(coverage.expires_at,(SELECT min(label_expires_at) FROM displayed)),{count}, + COALESCE((SELECT jsonb_agg(payload ORDER BY ordinal) FROM displayed),'[]'::jsonb) + FROM coverage"# + ); + Ok(QueryStatement { sql, parameters }) +} + +fn parameter(parameters: &mut Vec>, value: Option) -> String { + parameters.push(value); + format!("${}::text", parameters.len()) +} + +fn scope_predicate( + alias: &str, + scope: &ProjectionScope, + generation: ProjectionGeneration, + parameters: &mut Vec>, +) -> String { + let id = parameter(parameters, Some(scope.projection_id.clone())); + let epoch = parameter(parameters, Some(scope.epoch.to_string())); + let binding = parameter(parameters, Some(scope.binding_digest.clone())); + let grant = parameter(parameters, Some(scope.grant_digest.clone())); + let version = parameter(parameters, Some(generation.generation.to_string())); + format!("{alias}.projection_id={id} AND {alias}.epoch={epoch}::uuid AND {alias}.binding_digest={binding} AND {alias}.grant_digest={grant} AND {alias}.generation={version}::bigint AND {alias}.active") +} + +fn literal(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} + +fn project_fields(alias: &str, fields: &BTreeSet) -> String { + if fields.is_empty() { + return "'{}'::jsonb".to_owned(); + } + let fields = fields + .iter() + .map(|field| format!("{},({alias})->{}", literal(field), literal(field))) + .collect::>() + .join(","); + format!("CASE WHEN ({alias}) IS NULL THEN '{{}}'::jsonb ELSE jsonb_build_object({fields}) END") +} + +fn field_expression(alias: &str, field: &str, kind: ExternalFieldType) -> String { + let cast = sql_type(kind); + let expression = format!("(({alias})->>{})::{cast}", literal(field)); + if kind == ExternalFieldType::String { + format!("({expression}) COLLATE \"C\"") + } else { + expression + } +} + +fn sql_type(kind: ExternalFieldType) -> &'static str { + match kind { + ExternalFieldType::String => "text", + ExternalFieldType::Integer => "bigint", + ExternalFieldType::Boolean => "boolean", + ExternalFieldType::Decimal => "numeric", + ExternalFieldType::Date => "date", + ExternalFieldType::Uuid => "uuid", + } +} + +fn scalar(value: &Value, kind: ExternalFieldType) -> Result, ExternalQueryError> { + if value.is_null() { + return Ok(None); + } + let invalid = || ExternalQueryError::InvalidQuery; + let result = match kind { + ExternalFieldType::String => value + .as_str() + .filter(|s| s.len() <= 4096) + .map(str::to_owned) + .ok_or_else(invalid)?, + ExternalFieldType::Integer => value.as_i64().map(|v| v.to_string()).ok_or_else(invalid)?, + ExternalFieldType::Boolean => value.as_bool().map(|v| v.to_string()).ok_or_else(invalid)?, + ExternalFieldType::Decimal => { + let value = value.as_str().ok_or_else(invalid)?; + let digits = value.strip_prefix('-').unwrap_or(value); + let mut parts = digits.split('.'); + let integer = parts.next().unwrap_or(""); + let fraction = parts.next(); + if value.len() > 128 + || integer.is_empty() + || !integer.bytes().all(|b| b.is_ascii_digit()) + || fraction.is_some_and(|v| v.is_empty() || !v.bytes().all(|b| b.is_ascii_digit())) + || parts.next().is_some() + { + return Err(invalid()); + } + value.to_owned() + } + ExternalFieldType::Date => { + let value = value.as_str().ok_or_else(invalid)?; + if value.len() != 10 || chrono::NaiveDate::parse_from_str(value, "%Y-%m-%d").is_err() { + return Err(invalid()); + } + value.to_owned() + } + ExternalFieldType::Uuid => Uuid::parse_str(value.as_str().ok_or_else(invalid)?) + .map_err(|_| invalid())? + .to_string(), + }; + Ok(Some(result)) +} + +type ContinuationOrder<'a> = Option<(&'a str, ExternalFieldType, SortDirection)>; + +fn continuation_sql( + after: Option<(&PageAfter, ContinuationOrder<'_>)>, + parameters: &mut Vec>, +) -> Result { + let Some((after, order)) = after else { + return Ok("true".to_owned()); + }; + if after.source_key.len() > 1024 { + return Err(ExternalQueryError::InvalidQuery); + } + let root = parameter(parameters, Some(after.root_id.to_string())); + let key = parameter(parameters, Some(after.source_key.clone())); + let tie = format!("(root_id,source_key COLLATE \"C\")>({root}::uuid,{key} COLLATE \"C\")"); + let Some((column, kind, direction)) = order else { + if after.sort_value.is_some() { + return Err(ExternalQueryError::InvalidQuery); + } + return Ok(tie); + }; + match &after.sort_value { + None => Ok(format!("{column} IS NULL AND {tie}")), + Some(value) => { + let parsed = match kind { + ExternalFieldType::Integer => value + .parse::() + .map(Value::from) + .map_err(|_| ExternalQueryError::InvalidQuery)?, + ExternalFieldType::Boolean => value + .parse::() + .map(Value::from) + .map_err(|_| ExternalQueryError::InvalidQuery)?, + _ => Value::String(value.clone()), + }; + let value = scalar(&parsed, kind)?.ok_or(ExternalQueryError::InvalidQuery)?; + let value = parameter(parameters, Some(value)); + let cast = sql_type(kind); + Ok(format!("({column} {} {value}::{cast} OR {column} IS NULL OR ({column}={value}::{cast} AND {tie}))",direction.comparison())) + } + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use serde_json::json; + + pub(crate) fn registry() -> CompiledRegistry { + registry_with_external(true) + } + + pub(crate) fn registry_with_external(include_external: bool) -> CompiledRegistry { + let mut project = json!({ + "apiVersion":crate::compiler::AUTHORING_API_VERSION,"kind":"BRegProject", + "project":{"id":"external-query-test","version":"1","defaultLanguage":"en","canonicalBaseIri":"https://fixture.example.test"}, + "datasets":[{"id":"records"}], + "recordTypes":[{"id":"local-record","primaryDataset":"records","route":"records","mutationMode":"mutable","classification":"internal","fields":[ + {"id":"subject-key","type":"string","maximumLength":64,"classification":"internal"}, + {"id":"tenant","type":"string","maximumLength":32,"classification":"internal"} + ]}], + "externalRecordTypes":[{"id":"external-record","authority":"urn:test:authority","identifierScheme":"urn:test:keys","fields":[ + {"id":"label","type":"string","classification":"internal"}, + {"id":"hidden","type":"string","classification":"restricted"} + ]}], + "externalProjections":[{"id":"projection","recordType":"external-record","operation":"batch","operationVersion":"1","purpose":"consultation","scope":"eligible","selectorScheme":"urn:test:keys","mode":"indexed","cardinality":"one","cacheTtlSeconds":60,"fields":["label","hidden"]}], + "externalViews":[{"id":"view","recordType":"local-record","links":[{"id":"link","projection":"projection","fromField":"subject-key","identifierScheme":"urn:test:keys","fields":["label","hidden"]}]}], + "accessProfiles":[{"id":"reader","principalClaim":"registry_principal","requiredScopes":["registry.read"],"requiredPurposes":["consultation"],"permissions":{ + "records":[{"recordType":"local-record","operations":["get","list"],"allowCount":true,"rowBoundaries":[{"field":"tenant","claim":"tenant","operator":"equals"}]}], + "externalViews":[{"view":"view","links":[{"link":"link","readableFields":["label"],"filterableFields":["label"],"sortableFields":["label"],"allowCount":true}]}] + }}] + }); + if !include_external { + let root = project.as_object_mut().unwrap(); + root.remove("externalRecordTypes"); + root.remove("externalProjections"); + root.remove("externalViews"); + root.get_mut("accessProfiles") + .unwrap() + .as_array_mut() + .unwrap()[0]["permissions"] + .as_object_mut() + .unwrap() + .remove("externalViews"); + } + let project = + crate::contract::parse_project_json(&serde_json::to_vec(&project).unwrap()).unwrap(); + crate::compiler::compile_project(&project, &[], crate::compiler::CompileProfile::Authoring) + .unwrap() + } + + pub(super) fn request(scope: &ProjectionScope, generation: i64) -> IndexedViewQuery<'_> { + IndexedViewQuery { + profile_id: "reader", + view_id: "view", + link_id: "link", + scope, + generation: ProjectionGeneration { + epoch: scope.epoch, + generation, + }, + label_scope: None, + now: 1000, + root_id: None, + fields: BTreeSet::from(["label".to_owned()]), + filter: None, + order: None, + include_count: false, + page_size: 10, + after: None, + } + } + + #[test] + fn query_kernel_refuses_hidden_fields_in_selection_filter_and_order() { + let registry = registry(); + let scope = ProjectionScope::new("projection", Uuid::new_v4(), "binding", "grant", 60, 120) + .unwrap(); + let mut query = request(&scope, 0); + query.fields = BTreeSet::from(["hidden".to_owned()]); + assert!(matches!( + compile(®istry, &query), + Err(ExternalQueryError::NotPermitted) + )); + query.fields = BTreeSet::from(["label".to_owned()]); + query.filter = Some(ExactFilter { + field: "hidden".to_owned(), + value: json!("secret"), + }); + assert!(matches!( + compile(®istry, &query), + Err(ExternalQueryError::NotPermitted) + )); + query.filter = None; + query.order = Some(ExternalOrder { + field: "hidden".to_owned(), + direction: SortDirection::Ascending, + }); + assert!(matches!( + compile(®istry, &query), + Err(ExternalQueryError::NotPermitted) + )); + } + + #[test] + fn exact_values_are_typed_and_null_keeps_native_missing_semantics() { + assert_eq!( + scalar(&json!(null), ExternalFieldType::String).unwrap(), + None + ); + assert_eq!( + scalar(&json!(12), ExternalFieldType::Integer).unwrap(), + Some("12".into()) + ); + assert!(scalar(&json!("12"), ExternalFieldType::Integer).is_err()); + assert!(scalar(&json!("2024-02-30"), ExternalFieldType::Date).is_err()); + assert!(scalar(&json!("0; SELECT 1"), ExternalFieldType::Decimal).is_err()); + assert_eq!( + scalar(&json!("-12.50"), ExternalFieldType::Decimal).unwrap(), + Some("-12.50".into()) + ); + } + + #[test] + fn a_continuation_preserves_nulls_last_in_both_directions() { + let after = PageAfter { + root_id: Uuid::nil(), + source_key: "row".into(), + sort_value: Some("x".into()), + }; + let mut params = vec![]; + let asc = continuation_sql( + Some(( + &after, + Some(( + "sort_key", + ExternalFieldType::String, + SortDirection::Ascending, + )), + )), + &mut params, + ) + .unwrap(); + assert!(asc.contains("sort_key >")); + assert!(asc.contains("sort_key IS NULL")); + let desc = continuation_sql( + Some(( + &after, + Some(( + "sort_key", + ExternalFieldType::String, + SortDirection::Descending, + )), + )), + &mut params, + ) + .unwrap(); + assert!(desc.contains("sort_key <")); + assert!(desc.contains("sort_key IS NULL")); + } +} + +#[cfg(test)] +#[cfg(feature = "postgres-test")] +mod postgres_tests { + use super::super::external_projection::{configure_scope, install, INSTALL_SQL}; + use super::*; + use std::time::Duration; + use tokio_postgres::{Client, NoTls}; + + #[allow(dead_code)] + mod harness { + use crate as registry_breg; + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/support/postgres_harness.rs" + )); + } + + async fn run( + client: &Client, + registry: &CompiledRegistry, + request: &IndexedViewQuery<'_>, + ) -> tokio_postgres::Row { + run_statement(client, &compile(registry, request).unwrap()).await + } + + async fn run_statement(client: &Client, statement: &QueryStatement) -> tokio_postgres::Row { + let parameters = statement + .parameters + .iter() + .map(|value| { + ( + value as &(dyn tokio_postgres::types::ToSql + Sync), + Type::TEXT, + ) + }) + .collect::>(); + client + .query_typed_one(&statement.sql, ¶meters) + .await + .expect("compiled external SQL executes") + } + + async fn publish( + client: &Client, + scope: &ProjectionScope, + key: &str, + label: Option<&str>, + id: &str, + ) { + let sequence: i64 = client + .query_one( + "SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,$5)", + &[ + &scope.projection_id, + &scope.epoch, + &scope.binding_digest, + &scope.grant_digest, + &key, + ], + ) + .await + .unwrap() + .get(0); + let kind = if label.is_some() { "one" } else { "absent" }; + let rows = label + .map(|label| serde_json::json!([{"source_key":key,"fields":{"label":label}}])) + .unwrap_or_else(|| serde_json::json!([])); + let result:String=client.query_one("SELECT registry_internal.publish_external_projection($1,$2,$3,$4,$5,$6,$7,999,1059,1119,$8,$9)",&[&scope.projection_id,&scope.epoch,&scope.binding_digest,&scope.grant_digest,&sequence,&key,&id,&kind,&rows]).await.unwrap().get(0); + assert_eq!(result, "published"); + } + + #[tokio::test] + async fn root_rls_coverage_filter_count_and_bidirectional_null_paging_share_one_population() { + let database = harness::TestDatabase::create(2).await; + let (migration, migration_task) = database.connect_migration().await; + let registry = super::tests::registry(); + super::super::install_compiled_schema(&migration, ®istry, &database.runtime_role) + .await + .unwrap(); + install(&migration, &database.runtime_role).await.unwrap(); + migration + .batch_execute(&INSTALL_SQL.replace( + "floor(extract(epoch FROM clock_timestamp()))::bigint", + "1000", + )) + .await + .unwrap(); + let scope = ProjectionScope::new("projection", Uuid::new_v4(), "binding", "grant", 60, 120) + .unwrap(); + configure_scope(&migration, &scope).await.unwrap(); + let entity = ®istry.entities()["local-record"]; + let key = &entity + .stored_fields + .iter() + .find(|field| field.logical.id == "subject-key") + .unwrap() + .physical_name; + let tenant = &entity + .stored_fields + .iter() + .find(|field| field.logical.id == "tenant") + .unwrap() + .physical_name; + database.admin.batch_execute(&format!("INSERT INTO registry_data.\"{}\" (record_id,active_package_revision,\"{key}\",\"{tenant}\") VALUES ('00000000-0000-4000-8000-000000000001','fixture','first','allowed'),('00000000-0000-4000-8000-000000000002','fixture','absent','allowed'),('00000000-0000-4000-8000-000000000003','fixture','uncovered','allowed'),('00000000-0000-4000-8000-000000000004','fixture','hidden','other')",entity.physical_table)).await.unwrap(); + publish(&migration, &scope, "first", Some("A"), "first").await; + publish(&migration, &scope, "absent", None, "absent").await; + let (runtime, connection) = database + .runtime_config + .postgres() + .connect(NoTls) + .await + .unwrap(); + let runtime_task = tokio::spawn(async move { + connection.await.expect("query fixture connection"); + }); + runtime.batch_execute("SET registry.access_profile='reader'; SET registry.principal='staff'; SET registry.purpose='consultation'; SET registry.row_boundaries='[{\"field\":\"tenant\",\"operator\":\"equals\",\"values\":[\"allowed\"]}]';").await.unwrap(); + let mut request = super::tests::request(&scope, 2); + request.include_count = true; + request.filter = Some(ExactFilter { + field: "label".to_owned(), + value: Value::String("A".to_owned()), + }); + let incomplete = run(&runtime, ®istry, &request).await; + assert!(!incomplete.get::<_, bool>(1)); + assert_eq!(incomplete.get::<_, Option>(4), None); + assert_eq!(incomplete.get::<_, Value>(5), serde_json::json!([])); + publish(&migration, &scope, "uncovered", Some("B"), "third").await; + request.generation.generation = 3; + let complete = run(&runtime, ®istry, &request).await; + assert!(complete.get::<_, bool>(1)); + assert_eq!(complete.get::<_, Option>(4), Some(1)); + let rows: Vec = serde_json::from_value(complete.get(5)).unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].fields["label"], "A"); + request.filter = None; + request.order = Some(ExternalOrder { + field: "label".to_owned(), + direction: SortDirection::Descending, + }); + request.page_size = 1; + let first = run(&runtime, ®istry, &request).await; + assert_eq!(first.get::<_, Option>(4), Some(3)); + let rows: Vec = serde_json::from_value(first.get(5)).unwrap(); + assert_eq!(rows[0].fields["label"], "B"); + request.after = Some(PageAfter { + root_id: rows[0].root_id, + source_key: rows[0].source_key.clone(), + sort_value: rows[0].sort_value.clone(), + }); + let second = run(&runtime, ®istry, &request).await; + let rows: Vec = serde_json::from_value(second.get(5)).unwrap(); + assert_eq!(rows[0].fields["label"], "A"); + request.after = Some(PageAfter { + root_id: rows[0].root_id, + source_key: rows[0].source_key.clone(), + sort_value: rows[0].sort_value.clone(), + }); + let last = run(&runtime, ®istry, &request).await; + let rows: Vec = serde_json::from_value(last.get(5)).unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].availability, ExternalAvailability::Absent); + assert_eq!(rows[0].fields, serde_json::json!({})); + request.after = None; + request.order.as_mut().unwrap().direction = SortDirection::Ascending; + let ascending = run(&runtime, ®istry, &request).await; + let rows: Vec = serde_json::from_value(ascending.get(5)).unwrap(); + assert_eq!(rows[0].fields["label"], "A"); + // Pause the compiled query after it has read coverage. PostgreSQL + // must use that statement's snapshot even after another transaction + // commits a root that has no projection. + request.page_size = 10; + let mut gated = compile(®istry, &request).unwrap(); + assert!(gated.sql.contains("coverage AS MATERIALIZED (")); + assert!(gated.sql.contains("), matched AS MATERIALIZED (")); + gated.sql = gated + .sql + .replacen( + "coverage AS MATERIALIZED (", + "coverage_snapshot AS MATERIALIZED (", + 1, + ) + .replacen( + "), matched AS MATERIALIZED (", + "), coverage AS MATERIALIZED ( + SELECT coverage_snapshot.*, pg_catalog.pg_advisory_xact_lock(732995003) + FROM coverage_snapshot + ), matched AS MATERIALIZED (", + 1, + ); + let reader_pid: i32 = runtime + .query_one("SELECT pg_backend_pid()", &[]) + .await + .unwrap() + .get(0); + migration + .execute("SELECT pg_advisory_lock(732995003)", &[]) + .await + .unwrap(); + let insert_uncovered_root = async { + let waited = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let waiting: bool = database + .admin + .query_one( + "SELECT EXISTS (SELECT 1 FROM pg_locks + WHERE locktype='advisory' AND NOT granted AND pid=$1)", + &[&reader_pid], + ) + .await + .unwrap() + .get(0); + if waiting { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .is_ok(); + if waited { + database.admin.execute( + &format!("INSERT INTO registry_data.\"{}\" (record_id,active_package_revision,\"{key}\",\"{tenant}\") + VALUES ('00000000-0000-4000-8000-000000000006','fixture','new-uncovered','allowed')", entity.physical_table), + &[], + ).await.unwrap(); + } + migration + .execute("SELECT pg_advisory_unlock(732995003)", &[]) + .await + .unwrap(); + assert!(waited, "the query reached the coverage barrier"); + }; + let (racing_page, ()) = + tokio::join!(run_statement(&runtime, &gated), insert_uncovered_root); + assert!(racing_page.get::<_, bool>(1)); + assert_eq!(racing_page.get::<_, Option>(4), Some(3)); + let rows: Vec = serde_json::from_value(racing_page.get(5)).unwrap(); + assert_eq!(rows.len(), 3); + assert_eq!(rows[0].fields["label"], "A"); + let after_commit = run(&runtime, ®istry, &request).await; + assert!(!after_commit.get::<_, bool>(1)); + assert_eq!(after_commit.get::<_, Option>(4), None); + assert_eq!(after_commit.get::<_, Value>(5), serde_json::json!([])); + + // The eligibility read and publication share the same statement and + // real generated source relation, including its forced root RLS. + let relations = ReadRelations::statistics(entity, &["subject-key".to_owned()]).unwrap(); + let subject = relations.field_expression(entity, "subject-key").unwrap(); + let eligibility=format!("SELECT 1 FROM {} WHERE {}=ANY($13::uuid[]) AND ({})::text=$6::text AND current_setting('registry.access_profile',true)='reader'",relations.from_sql,relations.id_expression,subject.sql); + let sql = super::super::external_projection::publication_statement(&eligibility); + let ticket:i64=runtime.query_one("SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,'first')",&[&scope.projection_id,&scope.epoch,&scope.binding_digest,&scope.grant_digest]).await.unwrap().get(0); + let fields = serde_json::json!([{"source_key":"first","fields":{"label":"retained"}}]); + let roots = vec![Uuid::parse_str("00000000-0000-4000-8000-000000000001").unwrap()]; + let result: String = runtime + .query_one( + &sql, + &[ + &scope.projection_id, + &scope.epoch, + &scope.binding_digest, + &scope.grant_digest, + &ticket, + &"first", + &"association", + &1000_i64, + &1060_i64, + &1120_i64, + &"one", + &fields, + &roots, + ], + ) + .await + .unwrap() + .get(0); + assert_eq!(result, "published"); + let ticket:i64=runtime.query_one("SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,'first')",&[&scope.projection_id,&scope.epoch,&scope.binding_digest,&scope.grant_digest]).await.unwrap().get(0); + database.admin.batch_execute(&format!("INSERT INTO registry_data.\"{}\" (record_id,active_package_revision,\"{key}\",\"{tenant}\") VALUES ('00000000-0000-4000-8000-000000000005','fixture','first','allowed'); UPDATE registry_data.\"{}\" SET \"{key}\"='detached' WHERE record_id='00000000-0000-4000-8000-000000000001'; UPDATE registry_data.\"{}\" SET \"{key}\"='first' WHERE record_id='00000000-0000-4000-8000-000000000004';",entity.physical_table,entity.physical_table,entity.physical_table)).await.unwrap(); + for id in [ + "00000000-0000-4000-8000-000000000001", + "00000000-0000-4000-8000-000000000004", + ] { + let roots = vec![Uuid::parse_str(id).unwrap()]; + let result: String = runtime + .query_one( + &sql, + &[ + &scope.projection_id, + &scope.epoch, + &scope.binding_digest, + &scope.grant_digest, + &ticket, + &"first", + &"rejected", + &1000_i64, + &1060_i64, + &1120_i64, + &"one", + &fields, + &roots, + ], + ) + .await + .unwrap() + .get(0); + assert_eq!(result, "ineligible"); + } + assert_eq!(runtime.query_one("SELECT fields->>'label' FROM registry_internal.registry_external_projection_values WHERE projection_id='projection' AND subject_key='first'",&[]).await.unwrap().get::<_,String>(0),"retained"); + runtime_task.abort(); + migration_task.abort(); + drop(runtime); + drop(migration); + database.cleanup().await; + } +} diff --git a/crates/registry-breg/src/postgres/interlock.rs b/crates/registry-breg/src/postgres/interlock.rs index dcfd91de0d..dc33452122 100644 --- a/crates/registry-breg/src/postgres/interlock.rs +++ b/crates/registry-breg/src/postgres/interlock.rs @@ -2833,6 +2833,9 @@ async fn reconcile_runtime_acl_in( .await .map_err(|_| PostgresKernelError::Connection)?; } + super::external_projection::reconcile(transaction, registry, runtime_role) + .await + .map_err(super::external_projection::kernel_error)?; super::timer::install_record_timer_schema(transaction, registry, runtime_role).await?; reconcile_compiled_runtime_acl(transaction, registry, runtime_role).await } @@ -3039,6 +3042,9 @@ async fn execute_initial_package_ddl_in( ) .await?; } + super::external_projection::reconcile(transaction, registry, runtime_role) + .await + .map_err(super::external_projection::kernel_error)?; super::timer::install_record_timer_schema(transaction, registry, runtime_role).await?; reconcile_compiled_runtime_acl(transaction, registry, runtime_role).await?; super::timer::initialize_record_timer_bindings(transaction, registry, package_revision).await?; diff --git a/crates/registry-breg/src/postgres/mod.rs b/crates/registry-breg/src/postgres/mod.rs index 122691e13b..fc2e195920 100644 --- a/crates/registry-breg/src/postgres/mod.rs +++ b/crates/registry-breg/src/postgres/mod.rs @@ -7,6 +7,10 @@ pub(crate) mod cancellation; mod catalog; mod config; mod context; +pub(crate) mod external_label_acquisition; +pub(crate) mod external_projection; +pub(crate) mod external_runtime; +pub(crate) mod external_view_query; mod failure; mod history_read; mod interlock; diff --git a/crates/registry-breg/src/postgres/schema.rs b/crates/registry-breg/src/postgres/schema.rs index de3926a88c..8f9fb16c88 100644 --- a/crates/registry-breg/src/postgres/schema.rs +++ b/crates/registry-breg/src/postgres/schema.rs @@ -95,6 +95,9 @@ pub async fn install_compiled_schema( .await?; } + super::external_projection::reconcile(migration, registry, runtime_role) + .await + .map_err(super::external_projection::kernel_error)?; super::timer::install_record_timer_schema(migration, registry, runtime_role).await?; reconcile_compiled_runtime_acl(migration, registry, runtime_role).await diff --git a/crates/registry-breg/src/runtime_config.rs b/crates/registry-breg/src/runtime_config.rs index 506c120841..d6aed48495 100644 --- a/crates/registry-breg/src/runtime_config.rs +++ b/crates/registry-breg/src/runtime_config.rs @@ -327,6 +327,8 @@ pub enum RuntimeConfigError { Reader(ReaderRefusal), #[error("runtime configuration contains an invalid deployment binding")] InvalidBinding, + #[error("runtime configuration contains an invalid external source binding")] + InvalidExternalSources, #[error( "runtime configuration taskGrantStatus must contain at most 32 unique absolute source issuers; configure safe service/token URLs, non-empty client and assertion audience, absolute resource identifiers, one status scope token, and a status path with one {{grantId}} segment" )] @@ -410,6 +412,7 @@ impl RuntimeConfigError { match self { Self::Reader(refusal) => &refusal.deciding_diagnostic().code, Self::InvalidBinding => "breg.runtime.invalid-binding", + Self::InvalidExternalSources => "breg.runtime.invalid-external-sources", Self::InvalidTaskGrantStatus => "breg.runtime.invalid-task-grant-status", Self::InvalidInstanceId => "breg.runtime.invalid-instance-id", Self::EnvironmentIdentityConflict => "breg.runtime.environment-identity-conflict", @@ -450,6 +453,7 @@ impl RuntimeConfigError { match self { Self::Reader(refusal) => &refusal.deciding_diagnostic().path, Self::InvalidBinding | Self::Secret => "", + Self::InvalidExternalSources => "/externalSources", Self::InvalidTaskGrantStatus => "/taskGrantStatus", Self::InvalidInstanceId => "/identity/instanceId", Self::EnvironmentIdentityConflict => "/identity/databaseInitializationEnvironment", @@ -492,6 +496,9 @@ impl RuntimeConfigError { Self::InvalidTaskGrantStatus => { "Configure at most 32 distinct taskGrantStatus source issuers, safe service/token URLs, a client and assertion audience, absolute resource identifiers, one statusScope token, and a statusPath that starts with / and holds exactly one {grantId} segment." } + Self::InvalidExternalSources => { + "Use distinct projection IDs, a bounded absolute packFile, approved fields and an explicit grant cutoff. Set a refresh interval shorter than the permitted retention window." + } Self::InvalidBinding => { "Give every identity value as trimmed text without control characters, and declare in the runtime file, with a valid endpoint and credential, every task grant status source, Evidence provider, and review authority the package relies on." } @@ -924,6 +931,7 @@ pub struct RuntimeConfig { audit: AuditConfig, cursor: CursorConfig, event_destinations: EventDestinationConfigs, + external_sources: Vec, evidence_providers: std::collections::BTreeMap, review_authorities: BTreeMap, @@ -971,6 +979,8 @@ fn validate_pool_acquisition_budget( impl RuntimeConfig { fn from_raw(raw: RawRuntimeConfig) -> Result { validate_runtime_definition_ids(&raw)?; + crate::external_source::validate_bindings(&raw.external_sources) + .map_err(|_| RuntimeConfigError::InvalidExternalSources)?; let listener = ListenerConfig::from_raw(raw.listener)?; let identity = DeploymentIdentity::from_raw(raw.identity)?; let secret_providers = SecretProvidersConfig::from_raw(raw.secret_providers)?; @@ -1031,6 +1041,7 @@ impl RuntimeConfig { audit, cursor, event_destinations, + external_sources: raw.external_sources.into_vec(), evidence_providers: raw.evidence_providers, review_authorities, retention, @@ -1359,6 +1370,11 @@ impl RuntimeConfig { self.metrics_listener.as_ref() } + /// Operator bindings for the registry's separately governed external reads. + pub fn external_sources(&self) -> &[crate::external_source::ExternalSourceBinding] { + &self.external_sources + } + pub fn secret_resolver(&self) -> Result { self.secret_providers.resolver() } @@ -2943,6 +2959,10 @@ struct RawRuntimeConfig { /// destination. #[serde(default)] destinations: RawDestinationsConfig, + /// Independent acquisition bindings. Omission permits no source calls. + #[serde(default)] + external_sources: + registry_platform_yaml::UniqueIdList, #[serde(default, deserialize_with = "local_id_map")] #[cfg_attr( feature = "schema", diff --git a/crates/registry-breg/src/startup.rs b/crates/registry-breg/src/startup.rs index 25b60b3a86..b8f2ea6a44 100644 --- a/crates/registry-breg/src/startup.rs +++ b/crates/registry-breg/src/startup.rs @@ -141,6 +141,12 @@ pub enum StartupError { AuditDestination(String), #[error("the Registry cursor profile was refused")] Cursor, + #[error("an external source binding or its reviewed Adapter pack was refused")] + ExternalSources, + #[error( + "indexed external projections require separate migration and runtime PostgreSQL roles" + )] + ExternalProjectionRoleMode, #[error("the Registry OIDC key source was refused")] Oidc, #[error("the Registry authentication profile was refused")] @@ -302,19 +308,21 @@ pub enum BackgroundTask { ReviewWorker, SubjectAccessLogRetention, EvidenceRetention, + ExternalProjectionRetention, MetricsListener, } impl BackgroundTask { /// Every supervised task, used by exhaustive operational-log contract /// tests. - pub const ALL: [Self; 7] = [ + pub const ALL: [Self; 8] = [ Self::HookWorker, Self::RecordTimerWorker, Self::AttachmentVerificationWorker, Self::ReviewWorker, Self::SubjectAccessLogRetention, Self::EvidenceRetention, + Self::ExternalProjectionRetention, Self::MetricsListener, ]; } @@ -362,6 +370,12 @@ const fn background_task_stop_code(task: BackgroundTask, stop: BackgroundTaskSto (BackgroundTask::EvidenceRetention, BackgroundTaskStop::Returned) => { "evidence.retention.returned" } + (BackgroundTask::ExternalProjectionRetention, BackgroundTaskStop::Panicked) => { + "external_projection.retention.panicked" + } + (BackgroundTask::ExternalProjectionRetention, BackgroundTaskStop::Returned) => { + "external_projection.retention.returned" + } (BackgroundTask::MetricsListener, BackgroundTaskStop::Panicked) => { "metrics.listener.panicked" } @@ -751,6 +765,8 @@ impl StartupError { Self::Audit => "the Registry audit profile or destination was refused", Self::AuditDestination(_) => "the Registry audit destination was refused", Self::Cursor => "the Registry cursor profile was refused", + Self::ExternalSources => "an external source binding or its reviewed Adapter pack was refused", + Self::ExternalProjectionRoleMode => "indexed external projections require separate migration and runtime PostgreSQL roles", Self::Oidc => "the Registry OIDC key source was refused", Self::Authentication => "the Registry authentication profile was refused", Self::AuthenticationClientsUnlisted => { @@ -827,6 +843,10 @@ pub struct PreparedServer { review_worker: Option, access_log_retention: Option<(RuntimePool, Arc)>, evidence_retention: Option, + external_projection_retention: Option<( + Arc, + Arc, + )>, metrics: Option, #[cfg(feature = "wasm")] wasm_runtime: Option, @@ -909,6 +929,7 @@ impl PreparedServer { review_worker: None, access_log_retention: None, evidence_retention: None, + external_projection_retention: None, metrics: None, postgres_advisories: Vec::new(), role_mode: RoleMode::Split, @@ -949,6 +970,7 @@ impl PreparedServer { review_worker: None, access_log_retention: None, evidence_retention: None, + external_projection_retention: None, metrics: None, postgres_advisories: Vec::new(), role_mode: RoleMode::Split, @@ -1460,6 +1482,14 @@ async fn finish_prepared_server( .map_err(|_| StartupError::Oidc)?; let registry = Arc::new(startup.package().registry().clone()); + if crate::postgres::external_projection::enabled(®istry) + && RoleMode::from_roles( + config.database().roles().migration(), + config.database().roles().runtime(), + ) != RoleMode::Split + { + return Err(StartupError::ExternalProjectionRoleMode); + } if registry .queries() .operations @@ -1581,6 +1611,36 @@ async fn finish_prepared_server( } else { None }; + let external_views = + if registry.external_views().views.is_empty() && config.external_sources().is_empty() { + None + } else { + let sources = crate::external_source::ExternalSourceService::load( + ®istry, + config.external_sources(), + Arc::new( + config + .secret_resolver() + .map_err(|_| StartupError::ExternalSources)?, + ), + ) + .map_err(|_| StartupError::ExternalSources)?; + let external = crate::postgres::external_runtime::PostgresExternalViewService::new( + pool.clone(), + Arc::clone(®istry), + expected.clone(), + lock_key, + config.operational_timeouts().record_lock, + audit.clone(), + Arc::clone(&cursor_codec), + sources, + ); + external + .bind_configured_scopes() + .await + .map_err(|_| StartupError::ExternalSources)?; + Some(Arc::new(external)) + }; let records = PostgresRecordReadService::new( pool.clone(), Arc::clone(®istry), @@ -1803,11 +1863,18 @@ async fn finish_prepared_server( review_authorities, Arc::clone(&mutations), )); + let external_projection_retention = external_views + .as_ref() + .filter(|_| crate::postgres::external_projection::enabled(®istry)) + .map(|backend| (Arc::clone(backend), Arc::new(LastSuccess::default()))); let mut service = HttpService::new(registry, read_identity, records, readiness, cursor_codec) .with_postgres_revisions(revisions) .with_snapshots(snapshots) .with_postgres_mutations(mutations) .with_statistics(statistics); + if let Some(external_views) = external_views { + service = service.with_external_views(external_views); + } if let Some(receiver) = review_completion_receiver { service = service.with_review_completions(receiver); } @@ -1838,6 +1905,12 @@ async fn finish_prepared_server( if let Some(worker) = &review_worker { registry = registry.with_worker_progress(ProgressWorker::Review, worker.last_success()); } + if let Some((_, last_success)) = &external_projection_retention { + registry = registry.with_worker_progress( + ProgressWorker::ExternalProjectionRetention, + Arc::clone(last_success), + ); + } if let Some((_, last_success)) = &access_log_retention { registry = registry.with_worker_progress( ProgressWorker::SubjectAccessLogRetention, @@ -1868,6 +1941,7 @@ async fn finish_prepared_server( review_worker, access_log_retention, evidence_retention, + external_projection_retention, metrics, postgres_advisories, role_mode: RoleMode::from_roles( @@ -2105,6 +2179,7 @@ pub async fn serve_until_shutdown( review_worker, access_log_retention, evidence_retention, + external_projection_retention, metrics, #[cfg(feature = "wasm")] wasm_runtime: _wasm_runtime, @@ -2159,6 +2234,15 @@ pub async fn serve_until_shutdown( task_stopped_tx.clone(), ) }); + let mut external_projection_worker = + external_projection_retention.map(|(backend, last_success)| { + SupervisedTask::spawn( + BackgroundTask::ExternalProjectionRetention, + backend.run_retention(last_success, worker_shutdown_rx.clone()), + worker_shutdown_rx.clone(), + task_stopped_tx.clone(), + ) + }); let mut access_log_worker = access_log_retention.map(|(pool, last_success)| { SupervisedTask::spawn( BackgroundTask::SubjectAccessLogRetention, @@ -2247,6 +2331,9 @@ pub async fn serve_until_shutdown( if let Some(worker) = access_log_worker.as_mut() { worker.join().await; } + if let Some(worker) = external_projection_worker.as_mut() { + worker.join().await; + } if let Some(metrics_server) = metrics_server.as_mut() { metrics_server.join().await; } @@ -2280,6 +2367,9 @@ pub async fn serve_until_shutdown( if let Some(worker) = access_log_worker.as_mut() { worker.abort_and_join().await; } + if let Some(worker) = external_projection_worker.as_mut() { + worker.abort_and_join().await; + } Err(StartupError::Shutdown) } }; diff --git a/crates/registry-breg/tests/external_views.rs b/crates/registry-breg/tests/external_views.rs new file mode 100644 index 0000000000..c4e7f8a4ed --- /dev/null +++ b/crates/registry-breg/tests/external_views.rs @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: Apache-2.0 + +use registry_breg::contract::parse_project_yaml; +use registry_breg::{compile_project, CompileProfile}; + +const PROJECT: &str = r#" +apiVersion: id.registrystack.org/formats/breg/project/v1alpha1 +kind: BRegProject +project: + id: linked-example + version: "1" + defaultLanguage: en + canonicalBaseIri: https://registry.example.test +datasets: [{id: local-records}] +recordTypes: + - id: local-record + primaryDataset: local-records + route: local-records + mutationMode: mutable + classification: internal + fields: + - {id: subject-key, type: string, maximumLength: 32, classification: internal} + - {id: local-title, type: string, maximumLength: 120, classification: internal} +externalRecordTypes: + - id: external-subject + authority: urn:example:source + identifierScheme: urn:example:subject-key + fields: + - {id: label, type: string, classification: internal} + - {id: code, type: integer, classification: restricted} +externalProjections: + - id: approved-subjects + recordType: external-subject + operation: subjects-by-key + operationVersion: "1" + purpose: local-consultation + scope: approved-subjects + selectorScheme: urn:example:subject-key + mode: indexed + cardinality: one + cacheTtlSeconds: 600 + fields: [label, code] +externalViews: + - id: local-with-subject + recordType: local-record + links: + - id: subject + projection: approved-subjects + fromField: subject-key + identifierScheme: urn:example:subject-key + fields: [label, code] +accessProfiles: + - id: reader + principalClaim: registry_principal + requiredScopes: [registry.read] + requiredPurposes: [local-consultation] + permissions: + records: + - recordType: local-record + operations: [get, list] + readableFields: [local-title] + rowBoundaries: unrestricted + externalViews: + - view: local-with-subject + links: + - link: subject + readableFields: [label] + filterableFields: [label] +"#; + +fn compile(source: &str) -> Result { + let project = parse_project_yaml(source.as_bytes()).expect("external views parse"); + compile_project(&project, &[], CompileProfile::Authoring) +} + +fn rejects(source: &str, expected: &str) { + let refusal = compile(source).expect_err("invalid external authoring is refused"); + assert!( + refusal.diagnostics().iter().any(|d| d.code == expected), + "expected {expected}: {refusal:?}" + ); +} + +#[test] +fn external_views_are_compiled_separately_from_owned_fields_and_permissions() { + let registry = compile(PROJECT).expect("an explicitly granted external view compiles"); + let external = registry.external_views(); + let view = &external.views["local-with-subject"]; + let grant = &external.grants["reader"]["local-with-subject"].links[0]; + assert_eq!(view.links[0].from_field.as_str(), "subject-key"); + assert_eq!( + grant + .readable_fields + .iter() + .map(|id| id.as_str()) + .collect::>(), + ["label"] + ); + assert!(!registry.entities()["local-record"] + .fields + .contains_key("label")); + assert!(registry + .artifacts() + .get("compiled/external-views.json") + .is_some()); +} + +#[test] +fn direct_link_rejects_another_identifier_scheme() { + let original = " identifierScheme: urn:example:subject-key"; + let changed = " identifierScheme: urn:example:other-key"; + rejects( + &PROJECT.replace(original, changed), + "breg.external-view.identifier-scheme-mismatch", + ); +} + +#[test] +fn query_grant_cannot_use_a_hidden_external_field() { + rejects( + &PROJECT.replace("filterableFields: [label]", "filterableFields: [code]"), + "breg.external-view.processing-wider-than-read", + ); +} + +#[test] +fn live_projection_cannot_have_a_cache_or_query_grant() { + let live = PROJECT.replace("mode: indexed", "mode: live"); + rejects(&live, "breg.external-view.live-cache"); + rejects( + &live.replace(" cacheTtlSeconds: 600\n", ""), + "breg.external-view.live-query", + ); +} + +#[test] +fn refresh_requires_a_separate_scoped_service_profile() { + let source = PROJECT.replace( + " permissions:\n records:", + " permissions:\n refreshExternalProjections: [approved-subjects]\n records:", + ); + rejects(&source, "breg.external-view.refresh-service-profile"); +} + +#[test] +fn external_view_cannot_bypass_a_local_record_permission() { + let source = PROJECT.replace("operations: [get, list]", "operations: [list]"); + rejects(&source, "breg.external-view.root-read-missing"); +} + +#[test] +fn external_openapi_uses_named_view_and_field_grants() { + let registry = compile(PROJECT).expect("authorised view fixture compiles"); + let document: serde_json::Value = serde_json::from_slice( + ®istry + .artifacts() + .get("generated/openapi.json") + .unwrap() + .bytes, + ) + .unwrap(); + let operation = + &document["paths"]["/v1/external-views/local-with-subject/subject/records"]["get"]; + let schema = &operation["responses"]["200"]["content"]["application/json"]["schema"]; + let fields = &schema["properties"]["records"]["items"]["properties"]["fields"]["properties"]; + assert_eq!(fields["label"]["type"], "string"); + assert!(fields.get("code").is_none()); + assert!(schema["properties"].get("count").is_none()); + assert!(document["paths"]["/v1/external-views/local-with-subject/subject/refresh"].is_null()); +} diff --git a/crates/registry-breg/tests/external_views_example.rs b/crates/registry-breg/tests/external_views_example.rs new file mode 100644 index 0000000000..ab1eb2291b --- /dev/null +++ b/crates/registry-breg/tests/external_views_example.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: Apache-2.0 + +use registry_breg::contract::parse_project_yaml; +use registry_breg::external_views::{ExternalCardinality, ExternalProjectionMode}; +use registry_breg::{compile_project, CompileProfile}; + +#[test] +fn maintained_licensing_example_compiles_both_modes_and_keeps_owned_fields_separate() { + let project = parse_project_yaml(include_bytes!( + "../../../products/breg/examples/external-views/registry.yaml" + )) + .expect("the hand-authored example parses"); + let compiled = compile_project(&project, &[], CompileProfile::Authoring) + .expect("the hand-authored example compiles"); + let external = compiled.external_views(); + + for (id, mode, cardinality) in [ + ( + "person-live", + ExternalProjectionMode::Live, + ExternalCardinality::One, + ), + ( + "person-indexed", + ExternalProjectionMode::Indexed, + ExternalCardinality::One, + ), + ( + "holdings-live", + ExternalProjectionMode::Live, + ExternalCardinality::Many, + ), + ( + "holdings-indexed", + ExternalProjectionMode::Indexed, + ExternalCardinality::Many, + ), + ] { + let projection = &external.projections[id]; + assert_eq!( + (projection.mode, projection.cardinality), + (mode, cardinality) + ); + } + let holdings = &external.views["business-holdings-indexed"].links[0]; + let label = holdings.label.as_ref().expect("an explicit Person label"); + assert_eq!(label.from_field.as_str(), "person-key"); + assert_eq!(label.projection.as_str(), "person-indexed"); + assert!(!holdings + .fields + .iter() + .any(|field| field.as_str() == "person-key")); + assert!(!external.grants.contains_key("local-facts")); + assert!(external.refresh.contains_key("projection-refresh")); + assert!(!compiled.entities()["licence"] + .fields + .contains_key("family-name")); +} + +#[test] +fn holdings_label_refresh_requires_both_projection_grants() { + let source = include_str!("../../../products/breg/examples/external-views/registry.yaml"); + let labels = "/v1/external-views/business-holdings-indexed/holdings/refresh-labels"; + for (source, allowed) in [ + (source.to_owned(), true), + ( + source.replace("[person-indexed, holdings-indexed]", "[holdings-indexed]"), + false, + ), + ] { + let project = parse_project_yaml(source.as_bytes()).expect("authoring fixture parses"); + let compiled = compile_project(&project, &[], CompileProfile::Authoring) + .expect("authoring fixture compiles"); + let document: serde_json::Value = serde_json::from_slice( + &compiled + .artifacts() + .get("generated/openapi.json") + .unwrap() + .bytes, + ) + .unwrap(); + assert_eq!(document["paths"].get(labels).is_some(), allowed); + } +} diff --git a/crates/registry-breg/tests/startup_http.rs b/crates/registry-breg/tests/startup_http.rs index 4807e2bfd4..9e2d35bba8 100644 --- a/crates/registry-breg/tests/startup_http.rs +++ b/crates/registry-breg/tests/startup_http.rs @@ -506,7 +506,7 @@ async fn request_operational_log_has_only_closed_value_free_fields() { /// A description the audit writer gives for a torn audit file. const AUDIT_DESTINATION_REASON: &str = "the audit file could not be opened: audit file has an incomplete final entry; archive it and restart with a fresh path"; -fn startup_errors() -> [StartupError; 38] { +fn startup_errors() -> [StartupError; 40] { [ // The wrapped cause never changes the rendered operational message: it // only lets `bregctl doctor` name it. Any `RuntimeConfigError` variant @@ -549,6 +549,8 @@ fn startup_errors() -> [StartupError; 38] { StartupError::Audit, StartupError::AuditDestination(AUDIT_DESTINATION_REASON.to_owned()), StartupError::Cursor, + StartupError::ExternalSources, + StartupError::ExternalProjectionRoleMode, StartupError::Oidc, StartupError::Authentication, StartupError::AuthenticationClientsUnlisted, @@ -803,6 +805,12 @@ fn expected_startup_error(error: StartupError) -> &'static str { StartupError::Audit => "the Registry audit profile or destination was refused", StartupError::AuditDestination(_) => "the Registry audit destination was refused", StartupError::Cursor => "the Registry cursor profile was refused", + StartupError::ExternalSources => { + "an external source binding or its reviewed Adapter pack was refused" + } + StartupError::ExternalProjectionRoleMode => { + "indexed external projections require separate migration and runtime PostgreSQL roles" + } StartupError::Oidc => "the Registry OIDC key source was refused", StartupError::Authentication => "the Registry authentication profile was refused", StartupError::AuthenticationClientsUnlisted => { @@ -897,6 +905,12 @@ fn expected_background_task_stop_code( (BackgroundTask::EvidenceRetention, BackgroundTaskStop::Returned) => { "evidence.retention.returned" } + (BackgroundTask::ExternalProjectionRetention, BackgroundTaskStop::Panicked) => { + "external_projection.retention.panicked" + } + (BackgroundTask::ExternalProjectionRetention, BackgroundTaskStop::Returned) => { + "external_projection.retention.returned" + } (BackgroundTask::MetricsListener, BackgroundTaskStop::Panicked) => { "metrics.listener.panicked" } diff --git a/crates/registry-bregctl/src/dev/config.rs b/crates/registry-bregctl/src/dev/config.rs index 78c3110651..38e0d7cad0 100644 --- a/crates/registry-bregctl/src/dev/config.rs +++ b/crates/registry-bregctl/src/dev/config.rs @@ -2110,14 +2110,20 @@ pub(super) fn runtime(root: &Path, state: &State, clients: &Clients, test: bool) json!(allowed_clients) }; let assertion_issuers = assertion_issuers(state, clients)?; - // The local registry serves with the migration role, the one-role mode - // a small deployment runs in. The schema-test rehearsal stays split, - // because the package fingerprint is defined against a separate runtime - // role. - let (runtime_url, runtime_role) = if test { - ("test-runtime-database-url", RUNTIME_ROLE) - } else { - ("migration-database-url", MIGRATION_ROLE) + // Indexed projections require the bounded cache privileges of the + // separate role. The development database already has both roles and + // their URLs. Other local registries keep their one-role mode. + let indexed = compiled + .external_views() + .projections + .values() + .any(|projection| { + projection.mode == registry_breg::external_views::ExternalProjectionMode::Indexed + }); + let (runtime_url, runtime_role) = match (test, indexed) { + (true, _) => ("test-runtime-database-url", RUNTIME_ROLE), + (false, true) => ("runtime-database-url", RUNTIME_ROLE), + (false, false) => ("migration-database-url", MIGRATION_ROLE), }; let mut runtime = json!({ "apiVersion":"id.registrystack.org/formats/breg/runtime/v1alpha1","kind":"BRegRuntimeConfig", diff --git a/crates/registry-bregctl/src/doctor.rs b/crates/registry-bregctl/src/doctor.rs index 84d90a185a..ec1d2f5345 100644 --- a/crates/registry-bregctl/src/doctor.rs +++ b/crates/registry-bregctl/src/doctor.rs @@ -279,6 +279,16 @@ fn startup_diagnostic(error: StartupError) -> Diagnostic { "database", "the database was activated for a separate runtime role but the runtime file names one role: run bregctl apply --package DIR to activate it for one role", ), + StartupError::ExternalSources => ( + "startup.external-sources.refused", + "externalSources", + "review the indexed projection scope and the configured source grant and pinned Adapter pack", + ), + StartupError::ExternalProjectionRoleMode => ( + "startup.external-projections.role-mode", + "database.roles", + "indexed external projections require separate migration and runtime PostgreSQL roles", + ), StartupError::Audit => ( "startup.audit.refused", "audit", diff --git a/crates/registry-bregctl/src/lib.rs b/crates/registry-bregctl/src/lib.rs index 5aa9968203..92b163cf4a 100644 --- a/crates/registry-bregctl/src/lib.rs +++ b/crates/registry-bregctl/src/lib.rs @@ -626,13 +626,13 @@ struct GenerateArgs { #[arg(long)] production: bool, - /// New directory that will receive exactly the generated artifact inventory; - /// for identity-facts, a new YAML file. + /// Destination directory for generated artifacts. Use a new YAML file for + /// identity-facts or a new JSON file for external-source-facts. #[arg(long, value_name = "PATH")] output: PathBuf, - /// Runtime configuration file (identity-facts only): its authentication - /// block states the issuer, audience and admitted clients. + /// Runtime configuration file. Identity facts use its authentication block. + /// External source facts use its reviewed source bindings. #[arg(long, value_name = "ABSOLUTE_FILE")] runtime_config: Option, @@ -1632,6 +1632,8 @@ enum ArtifactSelector { /// The issuer, audience, clients, claims and access profiles this registry /// expects, for the hosted identity generator. Takes --runtime-config; --output is a new file. IdentityFacts, + /// Verified external-source capabilities and binding digests. Takes --runtime-config and a new JSON output file. + ExternalSourceFacts, } #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, ValueEnum)] @@ -8440,13 +8442,16 @@ fn generate_requested(args: &GenerateArgs) -> Result Result Result { + let fail = |code: &str, message: &str, artifact, action| { + source_failure( + "generate", + diagnostic(code, "arguments", message), + artifact, + action, + ) + }; + let Some(runtime_path) = args + .runtime_config + .as_ref() + .filter(|path| path.is_absolute()) + else { + return Err(fail( + "external-source-facts.arguments", + "generate external-source-facts requires an absolute --runtime-config path", + DiagnosticArtifact::CommandArguments, + SuggestedAction::CorrectCommandUsage, + )); + }; + if args.access_profile.is_some() + || args.entity.is_some() + || !args.selectors.is_empty() + || !args.fields.is_empty() + || args.source_id.is_some() + || args.connection.is_some() + || args.authentication.is_some() + { + return Err(fail( + "external-source-facts.arguments", + "generate external-source-facts accepts --runtime-config and --output", + DiagnosticArtifact::CommandArguments, + SuggestedAction::CorrectCommandUsage, + )); + } + let binding_refusal = || { + fail( + "external-source-facts.binding", + "review the runtime binding, source grant and pinned Adapter pack", + DiagnosticArtifact::RuntimeConfiguration, + SuggestedAction::CorrectRuntimeConfiguration, + ) + }; + let output_refusal = || { + fail( + "external-source-facts.output", + "provide a new output file in a writable directory", + DiagnosticArtifact::GeneratedArtifacts, + SuggestedAction::RetryArtifactGeneration, + ) + }; + let config = registry_breg::runtime_config::load_runtime_config(runtime_path) + .map_err(|error| runtime_config_failure("generate", error))?; + let profile = profile(args.production); + let (compiled, project) = compile_with_source(&args.project, profile, "generate")?; + let resolver = std::sync::Arc::new(config.secret_resolver().map_err(|_| binding_refusal())?); + let bindings = config.external_sources(); + let service = + registry_breg::external_source::ExternalSourceService::load(&compiled, bindings, resolver) + .map_err(|_| binding_refusal())?; + let inventory = compiled.external_views(); + let mut projections = Vec::with_capacity(bindings.len()); + for binding in bindings { + let id = binding.id.as_str(); + let compiled_projection = inventory.projections.get(id).ok_or_else(binding_refusal)?; + let record_type = inventory + .record_types + .get(compiled_projection.record_type.as_str()) + .ok_or_else(binding_refusal)?; + let bound = service.binding(id).ok_or_else(binding_refusal)?; + projections.push(json!({ + "id": id, + "mode": compiled_projection.mode, + "recordType": record_type.id, + "authority": record_type.authority, + "operation": compiled_projection.operation, + "operationVersion": compiled_projection.operation_version, + "selectorScheme": compiled_projection.selector_scheme, + "purpose": compiled_projection.purpose, + "scope": compiled_projection.scope, + "fields": bound.allowed_fields(), + "connection": binding.connection.id, + "bindingDigest": bound.binding_fingerprint(), + "grantDigest": bound.grant_fingerprint(), + "grantValidUntil": bound.grant_valid_until().to_rfc3339(), + "cacheTtlSeconds": bound.effective_cache_ttl_seconds(), + "maximumRetentionSeconds": bound.maximum_retention_seconds(), + "refreshIntervalSeconds": bound.refresh_interval_seconds(), + })); + } + let facts = json!({ + "apiVersion": "id.registrystack.org/formats/breg/external-source-facts/v1alpha1", + "kind": "BRegExternalSourceFacts", + "registryId": compiled.registry_id(), + "revision": compiled.revision(), + "projections": projections, + }); + let bytes = serde_json::to_vec_pretty(&facts).map_err(|_| output_refusal())?; + let parent = args + .output + .parent() + .filter(|path| !path.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + let mut pending = tempfile::NamedTempFile::new_in(parent).map_err(|_| output_refusal())?; + pending.write_all(&bytes).map_err(|_| output_refusal())?; + pending + .persist_noclobber(&args.output) + .map_err(|_| output_refusal())?; + Ok(SuccessReport { + ok: true, + command: "generate", + profile, + revision: Some(compiled.revision().to_owned()), + registry_revision: None, + package_digest: None, + findings: positioned_compiler_findings(&compiled, &project), + artifacts: vec![artifact_report( + &args.output.display().to_string(), + "application/json", + &bytes, + )], + explanation: Some(json!({"externalSourceFacts":{"projections":bindings.len()}})), + next_steps: Vec::new(), + }) +} + /// Write the identity facts of the registry project and runtime configuration. /// The report names the output file and counts; it carries no value read from /// either input. @@ -10319,7 +10453,8 @@ impl ArtifactSelector { ArtifactSelector::Sql => path == "generated/postgres/schema.sql", ArtifactSelector::EvidenceSource | ArtifactSelector::Odrl - | ArtifactSelector::IdentityFacts => false, + | ArtifactSelector::IdentityFacts + | ArtifactSelector::ExternalSourceFacts => false, ArtifactSelector::Sdk => { path.starts_with("generated/sdk/") || path.starts_with("generated/review-context-schemas/") @@ -10400,6 +10535,9 @@ fn explain_model(compiled: &CompiledRegistry) -> serde_json::Result { inspection["version"] = json!(compiled.version()); inspection["moduleOrder"] = json!(compiled.module_order()); inspection["moduleClosure"] = json!(compiled.module_closure()); + if !compiled.external_views().is_empty() { + inspection["externalViews"] = serde_json::to_value(compiled.external_views())?; + } Ok(inspection) } diff --git a/crates/registry-bregctl/tests/tool_schema.rs b/crates/registry-bregctl/tests/tool_schema.rs index ca432769dd..b4802bebfa 100644 --- a/crates/registry-bregctl/tests/tool_schema.rs +++ b/crates/registry-bregctl/tests/tool_schema.rs @@ -610,9 +610,9 @@ fn the_reader_accepts_every_word_the_migration_descriptor_schema_enumerates() { checked += 1; } } - // 3 change classes, 64 change codes, 15 target kinds, 1 recovery, + // 3 change classes, 66 change codes, 15 target kinds, 1 recovery, // 2 history choices, 4 object kinds, and 1 cursor protocol. - assert_eq!(checked, 90); + assert_eq!(checked, 92); let steps = &definitions["ReviewedMigrationStepDescriptor"]["oneOf"]; let types: Vec<&str> = steps diff --git a/crates/registry-cli-docs/catalog.json b/crates/registry-cli-docs/catalog.json index bfa607f99b..074dbf1312 100644 --- a/crates/registry-cli-docs/catalog.json +++ b/crates/registry-cli-docs/catalog.json @@ -3757,7 +3757,8 @@ "sql", "evidence-source", "sdk", - "identity-facts" + "identity-facts", + "external-source-facts" ], "environment": null }, @@ -3783,7 +3784,7 @@ }, { "display": "--output ", - "description": "New directory that will receive exactly the generated artifact inventory; for identity-facts, a new YAML file", + "description": "Destination directory for generated artifacts. Use a new YAML file for identity-facts or a new JSON file for external-source-facts", "always_required": true, "repeatable": false, "default_values": [], @@ -3792,7 +3793,7 @@ }, { "display": "--runtime-config ", - "description": "Runtime configuration file (identity-facts only): its authentication block states the issuer, audience and admitted clients. A symbolic link at any component of this path is refused.", + "description": "Runtime configuration file. Identity facts use its authentication block. External source facts use its reviewed source bindings. A symbolic link at any component of this path is refused.", "always_required": false, "repeatable": false, "default_values": [], diff --git a/products/adapter/CONCEPT.md b/products/adapter/CONCEPT.md index 3eeab49322..00ea760feb 100644 --- a/products/adapter/CONCEPT.md +++ b/products/adapter/CONCEPT.md @@ -2451,3 +2451,61 @@ diagnostics carry no values, and its row below passes. | Hooks | A forged, duplicate, or lost hook changes only when a pass runs | | Queue | A message is acknowledged only after acceptance; a crash redelivers it with no duplicate effect; a refused message reaches the dead-letter path | | Diagnostics and retention | No payload in the item table; no credentials or personal values in logs | + +## 18. External projection acquisition + +A pack can declare a `sourceRoutes[].projection` for a host that reads a +reviewed subset of records for a named purpose. This is a separate source +entry point. The facade, ingestion jobs and Evidence cannot bind it as one +of their own operations. The source authority's operation version, purpose, +eligible scope, selector scheme and record-key scheme must agree with the +host's governed project and operator binding. The host establishes the +current grant and the requested keys and fields before acquisition and is +responsible for terminal audit and any retained data. The operation must +declare `localDisclosure: host-profiles` for the host to disclose under its +local profiles when using the shared service identity. A route marked +`acquisition-only` provides no such authority. + +The acquisition uses the listener-free engine, existing endpoint bindings, +a reviewed connection policy, and a service credential. The operation is +read-only: HTTP GET or a named, reviewed safe POST. The pack may map source +protocols with its digest-pinned scripts. The host supplies a finite +operation, response-byte and deadline budget that spans all pages and +physical calls. The route declares limits on batch keys, collection pages +and total collection rows. + +Every source invocation receives a JSON object with `type` and `fields`. +An `exact` invocation supplies `key`; a `batch` supplies unique `keys`; +a `by-parent` invocation supplies `parentKey` and, after the first page, +the previous page's opaque `cursor`. Route results have the following +closed shapes: + +| Type | Members | +|---|---| +| `exact` | `record`, one keyed outcome for the requested key | +| `batch` | `records`, exactly one keyed outcome per requested key | +| `by-parent` | `parentKey`, `snapshot`, `rows`, `complete` and optional `nextCursor` | +| `unavailable` | No data members | + +An exact or batch record carries `key` and an `outcome`: `present` +includes `fields` and optional `sourceRevision`; `authoritative-absent` +may include `sourceRevision`; `unavailable` carries only the key. +Authoritative absence requires a source contract that permits it and a +successful source response explicitly confirming absence for that key. +Ambiguous matches, protected existence, HTTP denial and transport faults +cannot establish absence. The engine validates every returned field against +the digest-pinned closed JSON Schema and refuses fields the host did not +request. + +A collection row carries a stable unique `key`, the requested `parentKey`, +`fields`, and an optional `sourceRevision`. A snapshot preserves one +parent and snapshot identity across every page, terminates with explicit +`complete: true`, and never repeats a cursor or row key. A completed empty +snapshot is a result. An interruption yields no collection to the host. + +Live acquisitions have no storage in Adapter. Indexed acquisitions require +an explicit `retention: retain` operation before any source call. The engine +also checks the raw `Cache-Control` header on every data response before any +pack mapping. A `no-store` response leaves the acquisition usable for live +reads and unavailable for retention. That restriction survives mapping and +paging. Consultation's protected response headers stay unchanged. diff --git a/products/adapter/README.md b/products/adapter/README.md index 023ec35643..5952784827 100644 --- a/products/adapter/README.md +++ b/products/adapter/README.md @@ -36,6 +36,12 @@ source-assigned Create and strong conditional Update with separate domain input schemas and transformations. DHIS2 Write remains unavailable until its source can preserve those guarantees. +An embedding host such as BReg can run a [reviewed projection +operation](CONCEPT.md#18-external-projection-acquisition) for an exact read, +key batch or complete list for one parent. The host owns grants, disclosure +and any retention. See the [synthetic licensing +pack](examples/external-projections/README.md) for authoring and replay. + The [product concept](CONCEPT.md) is the contract: artifact model, engine, exchange contract, failure semantics, authorization, audit, configuration formats, and verification. The diff --git a/products/adapter/examples/external-projections/README.md b/products/adapter/examples/external-projections/README.md new file mode 100644 index 0000000000..f99f9459e2 --- /dev/null +++ b/products/adapter/examples/external-projections/README.md @@ -0,0 +1,36 @@ +# Synthetic licensing projections + +This pack supplies a licensing registry with Person names and a business's +shareholding rows. Person names are owned by the synthetic Population +Registry. Holdings are owned by the synthetic Business Registry. + +| Operation | Input | Source approval | +|---|---|---| +| `person-exact` | One Person key | Live only | +| `person-batch` | Unique Person keys | Indexed acquisition may retain approved fields | +| `holdings-by-business` | One Business key | Live or indexed; retained membership requires one complete snapshot | + +The fixture posts the closed operation input to a synthetic service, +including the approved requested fields. The Rhai mapping preserves the +identifiers and the completeness reported by that service. The pack pins +two JSON Schemas for the returned field objects. The source grant scope and +identifier schemes are fixture URNs or named synthetic scopes. No endpoint +or service credential is supplied by the pack. The embedding host uses an +operator-provided Adapter endpoint binding and keeps credentials in its +configured secret provider. + +The engine proof checks the requested inputs against the pack routes. +The sealed transcript exercises normalized outcomes; a loopback source +checks that the configured credential and approved fields arrive at the +source and that repeated raw cache directives are honored. Run from the +Registry Stack worktree with its own external Cargo target directory: + +```sh +cargo test --locked -p registry-adapter-engine --test projection_acquisition +``` + +The replay covers exact and batch identity, authoritative absence, +duplicate/missing batch keys, a company shareholder, complete/empty +collections, interruption, invalid collection membership and response +`no-store`. The host still needs an independently governed local grant, +audit and an authorized publication path before it can retain any result. diff --git a/products/adapter/examples/external-projections/pack.yaml b/products/adapter/examples/external-projections/pack.yaml new file mode 100644 index 0000000000..7636efcff7 --- /dev/null +++ b/products/adapter/examples/external-projections/pack.yaml @@ -0,0 +1,118 @@ +# yaml-language-server: $schema=../../schemas/pack.v1alpha1.schema.json +apiVersion: id.registrystack.org/formats/adapter/pack/v1alpha1 +kind: AdapterPack +id: synthetic-licensing-projections +version: "1" +testedVersions: [] +endpoints: + - id: synthetic-registry + component: http + authentication: + type: static-api-key + headerName: x-registry-service-key + attemptTimeoutMilliseconds: 5000 + maximumResponseBytes: 65536 + operations: + - id: person-exact + safeUpstream: true + - id: person-batch + safeUpstream: true + - id: holdings-by-business + safeUpstream: true +scripts: + - path: scripts/projection.rhai + digest: sha256:900698035e7268aeead23c1689542c11b16ad815c33acfb3183267b6ef632f36 +sourceRoutes: + - id: person-exact + projection: + operation: exact + operationVersion: "1" + authority: urn:example:population-register + purpose: licensing + scope: licensing-holders + selectorScheme: urn:example:person-number + recordKeyScheme: urn:example:person-number + retention: live-only + absence: authoritative + localDisclosure: host-profiles + fieldsSchema: + path: schemas/person.json + digest: sha256:1c1ad6f4c266d0470b8a28c054d360744c6a9f36ba61c379d6e75e0c791035b7 + maximumBatchKeys: 1 + maximumPages: 1 + maximumRows: 1 + steps: + - to: + uri: http:synthetic-registry + parameters: + httpMethod: POST + operation: person-exact + path: /people/exact + body: + jsonpath: "$" + - setBody: + rhai: + file: scripts/projection.rhai + entrypoint: exact + - id: person-batch + projection: + operation: batch + operationVersion: "1" + authority: urn:example:population-register + purpose: licensing + scope: licensing-holders + selectorScheme: urn:example:person-number + recordKeyScheme: urn:example:person-number + retention: retain + absence: authoritative + localDisclosure: host-profiles + fieldsSchema: + path: schemas/person.json + digest: sha256:1c1ad6f4c266d0470b8a28c054d360744c6a9f36ba61c379d6e75e0c791035b7 + maximumBatchKeys: 64 + maximumPages: 1 + maximumRows: 1 + steps: + - to: + uri: http:synthetic-registry + parameters: + httpMethod: POST + operation: person-batch + path: /people/batch + body: + jsonpath: "$" + - setBody: + rhai: + file: scripts/projection.rhai + entrypoint: batch + - id: holdings-by-business + projection: + operation: by-parent + operationVersion: "1" + authority: urn:example:business-register + purpose: licensing + scope: licensed-business-holdings + selectorScheme: urn:example:business-number + recordKeyScheme: urn:example:holding-number + retention: retain + absence: protected + localDisclosure: host-profiles + fieldsSchema: + path: schemas/holding.json + digest: sha256:9604300f5b394982df8f14c9336aa1bbc9de1dded4b0691e441637ae4bc9ca81 + maximumBatchKeys: 1 + maximumPages: 4 + maximumRows: 128 + steps: + - to: + uri: http:synthetic-registry + parameters: + httpMethod: POST + operation: holdings-by-business + path: /holdings/by-business + body: + jsonpath: "$" + - setBody: + rhai: + file: scripts/projection.rhai + entrypoint: holdings diff --git a/products/adapter/examples/external-projections/schemas/holding.json b/products/adapter/examples/external-projections/schemas/holding.json new file mode 100644 index 0000000000..d4c8ba0b3b --- /dev/null +++ b/products/adapter/examples/external-projections/schemas/holding.json @@ -0,0 +1,11 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "share-class": {"type": "string"}, + "share-count": {"type": "integer", "minimum": 0}, + "shareholder-key": {"type": "string"}, + "shareholder-type": {"type": "string", "enum": ["person", "business"]}, + "person-key": {"type": ["string", "null"]} + } +} diff --git a/products/adapter/examples/external-projections/schemas/person.json b/products/adapter/examples/external-projections/schemas/person.json new file mode 100644 index 0000000000..c0ac160b3f --- /dev/null +++ b/products/adapter/examples/external-projections/schemas/person.json @@ -0,0 +1,8 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "given-name": {"type": "string"}, + "family-name": {"type": "string"} + } +} diff --git a/products/adapter/examples/external-projections/scripts/projection.rhai b/products/adapter/examples/external-projections/scripts/projection.rhai new file mode 100644 index 0000000000..3bd56db6df --- /dev/null +++ b/products/adapter/examples/external-projections/scripts/projection.rhai @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: Apache-2.0 + +fn person(row) { + if row.state == "found" { + return #{outcome: "present", key: row.subject, fields: row.properties, sourceRevision: row.revision}; + } + if row.state == "confirmed-absent" { + return #{outcome: "authoritative-absent", key: row.subject, sourceRevision: row.revision}; + } + #{outcome: "unavailable", key: row.subject} +} + +fn exact(exchange) { + #{"type": "exact", record: person(exchange.body)} +} + +fn batch(exchange) { + let records = []; + for row in exchange.body.records { + records.push(person(row)); + } + #{"type": "batch", records: records} +} + +fn holdings(exchange) { + let page = exchange.body; + if page.state != "snapshot" { + return #{"type": "unavailable"}; + } + let rows = []; + for row in page.items { + rows.push(#{key: row.holding, parentKey: row.business, fields: row.properties, sourceRevision: row.revision}); + } + #{ + "type": "by-parent", + parentKey: page.business, + snapshot: page.snapshotId, + rows: rows, + complete: page.finished, + nextCursor: page.next + } +} diff --git a/products/adapter/schemas/pack.v1alpha1.schema.json b/products/adapter/schemas/pack.v1alpha1.schema.json index 677c71e35d..4196a1e43c 100644 --- a/products/adapter/schemas/pack.v1alpha1.schema.json +++ b/products/adapter/schemas/pack.v1alpha1.schema.json @@ -1291,6 +1291,110 @@ ], "type": "object" }, + "ProjectionAbsence": { + "enum": [ + "protected", + "authoritative" + ], + "type": "string" + }, + "ProjectionLocalDisclosure": { + "description": "Whether the source authorizes disclosure under the host's narrower\nprofiles when acquisition uses a shared service identity.", + "enum": [ + "acquisition-only", + "host-profiles" + ], + "type": "string" + }, + "ProjectionOperationKind": { + "enum": [ + "exact", + "batch", + "by-parent" + ], + "type": "string" + }, + "ProjectionRetention": { + "enum": [ + "live-only", + "retain" + ], + "type": "string" + }, + "ProjectionRouteContract": { + "additionalProperties": false, + "description": "A reviewed source operation's ceiling. The host still has to establish its\nown current project grant, connection and eligible selectors before I/O.", + "properties": { + "absence": { + "$ref": "#/$defs/ProjectionAbsence" + }, + "authority": { + "type": "string" + }, + "fieldsSchema": { + "$ref": "#/$defs/ScriptReference" + }, + "localDisclosure": { + "$ref": "#/$defs/ProjectionLocalDisclosure" + }, + "maximumBatchKeys": { + "format": "uint64", + "maximum": 1000, + "minimum": 1, + "type": "integer" + }, + "maximumPages": { + "format": "uint64", + "maximum": 100, + "minimum": 1, + "type": "integer" + }, + "maximumRows": { + "format": "uint64", + "maximum": 10000, + "minimum": 1, + "type": "integer" + }, + "operation": { + "$ref": "#/$defs/ProjectionOperationKind" + }, + "operationVersion": { + "type": "string" + }, + "purpose": { + "type": "string" + }, + "recordKeyScheme": { + "type": "string" + }, + "retention": { + "$ref": "#/$defs/ProjectionRetention" + }, + "scope": { + "type": "string" + }, + "selectorScheme": { + "type": "string" + } + }, + "required": [ + "operation", + "operationVersion", + "authority", + "purpose", + "scope", + "selectorScheme", + "recordKeyScheme", + "retention", + "absence", + "localDisclosure", + "fieldsSchema", + "maximumBatchKeys", + "maximumPages", + "maximumRows" + ], + "type": "object" + }, "RedeliveryOptions": { "additionalProperties": false, "description": "The redelivery option set, shared by the error handler and an\n`onException` clause (`products/adapter/CONCEPT.md` section 8.7; the\nnesting under `redeliveryPolicy` follows Camel's own YAML DSL,\n`manual/exception-clause.html` and `manual/exception-redelivery.html`).\nOption spellings stay ours (CFG-NAME); only the nesting is Camel's.\nOnly the non-default members serialize.", @@ -1643,7 +1747,7 @@ }, "SourceRouteDecl": { "additionalProperties": false, - "description": "One source route of the governed-ingestion mode\n(`products/adapter/CONCEPT.md` sections 17.1 and 17.2): a route\ndefinition's steps plus the item-contract declaration. A source route\nis an entry point of the job mode, never a subroutine: a facade\nbinding naming it, or a `direct:` send targeting it, refuses\n(`adapter.job.source-bound-refused`,\n`adapter.job.source-called-refused`). There is no `from`: the\n`from: file:` consumer form was rejected with the job's read step\n(section 17.2).", + "description": "One source entry point for an ingestion, Evidence or projection host.\nThe host's declaration supplies the operation contract alongside the\nsteps. A source route is never a subroutine: a facade binding naming it,\nor a `direct:` send targeting it, refuses\n(`adapter.job.source-bound-refused`,\n`adapter.job.source-called-refused`). There is no `from`: the\n`from: file:` consumer form was rejected with the job's read step\n(section 17.2).", "properties": { "edits": { "description": "How the source reports edits to items the scan has already read,\none of [`SOURCE_EDIT_VALUES`]: `ordered`, where an edit moves the\nitem past the read position or items never change so an ordinary\npass suffices, or `rescan`, where an edit can stay behind the\nposition and the declared `rescanDelaySeconds` starts an ordinary\npass from the first page. Mandatory; the checks refuse its absence\nunder `adapter.job.source-edits-required` and a value outside the\nclosed set under `adapter.job.source-edits-unknown`.", @@ -1655,12 +1759,16 @@ }, "id": { "$ref": "#/$defs/LocalId", - "description": "The source route identifier the job definition names." + "description": "The identifier the embedding host names." }, "nextRecovery": { "$ref": "#/$defs/SourceRecovery", "description": "The recovery semantics of `next`. Mandatory; the checks refuse its\nabsence under `adapter.job.source-recovery-required`." }, + "projection": { + "$ref": "#/$defs/ProjectionRouteContract", + "description": "Separately governed acquisition for a host that may retain a projection." + }, "readSupport": { "$ref": "#/$defs/SourceReadSupport", "description": "The source-neutral read semantics this route explicitly supports. A\nfacade host maps these native inputs and guarantees to its public\nprotocol; it never infers them from a route id or source brand." @@ -2611,7 +2719,7 @@ }, "sourceRoutes": { "default": [], - "description": "The governed-ingestion source routes the job mode runs\n(`products/adapter/CONCEPT.md` sections 17.1 and 17.2). Entry\npoints of the job mode; a facade route never reaches one.", + "description": "The source routes of the ingestion, Evidence and projection hosts.\nA facade route never reaches one.", "items": { "$ref": "#/$defs/SourceRouteDecl" }, diff --git a/products/adapter/scripts/check_dependency_direction.py b/products/adapter/scripts/check_dependency_direction.py index af41d7417b..17853c0253 100755 --- a/products/adapter/scripts/check_dependency_direction.py +++ b/products/adapter/scripts/check_dependency_direction.py @@ -13,11 +13,12 @@ by no facade or engine crate, and never reaches a product runtime, adopter tooling, supporting service, binding, or database crate. -`registry-adapter-engine` is the listener-free engine the ingestion, -facade, and Evidence hosts may link. Its resolved closure may reach only +`registry-adapter-engine` is the listener-free engine used by ingestion, +the facade, Evidence and BReg external projections. Its resolved closure may reach only `registry-adapter-core`, shared platform crates, and non-workspace support. The source check separately refuses facade-only dependencies, including -Axum and PostgreSQL clients. Evidence may reach Adapter core and engine only. +Axum and PostgreSQL clients. Evidence and BReg may reach only Adapter core +and engine. Edges are read with every feature enabled and on every target, so an optional dependency no default build enables still cannot escape the @@ -171,6 +172,15 @@ "registry-adapterctl", } ), + # BReg owns external-projection grants, storage and disclosure. Its + # acquisition host embeds only the listener-free Adapter boundary. + "registry-breg": frozenset( + {"registry-adapter-core", "registry-adapter-engine"} + ), + # The BReg authoring tool reaches the same code through registry-breg. + "registry-bregctl": frozenset( + {"registry-adapter-core", "registry-adapter-engine"} + ), # Evidence embeds the listener-free engine and core contract, never the # facade host, job host, connector, or Adapter tooling. "registry-evidence": frozenset( @@ -188,6 +198,11 @@ "registry-adapter-engine", } ), + # Casework's PostgreSQL proof can link BReg, and caseworkctl can reach + # that feature-enabled runtime. This adds no new Adapter host edge. + "registry-caseworkctl": frozenset( + {"registry-adapter-core", "registry-adapter-engine"} + ), "registry-discovery-client": frozenset( { "registry-adapter-core", diff --git a/products/adapter/scripts/test_dependency_direction.py b/products/adapter/scripts/test_dependency_direction.py index 004eb71fbe..adaeec4321 100755 --- a/products/adapter/scripts/test_dependency_direction.py +++ b/products/adapter/scripts/test_dependency_direction.py @@ -43,6 +43,7 @@ "cli-docs": "registry-cli-docs", "evidence": "registry-evidence", "casework": "registry-casework", + "caseworkctl": "registry-caseworkctl", "evidence-core": "registry-evidence-core", "coordinator-core": "registry-coordinator-core", "axum-alias": "axum", @@ -68,10 +69,10 @@ "platform-testing": ["platform"], "platform-script": ["serde"], "serde": [], - "breg": ["record", "platform"], + "breg": ["record", "platform", "shared-engine", "core"], "breg-client": ["record", "platform"], "bregctl": ["breg"], - "breg-mcp": ["breg-client", ("breg", "dev")], + "breg-mcp": ["breg-client"], "scheduling": ["platform"], "scheduling-client": ["platform"], "schedulingctl": ["scheduling"], @@ -83,7 +84,8 @@ "stack-client": ["breg-client", "coordinator-client"], "cli-docs": ["engine", "ctl", "job"], "evidence": ["shared-engine", "core", "platform"], - "casework": ["platform"], + "casework": ["platform", "breg"], + "caseworkctl": ["casework"], "evidence-core": ["platform"], "coordinator-core": ["platform"], "axum-alias": ["serde"], @@ -225,7 +227,7 @@ def test_a_dev_dependency_of_an_engine_crate_counts(self): def test_a_dev_dependency_of_a_reached_platform_crate_is_not_followed(self): self.assertEqual( MODULE.violations( - metadata(platform=["serde", ("casework", "dev")]) + metadata(platform=["serde", ("scheduling", "dev")]) ), [], ) @@ -312,6 +314,16 @@ def test_any_outside_crate_depending_on_an_adapter_crate_is_refused(self): failures(**{outside: [*INTENDED[outside], "connectors"]}), ) + def test_breg_projection_host_cannot_reach_the_adapter_facade_or_job_host(self): + for edge in ("engine", "job", "ctl", "connectors"): + with self.subTest(edge=edge): + found = failures(breg=[*INTENDED["breg"], edge]) + self.assertIn( + "registry-breg depends on Adapter package(s) it is not listed for:", + found, + ) + self.assertIn(NAMES[edge], found) + def test_evidence_cannot_reach_the_facade_host_even_for_tests(self): for edge in ("engine", ("engine", "dev")): with self.subTest(edge=edge): @@ -336,6 +348,12 @@ def test_cli_docs_carries_only_the_engine_command_trees(self): "registry-adapterctl", } ), + "registry-breg": frozenset( + {"registry-adapter-core", "registry-adapter-engine"} + ), + "registry-bregctl": frozenset( + {"registry-adapter-core", "registry-adapter-engine"} + ), "registry-evidence": frozenset( { "registry-adapter-core", @@ -348,6 +366,9 @@ def test_cli_docs_carries_only_the_engine_command_trees(self): "registry-adapter-engine", } ), + "registry-caseworkctl": frozenset( + {"registry-adapter-core", "registry-adapter-engine"} + ), "registry-discovery-client": frozenset( { "registry-adapter-core", diff --git a/products/breg/contracts/explain/ModelExplanation.schema.json b/products/breg/contracts/explain/ModelExplanation.schema.json index 69e2b3ebd2..b9bcc4866e 100644 --- a/products/breg/contracts/explain/ModelExplanation.schema.json +++ b/products/breg/contracts/explain/ModelExplanation.schema.json @@ -67,6 +67,26 @@ "revision": { "$ref": "#/$defs/Digest" }, + "externalViews": { + "type": "object", + "additionalProperties": false, + "required": ["recordTypes", "projections", "views", "grants", "refresh"], + "properties": { + "recordTypes": { "$ref": "#/$defs/ExternalDeclarations" }, + "projections": { "$ref": "#/$defs/ExternalDeclarations" }, + "views": { "$ref": "#/$defs/ExternalDeclarations" }, + "grants": { "$ref": "#/$defs/ExternalDeclarations" }, + "refresh": { + "type": "object", + "propertyNames": { "$ref": "#/$defs/LocalId" }, + "additionalProperties": { + "type": "array", + "uniqueItems": true, + "items": { "$ref": "#/$defs/LocalId" } + } + } + } + }, "recordTypes": { "type": "array", "items": { @@ -563,6 +583,12 @@ } }, "$defs": { + "ExternalDeclarations": { + "type": "object", + "propertyNames": { "$ref": "#/$defs/LocalId" }, + "additionalProperties": { "type": "object" }, + "x-registry-passthrough": "Normalized external view declarations and grants emitted by the compiler." + }, "Digest": { "description": "A SHA-256 digest: `sha256:` followed by 64 lowercase hex digits.", "pattern": "^sha256:[0-9a-f]{64}$", diff --git a/products/breg/contracts/security-invariant-matrix.yaml b/products/breg/contracts/security-invariant-matrix.yaml index 8b6c3b8657..58bed44a53 100644 --- a/products/breg/contracts/security-invariant-matrix.yaml +++ b/products/breg/contracts/security-invariant-matrix.yaml @@ -215,3 +215,7 @@ invariants: # Audited operator transition proof: crates/registry-breg/tests/postgres_record_timer_operator.rs::operator_retry_grants_one_technical_allowance_and_refuses_other_states - {"id": "BREG-SEC-202", "state": "enforced", "targetWave": "W5", "threat": "A timer operator read discloses Record values or an unaudited operator command changes timer control or retry state outside the current package and ledger fences.", "enforcementPoint": "Read-only operator transactions project a closed metadata-only occurrence and attempt model under verified active-package admission; pause, resume, and retry use paired value-free audit records and exact control, occurrence, attempt, binding, window, and state comparisons.", "refusal": "Return no Record field values, raw failure details, claim tokens, or unbounded history; refuse stale or ineligible operator transitions without mutation; refuse before mutation when the audit request cannot be recorded, and withhold a successful response as unresolved or unfinished when commit acknowledgement or the response audit is uncertain, without claiming reversal of a transition that may have committed.", "negativeId": "BREG-NEG-202", "negativeTest": {"path": "crates/registry-breg/tests/postgres_record_timer_operator.rs", "name": "operator_reads_pending_and_completed_occurrences_without_record_values"}} - {id: BREG-SEC-203, state: enforced, targetWave: W5, threat: "Requests from different OAuth clients for one principal are indistinguishable in the journal or expose client or bearer values.", enforcementPoint: "BReg request audit projection receives the OAuth client matched by the bearer verifier.", refusal: "Project an activation-scoped keyed client reference on attempts, terminal records, and refusals. Omit it when no client is matched and retain no raw client identifier or token.", negativeId: BREG-NEG-203, negativeTest: {path: crates/registry-breg/tests/postgres_access_log.rs, name: direct_client_is_distinguished_in_allowed_and_refused_request_audit}} + - {"id": "BREG-SEC-204", "state": "enforced", "targetWave": "W5", "threat": "An external operation uses a different source scope, identifier scheme or value contract, or retains data without source approval.", "enforcementPoint": "Startup binds the checked project to the pinned Adapter operation, reviewed source grant and bounded retention configuration.", "refusal": "Refuse mismatched operation metadata or indexed acquisition without approved retention.", "negativeId": "BREG-NEG-204", "negativeTest": {"path": "crates/registry-breg/src/external_source.rs", "name": "project_operation_cannot_change_source_scope_scheme_type_or_retention"}} + - {"id": "BREG-SEC-205", "state": "enforced", "targetWave": "W5", "threat": "An indexed external query leaks an ineligible root or reports a complete result over incomplete coverage.", "enforcementPoint": "Named view grants and PostgreSQL row policy determine eligibility. Coverage and page selection share a statement snapshot.", "refusal": "Refuse incomplete coverage before applying external predicates. Filter, sort, count and page over the same authorised population.", "negativeId": "BREG-NEG-205", "negativeTest": {"path": "crates/registry-breg/src/postgres/external_view_query.rs", "name": "root_rls_coverage_filter_count_and_bidirectional_null_paging_share_one_population"}} + - {"id": "BREG-SEC-206", "state": "enforced", "targetWave": "W5", "threat": "An older acquisition restores a value after a newer denial, retention refusal or other terminal outcome.", "enforcementPoint": "The projection publication function checks the current scope and ordered refresh receipts before replacing retained data.", "refusal": "Refuse stale publication while an earlier attempt is still live, including after the newer receipt expires or removes the snapshot.", "negativeId": "BREG-NEG-206", "negativeTest": {"path": "crates/registry-breg/src/postgres/external_projection.rs", "name": "terminal_receipts_fence_older_results_after_snapshot_removal"}} + - {"id": "BREG-SEC-207", "state": "enforced", "targetWave": "W5", "threat": "An expired or withdrawn projection is returned after normal serving or backup restoration.", "enforcementPoint": "Indexed reads enforce the current scope and expiry. The split-role store supports bounded purge and excludes its data from the documented backup.", "refusal": "Withhold expired results, remove expired and withdrawn rows through the guarded purge, and rebuild restored coverage under the current binding.", "negativeId": "BREG-NEG-207", "negativeTest": {"path": "crates/registry-breg/src/postgres/external_projection.rs", "name": "projection_expiry_purge_and_excluded_backup_restore_are_closed"}} diff --git a/products/breg/contracts/security-test-traceability.yaml b/products/breg/contracts/security-test-traceability.yaml index 60c6ccc316..aab38be61f 100644 --- a/products/breg/contracts/security-test-traceability.yaml +++ b/products/breg/contracts/security-test-traceability.yaml @@ -218,3 +218,7 @@ traceability: # Audited operator transition proof: crates/registry-breg/tests/postgres_record_timer_operator.rs::operator_retry_grants_one_technical_allowance_and_refuses_other_states. Encrypted-value non-disclosure proof: crates/registry-breg/tests/postgres_record_timers.rs::encrypted_predicate_write_opens_the_same_occurrence_and_maps_current_plaintext - {"id": "BREG-SEC-202", "state": "enforced", "negativeId": "BREG-NEG-202", "negativeTest": {"path": "crates/registry-breg/tests/postgres_record_timer_operator.rs", "name": "operator_reads_pending_and_completed_occurrences_without_record_values"}} - {id: BREG-SEC-203, state: enforced, negativeId: BREG-NEG-203, negativeTest: {path: crates/registry-breg/tests/postgres_access_log.rs, name: direct_client_is_distinguished_in_allowed_and_refused_request_audit}} + - {"id": "BREG-SEC-204", "state": "enforced", "negativeId": "BREG-NEG-204", "negativeTest": {"path": "crates/registry-breg/src/external_source.rs", "name": "project_operation_cannot_change_source_scope_scheme_type_or_retention"}} + - {"id": "BREG-SEC-205", "state": "enforced", "negativeId": "BREG-NEG-205", "negativeTest": {"path": "crates/registry-breg/src/postgres/external_view_query.rs", "name": "root_rls_coverage_filter_count_and_bidirectional_null_paging_share_one_population"}} + - {"id": "BREG-SEC-206", "state": "enforced", "negativeId": "BREG-NEG-206", "negativeTest": {"path": "crates/registry-breg/src/postgres/external_projection.rs", "name": "terminal_receipts_fence_older_results_after_snapshot_removal"}} + - {"id": "BREG-SEC-207", "state": "enforced", "negativeId": "BREG-NEG-207", "negativeTest": {"path": "crates/registry-breg/src/postgres/external_projection.rs", "name": "projection_expiry_purge_and_excluded_backup_restore_are_closed"}} diff --git a/products/breg/examples/README.md b/products/breg/examples/README.md index 6a24fe583c..e3536a8fc5 100644 --- a/products/breg/examples/README.md +++ b/products/breg/examples/README.md @@ -13,6 +13,10 @@ is governed configuration. Proposal lifecycle operations use promoted links. timer-only immediate Action. Its [walkthrough](future-activation/README.md) separates compiler inspection from the opt-in real-PostgreSQL native-process journey. +[`external-views/`](external-views/README.md) shows a hand-authored licensing +project with live and indexed external Person and shareholding views. +Its synthetic source and journey are acceptance fixtures for a disposable +deployment. The `minimal/` and `facility/` shipped starters include their own local clients and authored journeys. diff --git a/products/breg/examples/external-views/README.md b/products/breg/examples/external-views/README.md new file mode 100644 index 0000000000..7f665d3a5f --- /dev/null +++ b/products/breg/examples/external-views/README.md @@ -0,0 +1,169 @@ +# Synthetic licensing views + +The [native project](registry.yaml) owns licences and licensed businesses. +The Population Registry owns Person names. The Business Registry owns the +shareholding rows. BReg stores the local facts and the references in its +ordinary Record payloads. Live and indexed external views are separate +permissions on the same local root. + +| View | Root and source | Officer access | +|---|---|---| +| `licence-holder-live` | Licence → Population Registry Person | Live detail | +| `licence-holder-indexed` | Licence → Population Registry Person | Indexed detail, exact surname filter, ordering and count | +| `business-holdings-live` | Licensed business → Business Registry holdings | Live related detail | +| `business-holdings-indexed` | Licensed business → Business Registry holdings | Indexed collection, class filter, share count order, and a separately granted Person label | + +Two licences and a holding reuse the same indexed Person projection. +A company shareholder has a Business key and no Person key, so the +company row remains visible without a Person lookup. The restricted +profile reads the same local licences and businesses without any external +disclosure or query permission. The refresh profile uses a dedicated +service identity, has only the approved refresh scopes, and can list local +roots only in its assigned jurisdiction. + +Check the project with the candidate `bregctl`: + +```sh +bregctl check products/breg/examples/external-views +``` + +The [dev clients and seed records](dev-clients.yaml) provide synthetic +identities and roots for an isolated deployment. This directory is an +acceptance example and is not included in the shipped starter catalog. + +## Run the synthetic source + +Use separate owner-only files for the two fixture credentials. The +runtime's configured file secret provider must resolve the service key +file. Only the test driver receives the control key. Start the source: + +```sh +python3 products/breg/examples/external-views/synthetic_source.py \ + --service-key-file "$FIXTURE_SERVICE_KEY_FILE" \ + --control-key-file "$FIXTURE_CONTROL_KEY_FILE" \ + --runtime-fragment-file "$FIXTURE_RUNTIME_FRAGMENT" +``` + +The source binds only to numeric loopback, chooses a free port, and +prints its origin. The fragment pins the exact [Adapter pack](../../../adapter/examples/external-projections/pack.yaml), +binds its endpoint and scope for each projection, and names the service +key by secret reference. Merge the `externalSources` member into an +operator-owned runtime file before starting the candidate BReg. The +fragment grants an hour of synthetic source access and uses a five minute +indexed retention ceiling. Schedule refresh every minute for a continued +demo, or run the fixture refresh requests below. + +The fixture control identity can reset or rename the Person, replace or +empty holdings, interrupt the next page, apply `no-store` to a data +operation, or simulate denied, ambiguous and disconnected Person reads. +`hold-next` captures a response before pausing it; another request can +complete before the controller releases it. The control status reports +value-free operation counts and whether a response is held. The service +credential cannot use the control endpoints. + +Test the source and its snapshot discipline independently: + +```sh +python3 -B products/breg/examples/external-views/test_source.py +``` + +See the [operator guide](../../external-projection-storage.md) for refresh, retention, +scope changes and backup recovery. + +## Run with candidate binaries + +Indexed projections require separate PostgreSQL migration and runtime roles +for activation and serving. The candidate development configuration selects +those roles when the project declares an indexed projection. Use the fixture +runner to supply the synthetic source bindings and run the HTTP checks. + +Prepare two fresh databases on the same disposable loopback PostgreSQL TLS +server. Use names such as `breg_ext_rehearsal` and `breg_ext_runtime`. +Create two non-superuser login roles and configure them in both databases. +The migration role owns `registry_internal`, `registry_data`, +`registry_source`, `registry_derived` and `registry_context`. The runtime role +has no ownership, membership in the migration role, or `BYPASSRLS`. Install +`btree_gist` in each database, revoke public access to the databases and +managed schemas, and grant both roles database `CONNECT`. BReg grants the +runtime permissions during activation. The adopter workflow has the same +role and schema setup. [Existing adopter fixture](../../scripts/test-adopter-workflow.sh) + +Put the four PostgreSQL URLs in owner-only files named +`rehearsal-migration-url`, `rehearsal-runtime-url`, `migration-url`, and +`runtime-url`. Use the same respective role in each database. Each URL +needs an explicit loopback host, port, database and `?sslmode=require`. +Supply the local PostgreSQL CA PEM separately. + +```sh +python3 -B products/breg/examples/external-views/run_candidate.py \ + --breg-bin "$BREG_BIN" \ + --bregctl-bin "$BREGCTL_BIN" \ + --database-url-dir "$FIXTURE_DATABASE_URL_DIR" \ + --postgres-ca-file "$BREG_TEST_TLS_CA_PEM_PATH" \ + --work-dir "$FIXTURE_NEW_WORK_DIR" +``` + +The runner rehearses ordinary Record creation and packages the tested project. +Before activation it generates and validates an owner-only +`external-source-facts.json` for all four bindings. The facts can be +imported into Studio and the check verifies that generation made no source +requests. The runner then activates, serves and seeds the package with the +supplied binaries. A local EdDSA fixture signer issues short-lived tokens +that the candidate verifies +against static JWKS. The synthetic source runs on a loopback port for the +HTTP journey. The runner stops its own processes and leaves the two +databases and owner-only evidence directory available for inspection. A +repeat run needs fresh databases and a new work directory. + +To exercise an equivalent native project saved by Studio, add +`--project-dir "$FIXTURE_STUDIO_PROJECT_DIR"`. Supply the saved +`registry.yaml`, `dev-clients.yaml`, and `tests/journeys.yaml`. The runner +stages those three files unchanged and applies the same binding and HTTP +checks. + +Add `--recovery` to the small candidate runner to exercise migration-role +scope maintenance. It uses the staged migration connection and requires +`psql`. The runner pauses the first source response of a paged holdings +refresh, withdraws the inspected scope, and checks that the in-flight +refresh sends no later source requests or restores indexed data. It +rotates the scope using the checked synthetic binding, restarts BReg, +and verifies that indexed coverage remains incomplete until an explicit +fresh refresh. Owner-only `scope-*.json` and `recovery-*.json` reports +record the transition and operation counts. + +## Run the HTTP checks against an existing fixture + +Use the same split roles and seed the supplied records under the fixture +editor. Obtain short-lived officer, restricted and refresh tokens and keep +them in owner-only files. The refresh token must carry the verified service +actor and requester client `licensing-refresh`. Pass the local root IDs +for `licence-first` and `business-with-holdings` to the runner. + +```sh +python3 products/breg/examples/external-views/journey.py \ + --breg-origin "$FIXTURE_BREG_ORIGIN" \ + --source-origin "$FIXTURE_SOURCE_ORIGIN" \ + --officer-token-file "$FIXTURE_OFFICER_TOKEN_FILE" \ + --restricted-token-file "$FIXTURE_RESTRICTED_TOKEN_FILE" \ + --refresh-token-file "$FIXTURE_REFRESH_TOKEN_FILE" \ + --control-key-file "$FIXTURE_CONTROL_KEY_FILE" \ + --licence-root "$FIXTURE_LICENCE_ID" \ + --business-root "$FIXTURE_BUSINESS_ID" +``` + +The runner exercises live exact and related reads, role concealment before +and after refresh, surname count and paging across unopened licences, +authorized absence, label reuse, company shareholders, retention refusal, +complete snapshot replacement, and interrupted acquisition. It verifies that +a retained Person becomes unavailable after source denial, ambiguity or +disconnection and that recovery restores coverage. Source call counts check +that indexed and denied reads skip acquisition. Database races, retention +purges and scale have separate acceptance checks. + + +## Run the million-record fixture + +The scale exercise uses the same candidate binaries, split roles and TLS +setup with a larger synthetic corpus. It sets an explicit freshness and +request budget and records the actual native timings and PostgreSQL plans. +[Scale fixture instructions](SCALE.md) diff --git a/products/breg/examples/external-views/SCALE.md b/products/breg/examples/external-views/SCALE.md new file mode 100644 index 0000000000..d32fb8b56a --- /dev/null +++ b/products/breg/examples/external-views/SCALE.md @@ -0,0 +1,87 @@ +# Native external projection scale fixture + +Run this acceptance fixture against the same paired candidate binaries as +the [small licensing example](README.md). The native run uses two fresh +loopback PostgreSQL 18 TLS databases, distinct migration and runtime +roles, and a dedicated plan log. The predeclared machine is the shared +M5 Max workstation with local storage. Load and timing on the shared +machine can vary. + +| Measure | Predeclared target | +|---|---| +| Licence population | 1,000,000 authorized roots and 100,000 outside the runtime profile's row boundary | +| Holder reuse | 50,000 source keys, 20 roots per key, including 2,500 authorized absent keys | +| Related records | 120 rows for each of two allowed parents and two denied parents | +| Candidate HTTP | Each measured indexed read, including refusal, finishes within 5 seconds | +| Freshness and retention | Complete the sweep and the measured reads within 600 seconds from the first refresh. Retention ceiling is 900 seconds | +| Person source requests | At most 1,100 batch reads and no exact reads during the sweep | +| Holdings source requests | Six reads for the two allowed parents, no request to a denied parent | +| Indexed query source requests | Zero, including reads with missing coverage | + +The reviewed Adapter pack permits at most 64 Person keys per request and +128 rows per complete related snapshot. The fixture orders the first +50,000 local roots so that their keys are distinct. At 100 local roots per +refresh page, the first 500 pages should each make two Adapter requests. +Later pages should reuse published keys from this sweep. The 1,100-call +ceiling allows headroom above the 1,000 calls this layout predicts. The +native service must traverse 10,000 HTTP refresh pages. Each authorized +holding snapshot uses three source pages of 40 rows. The scale holdings +are company shareholders. The small native fixture covers reuse of a +Person label. + +Set the serving runtime role's PostgreSQL defaults to `work_mem = '64MB'` +and `statement_timeout = '10s'`. For the dedicated serving database, +configure the runtime role to load `auto_explain` at session start and +set `auto_explain.log_min_duration = '50ms'`, +`auto_explain.log_analyze = on`, `auto_explain.log_buffers = on`, and +`auto_explain.log_format = 'text'`. Route PostgreSQL logs to an existing +readable file with the prefix `[breg-plan] %d %u ` and keep it from rotating +during the run. Limit this configuration to the disposable fixture. The +runner extracts only appended plans from the named database and runtime +role after refresh. The plans must contain the candidate's actual +coverage and page nodes with buffers. + +```sh +python3 -B products/breg/examples/external-views/run_candidate.py \ + --breg-bin "$BREG_BIN" \ + --bregctl-bin "$BREGCTL_BIN" \ + --database-url-dir "$FIXTURE_DATABASE_URL_DIR" \ + --postgres-ca-file "$BREG_TEST_TLS_CA_PEM_PATH" \ + --work-dir "$FIXTURE_NEW_WORK_DIR" \ + --scale \ + --postgres-plan-log "$FIXTURE_POSTGRES_PLAN_LOG" +``` + +The launcher copies the authored fixture to its private work directory and +sets the two indexed cache lifetimes to 600 seconds. It rehearses and +activates that project using the candidate binaries. After activation, the +migration role uses PostgreSQL `generate_series` to bulk load only the +fixture's two empty owned Record tables under the fixture editor's row +policy. Table and field names come from the checked package inventory. This +setup does not measure Record ingestion or create Record history. Every +external snapshot is acquired through the candidate refresh endpoint and +the reviewed Adapter pack. + +The runner compares three first-page repetitions and every page of the +selected surname to an independently computed set of root IDs. It compares +both sort directions and their first continuations, authorized missing +values and their count, the unfiltered population count, and all related +rows plus filtered pages for both permitted parents. It inserts a new +authorized root through the ordinary HTTP route after the sweep and expects +a coverage refusal even though the requested surname would exclude that +root. It checks the two denied collection roots and compares source counters +before and after all indexed requests. + +Read `scale-report.json` for candidate binary SHA-256 values, the pack and +package inventory digests, PostgreSQL settings, refresh metrics, source +request counts and query wall times. The report includes partial measurements +on failure after the scale run begins. `scale-native-plans.log` contains the +matching native plans. Record the candidate repository revision and the +command beside these files. + +Test the deterministic corpus, source bounds and oracle without starting a +candidate or PostgreSQL: + +```sh +python3 -B products/breg/examples/external-views/test_scale.py +``` diff --git a/products/breg/examples/external-views/dev-clients.yaml b/products/breg/examples/external-views/dev-clients.yaml new file mode 100644 index 0000000000..520d419513 --- /dev/null +++ b/products/breg/examples/external-views/dev-clients.yaml @@ -0,0 +1,45 @@ +# yaml-language-server: $schema=https://id.registrystack.org/schemas/breg/dev-clients/dev-clients.v1alpha1.schema.json +apiVersion: id.registrystack.org/formats/breg/dev-clients/v1alpha1 +kind: BRegDevClients +clients: + - id: officer + allowHumanFixture: true + accessProfiles: [officer] + scopes: [registry:licensing:read] + claims: + registry_principal: synthetic-licensing-officer + registry_actor_kind: human + registry_purpose: licensing + jurisdiction: zone-a + - id: restricted + allowHumanFixture: true + accessProfiles: [local-facts] + scopes: [registry:licensing:read] + claims: + registry_principal: synthetic-licensing-restricted + registry_actor_kind: human + registry_purpose: licensing + jurisdiction: zone-a + - id: licensing-refresh + accessProfiles: [projection-refresh] + scopes: [registry:licensing:refresh] + claims: + registry_principal: synthetic-licensing-refresh + registry_purpose: licensing + jurisdiction: zone-a + - id: fixture-editor + accessProfiles: [fixture-editor] + scopes: [registry:licensing:write] + claims: + registry_principal: synthetic-licensing-editor + registry_purpose: licensing +seed: + - {id: licence-first, client: fixture-editor, recordType: licence, accessProfile: fixture-editor, data: {licenceNumber: L-001, holderKey: person-7, jurisdiction: zone-a}} + - {id: licence-reuses-holder, client: fixture-editor, recordType: licence, accessProfile: fixture-editor, data: {licenceNumber: L-002, holderKey: person-7, jurisdiction: zone-a}} + - {id: licence-unopened, client: fixture-editor, recordType: licence, accessProfile: fixture-editor, data: {licenceNumber: L-003, holderKey: person-8, jurisdiction: zone-a}} + - {id: licence-absent-holder, client: fixture-editor, recordType: licence, accessProfile: fixture-editor, data: {licenceNumber: L-004, holderKey: person-9, jurisdiction: zone-a}} + - {id: licence-fault-holder, client: fixture-editor, recordType: licence, accessProfile: fixture-editor, data: {licenceNumber: L-005, holderKey: person-10, jurisdiction: zone-a}} + - {id: licence-other-zone, client: fixture-editor, recordType: licence, accessProfile: fixture-editor, data: {licenceNumber: L-006, holderKey: person-11, jurisdiction: zone-b}} + - {id: business-with-holdings, client: fixture-editor, recordType: licensed-business, accessProfile: fixture-editor, data: {licenceNumber: B-001, businessKey: business-1, jurisdiction: zone-a}} + - {id: business-without-holdings, client: fixture-editor, recordType: licensed-business, accessProfile: fixture-editor, data: {licenceNumber: B-002, businessKey: business-2, jurisdiction: zone-a}} + - {id: business-other-zone, client: fixture-editor, recordType: licensed-business, accessProfile: fixture-editor, data: {licenceNumber: B-003, businessKey: business-3, jurisdiction: zone-b}} diff --git a/products/breg/examples/external-views/journey.py b/products/breg/examples/external-views/journey.py new file mode 100644 index 0000000000..ef65859385 --- /dev/null +++ b/products/breg/examples/external-views/journey.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 + +"""Check the projection journey against a fresh synthetic BReg deployment.""" + +import argparse +from concurrent.futures import ThreadPoolExecutor +import time +import json +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode, urlsplit +from urllib.request import HTTPRedirectHandler, ProxyHandler, Request, build_opener +from uuid import UUID + + +class JourneyError(RuntimeError): + pass + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, *_args, **_kwargs): + return None + + +HTTP = build_opener(ProxyHandler({}), NoRedirect()) + + +def expect(condition, name): + if not condition: + raise JourneyError(name) + + +def origin(value): + parsed = urlsplit(value) + if ( + parsed.scheme != "http" or parsed.hostname != "127.0.0.1" + or parsed.port is None or parsed.username is not None + or parsed.path not in ("", "/") or parsed.query or parsed.fragment + ): + raise argparse.ArgumentTypeError("use an explicit numeric loopback HTTP origin") + return value.rstrip("/") + + +def fixture_credential(path): + with path.open("rb") as stream: + value = stream.read(16_385).strip() + expect(0 < len(value) <= 16_384 and all(33 <= byte <= 126 for byte in value), + "a fixture credential file has an invalid shape") + return value.decode("ascii") + + +def request(base, path, headers, method="GET", query=None, body=None): + url = base + path + (("?" + urlencode(query)) if query else "") + data = None + if body is not None: + data = json.dumps(body).encode("utf-8") + headers = {**headers, "Content-Type": "application/json"} + target = Request(url, headers=headers, data=data, method=method) + try: + response = HTTP.open(target, timeout=15) + except HTTPError as error: + response = error + except URLError as error: + raise JourneyError("a loopback fixture service could not be reached") from error + with response: + payload = response.read(1_048_577) + expect(len(payload) <= 1_048_576, "fixture response exceeded its byte limit") + try: + document = json.loads(payload) if payload else None + except (UnicodeError, json.JSONDecodeError) as error: + raise JourneyError("fixture response was not JSON") from error + return response.status, document + + +class Journey: + def __init__(self, args): + self.args = args + self.auth = { + "officer": {"Authorization": "Bearer " + fixture_credential(args.officer_token_file)}, + "local-facts": {"Authorization": "Bearer " + fixture_credential(args.restricted_token_file)}, + "projection-refresh": {"Authorization": "Bearer " + fixture_credential(args.refresh_token_file)}, + } + self.control_auth = {"x-fixture-control-key": fixture_credential(args.control_key_file)} + + def view(self, view, link, root=None, profile="officer", query=None): + path = f"/v1/external-views/{view}/{link}/records" + if root is not None: + path += f"/{root}" + return request(self.args.breg_origin, path, self.auth[profile], + query={"accessProfile": profile, **(query or {})}) + + def refresh(self, view, link, *, labels=False): + operation = "refresh-labels" if labels else "refresh" + path = f"/v1/external-views/{view}/{link}/{operation}" + query = {"accessProfile": "projection-refresh"} + totals = {field: 0 for field in ("processed", "published", "unavailable")} + seen = set() + for _ in range(20): + status, report = request( + self.args.breg_origin, path, self.auth["projection-refresh"], + method="POST", query=query, + ) + expect(status == 200 and isinstance(report, dict), "a refresh request failed") + for field in totals: + count = report.get(field) + expect(type(count) is int and count >= 0, "refresh returned an invalid count") + totals[field] += count + cursor = report.get("nextCursor") + if cursor is None: + return totals + expect(isinstance(cursor, str) and cursor not in seen, "refresh cursor repeated") + seen.add(cursor) + query["$skiptoken"] = cursor + raise JourneyError("refresh exceeded the fixture's page budget") + + def control(self, action, **options): + status, report = request( + self.args.source_origin, "/fixture/control", self.control_auth, + method="POST", body={"action": action, **options}, + ) + expect(status == 200, "fixture control was refused") + return report + + def source_calls(self): + status, report = request( + self.args.source_origin, "/fixture/status", self.control_auth, + ) + expect(status == 200 and isinstance(report, dict), "fixture status was unavailable") + return report["operations"] + + def while_person_refresh_is_held(self, action): + self.control("hold-next", operation="person-batch") + with ThreadPoolExecutor(max_workers=1) as workers: + older = workers.submit(self.refresh, "licence-holder-indexed", "holder") + try: + deadline = time.monotonic() + 2 + while True: + status, report = request(self.args.source_origin, "/fixture/status", self.control_auth) + expect(status == 200, "fixture status was unavailable") + if report.get("held"): + break + expect(time.monotonic() < deadline, "the older refresh did not reach the source barrier") + time.sleep(0.02) + action() + finally: + self.control("release") + return older.result(timeout=5) + + def licence_filter(self, surname_literal, **extra): + return self.view("licence-holder-indexed", "holder", + query={"$filter": f"family-name eq {surname_literal}", "$count": "true", **extra}) + + def business(self, live=False): + return self.view("business-holdings-live" if live else "business-holdings-indexed", + "holdings", self.args.business_root) + + def run(self): + self.control("reset") + baseline = self.source_calls() + expect(self.licence_filter("'Lee'")[0] == 503, "unbuilt indexed view reported coverage") + expect(self.source_calls() == baseline, "indexed query contacted the source") + + restricted_root = request( + self.args.breg_origin, f"/v1/records/licences/{self.args.licence_root}", + self.auth["local-facts"], query={"accessProfile": "local-facts"}, + ) + expect(restricted_root[0] == 200, "restricted caller could not read the local licence") + expect(self.view("licence-holder-live", "holder", self.args.licence_root, + profile="local-facts")[0] == 404, "restricted caller reached a live view") + for query in ({"$filter": "family-name eq 'Lee'"}, {"$orderby": "family-name"}, {"$count": "true"}): + expect(self.view("licence-holder-indexed", "holder", profile="local-facts", query=query)[0] == 404, + "restricted caller processed an external field") + expect(self.source_calls() == baseline, "denied reads contacted the source") + + live = records(self.view("licence-holder-live", "holder", self.args.licence_root)) + expect(len(live) == 1 and live[0]["fields"]["family-name"] == "Lee", "live Person lookup failed") + live_holdings = records(self.business(live=True)) + expect({row["fields"]["shareholder-type"] for row in live_holdings} == {"person", "business"}, + "live list did not return both shareholder types") + expect(self.licence_filter("'Lee'")[0] == 503, "live read retained indexed data") + + expect(self.refresh("licence-holder-indexed", "holder")["published"] > 0, "Person refresh published nothing") + expect(self.refresh("business-holdings-indexed", "holdings")["published"] > 0, + "holdings refresh published nothing") + + reads_before = self.source_calls() + response = self.licence_filter("'Lee'", **{"$orderby": "family-name", "$top": "1"}) + first = records(response) + expect(len(first) == 1 and response[1].get("count") == 2, "surname filtering or count is incorrect") + cursor = response[1].get("nextCursor") + expect(isinstance(cursor, str) and cursor, "surname page has no cursor") + second = records(self.view("licence-holder-indexed", "holder", query={"$skiptoken": cursor})) + expect(len(second) == 1 and first[0]["rootId"] != second[0]["rootId"], "surname pages overlap") + unopened = self.licence_filter("'Rivera'") + expect(len(records(unopened)) == 1 and unopened[1].get("count") == 1, "unopened licence was not indexed") + absent = self.licence_filter("null") + expect(len(records(absent)) == 1 and absent[1].get("count") == 1, "keyed absence did not count as coverage") + expect(self.view("licence-holder-indexed", "holder", self.args.licence_root, + profile="local-facts")[0] == 404, "a cache hit bypassed the restricted profile") + after_refresh = request( + self.args.breg_origin, f"/v1/records/licences/{self.args.licence_root}", + self.auth["local-facts"], query={"accessProfile": "local-facts"}, + ) + expect(after_refresh == restricted_root, "refresh changed the owned licence response") + expect(self.source_calls() == reads_before, "indexed pages contacted the source") + + original_holdings = records(self.business()) + expect(len(original_holdings) == 2 and surname_from_label(original_holdings) == "Lee", + "the indexed holding did not reuse the Person projection") + + self.control("add-label-only") + self.refresh("business-holdings-indexed", "holdings") + indirect = next((row for row in records(self.business()) if row["fields"]["shareholder-key"] == "person-12"), {}) + expect(indirect.get("label", {}).get("availability") == "unavailable", + "a new related-only label was already populated") + calls = self.source_calls() + denied = request( + self.args.breg_origin, "/v1/external-views/business-holdings-indexed/holdings/refresh-labels", + self.auth["officer"], method="POST", query={"accessProfile": "officer"}, + ) + expect(denied[0] == 404 and self.source_calls() == calls, + "a reader acquired an independent label") + expect(self.refresh("business-holdings-indexed", "holdings", labels=True)["published"] >= 1, + "no authorised related label was published") + indirect = next((row for row in records(self.business()) if row["fields"]["shareholder-key"] == "person-12"), {}) + expect(indirect.get("label", {}).get("fields", {}).get("family-name") == "Solano", + "the approved label required a separate local licence root") + expect(not records(self.licence_filter("'Solano'")), + "a related label created a new local licence association") + + def refuse_newer_retention(): + self.control("rename-holder") + self.control("set-no-store", operation="person-batch", enabled=True) + expect(self.refresh("licence-holder-indexed", "holder")["unavailable"] > 0, + "the indexed operation did not report a no-store refusal") + + older = self.while_person_refresh_is_held(refuse_newer_retention) + expect(older["published"] == 0 and self.licence_filter("'Lee'")[0] == 503, + "a delayed response revived values after a later no-store refusal") + current = self.view("licence-holder-indexed", "holder", self.args.licence_root) + if current[0] == 200: + expect(all(row.get("fields", {}).get("family-name") != "Ng" for row in records(current)), + "a no-store value entered the indexed view") + else: + expect(current[0] == 503, "no-store lookup returned an unexpected outcome") + self.control("set-no-store", operation="person-batch", enabled=False) + expect(self.refresh("licence-holder-indexed", "holder")["published"] > 0, "Person recovery did not publish") + expect(len(records(self.licence_filter("'Ng'"))) == 2, "one Person refresh did not update both licences") + expect(surname_from_label(records(self.business())) == "Ng", "one Person refresh did not update the holding label") + expect(self.view("licence-holder-indexed", "holder", query={"$skiptoken": cursor})[0] == 400, + "a cursor survived a newer projection generation") + + def publish_newer_person(): + self.control("restore-holder") + expect(self.refresh("licence-holder-indexed", "holder")["published"] > 0, + "the newer Person refresh did not publish") + + older = self.while_person_refresh_is_held(publish_newer_person) + expect(older["published"] == 0 + and len(records(self.licence_filter("'Lee'"))) == 2 + and not records(self.licence_filter("'Ng'")), + "the older refresh replaced a newer Person publication") + + before_interruption = records(self.business()) + self.control("interrupt-collection", enabled=True) + expect(self.refresh("business-holdings-indexed", "holdings")["unavailable"] > 0, + "interrupted collection was not reported") + expect(records(self.business()) == before_interruption, + "interruption displaced the last complete holdings snapshot") + self.control("interrupt-collection", enabled=False) + self.control("replace-holdings") + self.refresh("business-holdings-indexed", "holdings") + remaining = records(self.business()) + expect(len(remaining) == 1 and remaining[0]["fields"]["shareholder-type"] == "business", + "complete refresh did not remove the old holding") + self.control("empty-holdings") + self.refresh("business-holdings-indexed", "holdings") + expect(not records(self.business()), "complete empty snapshot was not published") + + retained = records(self.licence_filter("'Doe'")) + expect(len(retained) == 1 and retained[0]["fields"]["family-name"] == "Doe", + "the fault fixture had no retained present value") + fault_root = retained[0]["rootId"] + for fault in ("denied", "ambiguous", "disconnect"): + self.control("set-person-fault", kind=fault) + report = self.refresh("licence-holder-indexed", "holder") + expect(report["unavailable"] > 0, f"{fault} was not reported by refresh") + for literal in ("'Doe'", "null"): + expect(self.licence_filter(literal)[0] == 503, + f"{fault} was treated as complete coverage") + detail = self.view("licence-holder-indexed", "holder", fault_root) + if detail[0] == 200: + rows = records(detail) + expect(detail[1]["status"] == "unavailable" + or (len(rows) == 1 and rows[0]["availability"] == "unavailable"), + f"{fault} did not withdraw the retained detail") + expect(all(not row["fields"] for row in rows), f"{fault} disclosed retained fields") + else: + expect(detail[0] == 503, f"{fault} produced an unexpected detail response") + self.control("set-person-fault", kind="none") + expect(self.refresh("licence-holder-indexed", "holder")["published"] > 0, + "recovered Person refresh did not publish") + recovered = records(self.licence_filter("'Doe'")) + expect(len(recovered) == 1 and recovered[0]["rootId"] == fault_root, + "recovered Person coverage did not restore the retained value") + + +def records(response): + status, document = response + expect(status == 200 and isinstance(document, dict) and isinstance(document.get("records"), list), + "a view did not return its records envelope") + expect(isinstance(document.get("authority"), str), "a view did not identify the source authority") + return document["records"] + + +def surname_from_label(holdings): + person = next((row for row in holdings if row["fields"]["shareholder-type"] == "person"), {}) + company = next((row for row in holdings if row["fields"]["shareholder-type"] == "business"), {}) + expect(company.get("label") is None, "a company shareholder was interpreted as a Person") + label = person.get("label") + if not isinstance(label, dict): + return None + fields = label.get("fields", label) + return fields.get("family-name") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--breg-origin", type=origin, required=True) + parser.add_argument("--source-origin", type=origin, required=True) + parser.add_argument("--officer-token-file", type=Path, required=True) + parser.add_argument("--restricted-token-file", type=Path, required=True) + parser.add_argument("--refresh-token-file", type=Path, required=True) + parser.add_argument("--control-key-file", type=Path, required=True) + parser.add_argument("--licence-root", type=UUID, required=True) + parser.add_argument("--business-root", type=UUID, required=True) + args = parser.parse_args() + try: + Journey(args).run() + except (JourneyError, KeyError, TypeError, OSError) as error: + detail = str(error) if isinstance(error, JourneyError) else "fixture result did not match its contract" + parser.exit(1, f"External view journey failed: {detail}\n") + print("External view journey passed") + + +if __name__ == "__main__": + main() diff --git a/products/breg/examples/external-views/recovery.py b/products/breg/examples/external-views/recovery.py new file mode 100644 index 0000000000..62529cc3f1 --- /dev/null +++ b/products/breg/examples/external-views/recovery.py @@ -0,0 +1,168 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise operator withdrawal and restart on the disposable native fixture.""" + +from concurrent.futures import ThreadPoolExecutor +import json +import os +from pathlib import Path +import re +import subprocess +from urllib.parse import unquote, urlsplit + +from journey import JourneyError, expect, records, request + +PROJECTION = "holdings-indexed" +VIEW = "business-holdings-indexed" +LINK = "holdings" +OPERATION = "holdings-by-business" +OPERATOR = Path(__file__).resolve().parents[4] / "scripts/external-projection-scope.sh" + + +def operator_report(output): + lines = [line.strip().removesuffix("+").rstrip() for line in output.decode().splitlines()] + expect(lines.count("{") == lines.count("}") == lines.count("COMMIT") == 1 + and "(1 row)" in lines, "the operator did not return one committed scope report") + try: + report = json.loads("\n".join(lines[lines.index("{"):lines.index("}") + 1])) + except ValueError as error: + raise JourneyError("the operator returned an invalid scope report") from error + expect(isinstance(report, dict), "the operator returned an invalid scope report") + return report + + +def verify_hidden(journey): + before = journey.source_calls() + status, body = journey.business() + expect(status == 200 and isinstance(body, dict) and body.get("status") == "unavailable" + and records((status, body)) == [], + "the holdings detail disclosed data without a complete current scope") + status, body = journey.view(VIEW, LINK, journey.args.business_root, query={"$count": "true"}) + expect(status == 503 and isinstance(body, dict), + "the holdings count accepted incomplete indexed coverage") + expect(journey.source_calls() == before, "serving incomplete holdings attempted acquisition") + + +def verify_holdings(journey): + rows = records(journey.business()) + expect(len(rows) == 2 and {row["fields"]["shareholder-type"] for row in rows} == {"person", "business"}, + "the fixture did not have a complete two-page holdings snapshot") + + +class ScopeRecovery: + def __init__(self, journey, fixture, work, facts, private_write, write_json): + self.journey = journey + self.fixture = fixture + self.work = work + self.write_json = write_json + self.fact = next(fact for fact in facts["projections"] if fact["id"] == PROJECTION) + keys = work / "secrets" + # The launcher validates these as loopback, split-role fixture URLs before staging them. + target = urlsplit((keys / "migration-url").read_text(encoding="ascii")) + service = keys / "recovery-pg-service.conf" + private_write(service, ( + f"[external-fixture-migration]\nhost={target.hostname}\nport={target.port}\n" + f"dbname={target.path[1:]}\nuser={unquote(target.username)}\n" + ).encode("ascii")) + self.env = {key: value for key, value in os.environ.items() if not key.startswith("PG")} + self.env.update(PGSERVICEFILE=str(service), PGSSLMODE="verify-full", + PGSSLROOTCERT=str(keys / "postgres-ca"), PGCONNECT_TIMEOUT="3", + PGPASSFILE="/dev/null", LC_ALL="C") + if target.password is not None: + self.env["PGPASSWORD"] = unquote(target.password) + + def operator(self, action, step, current=None, arguments=()): + identity = () if current is None else (current["epoch"], current["packageDigest"], current["activationId"]) + with (self.work / f"scope-{step}.stderr").open("xb") as error: + result = subprocess.run( + ["bash", OPERATOR, action, "external-fixture-migration", PROJECTION, *identity, *arguments], + cwd=self.work, env=self.env, stdout=subprocess.PIPE, stderr=error, timeout=20, check=False, + ) + expect(result.returncode == 0, "the migration-only scope operator failed; see the owner-only fixture log") + report = operator_report(result.stdout) + expect(report.get("projection") == PROJECTION and type(report.get("active")) is bool + and all(isinstance(report.get(key), str) for key in ("epoch", "packageDigest", "activationId")) + and all(re.fullmatch(r"[0-9a-f-]{36}", report[key]) for key in ("epoch", "activationId")) + and re.fullmatch(r"sha256:[0-9a-f]{64}", report["packageDigest"]), + "the operator did not identify the activated fixture scope") + if current is not None: + expect(all(report[key] == current[key] for key in ("packageDigest", "activationId")), + "operator maintenance changed the package activation identity") + self.write_json(self.work / f"scope-{step}.json", report) + return report + + def withdraw_and_rotate(self): + self.journey.control("reset") + self.journey.refresh(VIEW, LINK) + verify_holdings(self.journey) + active = self.operator("inspect", "before-withdrawal") + expect(active["active"] and active["snapshotCounts"]["fresh"] > 0, + "withdrawal fixture lacked a retained holdings snapshot") + before = self.journey.source_calls() + self.journey.control("hold-next", operation=OPERATION) + with ThreadPoolExecutor(max_workers=1) as workers: + inflight = workers.submit( + request, self.journey.args.breg_origin, f"/v1/external-views/{VIEW}/{LINK}/refresh", + self.journey.auth["projection-refresh"], method="POST", + query={"accessProfile": "projection-refresh"}, + ) + try: + expect(self.fixture.wait_for_hold(timeout=2), "the first holdings page did not reach the source barrier") + held = self.journey.source_calls() + expect(held == {**before, OPERATION: before.get(OPERATION, 0) + 1}, + "the source received more than the first holdings request before withdrawal") + withdrawn = self.operator("withdraw", "withdrawal", active) + expect(not withdrawn["active"] and withdrawn["epoch"] == active["epoch"], + "the operator did not withdraw the inspected scope") + expect(self.fixture.status()["held"], "scope withdrawal exceeded the source barrier deadline") + finally: + self.journey.control("release") + status, body = inflight.result(timeout=8) + expect(self.journey.source_calls() == held, "a subsequent Adapter request followed scope withdrawal") + expect(status == 503 or (status == 200 and body["published"] == 0 and body["unavailable"] > 0), + "the in-flight refresh did not fail closed after withdrawal") + verify_hidden(self.journey) + inactive = self.operator("inspect", "after-withdrawal") + expect(not inactive["active"] and inactive["epoch"] == withdrawn["epoch"] + and inactive["snapshotCounts"] == {"fresh": 0, "expired": 0, "invalidated": 0}, + "withdrawal did not purge the retained holdings snapshots") + self.write_json(self.work / "recovery-inflight.json", { + "refreshStatus": status, "sourceRequestsBefore": before, "sourceRequestsAtWithdrawal": held, + "sourceRequestsAfterRelease": self.journey.source_calls(), + }) + self.rotate(inactive) + + def rotate(self, inactive): + rotated = self.operator("rotate", "rotation", inactive, ( + self.fact["bindingDigest"], self.fact["grantDigest"], + str(self.fact["cacheTtlSeconds"]), str(self.fact["maximumRetentionSeconds"]), + )) + expect(rotated["active"] and rotated["epoch"] != inactive["epoch"] + and rotated["bindingDigest"] == self.fact["bindingDigest"] + and rotated["grantDigest"] == self.fact["grantDigest"] + and rotated["cacheTtlSeconds"] == self.fact["cacheTtlSeconds"] + and rotated["retentionSeconds"] == self.fact["maximumRetentionSeconds"], + "rotation did not install a fresh epoch for the current reviewed binding") + self.calls_at_rotation = self.journey.source_calls() + self.epoch = rotated["epoch"] + + def verify_rebuild(self): + verify_hidden(self.journey) + incomplete = self.operator("inspect", "after-restart") + expect(incomplete["active"] and incomplete["epoch"] == self.epoch + and incomplete["snapshotCounts"] == {"fresh": 0, "expired": 0, "invalidated": 0}, + "restart did not preserve the empty rotated holdings scope") + expect(self.journey.source_calls() == self.calls_at_rotation, + "the restarted candidate acquired holdings before an explicit refresh") + result = self.journey.refresh(VIEW, LINK) + expect(result["published"] > 0 and result["unavailable"] == 0, + "the rotated scope did not publish a fresh holdings refresh") + after_refresh = self.journey.source_calls() + expect(after_refresh.get(OPERATION, 0) >= self.calls_at_rotation.get(OPERATION, 0) + 2, + "rebuilding the scope did not acquire both holdings pages") + verify_holdings(self.journey) + expect(self.journey.source_calls() == after_refresh, "the rebuilt indexed read attempted acquisition") + self.write_json(self.work / "recovery-rebuild.json", { + "refresh": result, "sourceRequestsBeforeRefresh": self.calls_at_rotation, + "sourceRequestsAfterRefresh": after_refresh, + }) diff --git a/products/breg/examples/external-views/registry.yaml b/products/breg/examples/external-views/registry.yaml new file mode 100644 index 0000000000..7c0298fbd8 --- /dev/null +++ b/products/breg/examples/external-views/registry.yaml @@ -0,0 +1,231 @@ +# yaml-language-server: $schema=../../generated/authoring/registry-project.schema.json +apiVersion: id.registrystack.org/formats/breg/project/v1alpha1 +kind: BRegProject +project: + id: synthetic-licensing + version: "0.1.0" + defaultLanguage: en + canonicalBaseIri: https://licensing.example.test + display: + label: Synthetic licensing registry + description: Local licences and licensed businesses with separately granted external views. +package: + sourceRevision: synthetic-licensing-0.1.0 +datasets: + - id: licensing + display: {label: Licensing} + +recordTypes: + - id: licence + route: licences + primaryDataset: licensing + mutationMode: mutable + classification: internal + display: {label: Licence} + fields: + - {id: licence-number, type: string, required: true, maximumLength: 40, classification: internal} + - {id: holder-key, type: string, required: true, maximumLength: 40, classification: restricted} + - {id: jurisdiction, type: string, required: true, maximumLength: 40, classification: internal} + - id: licensed-business + route: licensed-businesses + primaryDataset: licensing + mutationMode: mutable + classification: internal + display: {label: Licensed business} + fields: + - {id: licence-number, type: string, required: true, maximumLength: 40, classification: internal} + - {id: business-key, type: string, required: true, maximumLength: 40, classification: restricted} + - {id: jurisdiction, type: string, required: true, maximumLength: 40, classification: internal} + +externalRecordTypes: + - id: person + authority: urn:example:population-register + identifierScheme: urn:example:person-number + display: {label: Person} + fields: + - {id: given-name, type: string, classification: restricted} + - {id: family-name, type: string, classification: restricted} + - id: holding + authority: urn:example:business-register + identifierScheme: urn:example:holding-number + display: {label: Shareholding} + fields: + - {id: share-class, type: string, classification: internal} + - {id: share-count, type: integer, classification: internal} + - {id: shareholder-type, type: string, classification: internal} + - {id: shareholder-key, type: string, classification: restricted} + - {id: person-key, type: string, classification: restricted, nullable: true} + +externalProjections: + - id: person-live + recordType: person + operation: person-exact + operationVersion: "1" + purpose: licensing + scope: licensing-holders + selectorScheme: urn:example:person-number + mode: live + cardinality: one + fields: [given-name, family-name] + - id: person-indexed + recordType: person + operation: person-batch + operationVersion: "1" + purpose: licensing + scope: licensing-holders + selectorScheme: urn:example:person-number + mode: indexed + cardinality: one + cacheTtlSeconds: 300 + fields: [given-name, family-name] + - id: holdings-live + recordType: holding + operation: holdings-by-business + operationVersion: "1" + purpose: licensing + scope: licensed-business-holdings + selectorScheme: urn:example:business-number + mode: live + cardinality: many + fields: [share-class, share-count, shareholder-type, shareholder-key] + - id: holdings-indexed + recordType: holding + operation: holdings-by-business + operationVersion: "1" + purpose: licensing + scope: licensed-business-holdings + selectorScheme: urn:example:business-number + mode: indexed + cardinality: many + cacheTtlSeconds: 300 + fields: [share-class, share-count, shareholder-type, shareholder-key, person-key] + +externalViews: + - id: licence-holder-live + recordType: licence + links: + - id: holder + projection: person-live + fromField: holder-key + identifierScheme: urn:example:person-number + fields: [given-name, family-name] + - id: licence-holder-indexed + recordType: licence + links: + - id: holder + projection: person-indexed + fromField: holder-key + identifierScheme: urn:example:person-number + fields: [given-name, family-name] + - id: business-holdings-live + recordType: licensed-business + links: + - id: holdings + projection: holdings-live + fromField: business-key + identifierScheme: urn:example:business-number + fields: [share-class, share-count, shareholder-type, shareholder-key] + - id: business-holdings-indexed + recordType: licensed-business + links: + - id: holdings + projection: holdings-indexed + fromField: business-key + identifierScheme: urn:example:business-number + fields: [share-class, share-count, shareholder-type, shareholder-key] + label: + projection: person-indexed + fromField: person-key + identifierScheme: urn:example:person-number + fields: [given-name, family-name] + +accessProfiles: + - id: officer + actorKind: human + requesterClients: [officer] + principalClaim: registry_principal + requiredScopes: [registry:licensing:read] + requiredPurposes: [licensing] + permissions: + records: + - recordType: licence + operations: [get, list] + readableFields: [licence-number, jurisdiction] + allowCount: true + rowBoundaries: [{field: jurisdiction, claim: jurisdiction, operator: equals}] + - recordType: licensed-business + operations: [get, list] + readableFields: [licence-number, jurisdiction] + allowCount: true + rowBoundaries: [{field: jurisdiction, claim: jurisdiction, operator: equals}] + externalViews: + - view: licence-holder-live + links: + - {link: holder, readableFields: [given-name, family-name]} + - view: licence-holder-indexed + links: + - link: holder + readableFields: [given-name, family-name] + filterableFields: [family-name] + sortableFields: [family-name] + allowCount: true + - view: business-holdings-live + links: + - {link: holdings, readableFields: [share-class, share-count, shareholder-type, shareholder-key]} + - view: business-holdings-indexed + links: + - link: holdings + readableFields: [share-class, share-count, shareholder-type, shareholder-key] + filterableFields: [share-class] + sortableFields: [share-count] + allowCount: true + labelFields: [given-name, family-name] + - id: local-facts + actorKind: human + requesterClients: [restricted] + principalClaim: registry_principal + requiredScopes: [registry:licensing:read] + requiredPurposes: [licensing] + permissions: + records: + - recordType: licence + operations: [get, list] + readableFields: [licence-number, jurisdiction] + rowBoundaries: [{field: jurisdiction, claim: jurisdiction, operator: equals}] + - recordType: licensed-business + operations: [get, list] + readableFields: [licence-number, jurisdiction] + rowBoundaries: [{field: jurisdiction, claim: jurisdiction, operator: equals}] + - id: projection-refresh + actorKind: service + requesterClients: [licensing-refresh] + principalClaim: registry_principal + requiredScopes: [registry:licensing:refresh] + requiredPurposes: [licensing] + permissions: + records: + - recordType: licence + operations: [list] + readableFields: [holder-key, jurisdiction] + rowBoundaries: [{field: jurisdiction, claim: jurisdiction, operator: equals}] + - recordType: licensed-business + operations: [list] + readableFields: [business-key, jurisdiction] + rowBoundaries: [{field: jurisdiction, claim: jurisdiction, operator: equals}] + refreshExternalProjections: [person-indexed, holdings-indexed] + - id: fixture-editor + principalClaim: registry_principal + requiredScopes: [registry:licensing:write] + requiredPurposes: [licensing] + permissions: + records: + - recordType: licence + operations: [create, get, patch] + readableFields: [licence-number, holder-key, jurisdiction] + writableFields: [licence-number, holder-key, jurisdiction] + rowBoundaries: unrestricted + - recordType: licensed-business + operations: [create, get, patch] + readableFields: [licence-number, business-key, jurisdiction] + writableFields: [licence-number, business-key, jurisdiction] + rowBoundaries: unrestricted diff --git a/products/breg/examples/external-views/run_candidate.py b/products/breg/examples/external-views/run_candidate.py new file mode 100644 index 0000000000..cb815952c1 --- /dev/null +++ b/products/breg/examples/external-views/run_candidate.py @@ -0,0 +1,415 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 + +"""Rehearse, package, activate and exercise the licensing example with candidate binaries.""" + +import argparse +import base64 +from contextlib import contextmanager +from datetime import datetime, timedelta, timezone +from http.server import ThreadingHTTPServer +import json +import os +from pathlib import Path +import re +from secrets import token_hex +import shutil +import socket +import subprocess +from threading import Thread +import time +from urllib.error import URLError +from urllib.parse import unquote, urlsplit +from uuid import UUID + +from journey import HTTP, Journey, JourneyError, expect, request +from synthetic_source import Fixture, handler, runtime_fragment + +PROJECT = Path(__file__).resolve().parent +ISSUER = "https://fixture-issuer.example.test/licensing" +AUDIENCE = "urn:breg:external-views-fixture" +KEY_ID = "external-views-fixture" +IDENTITIES = { + "officer": ("officer", "human", "read"), + "restricted": ("restricted", "human", "read"), + "licensing-refresh": ("refresh", "service", "refresh"), + "fixture-editor": ("editor", "service", "write"), +} +URL_FILES = ("rehearsal-migration-url", "rehearsal-runtime-url", "migration-url", "runtime-url") + + +def private_write(path, data): + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "wb") as stream: + stream.write(data) + + +def write_json(path, data): + private_write(path, json.dumps(data, indent=2).encode("utf-8") + b"\n") + + +def database_urls(directory, destination): + databases = [] + for name in URL_FILES: + with (directory / name).open("rb") as stream: + data = stream.read(4097).strip() + expect(0 < len(data) <= 4096, "a fixture database URL file has an invalid size") + try: + target = urlsplit(data.decode("ascii")) + port = target.port + except (UnicodeError, ValueError): + raise JourneyError("a fixture database URL is malformed") from None + role = unquote(target.username or "") + database = target.path.removeprefix("/") + expect( + target.scheme in ("postgresql", "postgres") + and target.hostname in ("localhost", "127.0.0.1", "::1") + and port is not None + and re.fullmatch(r"breg_ext_[a-z0-9_]{1,45}", database) + and re.fullmatch(r"[_a-z][_a-z0-9]{0,62}", role) + and target.query == "sslmode=require" + and not target.fragment, + "use explicit loopback TLS URLs for fresh breg_ext_ fixture databases", + ) + databases.append((target.hostname, port, database, role, data)) + rm, rr, dm, dr = databases + expect(rm[:3] == rr[:3] and dm[:3] == dr[:3] and rm[:2] == dm[:2] and rm[2] != dm[2] + and rm[3] == dm[3] and rr[3] == dr[3] and dm[3] != dr[3], + "the rehearsal and deployment must be separate databases using the same split roles") + for name, (*_, data) in zip(URL_FILES, databases): + private_write(destination / name, data) + return {"migration": dm[3], "runtime": dr[3]} + + +def command(work, name, args, report=False): + with (work / f"{name}.stderr").open("xb") as error: + result = subprocess.run(args, cwd=work, stdout=subprocess.PIPE, stderr=error, timeout=180, + check=False) + if report: + private_write(work / f"{name}.json", result.stdout) + expect(result.returncode == 0, f"{name} failed. See the owner-only fixture logs.") + return result.stdout + + +def b64(data): + return base64.urlsafe_b64encode(data).rstrip(b"=") + + +def create_fixture_identity(work, keys): + signer = work / "fixture-signing-key.pem" + command(work, "generate-fixture-key", ["openssl", "genpkey", "-algorithm", "ED25519", "-out", signer]) + signer.chmod(0o600) + public = command(work, "derive-fixture-jwk", ["openssl", "pkey", "-in", signer, "-pubout", "-outform", "DER"]) + ed25519_prefix = bytes.fromhex("302a300506032b6570032100") + expect(len(public) == len(ed25519_prefix) + 32 and public.startswith(ed25519_prefix), + "OpenSSL did not produce the expected Ed25519 public key") + write_json(keys / "oidc-jwks", {"keys": [{ + "alg": "EdDSA", "crv": "Ed25519", "kid": KEY_ID, "kty": "OKP", + "x": b64(public[len(ed25519_prefix):]).decode("ascii"), + }]}) + tokens = {} + now = int(time.time()) + for client, (person, actor, scope) in IDENTITIES.items(): + principal = f"synthetic-licensing-{person}" + claims = { + "iss": ISSUER, "aud": AUDIENCE, "sub": principal, "client_id": client, + "iat": now, "exp": now + 1800, "jti": token_hex(12), + "scope": f"registry:licensing:{scope}", "registry_actor_kind": actor, + "registry_principal": principal, "registry_purpose": "licensing", "jurisdiction": "zone-a", + } + signed = b".".join( + b64(json.dumps(value, separators=(",", ":")).encode("utf-8")) + for value in ({"alg": "EdDSA", "kid": KEY_ID, "typ": "JWT"}, claims) + ) + input_file = work / f"{client}.signing-input" + private_write(input_file, signed) + signature = command(work, f"sign-{client}", [ + "openssl", "pkeyutl", "-sign", "-rawin", "-inkey", signer, "-in", input_file, + ]) + token = keys / f"{client}-token" + private_write(token, signed + b"." + b64(signature)) + tokens[client] = token + return tokens + + +def runtime_config(work, roles, pack, listener, prefix="", **source): + return { + "apiVersion": "id.registrystack.org/formats/breg/runtime/v1alpha1", "kind": "BRegRuntimeConfig", + "listener": {"bind": listener}, + "identity": { + "environment": "acceptance", "instanceId": "external-views-fixture", + "databaseId": prefix + "external-views-fixture", "databaseInitializationEnvironment": "acceptance", + }, + "secretProviders": {"file": {"root": str(work / "secrets")}}, + "database": { + "migrationUrlRef": f"secret:file/{prefix}migration-url", + "runtimeUrlRef": f"secret:file/{prefix}runtime-url", + "trustedRootCertificateRef": "secret:file/postgres-ca", + "pool": {"maximumConnections": 4}, "roles": roles, + }, + "package": {"root": str(pack)}, + "authentication": { + "oidc": { + "issuer": ISSUER, "audience": AUDIENCE, "allowedAlgorithm": "EdDSA", + "accessTokenType": "JWT", "scopeClaim": "scope", "scopeSeparator": " ", + "allowedClients": sorted(IDENTITIES), "maximumTokenLifetimeSeconds": 1800, + "leewayMilliseconds": 30000, + "jwksSource": {"type": "static", "documentRef": "secret:file/oidc-jwks"}, + }, + "authorityClaims": {"principal": "registry_principal", "purpose": "registry_purpose"}, + }, + "audit": {"hashKeyRef": "secret:file/audit-key", "path": str(work / "audit" / f"{prefix}audit.jsonl")}, + "cursor": {"secretRef": "secret:file/cursor-key"}, + **source, + } + + +def verify_source_facts(facts, bindings, indexed_ttl): + projection_ids = {"person-live", "person-indexed", "holdings-live", "holdings-indexed"} + expect(isinstance(facts, dict) and set(facts) == {"apiVersion", "kind", "registryId", "revision", "projections"} + and facts["apiVersion"] == "id.registrystack.org/formats/breg/external-source-facts/v1alpha1" + and facts["kind"] == "BRegExternalSourceFacts" and facts["registryId"] == "synthetic-licensing" + and isinstance(facts["revision"], str) and re.fullmatch(r"[A-Za-z0-9:._-]+", facts["revision"]), + "source facts did not have the expected safe schema envelope") + projections = facts["projections"] + expect(isinstance(projections, list) and len(projections) == len(projection_ids) + and all(isinstance(fact, dict) for fact in projections) + and {fact.get("id") for fact in projections} == projection_ids, + "source facts did not cover the four expected projections") + expected = {binding["id"]: binding for binding in bindings["externalSources"]} + for fact in projections: + projection = fact["id"] + binding = expected[projection] + indexed = projection.endswith("-indexed") + person = projection.startswith("person-") + retention = binding.get("retention", {}) + checked = { + "id": projection, "mode": "indexed" if indexed else "live", + "recordType": "person" if person else "holding", + "authority": "urn:example:population-register" if person else "urn:example:business-register", + "operation": ("person-batch" if indexed else "person-exact") if person else "holdings-by-business", + "operationVersion": "1", + "selectorScheme": "urn:example:person-number" if person else "urn:example:business-number", + "purpose": "licensing", "scope": "licensing-holders" if person else "licensed-business-holdings", + "fields": binding["allowedFields"], "connection": "synthetic-licensing", + "cacheTtlSeconds": indexed_ttl if indexed else None, + "maximumRetentionSeconds": retention.get("maximumRetentionSeconds"), + "refreshIntervalSeconds": retention.get("refreshIntervalSeconds"), + } + expect(set(fact) == {*checked, "bindingDigest", "grantDigest", "grantValidUntil"} + and all(fact[key] == value for key, value in checked.items()) + and all(re.fullmatch(r"sha256:[0-9a-f]{64}", fact[key]) for key in ("bindingDigest", "grantDigest")) + and datetime.fromisoformat(fact["grantValidUntil"]) == datetime.fromisoformat(binding["grantValidUntil"]), + "source facts did not match the approved fixture operation and fields") + + +def ready(server, origin): + deadline = time.monotonic() + 30 + while server.poll() is None and time.monotonic() < deadline: + try: + with HTTP.open(origin + "/ready", timeout=1) as response: + if response.status == 200: + return + except (URLError, TimeoutError): + time.sleep(0.1) + raise JourneyError("the candidate runtime did not become ready; see the owner-only server log") + + +@contextmanager +def serving(breg, runtime, work, log_name, origin): + with (work / log_name).open("xb") as log: + with subprocess.Popen([breg, "--runtime-config", runtime], cwd=work, + stdout=log, stderr=subprocess.STDOUT) as server: + try: + ready(server, origin) + yield + finally: + if server.poll() is None: + server.terminate() + try: + server.wait(timeout=35) + except subprocess.TimeoutExpired: + server.kill() + server.wait() + + +def seed(origin, token): + def create(route, number, selector, subject, zone): + status, response = request( + origin, f"/v1/records/{route}", + {"Authorization": "Bearer " + token, "Idempotency-Key": f"external-view-fixture-{number}"}, + method="POST", query={"accessProfile": "fixture-editor"}, + body={"data": {"licenceNumber": number, selector: subject, "jurisdiction": zone}}, + ) + expect(status == 201, f"the candidate did not create the fixture {route} record") + return UUID(response["data"]["recordIdentifier"]) + + licences = [ + create("licences", f"L-{i:03}", "holderKey", key, "zone-b" if i == 6 else "zone-a") + for i, key in enumerate(("person-7", "person-7", "person-8", "person-9", "person-10", "person-11"), 1) + ] + businesses = [ + create("licensed-businesses", f"B-{i:03}", "businessKey", f"business-{i}", + "zone-b" if i == 3 else "zone-a") + for i in range(1, 4) + ] + return licences[0], businesses[0] + + +def stage_project(source, work): + paths = ("registry.yaml", "dev-clients.yaml", "tests/journeys.yaml") + contents = {} + for name in paths: + path = source / name + expect(not path.is_symlink() and not path.parent.is_symlink() and path.is_file(), + "the saved native fixture must contain registry, dev-client and rehearsal files") + with path.open("rb") as stream: + data = stream.read(1_048_577) + expect(0 < len(data) <= 1_048_576, "a saved native fixture file is empty or too large") + contents[name] = data + target = work / "saved-project" + (target / "tests").mkdir(parents=True, mode=0o700) + for name, data in contents.items(): + private_write(target / name, data) + return target + + +def candidate_binary(path): + expect(path.is_file() and os.access(path, os.X_OK), "a candidate binary is missing or is not executable") + return path.resolve(strict=True) + + +def run(options): + breg = candidate_binary(options.breg_bin) + ctl = [candidate_binary(options.bregctl_bin), "--format", "json"] + expect(shutil.which("openssl") is not None, "OpenSSL is required to generate the synthetic issuer") + if options.scale: + import scale_exercise as scale + expect(shutil.which("psql") is not None, "psql is required to load the scale corpus") + expect(options.postgres_plan_log is not None and options.postgres_plan_log.is_file(), + "the scale exercise requires an existing PostgreSQL plan log") + else: + scale = None + if options.recovery: + expect(shutil.which("psql") is not None, "psql is required for the recovery operator") + from recovery import ScopeRecovery + work = options.work_dir.resolve() + work.mkdir(mode=0o700) + os.umask(0o077) + project = stage_project(options.project_dir, work) if options.project_dir is not None else PROJECT + if scale: + project = scale.scale_project(project, work) + keys = work / "secrets" + keys.mkdir() + (work / "audit").mkdir() + (work / "empty-package").mkdir() + with options.postgres_ca_file.open("rb") as stream: + ca = stream.read(65537) + expect(0 < len(ca) <= 65536 and b"-----BEGIN CERTIFICATE-----" in ca, "the PostgreSQL CA file is invalid") + roles = database_urls(options.database_url_dir, keys) + private_write(keys / "postgres-ca", ca) + for name in ("audit-key", "cursor-key", "source-key", "control-key"): + private_write(keys / name, token_hex(32).encode("ascii")) + + tokens = create_fixture_identity(work, keys) + print("Rehearsing the fixture and building its checked package", flush=True) + rehearsal = work / "rehearsal-runtime.json" + write_json(rehearsal, runtime_config(work, roles, work / "empty-package", "127.0.0.1:0", prefix="rehearsal-")) + credentials = work / "rehearsal-credentials.json" + write_json(credentials, { + "apiVersion": "id.registrystack.org/formats/breg/schema-test-credentials/v1", + "kind": "BRegSchemaTestCredentials", + "bindings": [{"journeyId": "fixture-record-custody", "stepId": "create-licence", + "credential": {"type": "bearer", "tokenRef": "secret:file/fixture-editor-token"}}], + }) + receipt = work / "test-receipt.json" + command(work, "rehearsal", [*ctl, "test", project, "--runtime-config", rehearsal, + "--credentials", credentials, "--output", receipt], report=True) + command(work, "package", [*ctl, "package", project, "--test-receipt", receipt, + "--output", work / "build"], report=True) + + source_key = (keys / "source-key").read_text(encoding="ascii") + control_key = (keys / "control-key").read_text(encoding="ascii") + fixture = scale.ScaleFixture() if scale else Fixture() + with ThreadingHTTPServer(("127.0.0.1", 0), handler(fixture, source_key, control_key)) as source: + thread = Thread(target=source.serve_forever, kwargs={"poll_interval": 0.05}) + thread.start() + try: + source_origin = f"http://127.0.0.1:{source.server_port}" + cutoff = (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat() + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + address = f"127.0.0.1:{listener.getsockname()[1]}" + runtime = work / "runtime.json" + package = work / "build/package" + bindings = runtime_fragment(source_origin, "source-key", cutoff) + if scale: + bindings = scale.scale_bindings(bindings) + write_json(runtime, runtime_config(work, roles, package, address, **bindings)) + facts_path = work / "external-source-facts.json" + before_facts = fixture.status() + command(work, "source-facts", [*ctl, "generate", "external-source-facts", project, + "--runtime-config", runtime, "--output", facts_path]) + expect(facts_path.stat().st_mode & 0o077 == 0, "source facts were not owner-only") + facts = json.loads(facts_path.read_text()) + verify_source_facts(facts, bindings, scale.FRESHNESS_SECONDS if scale else 300) + expect(fixture.status() == before_facts, "source facts generation contacted the source") + print("Verified all four source bindings before activation", flush=True) + print("Activating the package with split PostgreSQL roles", flush=True) + command(work, "activation", [*ctl, "apply", "--runtime-config", runtime, + "--package", package, "--initial"], report=True) + origin = f"http://{address}" + with serving(breg, runtime, work, "breg.log", origin): + if scale: + scale.run_scale(work, package, origin, tokens, fixture, options.postgres_plan_log, + {"breg": breg, "bregctl": ctl[0]}) + return + print("Seeding ordinary records and checking the HTTP journey", flush=True) + licence, business = seed(origin, tokens["fixture-editor"].read_text(encoding="ascii")) + journey = Journey(argparse.Namespace( + breg_origin=origin, source_origin=source_origin, + officer_token_file=tokens["officer"], restricted_token_file=tokens["restricted"], + refresh_token_file=tokens["licensing-refresh"], control_key_file=keys / "control-key", + licence_root=licence, business_root=business, + )) + journey.run() + print("Basic external view HTTP checks passed", flush=True) + if options.recovery: + print("Withdrawing holdings scope during a paged refresh", flush=True) + recovery = ScopeRecovery(journey, fixture, work, facts, private_write, write_json) + recovery.withdraw_and_rotate() + if options.recovery: + print("Restarting and rebuilding the rotated holdings scope", flush=True) + with serving(breg, runtime, work, "breg-restarted.log", origin): + recovery.verify_rebuild() + finally: + source.shutdown() + thread.join() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--breg-bin", type=Path, required=True) + parser.add_argument("--bregctl-bin", type=Path, required=True) + parser.add_argument("--database-url-dir", type=Path, required=True, help="directory containing four fixture URL files") + parser.add_argument("--postgres-ca-file", type=Path, required=True) + parser.add_argument("--work-dir", type=Path, required=True, help="new private directory for fixture evidence") + parser.add_argument("--project-dir", type=Path, help="saved native project for the small fixture") + parser.add_argument("--recovery", action="store_true", help="exercise small fixture scope withdrawal and restart") + parser.add_argument("--scale", action="store_true", help="run the million-record acceptance fixture") + parser.add_argument("--postgres-plan-log", type=Path, help="dedicated PostgreSQL auto_explain log for --scale") + options = parser.parse_args() + if options.project_dir is not None and options.scale: + parser.error("--project-dir applies to the small fixture") + if options.recovery and options.scale: + parser.error("--recovery applies to the small fixture") + try: + run(options) + except (JourneyError, OSError, KeyError, ValueError, subprocess.TimeoutExpired) as error: + detail = str(error) if isinstance(error, JourneyError) else "fixture setup or response was refused" + parser.exit(1, f"Candidate journey failed: {detail}\n") + print("Candidate external view scale journey passed" if options.scale else "Candidate external view journey passed") + + +if __name__ == "__main__": + main() diff --git a/products/breg/examples/external-views/scale_exercise.py b/products/breg/examples/external-views/scale_exercise.py new file mode 100644 index 0000000000..abf04f7161 --- /dev/null +++ b/products/breg/examples/external-views/scale_exercise.py @@ -0,0 +1,475 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 + +"""Scale corpus, independent oracle and native HTTP acceptance driver.""" + +from hashlib import sha256 +import json +import os +import platform +import re +import shutil +import subprocess +import time +from urllib.parse import unquote, urlsplit + +from journey import JourneyError, expect, fixture_credential, records, request +from synthetic_source import Fixture, digest, require + +ELIGIBLE_ROOTS = 1_000_000 +DENIED_ROOTS = 100_000 +HOLDER_KEYS = 50_000 +ABSENT_KEYS = 2_500 +MATCH_KEYS = frozenset((5, 12, 134, 1027, 7132, 19007, 43821)) +REFRESH_PAGE_SIZE = 100 +BATCH_KEYS = 64 +PERSON_REQUEST_BUDGET = 1_100 +RELATED_REQUEST_BUDGET = 6 +RELATED_PAGE_ROWS = 40 +RELATED_ROWS = 120 +FRESHNESS_SECONDS = 600 +RETENTION_SECONDS = 900 +QUERY_SECONDS = 5 +READ_PAGE_SIZE = 50 +MAX_PLAN_BYTES = 16 * 1024 * 1024 +PERSON_VIEW = "/v1/external-views/licence-holder-indexed/holder" +HOLDINGS_VIEW = "/v1/external-views/business-holdings-indexed/holdings" + + +def root_id(index, business=False): + family = "4001" if business else "4000" + return f"00000000-0000-{family}-8000-{index:012x}" + + +def key_for_root(index): + return (index - 1) % HOLDER_KEYS + 1 + + +def person_fields(key): + if key % 20 == 0: + return None + surname = "Cohort" if key in MATCH_KEYS else f"Family-{key % 1024:04d}" + return {"family-name": surname, "given-name": f"Given-{key:06d}"} + + +def exact_oracle(surname): + matches = { + key for key in range(1, HOLDER_KEYS + 1) + if (person_fields(key) or {}).get("family-name") == surname + } + return [root_id(i) for i in range(1, ELIGIBLE_ROOTS + 1) if key_for_root(i) in matches] + + +def ordered_oracle(descending=False, roots=ELIGIBLE_ROOTS, limit=None): + groups = {} + for key in range(1, HOLDER_KEYS + 1): + surname = (person_fields(key) or {}).get("family-name") + groups.setdefault(surname, set()).add(key) + surnames = sorted((name for name in groups if name is not None), reverse=descending) + ordered = [] + for surname in [*surnames, None]: + keys = sorted(groups[surname]) + for offset in range(0, roots, HOLDER_KEYS): + for key in keys: + if offset + key > roots: + break + ordered.append(root_id(offset + key)) + if limit is not None and len(ordered) == limit: + return ordered + return ordered + + +def holding_oracle(parent, selected_class=None): + values = range((parent - 1) * RELATED_ROWS + 1, parent * RELATED_ROWS + 1) + return [count for count in values if selected_class is None or holding_class(count) == selected_class] + + +def holding_class(count): + return "common" if count % 2 == 0 else "preferred" + + +class ScaleFixture(Fixture): + def _reset(self): + super()._reset() + self._seen_people = set() + self._collection_calls = {} + self._largest_batch = 0 + self._denied_selectors = 0 + self._collections = {} + for parent in range(1, 5): + key = f"scale-business-{parent}" + items = [] + for count in holding_oracle(parent): + props = {"share-class": holding_class(count), "share-count": count, + "shareholder-type": "business", "shareholder-key": f"shareholder-{count}", + "person-key": None} + items.append({"holding": f"holding-{count}", "business": key, + "revision": digest(props), "properties": props}) + self._collections[key] = (digest([key, items]), items) + + def _one_person(self, key, fields): + match = re.fullmatch(r"scale-person-([0-9]{6})", key) if isinstance(key, str) else None + if not match or not 1 <= int(match[1]) <= HOLDER_KEYS + 1: + self._denied_selectors += 1 + require(False, "selector outside the approved scale scope", 403) + number = int(match[1]) + self._seen_people.add(number) + data = person_fields(number) + if data is None: + return {"subject": key, "state": "confirmed-absent"} + return {"subject": key, "state": "found", "revision": digest(data), + "properties": {field: data[field] for field in fields}} + + def _page(self, parent, cursor, fields): + if parent not in ("scale-business-1", "scale-business-2"): + self._denied_selectors += 1 + require(False, "parent outside the approved scale scope", 403) + self._collection_calls[parent] = self._collection_calls.get(parent, 0) + 1 + snapshot, rows = self._collections[parent] + offset = 0 + if cursor is not None: + match = re.fullmatch(r"([0-9a-f]{64}):([0-9]{1,3})", cursor) if isinstance(cursor, str) else None + require(match and match[1] == snapshot, "invalid scale continuation") + offset = int(match[2]) + require(offset in range(RELATED_PAGE_ROWS, len(rows), RELATED_PAGE_ROWS)) + end = min(offset + RELATED_PAGE_ROWS, len(rows)) + items = [{**row, "properties": {field: row["properties"][field] for field in fields}} + for row in rows[offset:end]] + return {"state": "snapshot", "business": parent, "snapshotId": snapshot, + "finished": end == len(rows), "next": None if end == len(rows) else f"{snapshot}:{end}", + "items": items} + + def acquire(self, operation, body): + result = super().acquire(operation, body) + if operation == "person-batch": + with self._lock: + self._largest_batch = max(self._largest_batch, len(body["keys"])) + return result + + def status(self): + with self._lock: + return {**super().status(), "uniquePersonKeys": len(self._seen_people), + "largestBatch": self._largest_batch, "deniedSelectors": self._denied_selectors, + "collections": dict(self._collection_calls)} + + +def scale_project(project, work): + target = work / "scale-project" + (target / "tests").mkdir(parents=True) + authored = (project / "registry.yaml").read_text() + expect(authored.count("cacheTtlSeconds: 300") == 2, "the fixture TTL declarations changed") + (target / "registry.yaml").write_text( + authored.replace("cacheTtlSeconds: 300", f"cacheTtlSeconds: {FRESHNESS_SECONDS}") + ) + shutil.copyfile(project / "tests/journeys.yaml", target / "tests/journeys.yaml") + return target + + +def scale_bindings(fragment): + for binding in fragment["externalSources"]: + binding.pop("allowedSelectors", None) + if "retention" in binding: + binding["retention"]["maximumRetentionSeconds"] = RETENTION_SECONDS + return fragment + + +def quoted(identifier): + expect(re.fullmatch(r"breg_[a-z0-9_]{1,58}", identifier), "the candidate physical-name inventory changed") + return f'"{identifier}"' + + +def seed_sql(inventory): + licence = inventory["entities"]["licence"] + business = inventory["entities"]["licensed-business"] + + def target(entity, selector): + table = "registry_data." + quoted(entity["table"]) + columns = ",".join(quoted(entity["fields"][name]) for name in ("licence-number", selector, "jurisdiction")) + return table, columns + + licences, licence_columns = target(licence, "holder-key") + businesses, business_columns = target(business, "business-key") + return f""" +BEGIN; +SET LOCAL statement_timeout='180s'; +SELECT set_config('registry.principal','synthetic-licensing-editor',true), + set_config('registry.access_profile','fixture-editor',true), + set_config('registry.purpose','licensing',true), + set_config('registry.row_boundaries','[]',true), + set_config('registry.recipients','[]',true), + set_config('registry.active_package_revision', + (SELECT active_activation_id::text FROM registry_internal.registry_state WHERE singleton),true); +DO $fixture$ BEGIN + IF EXISTS (SELECT 1 FROM {licences}) OR EXISTS (SELECT 1 FROM {businesses}) THEN + RAISE EXCEPTION 'scale fixture tables must be empty'; + END IF; +END; $fixture$; +INSERT INTO {licences} (record_id,{licence_columns}) +SELECT ('00000000-0000-4000-8000-'||lpad(to_hex(i),12,'0'))::uuid, 'L-scale-'||i, + CASE WHEN i<={ELIGIBLE_ROOTS} THEN 'scale-person-'||lpad((((i-1)%{HOLDER_KEYS})+1)::text,6,'0') + ELSE 'scale-denied-'||lpad((i-{ELIGIBLE_ROOTS})::text,6,'0') END, + CASE WHEN i<={ELIGIBLE_ROOTS} THEN 'zone-a' ELSE 'zone-b' END +FROM generate_series(1, {ELIGIBLE_ROOTS + DENIED_ROOTS}) AS i; +INSERT INTO {businesses} (record_id,{business_columns}) +SELECT ('00000000-0000-4001-8000-'||lpad(to_hex(i),12,'0'))::uuid, + 'B-scale-'||i,'scale-business-'||i,CASE WHEN i<=2 THEN 'zone-a' ELSE 'zone-b' END +FROM generate_series(1,4) AS i; +ANALYZE {licences}; +ANALYZE {businesses}; +COMMIT; +""" + + +def postgres(work, url_name, sql, name): + target = urlsplit((work / "secrets" / url_name).read_text()) + env = {key: value for key, value in os.environ.items() if not key.startswith("PG")} + env.update(PGHOST=target.hostname, PGPORT=str(target.port), PGUSER=unquote(target.username), + PGDATABASE=target.path[1:], PGSSLMODE="verify-full", + PGSSLROOTCERT=str(work / "secrets/postgres-ca"), PGCONNECT_TIMEOUT="5", PGPASSFILE="/dev/null") + if target.password is not None: + env["PGPASSWORD"] = unquote(target.password) + with (work / f"{name}.stderr").open("xb") as stderr: + result = subprocess.run(["psql", "-XqwAt", "-v", "ON_ERROR_STOP=1"], input=sql, text=True, + env=env, stderr=stderr, stdout=subprocess.PIPE, timeout=185, check=False) + expect(result.returncode == 0, f"{name} failed. See the owner-only PostgreSQL log.") + return result.stdout + + +def native_plans(log, database, role): + prefix = f"{database} {role} " + return [entry for entry in log.split("[breg-plan] ") if entry.startswith(prefix) + and "Query Text: WITH projection_state AS MATERIALIZED" in entry + and "CTE coverage" in entry and "CTE page" in entry and "Buffers:" in entry] + + +def capture_plans(path, start, database, role, work): + with path.open("rb") as stream: + info = os.fstat(stream.fileno()) + expect(info.st_ino == start.st_ino and info.st_size >= start.st_size, + "the PostgreSQL plan log rotated during the fixture") + stream.seek(start.st_size) + data = stream.read(MAX_PLAN_BYTES + 1) + expect(len(data) <= MAX_PLAN_BYTES, "the plan log slice exceeded the fixture limit") + entries = native_plans(data.decode("utf-8", errors="replace"), database, role) + if entries: + with (work / "scale-native-plans.log").open("x") as output: + output.write("".join("[breg-plan] " + entry for entry in entries)) + return len(entries) + + +def refresh(origin, path, headers, measurements, start, page_budget): + query = {"accessProfile": "projection-refresh", "$top": str(REFRESH_PAGE_SIZE)} + cursor = None + for _ in range(page_budget): + expect(time.monotonic() - start < FRESHNESS_SECONDS, "refresh exceeded the freshness window") + if cursor: + query = {"accessProfile": "projection-refresh", "$skiptoken": cursor} + status, doc = request(origin, path + "/refresh", headers, method="POST", query=query) + expect(status == 200 and isinstance(doc, dict), "a native refresh failed") + measurements["httpRequests"] += 1 + for field in ("processed", "published", "reused", "unavailable"): + value = doc.get(field) + expect(type(value) is int and value >= 0, "the refresh returned an invalid count") + measurements[field] += value + if measurements["httpRequests"] % 1_000 == 0: + print(f"Native refresh processed {measurements['processed']:,} roots", flush=True) + previous, cursor = cursor, doc.get("nextCursor") + if cursor is None: + return + expect(isinstance(cursor, str) and cursor != previous, "native refresh repeated its cursor") + expect(False, "native refresh exceeded its predeclared page budget") + + +def observe(origin, path, query, headers, timings, name): + started = time.monotonic() + result = request(origin, path, headers, query={"accessProfile": "officer", **query}) + elapsed = time.monotonic() - started + timings.append({"operation": name, "status": result[0], "wallSeconds": round(elapsed, 6)}) + expect(elapsed <= QUERY_SECONDS, f"{name} exceeded its HTTP response target") + return result + + +def person_page(response, expected): + rows = records(response) + expect([row["rootId"] for row in rows] == expected, "native Person page differed from the local-id oracle") + for row in rows: + index = int(row["rootId"][-12:], 16) + fields = person_fields(key_for_root(index)) + expect(row["fields"] == (fields or {}) and row["availability"] == ("available" if fields else "absent"), + "native Person value or absence differed from the fixture oracle") + return response[1] + + +def holding_page(response, expected): + rows = records(response) + expect([row["fields"]["share-count"] for row in rows] == expected, + "native holding filter or ordering differed from the independent oracle") + for row, count in zip(rows, expected): + expect(row["fields"]["share-class"] == holding_class(count) and row.get("label") is None, + "native holding class or company label differed from the fixture") + expect(isinstance(row.get("rowId"), str) and row["rowId"], "a holding row lacked its stable identity") + return response[1] + + +def sweep_counts(): + return dict.fromkeys(("httpRequests", "processed", "published", "reused", "unavailable"), 0) + + +def file_digest(path): + digestor = sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + digestor.update(chunk) + return digestor.hexdigest() + + +def run_scale(work, package, origin, tokens, source, plan_log, binaries): + report = { + "passed": False, "environment": {"host": platform.platform(), "cpuCount": os.cpu_count()}, + "candidateBinarySha256": { + name: file_digest(binary) for name, binary in binaries.items() + }, + "corpus": {"eligibleRoots": ELIGIBLE_ROOTS, "deniedRoots": DENIED_ROOTS, + "distinctHolderKeys": HOLDER_KEYS, "authorizedAbsentKeys": ABSENT_KEYS, + "authorizedRelatedRows": RELATED_ROWS * 2, "deniedRelatedRows": RELATED_ROWS * 2}, + "budgets": {"freshnessSeconds": FRESHNESS_SECONDS, "retentionSeconds": RETENTION_SECONDS, + "querySeconds": QUERY_SECONDS, "personSourceRequests": PERSON_REQUEST_BUDGET, + "relatedSourceRequests": RELATED_REQUEST_BUDGET, "adapterBatchKeys": BATCH_KEYS}, + "personRefresh": sweep_counts(), "holdingsRefresh": sweep_counts(), "queries": [], + "seeding": "Migration role inserted only fixture-owned base rows under fixture-editor RLS. No projection cache or revision history was fabricated.", + } + started = None + log_start = None + try: + runtime_url = urlsplit((work / "secrets/runtime-url").read_text()) + database, role = runtime_url.path[1:], unquote(runtime_url.username) + environment = postgres(work, "runtime-url", """ +SELECT jsonb_build_object('version',current_setting('server_version'), + 'workMem',current_setting('work_mem'),'statementTimeout',current_setting('statement_timeout'), + 'planLogging',current_setting('auto_explain.log_analyze',true)); +""", "scale-environment") + report["environment"]["postgres"] = json.loads(environment) + expect(report["environment"]["postgres"]["version"].split(".", 1)[0] == "18" + and report["environment"]["postgres"]["workMem"] == "64MB" + and report["environment"]["postgres"]["statementTimeout"] == "10s", + "runtime work_mem and statement_timeout do not match the predeclared environment") + report["packageInventorySha256"] = sha256((package / "inventories/physical-names.json").read_bytes()).hexdigest() + report["packDigest"] = json.loads((work / "runtime.json").read_text())["externalSources"][0]["packDigest"] + names = json.loads((package / "inventories/physical-names.json").read_text()) + print("Loading the scale corpus into its fixture-owned PostgreSQL tables", flush=True) + seeded_at = time.monotonic() + postgres(work, "migration-url", seed_sql(names), "scale-seed") + report["seedSeconds"] = round(time.monotonic() - seeded_at, 6) + officer = {"Authorization": "Bearer " + fixture_credential(tokens["officer"])} + refresh_actor = {"Authorization": "Bearer " + fixture_credential(tokens["licensing-refresh"])} + editor = {"Authorization": "Bearer " + fixture_credential(tokens["fixture-editor"])} + exact = exact_oracle("Cohort") + absent = exact_oracle(None) + ascending = ordered_oracle(limit=2 * READ_PAGE_SIZE) + descending = ordered_oracle(descending=True, limit=2 * READ_PAGE_SIZE) + + print("Refreshing indexed projections through the candidate HTTP service", flush=True) + started = time.monotonic() + refresh(origin, PERSON_VIEW, refresh_actor, report["personRefresh"], started, ELIGIBLE_ROOTS // REFRESH_PAGE_SIZE) + refresh(origin, HOLDINGS_VIEW, refresh_actor, report["holdingsRefresh"], started, 1) + report["refreshSeconds"] = round(time.monotonic() - started, 6) + before = source.status() + report["sourceAtEndOfRefresh"] = before + expect(report["personRefresh"]["processed"] == ELIGIBLE_ROOTS + and report["personRefresh"]["published"] == HOLDER_KEYS + and report["personRefresh"]["unavailable"] == 0 + and report["personRefresh"]["reused"] == ELIGIBLE_ROOTS - HOLDER_KEYS + and before["uniquePersonKeys"] == HOLDER_KEYS and before["largestBatch"] == BATCH_KEYS + and before["operations"].get("person-batch", 0) <= PERSON_REQUEST_BUDGET, + "Person refresh did not meet the predeclared population or source budget") + expect(report["holdingsRefresh"]["processed"] == 2 and report["holdingsRefresh"]["published"] == 2 + and report["holdingsRefresh"]["unavailable"] == 0 + and before["collections"] == {"scale-business-1": 3, "scale-business-2": 3} + and before["operations"].get("holdings-by-business", 0) == RELATED_REQUEST_BUDGET + and before["deniedSelectors"] == 0 and before["operations"].get("person-exact", 0) == 0, + "related refresh crossed its parent or request boundary") + + log_start = plan_log.stat() + timing = report["queries"] + print("Checking native indexed queries against the independent oracle", flush=True) + initial = {"$filter": "family-name eq 'Cohort'", "$orderby": "family-name", + "$count": "true", "$top": str(READ_PAGE_SIZE)} + for _ in range(3): + document = person_page(observe(origin, PERSON_VIEW + "/records", initial, officer, timing, "exact-first"), + exact[:READ_PAGE_SIZE]) + expect(document.get("count") == len(exact), "native surname count differed from the oracle") + for offset in range(READ_PAGE_SIZE, len(exact), READ_PAGE_SIZE): + document = person_page(observe(origin, PERSON_VIEW + "/records", {"$skiptoken": document["nextCursor"]}, + officer, timing, "exact-continuation"), + exact[offset:offset + READ_PAGE_SIZE]) + expect("nextCursor" not in document, "native surname results have additional rows") + for direction, expected in (("asc", ascending), ("desc", descending)): + query = {"$orderby": f"family-name {direction}", "$top": str(READ_PAGE_SIZE), "$count": "true"} + document = person_page(observe(origin, PERSON_VIEW + "/records", query, officer, timing, "ordered-first"), + expected[:READ_PAGE_SIZE]) + expect(document.get("count") == ELIGIBLE_ROOTS, "the native root count included denied roots") + person_page(observe(origin, PERSON_VIEW + "/records", {"$skiptoken": document["nextCursor"]}, + officer, timing, "ordered-continuation"), expected[READ_PAGE_SIZE:]) + document = person_page(observe(origin, PERSON_VIEW + "/records", { + "$filter": "family-name eq null", "$orderby": "family-name desc", + "$top": str(READ_PAGE_SIZE), "$count": "true", + }, officer, timing, "missing-sort-first"), absent[:READ_PAGE_SIZE]) + expect(document.get("count") == len(absent), "authorized missing-value count differed from the oracle") + person_page(observe(origin, PERSON_VIEW + "/records", {"$skiptoken": document["nextCursor"]}, + officer, timing, "missing-sort-continuation"), absent[READ_PAGE_SIZE:2 * READ_PAGE_SIZE]) + + for parent in (1, 2): + path = HOLDINGS_VIEW + "/records/" + root_id(parent, business=True) + all_counts = holding_oracle(parent) + related_query = {"$orderby": "share-count asc", "$count": "true", "$top": str(READ_PAGE_SIZE)} + row_ids = [] + for offset in range(0, len(all_counts), READ_PAGE_SIZE): + response = observe(origin, path, related_query, officer, timing, "related-snapshot") + document = holding_page(response, all_counts[offset:offset + READ_PAGE_SIZE]) + row_ids.extend(row["rowId"] for row in document["records"]) + if offset == 0: + expect(document.get("count") == len(all_counts), "complete related count differed from the oracle") + if offset + READ_PAGE_SIZE < len(all_counts): + related_query = {"$skiptoken": document["nextCursor"]} + expect("nextCursor" not in document and len(set(row_ids)) == len(all_counts), + "complete related pages repeated or omitted stable row identities") + expected = list(reversed(holding_oracle(parent, "common"))) + document = holding_page(observe(origin, path, {"$filter": "share-class eq 'common'", + "$orderby": "share-count desc", "$count": "true", + "$top": str(READ_PAGE_SIZE)}, officer, timing, "related-first"), + expected[:READ_PAGE_SIZE]) + expect(document.get("count") == len(expected), "related count differed from the parent oracle") + holding_page(observe(origin, path, {"$skiptoken": document["nextCursor"]}, officer, timing, + "related-continuation"), expected[READ_PAGE_SIZE:]) + for parent in (3, 4): + expect(observe(origin, HOLDINGS_VIEW + "/records/" + root_id(parent, business=True), {}, + officer, timing, "denied-parent")[0] == 404, "denied collection was disclosed") + + status, _ = request(origin, "/v1/records/licences", {**editor, "Idempotency-Key": "external-scale-deficit"}, + method="POST", query={"accessProfile": "fixture-editor"}, body={"data": { + "licenceNumber": "L-scale-deficit", "holderKey": f"scale-person-{HOLDER_KEYS + 1:06}", + "jurisdiction": "zone-a", + }}) + expect(status == 201, "the native fixture could not insert the uncovered root") + expect(observe(origin, PERSON_VIEW + "/records", initial, officer, timing, "coverage-deficit") + == (503, {"status": "incomplete"}), "the native predicate masked missing coverage") + expect(source.status() == before, "indexed or concealed reads contacted the upstream source") + expect(time.monotonic() - started < FRESHNESS_SECONDS, "indexed verification exceeded the freshness window") + report["nativePlans"] = capture_plans(plan_log, log_start, database, role, work) + expect(report["nativePlans"] >= 3, "the plan log lacks native coverage and page plans with buffers") + report["passed"] = True + except JourneyError as error: + report["failure"] = str(error) + raise + finally: + if log_start is not None and "nativePlans" not in report: + try: + report["nativePlans"] = capture_plans(plan_log, log_start, database, role, work) + except (JourneyError, OSError): + report["nativePlans"] = None + report["sourceFinal"] = source.status() + if started is not None: + report["elapsedFromRefreshSeconds"] = round(time.monotonic() - started, 6) + with (work / "scale-report.json").open("x") as output: + json.dump(report, output, indent=2) + output.write("\n") diff --git a/products/breg/examples/external-views/synthetic_source.py b/products/breg/examples/external-views/synthetic_source.py new file mode 100644 index 0000000000..edb4b7a4dc --- /dev/null +++ b/products/breg/examples/external-views/synthetic_source.py @@ -0,0 +1,400 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 + +"""Loopback source for the synthetic licensing projection journeys.""" + +import argparse +from collections import Counter +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +from hmac import compare_digest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +from pathlib import Path +from threading import Condition + +OPERATIONS = { + "/people/exact": "person-exact", + "/people/batch": "person-batch", + "/holdings/by-business": "holdings-by-business", +} +PERSON_FIELDS = {"given-name", "family-name"} +HOLDING_FIELDS = {"share-class", "share-count", "shareholder-type", "shareholder-key", "person-key"} +ALLOWED_PEOPLE = {"person-7", "person-8", "person-9", "person-10", "person-12"} +ALLOWED_BUSINESSES = {"business-1", "business-2"} +MAX_BODY_BYTES = 16_384 +PAGE_SIZE = 1 + + +class FixtureFault(ValueError): + def __init__(self, status, problem): + super().__init__(problem) + self.status = status + self.problem = problem + + +def require(condition, problem="invalid fixture request", status=400): + if not condition: + raise FixtureFault(status, problem) + + +def digest(value): + canonical = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return sha256(canonical.encode("ascii")).hexdigest() + + +def holding(key, shareholder_type, shareholder_key, share_class, share_count): + return { + "holding": key, + "business": "business-1", + "properties": { + "shareholder-type": shareholder_type, + "shareholder-key": shareholder_key, + "person-key": shareholder_key if shareholder_type == "person" else None, + "share-class": share_class, + "share-count": share_count, + }, + } + + +class Fixture: + def __init__(self): + self._lock = Condition() + self._operations = Counter() + self._reset() + + def _reset(self): + self._people = { + "person-7": {"given-name": "Ari", "family-name": "Lee"}, + "person-8": {"given-name": "Sam", "family-name": "Rivera"}, + "person-10": {"given-name": "Pat", "family-name": "Doe"}, + "person-12": {"given-name": "Mika", "family-name": "Solano"}, + } + self._holdings = [ + holding("holding-1", "person", "person-7", "common", 25), + holding("holding-2", "business", "business-2", "preferred", 75), + ] + self._fault = "none" + self._interrupt = False + self._no_store = {"person-exact"} + self._hold_next = None + self._held = False + + def _one_person(self, key, fields): + require( + isinstance(key, str) and key in ALLOWED_PEOPLE, + "selector outside the approved scope", + 403, + ) + if key == "person-9": + return {"subject": key, "state": "confirmed-absent"} + if key == "person-10": + if self._fault == "denied": + raise FixtureFault(403, "protected subject") + if self._fault == "disconnect": + raise FixtureFault(0, "fixture transport interruption") + if self._fault == "ambiguous": + return {"subject": key, "state": "ambiguous"} + record = self._people[key] + return { + "subject": key, + "state": "found", + "revision": digest(record), + "properties": {field: record[field] for field in fields}, + } + + def _page(self, parent, cursor, fields): + require( + isinstance(parent, str) and parent in ALLOWED_BUSINESSES, + "selector outside the approved scope", + 403, + ) + rows = self._holdings if parent == "business-1" else [] + snapshot = digest([parent, rows]) + offset = 0 + if cursor is not None: + require(isinstance(cursor, str) and len(cursor) <= 70 and ":" in cursor) + requested_snapshot, offset_text = cursor.split(":", 1) + require(compare_digest(requested_snapshot, snapshot), "snapshot changed", 409) + require(offset_text.isascii() and offset_text.isdecimal() and len(offset_text) <= 3) + offset = int(offset_text) + require(0 < offset < len(rows)) + require(not self._interrupt, "collection interrupted", 503) + items = [ + { + "holding": row["holding"], + "business": row["business"], + "revision": digest(row["properties"]), + "properties": {field: row["properties"][field] for field in fields}, + } + for row in rows[offset : offset + PAGE_SIZE] + ] + next_offset = offset + len(items) + finished = next_offset >= len(rows) + return { + "state": "snapshot", + "business": parent, + "snapshotId": snapshot, + "finished": finished, + "next": None if finished else f"{snapshot}:{next_offset}", + "items": items, + } + + def note_attempt(self, operation): + with self._lock: + self._operations[operation] += 1 + + def acquire(self, operation, request): + with self._lock: + self._operations[operation] += 1 + require(isinstance(request, dict)) + fields = request.get("fields") + permitted = HOLDING_FIELDS if operation == "holdings-by-business" else PERSON_FIELDS + require( + isinstance(fields, list) and fields + and all(isinstance(field, str) and field in permitted for field in fields) + and len(fields) == len(set(fields)), + "field outside the approved scope", + 403, + ) + if operation == "person-exact": + require(set(request) == {"type", "key", "fields"} and request["type"] == "exact") + result = self._one_person(request["key"], fields) + elif operation == "person-batch": + require(set(request) == {"type", "keys", "fields"} and request["type"] == "batch") + keys = request["keys"] + require( + isinstance(keys, list) and 1 <= len(keys) <= 64 + and all(isinstance(key, str) for key in keys) and len(keys) == len(set(keys)) + ) + result = {"records": [self._one_person(key, fields) for key in keys]} + elif operation == "holdings-by-business": + require( + set(request) in ({"type", "parentKey", "fields"}, {"type", "parentKey", "fields", "cursor"}) + and request["type"] == "by-parent" + ) + result = self._page(request["parentKey"], request.get("cursor"), fields) + else: + raise FixtureFault(404, "unknown fixture operation") + no_store = operation in self._no_store + if self._hold_next == operation: + self._hold_next = None + self._held = True + self._lock.notify_all() + if not self._lock.wait_for(lambda: not self._held, timeout=4): + self._held = False + raise FixtureFault(503, "fixture hold expired") + return result, no_store + + def wait_for_hold(self, timeout): + with self._lock: + return self._lock.wait_for(lambda: self._held, timeout=timeout) + + def status(self): + with self._lock: + return {"operations": dict(self._operations), "held": self._held} + + def control(self, body): + with self._lock: + require(isinstance(body, dict)) + action = body.get("action") + if action in ("reset", "rename-holder", "restore-holder", "add-label-only", "replace-holdings", + "empty-holdings", "release"): + require(set(body) == {"action"}) + if action == "reset": + self._reset() + self._operations.clear() + elif action == "rename-holder": + self._people["person-7"] = {"given-name": "Ari", "family-name": "Ng"} + elif action == "restore-holder": + self._people["person-7"] = {"given-name": "Ari", "family-name": "Lee"} + elif action == "add-label-only": + if not any(row["holding"] == "holding-3" for row in self._holdings): + self._holdings.append(holding("holding-3", "person", "person-12", "common", 15)) + elif action == "replace-holdings": + self._holdings = [holding("holding-2", "business", "business-2", "preferred", 100)] + elif action == "empty-holdings": + self._holdings = [] + else: + self._held = False + self._lock.notify_all() + elif action == "hold-next": + require(set(body) == {"action", "operation"} and body["operation"] in OPERATIONS.values()) + require(self._hold_next is None and not self._held, "fixture hold already active", 409) + self._hold_next = body["operation"] + elif action == "set-person-fault": + require(set(body) == {"action", "kind"} and body["kind"] in ("none", "ambiguous", "denied", "disconnect")) + self._fault = body["kind"] + elif action == "interrupt-collection": + require(set(body) == {"action", "enabled"} and isinstance(body["enabled"], bool)) + self._interrupt = body["enabled"] + elif action == "set-no-store": + require( + set(body) == {"action", "operation", "enabled"} + and body["operation"] in OPERATIONS.values() and isinstance(body["enabled"], bool) + ) + if body["enabled"]: + self._no_store.add(body["operation"]) + elif body["operation"] != "person-exact": + self._no_store.discard(body["operation"]) + else: + raise FixtureFault(400, "unknown fixture control") + return self.status() + + +def no_duplicate_members(pairs): + result = {} + for name, value in pairs: + require(name not in result, "duplicate fixture request member") + result[name] = value + return result + + +def handler(fixture, source_key, control_key): + class Handler(BaseHTTPRequestHandler): + def setup(self): + super().setup() + self.connection.settimeout(6) + + def log_message(self, *_): + return None + + def _require_key(self, header, key): + supplied = self.headers.get_all(header, []) + require( + key is not None and len(supplied) == 1 and supplied[0].isascii() + and compare_digest(supplied[0], key), + "fixture credential refused", + 403, + ) + + def _body(self): + lengths = self.headers.get_all("Content-Length", []) + require(len(lengths) == 1 and lengths[0].isascii() and lengths[0].isdecimal() and len(lengths[0]) <= 6) + length = int(lengths[0]) + require(0 < length <= MAX_BODY_BYTES and not self.headers.get_all("Transfer-Encoding")) + try: + return json.loads(self.rfile.read(length), object_pairs_hook=no_duplicate_members) + except (UnicodeError, json.JSONDecodeError, RecursionError) as error: + raise FixtureFault(400, "invalid fixture JSON") from error + + def _reply(self, status, value, no_store=False): + data = json.dumps(value, separators=(",", ":")).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + if no_store: + self.send_header("Cache-Control", "no-store") + self.end_headers() + try: + self.wfile.write(data) + except (BrokenPipeError, ConnectionResetError): + self.close_connection = True + + def _handle(self, operation): + try: + operation() + except FixtureFault as fault: + if fault.status: + self._reply(fault.status, {"problem": fault.problem}, no_store=True) + else: + self.close_connection = True + + def do_POST(self): + def execute(): + if self.path == "/fixture/control": + self._require_key("x-fixture-control-key", control_key) + self._reply(200, fixture.control(self._body()), no_store=True) + return + operation = OPERATIONS.get(self.path) + require(operation is not None, "unknown fixture route", 404) + try: + self._require_key("x-registry-service-key", source_key) + body = self._body() + except FixtureFault: + fixture.note_attempt(operation) + raise + result, no_store = fixture.acquire(operation, body) + self._reply(200, result, no_store) + self._handle(execute) + + def do_GET(self): + def execute(): + require(self.path == "/fixture/status", "unknown fixture route", 404) + self._require_key("x-fixture-control-key", control_key) + self._reply(200, fixture.status(), no_store=True) + self._handle(execute) + + return Handler + + +def source_key_from_file(path): + with path.open("rb") as stream: + value = stream.read(257).strip() + require(8 <= len(value) <= 256 and all(33 <= byte <= 126 for byte in value), + "fixture key file has an invalid shape") + return value.decode("ascii") + + +def runtime_fragment(origin, key_file_name, grant_cutoff): + pack = Path(__file__).resolve().parents[3] / "adapter/examples/external-projections/pack.yaml" + pack_digest = "sha256:" + sha256(pack.read_bytes()).hexdigest() + connection = { + "id": "synthetic-licensing", + "endpoints": [{ + "endpoint": "synthetic-registry", "baseUrl": origin, + "credential": {"type": "static-api-key", "valueRef": f"secret:file/{key_file_name}"}, + }], + } + policy = [{ + "endpoint": "synthetic-registry", "baseUrl": origin, "clientIdentity": "forbidden", + "authentication": {"type": "static-api-key"}, + }] + bindings = [] + for binding, fields, selectors in ( + ("person-live", PERSON_FIELDS, ALLOWED_PEOPLE), + ("person-indexed", PERSON_FIELDS, ALLOWED_PEOPLE), + ("holdings-live", HOLDING_FIELDS - {"person-key"}, ALLOWED_BUSINESSES), + ("holdings-indexed", HOLDING_FIELDS, ALLOWED_BUSINESSES), + ): + entry = { + "id": binding, "packFile": str(pack), "packDigest": pack_digest, + "connection": connection, "endpointPolicies": policy, + "allowedFields": sorted(fields), "allowedSelectors": sorted(selectors), + "grantValidUntil": grant_cutoff, + "acquisition": {"maximumDataOperations": 8, "timeoutMilliseconds": 8000, + "maximumResponseBytes": 262144}, + } + if binding.endswith("-indexed"): + entry["retention"] = {"maximumRetentionSeconds": 300, "refreshIntervalSeconds": 60} + bindings.append(entry) + return {"externalSources": bindings} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--port", type=int, default=0, help="numeric loopback port, default ephemeral") + parser.add_argument("--service-key-file", type=Path, required=True) + parser.add_argument("--control-key-file", type=Path) + parser.add_argument("--runtime-fragment-file", type=Path) + options = parser.parse_args() + source_key = source_key_from_file(options.service_key_file) + control_key = source_key_from_file(options.control_key_file) if options.control_key_file else None + if control_key is not None and compare_digest(source_key, control_key): + parser.error("the read service and control identities must differ") + with ThreadingHTTPServer(("127.0.0.1", options.port), handler(Fixture(), source_key, control_key)) as server: + origin = f"http://127.0.0.1:{server.server_port}" + if options.runtime_fragment_file: + cutoff = (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat() + fragment = runtime_fragment(origin, options.service_key_file.name, cutoff) + fd = os.open(options.runtime_fragment_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as stream: + json.dump(fragment, stream, indent=2) + stream.write("\n") + print(origin, flush=True) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/products/breg/examples/external-views/test_scale.py b/products/breg/examples/external-views/test_scale.py new file mode 100644 index 0000000000..5200a9dfd9 --- /dev/null +++ b/products/breg/examples/external-views/test_scale.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 + +"""Deterministic checks for the native scale fixture and independent oracle.""" + +import unittest + +from scale_exercise import ( + ABSENT_KEYS, BATCH_KEYS, DENIED_ROOTS, ELIGIBLE_ROOTS, HOLDER_KEYS, + MATCH_KEYS, PERSON_REQUEST_BUDGET, REFRESH_PAGE_SIZE, ScaleFixture, + exact_oracle, holding_oracle, native_plans, ordered_oracle, root_id, seed_sql, +) +from synthetic_source import FixtureFault + + +class ScaleFixtureTests(unittest.TestCase): + def test_corpus_and_current_pagination_have_a_predeclared_budget(self): + self.assertEqual(ELIGIBLE_ROOTS, 1_000_000) + self.assertEqual(DENIED_ROOTS, 100_000) + self.assertEqual(ABSENT_KEYS, 2_500) + self.assertEqual(ELIGIBLE_ROOTS // HOLDER_KEYS, 20) + self.assertEqual((REFRESH_PAGE_SIZE, BATCH_KEYS), (100, 64)) + self.assertEqual(PERSON_REQUEST_BUDGET, 1_100) + self.assertEqual(len(exact_oracle("Cohort")), len(MATCH_KEYS) * 20) + self.assertEqual(len(exact_oracle(None)), 50_000) + + def test_oracle_orders_by_source_value_then_immutable_local_id(self): + ascending = ordered_oracle(roots=500) + descending = ordered_oracle(descending=True, roots=500) + self.assertEqual(len(ascending), 500) + self.assertEqual(len(set(ascending)), 500) + self.assertEqual(ascending[-25:], descending[-25:]) + self.assertEqual(ascending[-1], root_id(500)) + self.assertNotEqual(ascending[0], descending[0]) + + def test_source_enforces_batch_ceiling_and_denied_subject_boundary(self): + source = ScaleFixture() + query = {"type": "batch", "fields": ["family-name"], "keys": ["scale-person-000005", "scale-person-000020"]} + body, no_store = source.acquire("person-batch", query) + self.assertFalse(no_store) + self.assertEqual([row["state"] for row in body["records"]], ["found", "confirmed-absent"]) + self.assertEqual(body["records"][0]["properties"]["family-name"], "Cohort") + with self.assertRaises(FixtureFault): + source.acquire("person-batch", {**query, "keys": ["scale-denied-000001"]}) + with self.assertRaises(FixtureFault): + source.acquire("person-batch", {**query, "keys": [f"scale-person-{i:06}" for i in range(1, 66)]}) + self.assertEqual(source.status()["uniquePersonKeys"], 2) + self.assertEqual(source.status()["deniedSelectors"], 1) + + def test_collection_snapshots_fit_the_pack_and_cover_two_allowed_parents(self): + source = ScaleFixture() + for parent in range(1, 3): + request = {"type": "by-parent", "parentKey": f"scale-business-{parent}", "fields": ["share-count"]} + counts = [] + snapshots = set() + while True: + page, _ = source.acquire("holdings-by-business", request) + counts.extend(row["properties"]["share-count"] for row in page["items"]) + snapshots.add(page["snapshotId"]) + if page["finished"]: + break + request["cursor"] = page["next"] + self.assertEqual(counts, holding_oracle(parent)) + self.assertEqual(len(snapshots), 1) + with self.assertRaises(FixtureFault): + source.acquire("holdings-by-business", {**request, "parentKey": "scale-business-3"}) + self.assertEqual(source.status()["collections"], {"scale-business-1": 3, "scale-business-2": 3}) + + def test_bulk_seed_is_confined_to_owned_tables_and_requires_empty_fixture(self): + entity = {"table": "breg_e_licence_1", "fields": {"licence-number": "breg_f_number_1", + "holder-key": "breg_f_holder_1", "jurisdiction": "breg_f_zone_1"}} + business = {"table": "breg_e_business_1", "fields": {"licence-number": "breg_f_number_2", + "business-key": "breg_f_business_2", "jurisdiction": "breg_f_zone_2"}} + sql = seed_sql({"entities": {"licence": entity, "licensed-business": business}}) + self.assertIn("generate_series(1, 1100000)", sql) + self.assertIn("scale fixture tables must be empty", sql) + self.assertIn("active_activation_id::text", sql) + self.assertNotIn("registry_external_projection_", sql) + + def test_plans_require_native_coverage_page_and_buffer_evidence(self): + entry = """[breg-plan] breg_ext_runtime runtime LOG: +Query Text: WITH projection_state AS MATERIALIZED (SELECT 1) + CTE coverage + CTE page + Buffers: shared hit=1 +""" + self.assertEqual(len(native_plans(entry, "breg_ext_runtime", "runtime")), 1) + self.assertEqual(native_plans(entry.replace("CTE page", "CTE other"), "breg_ext_runtime", "runtime"), []) + self.assertEqual(native_plans(entry, "breg_ext_other", "runtime"), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/products/breg/examples/external-views/test_source.py b/products/breg/examples/external-views/test_source.py new file mode 100644 index 0000000000..3fd7187fd9 --- /dev/null +++ b/products/breg/examples/external-views/test_source.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 + +from http.server import ThreadingHTTPServer +import json +from pathlib import Path +import tempfile +from threading import Thread +import unittest +from urllib.error import HTTPError +from urllib.request import ProxyHandler, Request, build_opener + +from journey import JourneyError +from recovery import operator_report +from run_candidate import URL_FILES, database_urls, stage_project, verify_source_facts +from synthetic_source import Fixture, FixtureFault, handler, runtime_fragment + + +class RecoveryTests(unittest.TestCase): + def test_operator_report_reads_the_psql_json_cell(self): + output = b"""BEGIN +SET +SET + jsonb_pretty +---------------------------------------------------- + { + + "projection": "holdings-indexed", + + "active": false + + } +(1 row) + +COMMIT +""" + self.assertEqual(operator_report(output), {"projection": "holdings-indexed", "active": False}) + for invalid in (output.replace(b"COMMIT", b"ROLLBACK"), b"COMMIT", output + output): + with self.assertRaisesRegex(JourneyError, "scope report"): + operator_report(invalid) + + +class FixtureTests(unittest.TestCase): + def setUp(self): + self.source = Fixture() + + def test_person_fields_and_keyed_absence(self): + result, no_store = self.source.acquire( + "person-batch", + {"type": "batch", "keys": ["person-7", "person-9"], "fields": ["family-name"]}, + ) + self.assertFalse(no_store) + self.assertEqual(result["records"][0]["properties"], {"family-name": "Lee"}) + self.assertEqual(result["records"][1], {"subject": "person-9", "state": "confirmed-absent"}) + with self.assertRaisesRegex(FixtureFault, "outside the approved scope"): + self.source.acquire( + "person-exact", + {"type": "exact", "key": "person-11", "fields": ["family-name"]}, + ) + + def test_a_shareholder_can_be_reached_only_through_a_holding(self): + self.source.control({"action": "add-label-only"}) + rows = [] + request = {"type": "by-parent", "parentKey": "business-1", "fields": ["person-key"]} + for _ in range(3): + page, _ = self.source.acquire("holdings-by-business", request) + rows.extend(page["items"]) + if page["finished"]: + break + request["cursor"] = page["next"] + self.assertTrue(page["finished"]) + keys = {row["properties"]["person-key"] for row in rows} + self.assertEqual(keys, {"person-7", "person-12", None}) + label, _ = self.source.acquire( + "person-batch", {"type": "batch", "keys": ["person-12"], "fields": ["family-name"]}, + ) + self.assertEqual(label["records"][0]["properties"], {"family-name": "Solano"}) + + def test_company_rows_have_no_person_identifier(self): + first, _ = self.source.acquire( + "holdings-by-business", + {"type": "by-parent", "parentKey": "business-1", "fields": ["person-key", "shareholder-type"]}, + ) + second, _ = self.source.acquire( + "holdings-by-business", + {"type": "by-parent", "parentKey": "business-1", "fields": ["person-key", "shareholder-type"], + "cursor": first["next"]}, + ) + self.assertEqual(first["snapshotId"], second["snapshotId"]) + self.assertEqual(first["items"][0]["properties"]["person-key"], "person-7") + self.assertEqual(second["items"][0]["properties"], {"person-key": None, "shareholder-type": "business"}) + self.assertTrue(second["finished"]) + self.source.control({"action": "replace-holdings"}) + with self.assertRaisesRegex(FixtureFault, "snapshot changed"): + self.source.acquire( + "holdings-by-business", + {"type": "by-parent", "parentKey": "business-1", "fields": ["shareholder-type"], + "cursor": first["next"]}, + ) + + def test_a_paused_source_call_holds_the_snapshot_while_new_calls_progress(self): + request = {"type": "batch", "keys": ["person-7"], "fields": ["family-name"]} + earlier = [] + self.source.control({"action": "hold-next", "operation": "person-batch"}) + waiter = Thread(target=lambda: earlier.append(self.source.acquire("person-batch", request))) + waiter.start() + self.assertTrue(self.source.wait_for_hold(2)) + self.source.control({"action": "rename-holder"}) + later, _ = self.source.acquire("person-batch", request) + self.source.control({"action": "release"}) + waiter.join(2) + self.assertFalse(waiter.is_alive()) + self.assertEqual(earlier[0][0]["records"][0]["properties"], {"family-name": "Lee"}) + self.assertEqual(later["records"][0]["properties"], {"family-name": "Ng"}) + + def test_no_store_and_interruption_are_separate_controls(self): + request = {"type": "batch", "keys": ["person-7"], "fields": ["family-name"]} + self.source.control({"action": "set-no-store", "operation": "person-batch", "enabled": True}) + self.assertTrue(self.source.acquire("person-batch", request)[1]) + self.source.control({"action": "interrupt-collection", "enabled": True}) + parent = {"type": "by-parent", "parentKey": "business-1", "fields": ["share-class"]} + first, _ = self.source.acquire("holdings-by-business", parent) + with self.assertRaisesRegex(FixtureFault, "collection interrupted"): + self.source.acquire("holdings-by-business", {**parent, "cursor": first["next"]}) + + +class CandidateFixtureTests(unittest.TestCase): + def test_saved_native_project_is_staged_byte_for_byte_and_requires_all_three_files(self): + with tempfile.TemporaryDirectory() as scratch: + work = Path(scratch) + source = work / "studio" + (source / "tests").mkdir(parents=True) + files = {"registry.yaml": b"registry", "dev-clients.yaml": b"clients", + "tests/journeys.yaml": b"journeys"} + for name, data in files.items(): + (source / name).write_bytes(data) + staged = stage_project(source, work) + for name, data in files.items(): + self.assertEqual((staged / name).read_bytes(), data) + self.assertEqual((source / name).read_bytes(), data) + incomplete = work / "incomplete" + incomplete.mkdir() + (incomplete / "registry.yaml").write_bytes(b"registry") + with self.assertRaises(JourneyError): + stage_project(incomplete, work) + + def test_source_facts_require_all_projections_and_refuse_connection_details(self): + bindings = runtime_fragment("http://127.0.0.1:8001", "synthetic", "2030-01-01T00:00:00Z") + projections = [] + for binding in bindings["externalSources"]: + person = binding["id"].startswith("person") + indexed = binding["id"].endswith("-indexed") + projections.append({ + "id": binding["id"], "mode": "indexed" if indexed else "live", + "recordType": "person" if person else "holding", + "authority": "urn:example:population-register" if person else "urn:example:business-register", + "operation": ("person-batch" if indexed else "person-exact") if person else "holdings-by-business", + "operationVersion": "1", + "selectorScheme": "urn:example:person-number" if person else "urn:example:business-number", + "purpose": "licensing", "scope": "licensing-holders" if person else "licensed-business-holdings", + "fields": binding["allowedFields"], "connection": "synthetic-licensing", + "bindingDigest": "sha256:" + "1" * 64, "grantDigest": "sha256:" + "2" * 64, + "grantValidUntil": "2030-01-01T00:00:00+00:00", + "cacheTtlSeconds": 300 if indexed else None, + "maximumRetentionSeconds": 300 if indexed else None, + "refreshIntervalSeconds": 60 if indexed else None, + }) + facts = { + "apiVersion": "id.registrystack.org/formats/breg/external-source-facts/v1alpha1", + "kind": "BRegExternalSourceFacts", "registryId": "synthetic-licensing", + "revision": "fixture-revision", "projections": projections, + } + verify_source_facts(facts, bindings, 300) + with self.assertRaises(JourneyError): + verify_source_facts({**facts, "projections": projections[:-1]}, bindings, 300) + with self.assertRaises(JourneyError): + verify_source_facts({**facts, "projections": [{**projections[0], "baseUrl": "private"}, *projections[1:]]}, + bindings, 300) + with self.assertRaises(JourneyError): + verify_source_facts({**facts, "projections": [{**projections[0], "fields": ["other"]}, *projections[1:]]}, + bindings, 300) + + def test_url_files_must_describe_fresh_split_loopback_databases(self): + valid = [ + "postgresql://migrator@localhost:6543/breg_ext_rehearsal?sslmode=require", + "postgresql://server@localhost:6543/breg_ext_rehearsal?sslmode=require", + "postgresql://migrator@localhost:6543/breg_ext_serving?sslmode=require", + "postgresql://server@localhost:6543/breg_ext_serving?sslmode=require", + ] + with tempfile.TemporaryDirectory() as scratch: + root = Path(scratch) + output = root / "output" + output.mkdir() + for invalid in (valid[2], valid[3] + "&host=example.test"): + for name, url in zip(URL_FILES, [*valid[:3], invalid]): + (root / name).write_text(url, encoding="ascii") + with self.assertRaises(JourneyError): + database_urls(root, output) + self.assertEqual(list(output.iterdir()), []) + for name, url in zip(URL_FILES, valid): + (root / name).write_text(url, encoding="ascii") + self.assertEqual(database_urls(root, output), {"migration": "migrator", "runtime": "server"}) + self.assertEqual({path.name for path in output.iterdir()}, set(URL_FILES)) + + +class HttpFixtureTests(unittest.TestCase): + def test_source_and_control_identities_are_separate(self): + source_key, control_key = "synthetic-source-key", "synthetic-control-key" + with ThreadingHTTPServer(("127.0.0.1", 0), handler(Fixture(), source_key, control_key)) as server: + thread = Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}) + thread.start() + try: + origin = f"http://127.0.0.1:{server.server_port}" + client = build_opener(ProxyHandler({})) + data = json.dumps({"type": "exact", "key": "person-7", "fields": ["family-name"]}).encode() + with client.open(Request( + origin + "/people/exact", data=data, headers={"x-registry-service-key": source_key}, + ), timeout=2) as response: + self.assertEqual(response.headers["Cache-Control"], "no-store") + self.assertEqual(json.load(response)["properties"], {"family-name": "Lee"}) + with self.assertRaises(HTTPError) as denied: + client.open(Request(origin + "/fixture/status", headers={"x-fixture-control-key": source_key}), timeout=2) + self.assertEqual(denied.exception.code, 403) + denied.exception.close() + with self.assertRaises(HTTPError) as wrong_identity: + client.open(Request( + origin + "/people/exact", data=data, headers={"x-registry-service-key": control_key}, + ), timeout=2) + self.assertEqual(wrong_identity.exception.code, 403) + wrong_identity.exception.close() + with client.open(Request( + origin + "/fixture/status", headers={"x-fixture-control-key": control_key}, + ), timeout=2) as response: + self.assertEqual(json.load(response)["operations"], {"person-exact": 2}) + finally: + server.shutdown() + thread.join(2) + + def test_generated_fragment_is_scoped_and_pins_the_maintained_pack(self): + config = runtime_fragment("http://127.0.0.1:8001", "projection-source-key", "2030-01-01T00:00:00Z") + bindings = {entry["id"]: entry for entry in config["externalSources"]} + self.assertEqual(set(bindings), {"person-live", "person-indexed", "holdings-live", "holdings-indexed"}) + self.assertNotIn("retention", bindings["person-live"]) + self.assertIn("retention", bindings["person-indexed"]) + self.assertIn("person-key", bindings["holdings-indexed"]["allowedFields"]) + self.assertNotIn("person-key", bindings["holdings-live"]["allowedFields"]) + self.assertTrue(bindings["person-indexed"]["packDigest"].startswith("sha256:")) + self.assertEqual(bindings["person-live"]["packDigest"], bindings["holdings-indexed"]["packDigest"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/products/breg/examples/external-views/tests/journeys.yaml b/products/breg/examples/external-views/tests/journeys.yaml new file mode 100644 index 0000000000..cf5377c4ff --- /dev/null +++ b/products/breg/examples/external-views/tests/journeys.yaml @@ -0,0 +1,20 @@ +# yaml-language-server: $schema=https://id.registrystack.org/schemas/breg/journeys/journeys.v1.schema.json +apiVersion: id.registrystack.org/formats/breg/journeys/v1 +kind: BRegJourneys +journeys: + - id: fixture-record-custody + steps: + - id: create-licence + recordType: licence + accessProfile: fixture-editor + claims: + principal: synthetic-licensing-editor + scopes: [registry:licensing:write] + purpose: licensing + request: + type: create + data: {licenceNumber: L-REHEARSAL, holderKey: person-7, jurisdiction: zone-a} + expect: + outcome: success + status: 201 + fields: {licenceNumber: L-REHEARSAL, holderKey: person-7, jurisdiction: zone-a} diff --git a/products/breg/external-projection-storage.md b/products/breg/external-projection-storage.md new file mode 100644 index 0000000000..e4b0c30256 --- /dev/null +++ b/products/breg/external-projection-storage.md @@ -0,0 +1,81 @@ +# External projection storage and recovery + +When you operate indexed external views, keep the projection cache separate from the registry's own records and backups. An external value becomes readable only under the current package, deployment binding, source grant, local profile, and expiry. Refreshing the cache does not create a Record revision. + +## Provisioning and binding + +Indexed projections require separate migration and runtime PostgreSQL roles. Activation and startup refuse a shared role; packages without indexed projections can keep their existing role layout. The migration role installs the projection store when the compiled project contains an indexed projection. A project with live projections only, or without external projections, does not install this store. The runtime role can read the four tables and execute their bounded functions. It cannot insert, update, or delete their rows directly. + +The migration-installed binding function contains the compiled projection allowlist. Its definition is part of the managed catalog fingerprint. At startup, a current reviewed deployment binding can fill an empty allowlisted slot under the exact active package and activation. A restart with the same binding and source grant reuses the stored epoch. + +A different binding, source grant, or retention policy cannot replace an already bound slot through runtime startup. A withdrawn slot also stays withdrawn. These changes require operator maintenance under migration authority. Changing the compiled external contract invalidates its previous associations and requires a fresh binding and acquisition. Removing the final indexed projection removes its rebuildable store during package activation. + +The current link requires the local reference scheme to match the source selector scheme. +The named link reserves `identifierBinding` for a future versioned resolver contract. +That resolver would take the authorised local reference and return a source-scoped handle +before Adapter acquisition. The current checker refuses unknown binding configuration. + +## Freshness and retention + +`cacheTtlSeconds` bounds the age of a usable source result. The effective deployment TTL may shorten, but never exceed, the authored TTL. The effective read expiry is the earlier of that lifetime and the source result's expiry. The retention bound is measured from the original source observation time, not from refresh completion and not as an extra grace period after expiry. Source retention permission can shorten that bound. + +A captured snapshot stores the observation time and a keyed revision reference when the source supplies a revision. The reference is scoped to the bound source. + +Replaying an accepted result keeps its original expiry. A failed exact-key refresh removes the previous one/absent snapshot, making coverage unproven. An interrupted related-collection refresh preserves its previous verified complete snapshot without extending its lifetime. Explicit grant denial or no-store removes either kind, fenced against newer publications. A source response marked `no-store` cannot enter the indexed store. If it supersedes an earlier retained result for the same subject, the guarded retention-refusal path removes that result; an older refresh cannot erase a newer publication. + +Reads check expiry independently of the refresh job. Indexed predicates and counts refuse incomplete coverage before selecting matching rows. A complete empty related collection is covered; a partial collection, source failure, or protected-existence response is not evidence of emptiness or absence. + +The bounded purge selects snapshots at freshness expiry, before their maximum retention deadline where that deadline allows headroom. Run it regularly, size capacity for the backlog, and configure an erasure margin greater than the worker interval and expected recovery delay. Purge also removes data from withdrawn or replaced epochs. Freshness enforcement keeps expired values out of responses even when refresh or purge is delayed, but a delayed purge can still breach a storage obligation. Monitor purge failures, expiry backlog, and database downtime separately. No physical erasure guarantee holds during a database outage or when backlog exceeds the configured margin. Database WAL and physical snapshots need their own retention policy. + +## Schedule refresh and inspect coverage + +Run the named view’s `POST /v1/external-views/{view}/{link}/refresh` with the approved machine profile. Continue with the returned `nextCursor` until the sweep ends. For independent related labels, run `POST /v1/external-views/{view}/{link}/refresh-labels` after the related collection refresh. That machine profile needs explicit refresh grants on both projections under the same purpose. The label sweep selects keys only from authorised roots with fresh complete collections. A changed collection generation requires a fresh sweep. + +The refresh report counts processed roots or related rows, published values, reused keys and unavailable results. The migration role’s scope inspection reports receipt counts by outcome and fresh, expired and invalidated snapshot counts. Use an authorised indexed query to test readiness for its local root scope. + +## Operator withdrawal and grant renewal + +Use [external-projection-scope.sh](../../scripts/external-projection-scope.sh) with a PostgreSQL service profile for the migration role. The script never requests source values. Runtime cannot execute its maintenance function. It takes the package advisory lock and checks the exact inspected package, activation, and scope epoch before mutation. + +1. Stop dependent acquisition and update or disable the deployment binding first. Record the source withdrawal in deployment configuration or the source authority so it remains effective after a backup restore. +2. Run `bash scripts/external-projection-scope.sh inspect breg-migration PROJECTION`. Preserve the returned epoch, packageDigest, and activationId for the next command. A null epoch is passed as `unbound`. +3. Withdraw with `bash scripts/external-projection-scope.sh withdraw breg-migration PROJECTION EPOCH PACKAGE_DIGEST ACTIVATION_UUID`. This disables the scope and erases its retained values and refresh receipts. Runtime startup cannot reactivate it. +4. After reviewing a renewed binding and grant, obtain its validated fingerprints and effective limits with `bregctl generate external-source-facts --runtime-config ABSOLUTE_FILE --output NEW_FILE PROJECT`. This reads deployment configuration without acquiring source data. Use the chosen projection entry's `bindingDigest`, `grantDigest`, `cacheTtlSeconds`, and `maximumRetentionSeconds`; do not calculate the digests by hand. Inspect again, then run `bash scripts/external-projection-scope.sh rotate breg-migration PROJECTION EPOCH PACKAGE_DIGEST ACTIVATION_UUID BINDING_DIGEST GRANT_DIGEST TTL_SECONDS RETENTION_SECONDS`. Use the exact digests and effective bounds from the reviewed deployment configuration. Rotation allocates a fresh epoch and removes previous cache contents. A changed package or epoch refuses the operation; inspect and review again instead of retrying old values. +5. Start with the matching reviewed configuration and acquire fresh authorized data. Rotation does not assert coverage. + +These operations do not change the governed schema or package digest. They mutate only the rebuildable projection cache. A database-only withdrawal cannot be durable across a backup that excludes those rows; the current deployment/source authority is the durable cutoff. + +## Logical backups + +Exclude data from all four projection tables. Keep their schema and functions so a restore retains the package-installed allowlist but starts with no source values, old bindings, or refresh receipts. For example, with PostgreSQL service profiles supplying your connections and credentials: + +```bash +pg_dump --dbname='service=breg-backup' --format=custom \ + --exclude-table-data=registry_internal.registry_external_projection_scopes \ + --exclude-table-data=registry_internal.registry_external_projection_refreshes \ + --exclude-table-data=registry_internal.registry_external_projection_snapshots \ + --exclude-table-data=registry_internal.registry_external_projection_values \ + --file=registry.backup +``` + +The four exclusions do not remove the local identifiers that your own Records hold. Those identifiers remain part of the registry backup according to its ordinary policy. Do not use `--exclude-table`: the projection table definitions, privileges, and functions belong to the governed database schema. + +Use a PostgreSQL dump client compatible with the server version. Protect the backup according to the sensitivity of the registry's own records. Physical backups and storage snapshots do not provide these table-data exclusions, so they need a separately approved erasure and restore procedure before they can be used for an indexed grant. + +## Restore + +Restore with the registry's ordinary PostgreSQL and instance-claim procedure. Fence the original instance and complete explicit instance adoption before serving the restored database. Preserve the exact package and verify the restored managed catalog and runtime privileges. + +Before startup, check the current source grant and deployment binding. A withdrawal stored only in excluded database rows does not survive the restore. Reflect withdrawn source authority in the current deployment configuration or at the source before allowing reacquisition. The absence of cached values prevents old bytes from being served, but it is not permission for a new acquisition. + +Startup can bind empty slots using the current reviewed deployment configuration. Indexed queries remain incomplete until authorized refresh covers their eligible roots. Restore does not mark a projection ready or extend any source lifetime. Live reads continue to use their own current source contract and retain no projection copy. + +## Implementation references + +- [Guarded projection storage and binding](../../crates/registry-breg/src/postgres/external_projection.rs) +- [Indexed coverage and query kernel](../../crates/registry-breg/src/postgres/external_view_query.rs) +- [Managed schema installation](../../crates/registry-breg/src/postgres/schema.rs) +- [Exact catalog and runtime privilege checks](../../crates/registry-breg/src/postgres/catalog.rs) +- [Instance adoption contract](../../crates/registry-breg/src/instance_claim.rs) + +The focused PostgreSQL tests exercise publication fencing, replay, expiry, collection replacement, empty-cache restore provisioning, binding refusal, and runtime ACLs. The query tests exercise the compiled root read policy, coverage, filtering, counts, and paging. Full package activation, source-adapter, HTTP, and backup/restore acceptance remain separate integration checks. diff --git a/products/breg/generated/authoring/registry-project.schema.json b/products/breg/generated/authoring/registry-project.schema.json index f4c0119406..e27a99885c 100644 --- a/products/breg/generated/authoring/registry-project.schema.json +++ b/products/breg/generated/authoring/registry-project.schema.json @@ -2639,6 +2639,48 @@ ], "type": "object" }, + "ExternalCardinality": { + "enum": [ + "one", + "many" + ], + "type": "string" + }, + "ExternalFieldSource": { + "additionalProperties": false, + "properties": { + "classification": { + "$ref": "#/$defs/Classification" + }, + "id": { + "$ref": "#/$defs/LocalId" + }, + "nullable": { + "default": false, + "type": "boolean" + }, + "type": { + "$ref": "#/$defs/ExternalFieldType" + } + }, + "required": [ + "id", + "type", + "classification" + ], + "type": "object" + }, + "ExternalFieldType": { + "enum": [ + "string", + "integer", + "boolean", + "decimal", + "date", + "uuid" + ], + "type": "string" + }, "ExternalId": { "description": "An identifier another system issues, written exactly as its issuer gives it.", "maxLength": 512, @@ -2646,6 +2688,211 @@ "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", "type": "string" }, + "ExternalLabelSource": { + "additionalProperties": false, + "properties": { + "fields": { + "items": { + "$ref": "#/$defs/LocalId" + }, + "type": "array", + "uniqueItems": true + }, + "fromField": { + "$ref": "#/$defs/LocalId" + }, + "identifierScheme": { + "$ref": "#/$defs/ExternalId" + }, + "projection": { + "$ref": "#/$defs/LocalId" + } + }, + "required": [ + "projection", + "fromField", + "identifierScheme", + "fields" + ], + "type": "object" + }, + "ExternalLinkPermissionSource": { + "additionalProperties": false, + "properties": { + "allowCount": { + "default": false, + "type": "boolean" + }, + "filterableFields": { + "items": { + "$ref": "#/$defs/LocalId" + }, + "type": "array", + "uniqueItems": true + }, + "labelFields": { + "description": "A separate grant for label fields reached through the related row.", + "items": { + "$ref": "#/$defs/LocalId" + }, + "type": "array", + "uniqueItems": true + }, + "link": { + "$ref": "#/$defs/LocalId" + }, + "readableFields": { + "items": { + "$ref": "#/$defs/LocalId" + }, + "type": "array", + "uniqueItems": true + }, + "sortableFields": { + "items": { + "$ref": "#/$defs/LocalId" + }, + "type": "array", + "uniqueItems": true + } + }, + "required": [ + "link", + "readableFields" + ], + "type": "object" + }, + "ExternalLinkSource": { + "additionalProperties": false, + "properties": { + "fields": { + "items": { + "$ref": "#/$defs/LocalId" + }, + "type": "array", + "uniqueItems": true + }, + "fromField": { + "$ref": "#/$defs/LocalId" + }, + "id": { + "$ref": "#/$defs/LocalId" + }, + "identifierScheme": { + "$ref": "#/$defs/ExternalId" + }, + "label": { + "$ref": "#/$defs/ExternalLabelSource", + "description": "An optional to-one label on a row of an indexed related collection." + }, + "projection": { + "$ref": "#/$defs/LocalId" + } + }, + "required": [ + "id", + "projection", + "fromField", + "identifierScheme", + "fields" + ], + "type": "object" + }, + "ExternalProjectionMode": { + "enum": [ + "live", + "indexed" + ], + "type": "string" + }, + "ExternalProjectionSource": { + "additionalProperties": false, + "properties": { + "cacheTtlSeconds": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "cardinality": { + "$ref": "#/$defs/ExternalCardinality" + }, + "fields": { + "items": { + "$ref": "#/$defs/LocalId" + }, + "type": "array", + "uniqueItems": true + }, + "id": { + "$ref": "#/$defs/LocalId" + }, + "mode": { + "$ref": "#/$defs/ExternalProjectionMode" + }, + "operation": { + "$ref": "#/$defs/LocalId" + }, + "operationVersion": { + "$ref": "#/$defs/ExternalId" + }, + "purpose": { + "$ref": "#/$defs/ExternalId" + }, + "recordType": { + "$ref": "#/$defs/LocalId" + }, + "scope": { + "$ref": "#/$defs/ExternalId" + }, + "selectorScheme": { + "$ref": "#/$defs/ExternalId" + } + }, + "required": [ + "id", + "recordType", + "operation", + "operationVersion", + "purpose", + "scope", + "selectorScheme", + "mode", + "cardinality", + "fields" + ], + "type": "object" + }, + "ExternalRecordTypeSource": { + "additionalProperties": false, + "properties": { + "authority": { + "$ref": "#/$defs/ExternalId" + }, + "display": { + "$ref": "#/$defs/DisplaySource", + "default": {} + }, + "fields": { + "items": { + "$ref": "#/$defs/ExternalFieldSource" + }, + "type": "array" + }, + "id": { + "$ref": "#/$defs/LocalId" + }, + "identifierScheme": { + "$ref": "#/$defs/ExternalId" + } + }, + "required": [ + "id", + "authority", + "identifierScheme", + "fields" + ], + "type": "object" + }, "ExternalReferenceSource": { "additionalProperties": false, "properties": { @@ -2659,6 +2906,48 @@ ], "type": "object" }, + "ExternalViewPermissionSource": { + "additionalProperties": false, + "properties": { + "links": { + "items": { + "$ref": "#/$defs/ExternalLinkPermissionSource" + }, + "type": "array" + }, + "view": { + "$ref": "#/$defs/LocalId" + } + }, + "required": [ + "view", + "links" + ], + "type": "object" + }, + "ExternalViewSource": { + "additionalProperties": false, + "properties": { + "id": { + "$ref": "#/$defs/LocalId" + }, + "links": { + "items": { + "$ref": "#/$defs/ExternalLinkSource" + }, + "type": "array" + }, + "recordType": { + "$ref": "#/$defs/LocalId" + } + }, + "required": [ + "id", + "recordType", + "links" + ], + "type": "object" + }, "FieldGenerationSource": { "description": "Server-side generation requested for a persisted business identifier.\n\nThe canonical record UUID remains independent. On a sequence, `scope`\nnames required, immutable stored fields whose values partition the\nnumbering. A random code has one namespace per field.", "oneOf": [ @@ -3851,6 +4140,12 @@ }, "type": "array" }, + "externalViews": { + "items": { + "$ref": "#/$defs/ExternalViewPermissionSource" + }, + "type": "array" + }, "proposals": { "default": [], "items": { @@ -3865,6 +4160,13 @@ "$ref": "#/$defs/EntityPermissionSource" }, "type": "array" + }, + "refreshExternalProjections": { + "items": { + "$ref": "#/$defs/LocalId" + }, + "type": "array", + "uniqueItems": true } }, "type": "object" @@ -5688,6 +5990,24 @@ }, "type": "array" }, + "externalProjections": { + "items": { + "$ref": "#/$defs/ExternalProjectionSource" + }, + "type": "array" + }, + "externalRecordTypes": { + "items": { + "$ref": "#/$defs/ExternalRecordTypeSource" + }, + "type": "array" + }, + "externalViews": { + "items": { + "$ref": "#/$defs/ExternalViewSource" + }, + "type": "array" + }, "functions": { "default": [], "items": { diff --git a/products/breg/generated/runtime/runtime.schema.json b/products/breg/generated/runtime/runtime.schema.json index b460d4aec7..585ee8e74f 100644 --- a/products/breg/generated/runtime/runtime.schema.json +++ b/products/breg/generated/runtime/runtime.schema.json @@ -10,6 +10,41 @@ ], "type": "string" }, + "AdapterEndpointBindings": { + "additionalProperties": false, + "description": "The endpoint bindings of one deployment: the runtime members every\nmode shares (`products/adapter/CONCEPT.md` sections 10.2 and 17).\nThe ingestion job's runtime document (`AdapterJobRuntimeConfig`)\nstates these members beside its own `sources`, `store`,\n`destinations`, `secretProviders`, and `hookIngress` members; the\nfacade's document states them beside the facade-only members\n(`secretProviders`, `listener`, `database`, `oidc`, `allowedClients`,\n`audit`, `idempotency`, and the cursor custody), which\n[`AdapterRuntimeConfig::endpoint_bindings`] sets aside.", + "properties": { + "allowedPrivateCidrs": { + "default": [], + "description": "The private ranges egress may reach. Omission is a secure empty\nallowlist for a listener-free program with no network endpoints.\nThe facade's `AdapterRuntimeConfig` retains its nonempty requirement.", + "items": { + "$ref": "#/$defs/Cidr" + }, + "type": "array", + "uniqueItems": true + }, + "endpoints": { + "description": "One binding per configured endpoint.", + "items": { + "$ref": "#/$defs/EndpointBinding" + }, + "type": "array" + }, + "id": { + "$ref": "#/$defs/LocalId", + "description": "The deployment this binding configures." + }, + "outboundTls": { + "$ref": "#/$defs/OutboundTls", + "description": "The private CAs endpoints may trust beside the system roots;\nabsent, every endpoint trusts the system roots alone." + } + }, + "required": [ + "id", + "endpoints" + ], + "type": "object" + }, "AllowedClients": { "anyOf": [ { @@ -33,6 +68,11 @@ ], "description": "The OAuth clients whose tokens a runtime accepts: the keyword\n`unrestricted`, or a list of at least one client." }, + "AnonymousHttpsAcknowledgement": { + "const": "operator-approved", + "description": "The operator reviewed and approved this fixed anonymous source.", + "type": "string" + }, "AuditDestinationKind": { "description": "The configured audit destination kind, as written in product configuration.", "oneOf": [ @@ -48,6 +88,11 @@ } ] }, + "Cidr": { + "description": "A private network range in `addr/prefix` form, for example 10.0.0.0/8.", + "pattern": "^[0-9a-fA-F.:]+/[0-9]{1,3}$", + "type": "string" + }, "ClaimName": { "description": "The name of one top-level claim of an access token. The reader accepts any\ntext; the runtime then refuses a name that is not a valid claim name, one\nthat repeats another contextual name, and one that is a registered or\nauthentication claim.", "type": "string" @@ -60,11 +105,83 @@ ], "type": "string" }, + "CredentialPlacement": { + "description": "Where a shared client secret travels, Evidence's `CredentialPlacement`\nspelling.", + "oneOf": [ + { + "const": "basic-header", + "description": "`Authorization: Basic` on the token request.", + "type": "string" + }, + { + "const": "form-body", + "description": "A form parameter on the token request.", + "type": "string" + } + ] + }, "Digest": { "description": "A SHA-256 digest: `sha256:` followed by 64 lowercase hex digits.", "pattern": "^sha256:[0-9a-f]{64}$", "type": "string" }, + "EndpointBinding": { + "additionalProperties": false, + "description": "The environment binding of one pack endpoint: its base URL, its\ncredentials matching the endpoint's declared kind, and the optional TLS\nadditions (`products/adapter/CONCEPT.md` section 10.2).", + "properties": { + "anonymousHttps": { + "$ref": "#/$defs/AnonymousHttpsAcknowledgement", + "description": "The operator's explicit acknowledgement that a credential-free\nsource is intentionally served over verified HTTPS. Omission keeps\nthe credential-free policy closed outside numeric loopback and mTLS." + }, + "attemptTimeoutMilliseconds": { + "description": "A finite attempt-timeout override in milliseconds. Evidence may raise\nor lower its pack default under the acquisition's remaining deadline;\nfacade and job admit only a lower value within their ten-second policy.", + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 1, + "type": "integer" + }, + "baseUrl": { + "$ref": "#/$defs/Url", + "description": "The upstream origin; a fixed destination the engine never derives\nfrom a response." + }, + "credential": { + "$ref": "#/$defs/RuntimeCredential", + "description": "The credential; its kind must match the endpoint's declared kind." + }, + "endpoint": { + "$ref": "#/$defs/ExternalId", + "description": "The pack endpoint this binding configures." + }, + "maximumConcurrency": { + "default": 8, + "description": "The maximum simultaneous physical data operations admitted for this\nendpoint. Evidence programs selecting the same named connection share\none permit pool; token acquisition retains its own cache/single-flight\ncontrols.", + "format": "uint64", + "maximum": 256, + "minimum": 1, + "type": "integer" + }, + "maximumResponseBytes": { + "description": "A lower response bound than the pack declares, in bytes (1 to\n16777216, the largest bound a pack endpoint may declare); absent, the\ndeclared one applies. A value above the declared one is refused when\nthe deployment loads.", + "format": "uint64", + "maximum": 16777216, + "minimum": 1, + "type": "integer" + }, + "mtlsIdentity": { + "$ref": "#/$defs/MtlsIdentity", + "description": "The mTLS client identity, combinable with any authentication kind." + }, + "tlsTrustProfile": { + "$ref": "#/$defs/LocalId", + "description": "A trust profile under `outboundTls.trustProfiles` whose CA bundle\nthe endpoint trusts beside the system roots; absent, the endpoint\ntrusts the system roots alone." + } + }, + "required": [ + "endpoint", + "baseUrl" + ], + "type": "object" + }, "EnvironmentSecretProviderConfig": { "additionalProperties": false, "description": "The environment secret provider. It takes no settings.", @@ -84,6 +201,171 @@ ], "type": "string" }, + "EvidenceAuthenticationPolicy": { + "description": "The non-secret authentication authority reviewed with an Evidence source\nconnection. Pack endpoints still own API-intrinsic fields such as an API\nkey header name or authorization scheme; runtime bindings own only secret\nreferences and operational cache bounds.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "none": { + "additionalProperties": false, + "type": "object" + } + }, + "required": [ + "none" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "basic": { + "additionalProperties": false, + "type": "object" + } + }, + "required": [ + "basic" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "static-authorization": { + "additionalProperties": false, + "type": "object" + } + }, + "required": [ + "static-authorization" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "static-api-key": { + "additionalProperties": false, + "type": "object" + } + }, + "required": [ + "static-api-key" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "oauth2-client-credentials": { + "additionalProperties": false, + "properties": { + "audience": { + "type": "string" + }, + "credentialPlacement": { + "$ref": "#/$defs/CredentialPlacement" + }, + "resource": { + "type": "string" + }, + "scope": { + "type": "string" + }, + "tokenEndpoint": { + "$ref": "#/$defs/Url" + } + }, + "required": [ + "tokenEndpoint", + "credentialPlacement" + ], + "type": "object" + } + }, + "required": [ + "oauth2-client-credentials" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "oauth2-private-key-jwt": { + "additionalProperties": false, + "properties": { + "audience": { + "type": "string" + }, + "clientAssertionAudience": { + "type": "string" + }, + "resource": { + "type": "string" + }, + "scope": { + "type": "string" + }, + "tokenEndpoint": { + "$ref": "#/$defs/Url" + } + }, + "required": [ + "tokenEndpoint" + ], + "type": "object" + } + }, + "required": [ + "oauth2-private-key-jwt" + ], + "type": "object" + } + ] + }, + "EvidenceClientIdentityPolicy": { + "description": "The reviewed client-identity posture of one Evidence connection.", + "enum": [ + "forbidden", + "required" + ], + "type": "string" + }, + "EvidenceConnectionPolicy": { + "additionalProperties": false, + "description": "One reviewed Evidence connection policy from the governed bundle. Runtime\nbindings supply material and deployment values, but may not select a\ndifferent destination or trust posture from this policy.", + "properties": { + "authentication": { + "$ref": "#/$defs/EvidenceAuthenticationPolicy", + "description": "The reviewed non-secret authentication authority. Credential material\nand cache tuning stay in the runtime binding, but an OAuth token\ndestination or request audience cannot be selected there alone." + }, + "baseUrl": { + "$ref": "#/$defs/Url", + "description": "The exact reviewed origin and base path." + }, + "clientIdentity": { + "$ref": "#/$defs/EvidenceClientIdentityPolicy", + "description": "Whether the runtime must or must not supply an mTLS identity." + }, + "endpoint": { + "$ref": "#/$defs/LocalId", + "description": "The pack endpoint whose reviewed authentication kind applies." + }, + "tlsTrustProfile": { + "$ref": "#/$defs/LocalId", + "description": "The exact private trust profile selection; `None` means system roots." + } + }, + "required": [ + "endpoint", + "baseUrl", + "clientIdentity", + "authentication" + ], + "type": "object" + }, "EvidencePrivateKeyJwtConfig": { "additionalProperties": false, "properties": { @@ -176,6 +458,117 @@ "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", "type": "string" }, + "ExternalSourceAcquisition": { + "additionalProperties": false, + "properties": { + "maximumDataOperations": { + "format": "uint64", + "maximum": 1000, + "minimum": 1, + "type": "integer" + }, + "maximumResponseBytes": { + "format": "uint64", + "maximum": 16777216, + "minimum": 1, + "type": "integer" + }, + "timeoutMilliseconds": { + "format": "uint64", + "maximum": 60000, + "minimum": 1, + "type": "integer" + } + }, + "required": [ + "maximumDataOperations", + "timeoutMilliseconds", + "maximumResponseBytes" + ], + "type": "object" + }, + "ExternalSourceBinding": { + "additionalProperties": false, + "description": "Operator binding for one already declared projection. The reviewed pack\nprovides the acquisition ceiling; runtime configuration can narrow it.", + "properties": { + "acquisition": { + "$ref": "#/$defs/ExternalSourceAcquisition" + }, + "allowedFields": { + "items": { + "$ref": "#/$defs/LocalId" + }, + "type": "array", + "uniqueItems": true + }, + "allowedSelectors": { + "description": "An optional further restriction on locally authorized selectors.", + "items": { + "type": "string" + }, + "type": "array", + "uniqueItems": true + }, + "connection": { + "$ref": "#/$defs/AdapterEndpointBindings" + }, + "endpointPolicies": { + "items": { + "$ref": "#/$defs/EvidenceConnectionPolicy" + }, + "type": "array" + }, + "grantValidUntil": { + "description": "The reviewed source grant cutoff, as an RFC 3339 timestamp.", + "type": "string" + }, + "id": { + "$ref": "#/$defs/LocalId" + }, + "packDigest": { + "$ref": "#/$defs/Digest" + }, + "packFile": { + "type": "string" + }, + "retention": { + "$ref": "#/$defs/ExternalSourceRetention" + } + }, + "required": [ + "id", + "packFile", + "packDigest", + "connection", + "endpointPolicies", + "allowedFields", + "grantValidUntil", + "acquisition" + ], + "type": "object" + }, + "ExternalSourceRetention": { + "additionalProperties": false, + "properties": { + "maximumRetentionSeconds": { + "format": "uint64", + "maximum": 31536000, + "minimum": 1, + "type": "integer" + }, + "refreshIntervalSeconds": { + "format": "uint64", + "maximum": 31536000, + "minimum": 1, + "type": "integer" + } + }, + "required": [ + "maximumRetentionSeconds", + "refreshIntervalSeconds" + ], + "type": "object" + }, "FileSecretProviderConfig": { "additionalProperties": false, "description": "The file secret provider.", @@ -266,6 +659,43 @@ ], "type": "string" }, + "MtlsIdentity": { + "additionalProperties": false, + "description": "The mTLS client identity of an endpoint, orthogonal to the\nauthentication kind (`products/adapter/CONCEPT.md` section 10.2). It is\none way a `none` endpoint reaches a non-loopback source. An operator may\ninstead acknowledge a credential-free HTTPS source on its fixed runtime\nbinding.", + "properties": { + "certificateRef": { + "$ref": "#/$defs/SecretReference", + "description": "The client certificate secret reference." + }, + "keyRef": { + "$ref": "#/$defs/SecretReference", + "description": "The client private key secret reference." + } + }, + "required": [ + "certificateRef", + "keyRef" + ], + "type": "object" + }, + "OutboundTls": { + "additionalProperties": false, + "description": "The private certificate authorities endpoints may trust beside the\nsystem roots (`products/adapter/CONCEPT.md` section 10.2), in the shape\nEvidence ships: a named profile is a process-local PEM file. A CA\nbundle is not a secret, so the profile names a file, never a secret\nreference.", + "properties": { + "trustProfiles": { + "additionalProperties": { + "$ref": "#/$defs/TrustProfile" + }, + "default": {}, + "description": "The named private CA bundles.", + "propertyNames": { + "$ref": "#/$defs/LocalId" + }, + "type": "object" + } + }, + "type": "object" + }, "RawAttachmentStorageConfig": { "oneOf": [ { @@ -1375,6 +1805,223 @@ }, "type": "object" }, + "RuntimeCredential": { + "description": "The credential binding of one endpoint in the runtime config\n(`products/adapter/CONCEPT.md` section 10.2): Evidence's\n`SourceAuthentication` variant field for field, minus the pack-side\n`headerName` and `scheme`. The kind must match the endpoint's declared\nkind; the checks refuse a mismatch.", + "oneOf": [ + { + "additionalProperties": false, + "description": "No credential; matches a `none` endpoint.", + "properties": { + "none": { + "additionalProperties": false, + "type": "object" + } + }, + "required": [ + "none" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The Basic credential pair.", + "properties": { + "basic": { + "additionalProperties": false, + "properties": { + "passwordRef": { + "$ref": "#/$defs/SecretReference", + "description": "The password secret reference." + }, + "usernameRef": { + "$ref": "#/$defs/SecretReference", + "description": "The username secret reference." + } + }, + "required": [ + "usernameRef", + "passwordRef" + ], + "type": "object" + } + }, + "required": [ + "basic" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The static token, presented under the endpoint's declared `scheme`.", + "properties": { + "static-authorization": { + "additionalProperties": false, + "properties": { + "tokenRef": { + "$ref": "#/$defs/SecretReference", + "description": "The token secret reference." + } + }, + "required": [ + "tokenRef" + ], + "type": "object" + } + }, + "required": [ + "static-authorization" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The static API key, sent in the endpoint's declared header.", + "properties": { + "static-api-key": { + "additionalProperties": false, + "properties": { + "valueRef": { + "$ref": "#/$defs/SecretReference", + "description": "The key secret reference." + } + }, + "required": [ + "valueRef" + ], + "type": "object" + } + }, + "required": [ + "static-api-key" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The client-credentials binding, Evidence's fields: the token\nendpoint, the client id, and the shared secret, all as references\nor fixed strings.", + "properties": { + "oauth2-client-credentials": { + "additionalProperties": false, + "properties": { + "assumedLifetimeSeconds": { + "description": "Lifetime assumed when the provider omits `expires_in`.", + "format": "uint64", + "maximum": 86400, + "minimum": 1, + "type": "integer" + }, + "audience": { + "description": "Fixed `audience` form parameter.", + "type": "string" + }, + "clientIdRef": { + "$ref": "#/$defs/SecretReference", + "description": "The client id secret reference." + }, + "clientSecretRef": { + "$ref": "#/$defs/SecretReference", + "description": "The shared client secret reference." + }, + "credentialPlacement": { + "$ref": "#/$defs/CredentialPlacement", + "description": "Where the shared client secret travels." + }, + "maximumCacheSeconds": { + "description": "The token cache ceiling, Evidence's bound.", + "format": "uint64", + "maximum": 86400, + "minimum": 0, + "type": "integer" + }, + "resource": { + "description": "RFC 8707 resource indicator sent as a token-request form\nparameter.", + "type": "string" + }, + "scope": { + "description": "The token-request scope parameter.", + "type": "string" + }, + "tokenEndpoint": { + "$ref": "#/$defs/Url", + "description": "The token endpoint, a fixed destination never derived from a\nresponse; an absolute `http`/`https` URL the origin check\nconfines like any other destination." + } + }, + "required": [ + "tokenEndpoint", + "clientIdRef", + "clientSecretRef", + "credentialPlacement", + "maximumCacheSeconds" + ], + "type": "object" + } + }, + "required": [ + "oauth2-client-credentials" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The private-key-jwt binding, Evidence's fields. The signed\nassertion always travels in the form body; there is no placement\nparameter.", + "properties": { + "oauth2-private-key-jwt": { + "additionalProperties": false, + "properties": { + "assumedLifetimeSeconds": { + "format": "uint64", + "maximum": 86400, + "minimum": 1, + "type": "integer" + }, + "audience": { + "type": "string" + }, + "clientAssertionAudience": { + "description": "Audience claim of the signed assertion, defaulting to the\ntoken endpoint.", + "type": "string" + }, + "clientAssertionKeyRef": { + "$ref": "#/$defs/SecretReference", + "description": "The private JWK the client assertion is signed with." + }, + "clientIdRef": { + "$ref": "#/$defs/SecretReference", + "description": "The client id secret reference." + }, + "maximumCacheSeconds": { + "format": "uint64", + "maximum": 86400, + "minimum": 0, + "type": "integer" + }, + "resource": { + "type": "string" + }, + "scope": { + "type": "string" + }, + "tokenEndpoint": { + "$ref": "#/$defs/Url", + "description": "The token endpoint, a fixed destination never derived from a\nresponse; an absolute `http`/`https` URL the origin check\nconfines like any other destination." + } + }, + "required": [ + "tokenEndpoint", + "clientIdRef", + "clientAssertionKeyRef", + "maximumCacheSeconds" + ], + "type": "object" + } + }, + "required": [ + "oauth2-private-key-jwt" + ], + "type": "object" + } + ] + }, "SecretProvidersConfig": { "additionalProperties": false, "anyOf": [ @@ -1471,6 +2118,20 @@ ], "type": "object" }, + "TrustProfile": { + "additionalProperties": false, + "description": "One named private CA bundle: the absolute path of the process-local PEM\nfile the runtime reads.", + "properties": { + "caBundleFile": { + "description": "The absolute path of the PEM file holding the bundle's certificates.", + "type": "string" + } + }, + "required": [ + "caBundleFile" + ], + "type": "object" + }, "Url": { "description": "An absolute http or https URL with a host preserved as written, no user information, and no backslash, whitespace or control character. DNS case and equivalent IPv6 literals are permitted.", "format": "uri", @@ -1527,6 +2188,14 @@ }, "type": "object" }, + "externalSources": { + "default": [], + "description": "Independent acquisition bindings. Omission permits no source calls.", + "items": { + "$ref": "#/$defs/ExternalSourceBinding" + }, + "type": "array" + }, "fieldEncryption": { "$ref": "#/$defs/RawFieldEncryptionConfig", "default": {}, diff --git a/products/breg/generated/tools/migration-descriptor.v1alpha1.schema.json b/products/breg/generated/tools/migration-descriptor.v1alpha1.schema.json index d6f5060b4a..a600886120 100644 --- a/products/breg/generated/tools/migration-descriptor.v1alpha1.schema.json +++ b/products/breg/generated/tools/migration-descriptor.v1alpha1.schema.json @@ -96,7 +96,8 @@ "recipient-group-changed", "statistical-dataset-added", "statistical-dataset-removed", - "statistical-dataset-changed" + "statistical-dataset-changed", + "external-views-changed" ], "type": "string" }, @@ -133,7 +134,8 @@ "timer", "action", "recipient", - "statistical-dataset" + "statistical-dataset", + "external-views" ], "type": "string" }, diff --git a/products/breg/scripts/check-contracts.sh b/products/breg/scripts/check-contracts.sh index 9df1a424fe..1fb362b70e 100755 --- a/products/breg/scripts/check-contracts.sh +++ b/products/breg/scripts/check-contracts.sh @@ -17,6 +17,8 @@ python3 -m unittest \ "$script_dir/test_generated_gates.py" \ "$script_dir/test_quickstart.py" \ "$script_dir/../examples/webhook-receiver/test_receiver.py" +python3 "$script_dir/../examples/external-views/test_source.py" +python3 "$script_dir/../examples/external-views/test_scale.py" python3 -m unittest discover -s "$script_dir/../acceptance/reviewed-evidence-journey/tests" echo "Base Registry Engine product contracts passed" diff --git a/products/breg/scripts/test_validate_product.py b/products/breg/scripts/test_validate_product.py index 9004bf94ca..4c79a40e3c 100644 --- a/products/breg/scripts/test_validate_product.py +++ b/products/breg/scripts/test_validate_product.py @@ -77,7 +77,7 @@ def test_security_range_is_closed_through_derived_sql_invariants(self) -> None: ) extension_rows = matrix["invariants"][24:] self.assertEqual( - [f"BREG-NEG-{index:02d}" for index in range(25, 204) if index not in {61, 62}], + [f"BREG-NEG-{index:02d}" for index in range(25, 208) if index not in {61, 62}], [invariant["negativeId"] for invariant in extension_rows], ) for invariant in extension_rows: @@ -125,6 +125,8 @@ def test_w0_crate_boundary_includes_clients_and_opt_in_runtime(self) -> None: "dep:ipnet", "dep:jsonwebtoken", "dep:reqwest", + "dep:registry-adapter-core", + "dep:registry-adapter-engine", "dep:registry-platform-audit", "dep:registry-platform-authcommon", "dep:registry-platform-buildinfo", diff --git a/products/breg/scripts/validate_product.py b/products/breg/scripts/validate_product.py index 8780922c02..9a16acd0d5 100644 --- a/products/breg/scripts/validate_product.py +++ b/products/breg/scripts/validate_product.py @@ -30,7 +30,7 @@ CONTRACT_STATES = {"enforced", "partial", "planned"} V1_REQUIREMENT_IDS = tuple(f"BREG-V1-{index:02d}" for index in range(1, 45)) ACCEPTANCE_JOURNEY_IDS = tuple(f"BREG-J{index:02d}" for index in range(1, 25)) -SECURITY_INVARIANT_IDS = tuple(f"BREG-SEC-{index:02d}" for index in range(1, 204) if index not in {61, 62}) +SECURITY_INVARIANT_IDS = tuple(f"BREG-SEC-{index:02d}" for index in range(1, 208) if index not in {61, 62}) ACCEPTANCE_FIXTURES = { "BREG-J01": ("asset-site-placement", "acceptance/asset-site-placement"), "BREG-J02": ("asset-site-placement", "acceptance/asset-site-placement"), diff --git a/scripts/external-projection-scope.sh b/scripts/external-projection-scope.sh new file mode 100755 index 0000000000..8d5e3685fb --- /dev/null +++ b/scripts/external-projection-scope.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + printf '%s\n' 'Usage: external-projection-scope.sh inspect SERVICE PROJECTION' \ + ' external-projection-scope.sh withdraw SERVICE PROJECTION EPOCH PACKAGE_DIGEST ACTIVATION_UUID' \ + ' external-projection-scope.sh rotate SERVICE PROJECTION EPOCH PACKAGE_DIGEST ACTIVATION_UUID BINDING_DIGEST GRANT_DIGEST TTL_SECONDS RETENTION_SECONDS' \ + 'Use unbound for EPOCH only when inspect reports a null epoch. SERVICE must use the migration role.' >&2 + exit 2 +} +[[ $# -ge 3 ]] || usage +action=$1 +service=$2 +projection=$3 +[[ "$service" =~ ^[A-Za-z0-9_-]+$ ]] || usage +case "$action" in +inspect) [[ $# -eq 3 ]] || usage ;; +withdraw) [[ $# -eq 6 ]] || usage ;; +rotate) [[ $# -eq 10 ]] || usage ;; +*) usage ;; +esac +psql --no-psqlrc --dbname="service=$service" --set=ON_ERROR_STOP=1 \ + --set=action="$action" --set=projection="$projection" --set=epoch="${4:-unbound}" \ + --set=package="${5:-}" --set=activation="${6:-}" --set=binding="${7:-}" \ + --set=grant="${8:-}" --set=ttl="${9:-}" --set=retention="${10:-}" <<'SQL' +BEGIN; +SET LOCAL lock_timeout='10s'; +SET LOCAL statement_timeout='30s'; +SELECT jsonb_pretty(registry_internal.maintain_external_projection_scope( + :'action', :'projection', NULLIF(:'epoch','unbound')::uuid, + NULLIF(:'package',''), NULLIF(:'activation','')::uuid, + NULLIF(:'binding',''), NULLIF(:'grant',''), + NULLIF(:'ttl','')::bigint, NULLIF(:'retention','')::bigint)); +COMMIT; +SQL From 8bb3c680c796e9b7dc125afef9057f0c055ae6d0 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 09:46:40 +0700 Subject: [PATCH 02/64] refactor(breg): move external view runners out of the examples folder Signed-off-by: Jeremi Joslin --- .../breg/examples/external-views/README.md | 11 +- .../examples/external-views/test_source.py | 105 ---------------- .../external-views/SCALE.md | 7 +- .../loadtest/external-views/fixture_path.py | 10 ++ .../external-views/recovery.py | 1 + .../external-views/run_candidate.py | 3 +- .../external-views/scale_exercise.py | 1 + .../loadtest/external-views/test_runners.py | 118 ++++++++++++++++++ .../external-views/test_scale.py | 1 + products/breg/scripts/check-contracts.sh | 3 +- 10 files changed, 146 insertions(+), 114 deletions(-) rename products/breg/{examples => loadtest}/external-views/SCALE.md (94%) create mode 100644 products/breg/loadtest/external-views/fixture_path.py rename products/breg/{examples => loadtest}/external-views/recovery.py (99%) rename products/breg/{examples => loadtest}/external-views/run_candidate.py (99%) rename products/breg/{examples => loadtest}/external-views/scale_exercise.py (99%) create mode 100644 products/breg/loadtest/external-views/test_runners.py rename products/breg/{examples => loadtest}/external-views/test_scale.py (99%) diff --git a/products/breg/examples/external-views/README.md b/products/breg/examples/external-views/README.md index 7f665d3a5f..3a0aa9d60f 100644 --- a/products/breg/examples/external-views/README.md +++ b/products/breg/examples/external-views/README.md @@ -74,8 +74,9 @@ scope changes and backup recovery. Indexed projections require separate PostgreSQL migration and runtime roles for activation and serving. The candidate development configuration selects -those roles when the project declares an indexed projection. Use the fixture -runner to supply the synthetic source bindings and run the HTTP checks. +those roles when the project declares an indexed projection. Use the +[candidate runner](../../loadtest/external-views/run_candidate.py) to supply the +synthetic source bindings and run the HTTP checks. Prepare two fresh databases on the same disposable loopback PostgreSQL TLS server. Use names such as `breg_ext_rehearsal` and `breg_ext_runtime`. @@ -95,7 +96,7 @@ needs an explicit loopback host, port, database and `?sslmode=require`. Supply the local PostgreSQL CA PEM separately. ```sh -python3 -B products/breg/examples/external-views/run_candidate.py \ +python3 -B products/breg/loadtest/external-views/run_candidate.py \ --breg-bin "$BREG_BIN" \ --bregctl-bin "$BREGCTL_BIN" \ --database-url-dir "$FIXTURE_DATABASE_URL_DIR" \ @@ -163,7 +164,7 @@ purges and scale have separate acceptance checks. ## Run the million-record fixture -The scale exercise uses the same candidate binaries, split roles and TLS +The scale exercise is load-test tooling and lives beside the other BReg load tests. It uses the same candidate binaries, split roles and TLS setup with a larger synthetic corpus. It sets an explicit freshness and request budget and records the actual native timings and PostgreSQL plans. -[Scale fixture instructions](SCALE.md) +[Scale fixture instructions](../../loadtest/external-views/SCALE.md) diff --git a/products/breg/examples/external-views/test_source.py b/products/breg/examples/external-views/test_source.py index 3fd7187fd9..f9d10d0c53 100644 --- a/products/breg/examples/external-views/test_source.py +++ b/products/breg/examples/external-views/test_source.py @@ -3,40 +3,14 @@ from http.server import ThreadingHTTPServer import json -from pathlib import Path -import tempfile from threading import Thread import unittest from urllib.error import HTTPError from urllib.request import ProxyHandler, Request, build_opener -from journey import JourneyError -from recovery import operator_report -from run_candidate import URL_FILES, database_urls, stage_project, verify_source_facts from synthetic_source import Fixture, FixtureFault, handler, runtime_fragment -class RecoveryTests(unittest.TestCase): - def test_operator_report_reads_the_psql_json_cell(self): - output = b"""BEGIN -SET -SET - jsonb_pretty ----------------------------------------------------- - { + - "projection": "holdings-indexed", + - "active": false + - } -(1 row) - -COMMIT -""" - self.assertEqual(operator_report(output), {"projection": "holdings-indexed", "active": False}) - for invalid in (output.replace(b"COMMIT", b"ROLLBACK"), b"COMMIT", output + output): - with self.assertRaisesRegex(JourneyError, "scope report"): - operator_report(invalid) - - class FixtureTests(unittest.TestCase): def setUp(self): self.source = Fixture() @@ -121,85 +95,6 @@ def test_no_store_and_interruption_are_separate_controls(self): self.source.acquire("holdings-by-business", {**parent, "cursor": first["next"]}) -class CandidateFixtureTests(unittest.TestCase): - def test_saved_native_project_is_staged_byte_for_byte_and_requires_all_three_files(self): - with tempfile.TemporaryDirectory() as scratch: - work = Path(scratch) - source = work / "studio" - (source / "tests").mkdir(parents=True) - files = {"registry.yaml": b"registry", "dev-clients.yaml": b"clients", - "tests/journeys.yaml": b"journeys"} - for name, data in files.items(): - (source / name).write_bytes(data) - staged = stage_project(source, work) - for name, data in files.items(): - self.assertEqual((staged / name).read_bytes(), data) - self.assertEqual((source / name).read_bytes(), data) - incomplete = work / "incomplete" - incomplete.mkdir() - (incomplete / "registry.yaml").write_bytes(b"registry") - with self.assertRaises(JourneyError): - stage_project(incomplete, work) - - def test_source_facts_require_all_projections_and_refuse_connection_details(self): - bindings = runtime_fragment("http://127.0.0.1:8001", "synthetic", "2030-01-01T00:00:00Z") - projections = [] - for binding in bindings["externalSources"]: - person = binding["id"].startswith("person") - indexed = binding["id"].endswith("-indexed") - projections.append({ - "id": binding["id"], "mode": "indexed" if indexed else "live", - "recordType": "person" if person else "holding", - "authority": "urn:example:population-register" if person else "urn:example:business-register", - "operation": ("person-batch" if indexed else "person-exact") if person else "holdings-by-business", - "operationVersion": "1", - "selectorScheme": "urn:example:person-number" if person else "urn:example:business-number", - "purpose": "licensing", "scope": "licensing-holders" if person else "licensed-business-holdings", - "fields": binding["allowedFields"], "connection": "synthetic-licensing", - "bindingDigest": "sha256:" + "1" * 64, "grantDigest": "sha256:" + "2" * 64, - "grantValidUntil": "2030-01-01T00:00:00+00:00", - "cacheTtlSeconds": 300 if indexed else None, - "maximumRetentionSeconds": 300 if indexed else None, - "refreshIntervalSeconds": 60 if indexed else None, - }) - facts = { - "apiVersion": "id.registrystack.org/formats/breg/external-source-facts/v1alpha1", - "kind": "BRegExternalSourceFacts", "registryId": "synthetic-licensing", - "revision": "fixture-revision", "projections": projections, - } - verify_source_facts(facts, bindings, 300) - with self.assertRaises(JourneyError): - verify_source_facts({**facts, "projections": projections[:-1]}, bindings, 300) - with self.assertRaises(JourneyError): - verify_source_facts({**facts, "projections": [{**projections[0], "baseUrl": "private"}, *projections[1:]]}, - bindings, 300) - with self.assertRaises(JourneyError): - verify_source_facts({**facts, "projections": [{**projections[0], "fields": ["other"]}, *projections[1:]]}, - bindings, 300) - - def test_url_files_must_describe_fresh_split_loopback_databases(self): - valid = [ - "postgresql://migrator@localhost:6543/breg_ext_rehearsal?sslmode=require", - "postgresql://server@localhost:6543/breg_ext_rehearsal?sslmode=require", - "postgresql://migrator@localhost:6543/breg_ext_serving?sslmode=require", - "postgresql://server@localhost:6543/breg_ext_serving?sslmode=require", - ] - with tempfile.TemporaryDirectory() as scratch: - root = Path(scratch) - output = root / "output" - output.mkdir() - for invalid in (valid[2], valid[3] + "&host=example.test"): - for name, url in zip(URL_FILES, [*valid[:3], invalid]): - (root / name).write_text(url, encoding="ascii") - with self.assertRaises(JourneyError): - database_urls(root, output) - self.assertEqual(list(output.iterdir()), []) - for name, url in zip(URL_FILES, valid): - (root / name).write_text(url, encoding="ascii") - self.assertEqual(database_urls(root, output), {"migration": "migrator", "runtime": "server"}) - self.assertEqual({path.name for path in output.iterdir()}, set(URL_FILES)) - - class HttpFixtureTests(unittest.TestCase): def test_source_and_control_identities_are_separate(self): source_key, control_key = "synthetic-source-key", "synthetic-control-key" diff --git a/products/breg/examples/external-views/SCALE.md b/products/breg/loadtest/external-views/SCALE.md similarity index 94% rename from products/breg/examples/external-views/SCALE.md rename to products/breg/loadtest/external-views/SCALE.md index d32fb8b56a..30bc230ba6 100644 --- a/products/breg/examples/external-views/SCALE.md +++ b/products/breg/loadtest/external-views/SCALE.md @@ -18,6 +18,9 @@ machine can vary. | Holdings source requests | Six reads for the two allowed parents, no request to a denied parent | | Indexed query source requests | Zero, including reads with missing coverage | +The freshness target covers reads made after the refresh sweep completes. The +exercise does not measure reads while refresh is still running continuously. + The reviewed Adapter pack permits at most 64 Person keys per request and 128 rows per complete related snapshot. The fixture orders the first 50,000 local roots so that their keys are distinct. At 100 local roots per @@ -42,7 +45,7 @@ role after refresh. The plans must contain the candidate's actual coverage and page nodes with buffers. ```sh -python3 -B products/breg/examples/external-views/run_candidate.py \ +python3 -B products/breg/loadtest/external-views/run_candidate.py \ --breg-bin "$BREG_BIN" \ --bregctl-bin "$BREGCTL_BIN" \ --database-url-dir "$FIXTURE_DATABASE_URL_DIR" \ @@ -83,5 +86,5 @@ Test the deterministic corpus, source bounds and oracle without starting a candidate or PostgreSQL: ```sh -python3 -B products/breg/examples/external-views/test_scale.py +python3 -B products/breg/loadtest/external-views/test_scale.py ``` diff --git a/products/breg/loadtest/external-views/fixture_path.py b/products/breg/loadtest/external-views/fixture_path.py new file mode 100644 index 0000000000..994892067b --- /dev/null +++ b/products/breg/loadtest/external-views/fixture_path.py @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""Make the synthetic source and journey helpers in the example importable.""" + +import sys +from pathlib import Path + +EXAMPLE = Path(__file__).resolve().parents[2] / "examples/external-views" +if str(EXAMPLE) not in sys.path: + sys.path.insert(0, str(EXAMPLE)) diff --git a/products/breg/examples/external-views/recovery.py b/products/breg/loadtest/external-views/recovery.py similarity index 99% rename from products/breg/examples/external-views/recovery.py rename to products/breg/loadtest/external-views/recovery.py index 62529cc3f1..7ae951ddb1 100644 --- a/products/breg/examples/external-views/recovery.py +++ b/products/breg/loadtest/external-views/recovery.py @@ -10,6 +10,7 @@ import subprocess from urllib.parse import unquote, urlsplit +import fixture_path # noqa: F401 from journey import JourneyError, expect, records, request PROJECTION = "holdings-indexed" diff --git a/products/breg/examples/external-views/run_candidate.py b/products/breg/loadtest/external-views/run_candidate.py similarity index 99% rename from products/breg/examples/external-views/run_candidate.py rename to products/breg/loadtest/external-views/run_candidate.py index cb815952c1..d5b3c302b1 100644 --- a/products/breg/examples/external-views/run_candidate.py +++ b/products/breg/loadtest/external-views/run_candidate.py @@ -22,10 +22,11 @@ from urllib.parse import unquote, urlsplit from uuid import UUID +import fixture_path # noqa: F401 from journey import HTTP, Journey, JourneyError, expect, request from synthetic_source import Fixture, handler, runtime_fragment -PROJECT = Path(__file__).resolve().parent +PROJECT = Path(__file__).resolve().parents[2] / "examples/external-views" ISSUER = "https://fixture-issuer.example.test/licensing" AUDIENCE = "urn:breg:external-views-fixture" KEY_ID = "external-views-fixture" diff --git a/products/breg/examples/external-views/scale_exercise.py b/products/breg/loadtest/external-views/scale_exercise.py similarity index 99% rename from products/breg/examples/external-views/scale_exercise.py rename to products/breg/loadtest/external-views/scale_exercise.py index abf04f7161..f54a5c7102 100644 --- a/products/breg/examples/external-views/scale_exercise.py +++ b/products/breg/loadtest/external-views/scale_exercise.py @@ -13,6 +13,7 @@ import time from urllib.parse import unquote, urlsplit +import fixture_path # noqa: F401 from journey import JourneyError, expect, fixture_credential, records, request from synthetic_source import Fixture, digest, require diff --git a/products/breg/loadtest/external-views/test_runners.py b/products/breg/loadtest/external-views/test_runners.py new file mode 100644 index 0000000000..0555548ca7 --- /dev/null +++ b/products/breg/loadtest/external-views/test_runners.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 + +"""Deterministic checks for the candidate runner helpers.""" + +from pathlib import Path +import tempfile +import unittest + +import fixture_path # noqa: F401 +from journey import JourneyError +from recovery import operator_report +from run_candidate import URL_FILES, database_urls, stage_project, verify_source_facts +from synthetic_source import runtime_fragment + + +class RecoveryTests(unittest.TestCase): + def test_operator_report_reads_the_psql_json_cell(self): + output = b"""BEGIN +SET +SET + jsonb_pretty +---------------------------------------------------- + { + + "projection": "holdings-indexed", + + "active": false + + } +(1 row) + +COMMIT +""" + self.assertEqual(operator_report(output), {"projection": "holdings-indexed", "active": False}) + for invalid in (output.replace(b"COMMIT", b"ROLLBACK"), b"COMMIT", output + output): + with self.assertRaisesRegex(JourneyError, "scope report"): + operator_report(invalid) + + +class CandidateFixtureTests(unittest.TestCase): + def test_saved_native_project_is_staged_byte_for_byte_and_requires_all_three_files(self): + with tempfile.TemporaryDirectory() as scratch: + work = Path(scratch) + source = work / "studio" + (source / "tests").mkdir(parents=True) + files = {"registry.yaml": b"registry", "dev-clients.yaml": b"clients", + "tests/journeys.yaml": b"journeys"} + for name, data in files.items(): + (source / name).write_bytes(data) + staged = stage_project(source, work) + for name, data in files.items(): + self.assertEqual((staged / name).read_bytes(), data) + self.assertEqual((source / name).read_bytes(), data) + incomplete = work / "incomplete" + incomplete.mkdir() + (incomplete / "registry.yaml").write_bytes(b"registry") + with self.assertRaises(JourneyError): + stage_project(incomplete, work) + + def test_source_facts_require_all_projections_and_refuse_connection_details(self): + bindings = runtime_fragment("http://127.0.0.1:8001", "synthetic", "2030-01-01T00:00:00Z") + projections = [] + for binding in bindings["externalSources"]: + person = binding["id"].startswith("person") + indexed = binding["id"].endswith("-indexed") + projections.append({ + "id": binding["id"], "mode": "indexed" if indexed else "live", + "recordType": "person" if person else "holding", + "authority": "urn:example:population-register" if person else "urn:example:business-register", + "operation": ("person-batch" if indexed else "person-exact") if person else "holdings-by-business", + "operationVersion": "1", + "selectorScheme": "urn:example:person-number" if person else "urn:example:business-number", + "purpose": "licensing", "scope": "licensing-holders" if person else "licensed-business-holdings", + "fields": binding["allowedFields"], "connection": "synthetic-licensing", + "bindingDigest": "sha256:" + "1" * 64, "grantDigest": "sha256:" + "2" * 64, + "grantValidUntil": "2030-01-01T00:00:00+00:00", + "cacheTtlSeconds": 300 if indexed else None, + "maximumRetentionSeconds": 300 if indexed else None, + "refreshIntervalSeconds": 60 if indexed else None, + }) + facts = { + "apiVersion": "id.registrystack.org/formats/breg/external-source-facts/v1alpha1", + "kind": "BRegExternalSourceFacts", "registryId": "synthetic-licensing", + "revision": "fixture-revision", "projections": projections, + } + verify_source_facts(facts, bindings, 300) + with self.assertRaises(JourneyError): + verify_source_facts({**facts, "projections": projections[:-1]}, bindings, 300) + with self.assertRaises(JourneyError): + verify_source_facts({**facts, "projections": [{**projections[0], "baseUrl": "private"}, *projections[1:]]}, + bindings, 300) + with self.assertRaises(JourneyError): + verify_source_facts({**facts, "projections": [{**projections[0], "fields": ["other"]}, *projections[1:]]}, + bindings, 300) + + def test_url_files_must_describe_fresh_split_loopback_databases(self): + valid = [ + "postgresql://migrator@localhost:6543/breg_ext_rehearsal?sslmode=require", + "postgresql://server@localhost:6543/breg_ext_rehearsal?sslmode=require", + "postgresql://migrator@localhost:6543/breg_ext_serving?sslmode=require", + "postgresql://server@localhost:6543/breg_ext_serving?sslmode=require", + ] + with tempfile.TemporaryDirectory() as scratch: + root = Path(scratch) + output = root / "output" + output.mkdir() + for invalid in (valid[2], valid[3] + "&host=example.test"): + for name, url in zip(URL_FILES, [*valid[:3], invalid]): + (root / name).write_text(url, encoding="ascii") + with self.assertRaises(JourneyError): + database_urls(root, output) + self.assertEqual(list(output.iterdir()), []) + for name, url in zip(URL_FILES, valid): + (root / name).write_text(url, encoding="ascii") + self.assertEqual(database_urls(root, output), {"migration": "migrator", "runtime": "server"}) + self.assertEqual({path.name for path in output.iterdir()}, set(URL_FILES)) + + +if __name__ == "__main__": + unittest.main() diff --git a/products/breg/examples/external-views/test_scale.py b/products/breg/loadtest/external-views/test_scale.py similarity index 99% rename from products/breg/examples/external-views/test_scale.py rename to products/breg/loadtest/external-views/test_scale.py index 5200a9dfd9..2f15535b1e 100644 --- a/products/breg/examples/external-views/test_scale.py +++ b/products/breg/loadtest/external-views/test_scale.py @@ -5,6 +5,7 @@ import unittest +import fixture_path # noqa: F401 from scale_exercise import ( ABSENT_KEYS, BATCH_KEYS, DENIED_ROOTS, ELIGIBLE_ROOTS, HOLDER_KEYS, MATCH_KEYS, PERSON_REQUEST_BUDGET, REFRESH_PAGE_SIZE, ScaleFixture, diff --git a/products/breg/scripts/check-contracts.sh b/products/breg/scripts/check-contracts.sh index 1fb362b70e..f61207cac2 100755 --- a/products/breg/scripts/check-contracts.sh +++ b/products/breg/scripts/check-contracts.sh @@ -18,7 +18,8 @@ python3 -m unittest \ "$script_dir/test_quickstart.py" \ "$script_dir/../examples/webhook-receiver/test_receiver.py" python3 "$script_dir/../examples/external-views/test_source.py" -python3 "$script_dir/../examples/external-views/test_scale.py" +python3 "$script_dir/../loadtest/external-views/test_scale.py" +python3 "$script_dir/../loadtest/external-views/test_runners.py" python3 -m unittest discover -s "$script_dir/../acceptance/reviewed-evidence-journey/tests" echo "Base Registry Engine product contracts passed" From 621f62b7ae804478e7cc2157267ee19a73ef4e8f Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 10:05:26 +0700 Subject: [PATCH 03/64] fix(breg): keep external projection withdrawal with migration authority Signed-off-by: Jeremi Joslin --- .../src/postgres/external_projection.rs | 53 ++++++++++++++----- 1 file changed, 40 insertions(+), 13 deletions(-) diff --git a/crates/registry-breg/src/postgres/external_projection.rs b/crates/registry-breg/src/postgres/external_projection.rs index 695ccf844f..75c2965835 100644 --- a/crates/registry-breg/src/postgres/external_projection.rs +++ b/crates/registry-breg/src/postgres/external_projection.rs @@ -14,6 +14,10 @@ use super::{ExpectedRegistryIdentity, GuardedTransaction, RuntimeRevoke, SqlIden pub(crate) const BIND_ONCE_FUNCTION: &str = "registry_internal.bind_external_projection_once(text,uuid,text,text,bigint,bigint,text,text,uuid)"; pub(crate) const MAINTENANCE_FUNCTION:&str="registry_internal.maintain_external_projection_scope(text,text,uuid,text,uuid,text,text,bigint,bigint)"; +/// Withdrawal stays with migration authority: the runtime role holds no +/// EXECUTE grant on it, so a compromised runtime cannot purge a scope. +pub(crate) const WITHDRAW_FUNCTION: &str = + "registry_internal.withdraw_external_projection(text,uuid,text,text)"; /// Exact catalog identities include PostgreSQL argument names and spacing. /// ACL/install signatures above remain type-only SQL call signatures. /// Entries are (identity, result type, returns set, security definer, runtime execute). @@ -21,7 +25,7 @@ pub(crate) const MANAGED_FUNCTIONS: &[(&str,&str,bool,bool,bool)] = &[ ("registry_internal.allocate_external_projection_refresh(p_projection text, p_epoch uuid, p_binding text, p_grant text, p_subject text)","bigint",false,true,true), ("registry_internal.publish_external_projection(p_projection text, p_epoch uuid, p_binding text, p_grant text, p_sequence bigint, p_subject text, p_snapshot text, p_observed bigint, p_expires bigint, p_purge bigint, p_kind text, p_rows jsonb)","text",false,true,true), ("registry_internal.fail_external_projection_refresh(p_projection text, p_epoch uuid, p_sequence bigint, p_subject text, p_reason text)","boolean",false,true,true), - ("registry_internal.withdraw_external_projection(p_projection text, p_epoch uuid, p_binding text, p_grant text)","boolean",false,true,true), + ("registry_internal.withdraw_external_projection(p_projection text, p_epoch uuid, p_binding text, p_grant text)","boolean",false,true,false), ("registry_internal.purge_external_projections(p_limit integer)","record",true,true,true), ("registry_internal.lock_external_projection_generation(p_projection text, p_epoch uuid, p_binding text, p_grant text)","bigint",false,true,true), ("registry_internal.bind_external_projection_once(p_projection text, p_epoch uuid, p_binding text, p_grant text, p_ttl bigint, p_retention bigint, p_contract text, p_package text, p_activation uuid)","uuid",false,true,true), @@ -72,11 +76,11 @@ pub(crate) async fn reconcile( if !enabled(registry) { let exists: bool=migration.query_one("SELECT to_regclass('registry_internal.registry_external_projection_scopes') IS NOT NULL",&[]).await.map_err(|_|ProjectionError::Unavailable)?.get(0); if exists { - for function in RUNTIME_FUNCTIONS - .iter() - .copied() - .chain([BIND_ONCE_FUNCTION, MAINTENANCE_FUNCTION]) - { + for function in RUNTIME_FUNCTIONS.iter().copied().chain([ + WITHDRAW_FUNCTION, + BIND_ONCE_FUNCTION, + MAINTENANCE_FUNCTION, + ]) { migration .batch_execute(&format!("DROP FUNCTION IF EXISTS {function};")) .await @@ -230,7 +234,6 @@ pub(crate) const RUNTIME_FUNCTIONS: &[&str] = &[ "registry_internal.allocate_external_projection_refresh(text,uuid,text,text,text)", "registry_internal.publish_external_projection(text,uuid,text,text,bigint,text,text,bigint,bigint,bigint,text,jsonb)", "registry_internal.fail_external_projection_refresh(text,uuid,bigint,text,text)", - "registry_internal.withdraw_external_projection(text,uuid,text,text)", "registry_internal.purge_external_projections(integer)", "registry_internal.lock_external_projection_generation(text,uuid,text,text)", ]; @@ -602,6 +605,12 @@ pub(crate) async fn install( for function in RUNTIME_FUNCTIONS { migration.batch_execute(&format!("REVOKE ALL ON FUNCTION {function} FROM {revoke}; GRANT EXECUTE ON FUNCTION {function} TO {};", runtime_role.quoted())).await.map_err(|_| ProjectionError::Unavailable)?; } + migration + .batch_execute(&format!( + "REVOKE ALL ON FUNCTION {WITHDRAW_FUNCTION} FROM {revoke};" + )) + .await + .map_err(|_| ProjectionError::Unavailable)?; Ok(()) } @@ -882,13 +891,14 @@ pub(crate) async fn lock_generation( }) } +/// Tests withdraw under migration authority; operators use the maintenance +/// function, and the runtime role holds no EXECUTE grant on either. #[cfg(all(test, feature = "postgres-test"))] pub(crate) async fn withdraw_scope( - transaction: &GuardedTransaction<'_>, + migration: &impl GenericClient, scope: &ProjectionScope, ) -> Result<(), ProjectionError> { - let row = transaction - .transaction() + let row = migration .query_one( "SELECT registry_internal.withdraw_external_projection($1,$2,$3,$4)", &[ @@ -1707,7 +1717,16 @@ mod postgres_tests { .await .unwrap() .contains("subject")); - withdraw_scope(&transaction, &bound).await.unwrap(); + transaction.commit().await.unwrap(); + withdraw_scope(&migration, &bound).await.unwrap(); + let transaction = super::super::begin_registry_transaction( + &mut runtime, + RegistryLockKey::derive(registry.registry_id()).unwrap(), + std::time::Duration::from_secs(2), + &expected, + ) + .await + .unwrap(); assert_eq!( refresh_due_keys(&transaction, &bound, &[], baseline, now) .await @@ -1988,7 +2007,9 @@ mod postgres_tests { assert_eq!(runtime.query_one("SELECT kind FROM registry_internal.registry_external_projection_snapshots WHERE subject_key='parent'", &[]).await.unwrap().get::<_,String>(0), "complete-collection"); let pending = allocate_inner(&runtime, &scope, "subject").await.unwrap(); - runtime + // Withdrawal is operator authority: a runtime credential cannot end a + // scope, and so cannot force the next start to rebind it. + let refused = runtime .query_one( "SELECT registry_internal.withdraw_external_projection($1,$2,$3,$4)", &[ @@ -1999,7 +2020,13 @@ mod postgres_tests { ], ) .await - .unwrap(); + .unwrap_err(); + assert_eq!( + refused.code(), + Some(&tokio_postgres::error::SqlState::INSUFFICIENT_PRIVILEGE) + ); + assert!(migration.query_one("SELECT active FROM registry_internal.registry_external_projection_scopes WHERE projection_id=$1",&[&scope.projection_id]).await.unwrap().get::<_,bool>(0)); + withdraw_scope(&migration, &scope).await.unwrap(); assert_eq!( publish_inner( &runtime, From 83dd5bac6a6dfc6b49b5cf3eb4de61136dafc8cc Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 10:24:14 +0700 Subject: [PATCH 04/64] refactor(breg): give external projections one cache lifetime A cached external value now lives for the authored cacheTtlSeconds, which a deployment binding may only shorten, and is purged when it expires. A separate retention window, refresh interval and purge deadline added configuration without adding protection, so they are gone from the binding, the store and the source facts. The deployment allowedSelectors list is also removed. Local row authority already decides which roots may be acquired. Signed-off-by: Jeremi Joslin --- crates/registry-breg/src/external_source.rs | 137 +++++++++-------- .../postgres/external_label_acquisition.rs | 27 ++-- .../src/postgres/external_projection.rs | 141 +++++++----------- .../src/postgres/external_runtime.rs | 10 +- .../src/postgres/external_view_query.rs | 14 +- crates/registry-breg/src/runtime_config.rs | 2 +- crates/registry-bregctl/src/lib.rs | 2 - .../contracts/security-invariant-matrix.yaml | 2 +- .../breg/examples/external-views/README.md | 6 +- .../external-views/synthetic_source.py | 14 +- .../examples/external-views/test_source.py | 6 +- products/breg/external-projection-storage.md | 10 +- .../generated/runtime/runtime.schema.json | 38 +---- .../breg/loadtest/external-views/SCALE.md | 2 +- .../breg/loadtest/external-views/recovery.py | 5 +- .../loadtest/external-views/run_candidate.py | 5 - .../loadtest/external-views/scale_exercise.py | 11 +- .../loadtest/external-views/test_runners.py | 2 - scripts/external-projection-scope.sh | 8 +- 19 files changed, 176 insertions(+), 266 deletions(-) diff --git a/crates/registry-breg/src/external_source.rs b/crates/registry-breg/src/external_source.rs index 1de1c93bb4..12b848882d 100644 --- a/crates/registry-breg/src/external_source.rs +++ b/crates/registry-breg/src/external_source.rs @@ -14,10 +14,9 @@ use std::{ use chrono::{DateTime, Utc}; use registry_adapter_core::{ - projection_token_valid, AdapterEndpointBindings, AdapterPack, EvidenceConnectionPolicy, - ProjectionLocalDisclosure, ProjectionMode, ProjectionOperationKind, ProjectionRecord, - ProjectionRequest, ProjectionResult, ProjectionRetention, ADAPTER_PACK_API_VERSION, - ADAPTER_PACK_KIND, + AdapterEndpointBindings, AdapterPack, EvidenceConnectionPolicy, ProjectionLocalDisclosure, + ProjectionMode, ProjectionOperationKind, ProjectionRecord, ProjectionRequest, ProjectionResult, + ProjectionRetention, ADAPTER_PACK_API_VERSION, ADAPTER_PACK_KIND, }; use registry_adapter_engine::{ artifacts::PackArtifactBytes, @@ -51,7 +50,7 @@ const MAXIMUM_CLOSURE_BYTES: usize = 16 * 1_048_576; const MAXIMUM_RESPONSE_BYTES: u64 = 16 * 1_048_576; const MAXIMUM_CA_BYTES: usize = registry_platform_httputil::destination::MAX_DESTINATION_CA_BUNDLE_BYTES; -const MAXIMUM_RETENTION_SECONDS: u64 = 31_536_000; +const MAXIMUM_CACHE_TTL_SECONDS: u64 = 31_536_000; /// Operator binding for one already declared projection. The reviewed pack /// provides the acquisition ceiling; runtime configuration can narrow it. @@ -67,11 +66,11 @@ pub struct ExternalSourceBinding { pub allowed_fields: UniqueList, /// The reviewed source grant cutoff, as an RFC 3339 timestamp. pub grant_valid_until: String, - /// An optional further restriction on locally authorized selectors. + /// An optional shorter cache lifetime for an indexed projection. The + /// cache keeps a value for the shorter of this and the authored + /// cacheTtlSeconds, then purges it. A live projection refuses it. #[serde(default, skip_serializing_if = "Option::is_none")] - pub allowed_selectors: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub retention: Option, + pub cache_ttl_seconds: Option>, pub acquisition: ExternalSourceAcquisition, } @@ -81,14 +80,6 @@ impl Identified for ExternalSourceBinding { } } -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Clone, Copy, Deserialize, Serialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct ExternalSourceRetention { - pub maximum_retention_seconds: BoundedU64<1, MAXIMUM_RETENTION_SECONDS>, - pub refresh_interval_seconds: BoundedU64<1, MAXIMUM_RETENTION_SECONDS>, -} - #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[derive(Clone, Copy, Deserialize, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] @@ -124,13 +115,11 @@ pub struct ExternalSourceService { pub struct BoundExternalSource { program: ProjectionProgram, fields: BTreeSet, - allowed_selectors: Option>, mode: ExternalProjectionMode, grant_valid_until: DateTime, binding_fingerprint: String, grant_fingerprint: String, cache_ttl_seconds: Option, - retention: Option, field_types: BTreeMap, } @@ -155,16 +144,6 @@ impl BoundExternalSource { self.cache_ttl_seconds } - pub fn maximum_retention_seconds(&self) -> Option { - self.retention - .map(|retention| retention.maximum_retention_seconds.get() as u32) - } - - pub fn refresh_interval_seconds(&self) -> Option { - self.retention - .map(|retention| retention.refresh_interval_seconds.get() as u32) - } - pub fn maximum_batch_keys(&self) -> usize { self.program.contract().maximum_batch_keys.get() as usize } @@ -208,12 +187,9 @@ impl ExternalSourceService { .map_err(|_| ExternalSourceError::Contract)?; check_pack_contract(projection, record, &program)?; let (fields, field_types) = project_fields(projection, record, binding)?; - let cache_ttl_seconds = effective_ttl(projection, binding.retention)?; + let cache_ttl_seconds = + effective_ttl(projection, binding.cache_ttl_seconds.map(|ttl| ttl.get()))?; let grant_valid_until = parse_grant_cutoff(&binding.grant_valid_until)?; - let allowed_selectors = binding - .allowed_selectors - .as_ref() - .map(|list| list.iter().cloned().collect::>()); let ca_bundles = capture_ca_bundles(&binding.connection)?; let ca_digests: BTreeMap<_, _> = ca_bundles @@ -261,22 +237,18 @@ impl ExternalSourceService { "externalType":record, "sourceGrant":program.contract(), "allowedFields":fields, - "allowedSelectors":allowed_selectors, - "grantValidUntil":grant_valid_until.to_rfc3339(), - "retention":binding.retention + "grantValidUntil":grant_valid_until.to_rfc3339() }))?; result.bindings.insert( id.to_owned(), BoundExternalSource { program, fields, - allowed_selectors, mode: projection.mode, grant_valid_until, binding_fingerprint, grant_fingerprint, cache_ttl_seconds, - retention: binding.retention, field_types, }, ); @@ -302,11 +274,7 @@ impl ExternalSourceService { .binding(projection) .ok_or(ExternalSourceError::Unbound)?; let now = clock(); - if now >= bound.grant_valid_until - || fields.is_empty() - || !fields.is_subset(&bound.fields) - || !selectors_allowed(&request, bound.allowed_selectors.as_ref()) - { + if now >= bound.grant_valid_until || fields.is_empty() || !fields.is_subset(&bound.fields) { return Err(ExternalSourceError::Grant); } let mode = match bound.mode { @@ -351,13 +319,6 @@ pub(crate) fn validate_bindings( || binding.allowed_fields.is_empty() || binding.allowed_fields.len() > 256 || binding.endpoint_policies.is_empty() - || binding.allowed_selectors.as_ref().is_some_and(|selectors| { - selectors.len() > 10_000 || selectors.iter().any(|key| !projection_token_valid(key)) - }) - || binding.retention.is_some_and(|retention| { - retention.refresh_interval_seconds.get() - >= retention.maximum_retention_seconds.get() - }) { return Err(ExternalSourceError::Configuration); } @@ -472,34 +433,27 @@ fn project_fields( Ok((fields, types)) } +/// One cache lifetime: the authored cacheTtlSeconds, which a deployment may +/// only shorten. A cached value is purged when it expires. fn effective_ttl( projection: &ExternalProjectionSource, - retention: Option, + deployment: Option, ) -> Result, ExternalSourceError> { - match (projection.mode, projection.cache_ttl_seconds, retention) { + match (projection.mode, projection.cache_ttl_seconds, deployment) { (ExternalProjectionMode::Live, None, None) => Ok(None), - (ExternalProjectionMode::Indexed, Some(ttl), Some(retention)) => { - let effective = ttl.min(retention.maximum_retention_seconds.get() as u32); - if u64::from(effective) <= retention.refresh_interval_seconds.get() { + (ExternalProjectionMode::Indexed, Some(authored), deployment) => { + let shortened = deployment.map_or(authored, |ttl| { + u32::try_from(ttl).map_or(authored, |ttl| authored.min(ttl)) + }); + if shortened == 0 || u64::from(shortened) > MAXIMUM_CACHE_TTL_SECONDS { return Err(ExternalSourceError::Configuration); } - Ok(Some(effective)) + Ok(Some(shortened)) } _ => Err(ExternalSourceError::Configuration), } } -fn selectors_allowed(request: &ProjectionRequest, allowed: Option<&BTreeSet>) -> bool { - let Some(allowed) = allowed else { - return true; - }; - match request { - ProjectionRequest::Exact { key } => allowed.contains(key), - ProjectionRequest::Batch { keys } => keys.iter().all(|key| allowed.contains(key)), - ProjectionRequest::ByParent { parent_key } => allowed.contains(parent_key), - } -} - fn check_result(result: &ProjectionResult, types: &FieldTypes) -> Result<(), ExternalSourceError> { let check = |fields: &Map| { fields.iter().all(|(name, value)| { @@ -846,13 +800,11 @@ mod tests { BoundExternalSource { program, fields: fields.clone(), - allowed_selectors: None, mode: ExternalProjectionMode::Live, grant_valid_until: cutoff, binding_fingerprint: "binding".to_owned(), grant_fingerprint: "grant".to_owned(), cache_ttl_seconds: None, - retention: None, field_types: BTreeMap::from([( "family-name".to_owned(), (ExternalFieldType::String, false), @@ -881,4 +833,49 @@ mod tests { assert!(matches!(result, Err(ExternalSourceError::Grant))); transcript.finish().unwrap(); } + + #[test] + fn a_deployment_cache_lifetime_only_shortens_the_authored_one() { + let (live, _) = project_contracts(); + assert!(matches!(effective_ttl(&live, None), Ok(None))); + assert!(matches!( + effective_ttl(&live, Some(60)), + Err(ExternalSourceError::Configuration) + )); + let mut indexed = live; + indexed.mode = ExternalProjectionMode::Indexed; + indexed.cache_ttl_seconds = Some(300); + assert!(matches!(effective_ttl(&indexed, None), Ok(Some(300)))); + assert!(matches!(effective_ttl(&indexed, Some(60)), Ok(Some(60)))); + assert!(matches!(effective_ttl(&indexed, Some(900)), Ok(Some(300)))); + } + + #[test] + fn retired_retention_and_selector_members_are_refused() { + let binding = json!({ + "id":"holder","packFile":"/packs/pack.yaml", + "packDigest":"sha256:0000000000000000000000000000000000000000000000000000000000000000", + "connection":{"id":"source","endpoints":[{"endpoint":"synthetic-registry", + "baseUrl":"https://source.example.test", + "credential":{"type":"static-api-key","valueRef":"secret:file/source-key"}}]}, + "endpointPolicies":[{"endpoint":"synthetic-registry","baseUrl":"https://source.example.test", + "clientIdentity":"forbidden","authentication":{"type":"static-api-key"}}], + "allowedFields":["family-name"],"grantValidUntil":"2030-01-01T00:00:00Z", + "cacheTtlSeconds":60, + "acquisition":{"maximumDataOperations":8,"timeoutMilliseconds":8000,"maximumResponseBytes":262144} + }); + let parsed: ExternalSourceBinding = serde_json::from_value(binding.clone()).unwrap(); + assert_eq!(parsed.cache_ttl_seconds.map(|ttl| ttl.get()), Some(60)); + for (member, value) in [ + ("allowedSelectors", json!(["person-7"])), + ( + "retention", + json!({"maximumRetentionSeconds":300,"refreshIntervalSeconds":60}), + ), + ] { + let mut retired = binding.clone(); + retired[member] = value; + assert!(serde_json::from_value::(retired).is_err()); + } + } } diff --git a/crates/registry-breg/src/postgres/external_label_acquisition.rs b/crates/registry-breg/src/postgres/external_label_acquisition.rs index b93040b245..a019042114 100644 --- a/crates/registry-breg/src/postgres/external_label_acquisition.rs +++ b/crates/registry-breg/src/postgres/external_label_acquisition.rs @@ -173,7 +173,7 @@ impl LabelAcquisition { } } - /// Use with publish_for_roots: $6 is the label selector, $13 the admitted + /// Use with publish_for_roots: $6 is the label selector, $12 the admitted /// root IDs. Current parent generation and member association are checked /// in that same statement, without interpolating source keys into SQL. pub(crate) fn publication_eligibility(&self) -> &str { @@ -263,7 +263,7 @@ SELECT EXISTS(SELECT 1 FROM parent_state),COALESCE((SELECT jsonb_agg(jsonb_build const PUBLICATION_SQL: &str = r#" WITH parent_state AS MATERIALIZED (__STATE_SQL__) SELECT 1 FROM (__ELIGIBLE_SQL__) AS selected -WHERE selected.root_id=ANY($13::uuid[]) AND selected.selector=$6::text +WHERE selected.root_id=ANY($12::uuid[]) AND selected.selector=$6::text "#; #[cfg(test)] @@ -306,8 +306,7 @@ mod tests { #[test] fn separate_machine_grants_and_same_purpose_are_required() { - let scope = - ProjectionScope::new("parent", Uuid::new_v4(), "binding", "grant", 60, 120).unwrap(); + let scope = ProjectionScope::new("parent", Uuid::new_v4(), "binding", "grant", 60).unwrap(); let generation = ProjectionGeneration { epoch: scope.epoch, generation: 1, @@ -346,7 +345,7 @@ mod tests { ) .unwrap(); assert!(plan.publication_eligibility().contains("$6::text")); - assert!(plan.publication_eligibility().contains("$13::uuid[]")); + assert!(plan.publication_eligibility().contains("$12::uuid[]")); assert!(plan .publication_eligibility() .contains("complete-collection")); @@ -422,16 +421,10 @@ mod postgres_tests { .await .unwrap(); let parent = - ProjectionScope::new("parent", Uuid::new_v4(), "binding", "grant", 60, 120).unwrap(); - let label = ProjectionScope::new( - "label", - Uuid::new_v4(), - "label-binding", - "label-grant", - 60, - 120, - ) - .unwrap(); + ProjectionScope::new("parent", Uuid::new_v4(), "binding", "grant", 60).unwrap(); + let label = + ProjectionScope::new("label", Uuid::new_v4(), "label-binding", "label-grant", 60) + .unwrap(); configure_scope(&migration, &parent).await.unwrap(); configure_scope(&migration, &label).await.unwrap(); let entity = ®istry.entities()["local-record"]; @@ -464,7 +457,7 @@ mod postgres_tests { .unwrap() .get(0); let rows = json!([{"source_key":"a","fields":{"holder":"indirect-person"}},{"source_key":"b","fields":{"holder":null}},{"source_key":"c","fields":{}},{"source_key":"d","fields":{"holder":"second-person"}}]); - let answer:String=migration.query_one("SELECT registry_internal.publish_external_projection($1,$2,$3,$4,$5,$6,'members',990,1050,1110,'complete-collection',$7)",&[&parent.projection_id,&parent.epoch,&parent.binding_digest,&parent.grant_digest,&ticket,&subject,&rows]).await.unwrap().get(0); + let answer:String=migration.query_one("SELECT registry_internal.publish_external_projection($1,$2,$3,$4,$5,$6,'members',990,1050,'complete-collection',$7)",&[&parent.projection_id,&parent.epoch,&parent.binding_digest,&parent.grant_digest,&ticket,&subject,&rows]).await.unwrap().get(0); assert_eq!(answer, "published"); } migration.execute("UPDATE registry_internal.registry_external_projection_snapshots SET expires_at=999 WHERE projection_id='parent' AND subject_key='expired-parent'",&[]).await.unwrap(); @@ -530,7 +523,6 @@ mod postgres_tests { &"label", &999_i64, &1059_i64, - &1119_i64, &"one", &rows, &roots, @@ -554,7 +546,6 @@ mod postgres_tests { &"label", &999_i64, &1059_i64, - &1119_i64, &"one", &rows, &roots, diff --git a/crates/registry-breg/src/postgres/external_projection.rs b/crates/registry-breg/src/postgres/external_projection.rs index 75c2965835..babfd37bf4 100644 --- a/crates/registry-breg/src/postgres/external_projection.rs +++ b/crates/registry-breg/src/postgres/external_projection.rs @@ -12,8 +12,9 @@ use uuid::Uuid; use super::{ExpectedRegistryIdentity, GuardedTransaction, RuntimeRevoke, SqlIdentifier}; -pub(crate) const BIND_ONCE_FUNCTION: &str = "registry_internal.bind_external_projection_once(text,uuid,text,text,bigint,bigint,text,text,uuid)"; -pub(crate) const MAINTENANCE_FUNCTION:&str="registry_internal.maintain_external_projection_scope(text,text,uuid,text,uuid,text,text,bigint,bigint)"; +pub(crate) const BIND_ONCE_FUNCTION: &str = + "registry_internal.bind_external_projection_once(text,uuid,text,text,bigint,text,text,uuid)"; +pub(crate) const MAINTENANCE_FUNCTION:&str="registry_internal.maintain_external_projection_scope(text,text,uuid,text,uuid,text,text,bigint)"; /// Withdrawal stays with migration authority: the runtime role holds no /// EXECUTE grant on it, so a compromised runtime cannot purge a scope. pub(crate) const WITHDRAW_FUNCTION: &str = @@ -23,13 +24,13 @@ pub(crate) const WITHDRAW_FUNCTION: &str = /// Entries are (identity, result type, returns set, security definer, runtime execute). pub(crate) const MANAGED_FUNCTIONS: &[(&str,&str,bool,bool,bool)] = &[ ("registry_internal.allocate_external_projection_refresh(p_projection text, p_epoch uuid, p_binding text, p_grant text, p_subject text)","bigint",false,true,true), - ("registry_internal.publish_external_projection(p_projection text, p_epoch uuid, p_binding text, p_grant text, p_sequence bigint, p_subject text, p_snapshot text, p_observed bigint, p_expires bigint, p_purge bigint, p_kind text, p_rows jsonb)","text",false,true,true), + ("registry_internal.publish_external_projection(p_projection text, p_epoch uuid, p_binding text, p_grant text, p_sequence bigint, p_subject text, p_snapshot text, p_observed bigint, p_expires bigint, p_kind text, p_rows jsonb)","text",false,true,true), ("registry_internal.fail_external_projection_refresh(p_projection text, p_epoch uuid, p_sequence bigint, p_subject text, p_reason text)","boolean",false,true,true), ("registry_internal.withdraw_external_projection(p_projection text, p_epoch uuid, p_binding text, p_grant text)","boolean",false,true,false), ("registry_internal.purge_external_projections(p_limit integer)","record",true,true,true), ("registry_internal.lock_external_projection_generation(p_projection text, p_epoch uuid, p_binding text, p_grant text)","bigint",false,true,true), - ("registry_internal.bind_external_projection_once(p_projection text, p_epoch uuid, p_binding text, p_grant text, p_ttl bigint, p_retention bigint, p_contract text, p_package text, p_activation uuid)","uuid",false,true,true), - ("registry_internal.maintain_external_projection_scope(p_action text, p_projection text, p_epoch uuid, p_package text, p_activation uuid, p_binding text, p_grant text, p_ttl bigint, p_retention bigint)","jsonb",false,false,false), + ("registry_internal.bind_external_projection_once(p_projection text, p_epoch uuid, p_binding text, p_grant text, p_ttl bigint, p_contract text, p_package text, p_activation uuid)","uuid",false,true,true), + ("registry_internal.maintain_external_projection_scope(p_action text, p_projection text, p_epoch uuid, p_package text, p_activation uuid, p_binding text, p_grant text, p_ttl bigint)","jsonb",false,false,false), ]; pub(crate) const INDEXES: &[&str] = &[ @@ -187,14 +188,13 @@ pub(crate) async fn bind_initial_scope( let row = transaction .transaction() .query_one( - "SELECT registry_internal.bind_external_projection_once($1,$2,$3,$4,$5,$6,$7,$8,$9)", + "SELECT registry_internal.bind_external_projection_once($1,$2,$3,$4,$5,$6,$7,$8)", &[ &requested.projection_id, &requested.epoch, &requested.binding_digest, &requested.grant_digest, &requested.cache_ttl_seconds, - &requested.retention_seconds, &contract, &expected.package_digest, &activation, @@ -232,7 +232,7 @@ pub(crate) const VALUES_TABLE: &str = BACKUP_EXCLUDED_TABLES[3]; pub(crate) const RUNTIME_FUNCTIONS: &[&str] = &[ "registry_internal.allocate_external_projection_refresh(text,uuid,text,text,text)", - "registry_internal.publish_external_projection(text,uuid,text,text,bigint,text,text,bigint,bigint,bigint,text,jsonb)", + "registry_internal.publish_external_projection(text,uuid,text,text,bigint,text,text,bigint,bigint,text,jsonb)", "registry_internal.fail_external_projection_refresh(text,uuid,bigint,text,text)", "registry_internal.purge_external_projections(integer)", "registry_internal.lock_external_projection_generation(text,uuid,text,text)", @@ -267,16 +267,16 @@ pub(crate) fn kernel_error(error: ProjectionError) -> super::PostgresKernelError } /// Current deployment authority. An epoch is never reused after withdrawal or -/// a changed source grant, binding, acquisition scope, or retention policy. +/// a changed source grant or binding. #[derive(Clone)] pub(crate) struct ProjectionScope { pub(crate) projection_id: String, pub(crate) epoch: Uuid, pub(crate) binding_digest: String, pub(crate) grant_digest: String, + /// The one cache lifetime from snapshot time. A value is purged when it + /// expires; there is no retention window after expiry. pub(crate) cache_ttl_seconds: i64, - /// Maximum retention from snapshot time, not extra time after expiry. - pub(crate) retention_seconds: i64, } impl fmt::Debug for ProjectionScope { @@ -292,14 +292,11 @@ impl ProjectionScope { binding_digest: &str, grant_digest: &str, cache_ttl_seconds: i64, - retention_seconds: i64, ) -> Result { if !valid_key(projection_id) || !valid_key(binding_digest) || !valid_key(grant_digest) - || cache_ttl_seconds < 1 - || retention_seconds < cache_ttl_seconds - || retention_seconds > 31_536_000 + || !(1..=31_536_000).contains(&cache_ttl_seconds) { return Err(ProjectionError::InvalidScope); } @@ -309,7 +306,6 @@ impl ProjectionScope { binding_digest: binding_digest.to_owned(), grant_digest: grant_digest.to_owned(), cache_ttl_seconds, - retention_seconds, }) } } @@ -373,7 +369,6 @@ pub(crate) struct AcceptedSnapshot { snapshot_id: String, observed_at: i64, source_expires_at: i64, - retain_until: i64, kind: SnapshotKind, rows: Vec, } @@ -458,6 +453,8 @@ impl AcceptedSnapshot { kind: SnapshotKind, mut rows: Vec, ) -> Result { + // The source must permit storage until its own expiry. The cache + // never keeps a value past that expiry, so no later deadline is held. let RetentionPermission::PermittedUntil(retain_until) = retention else { return Err(ProjectionError::RetentionRefused); }; @@ -491,24 +488,18 @@ impl AcceptedSnapshot { snapshot_id: snapshot_id.to_owned(), observed_at, source_expires_at, - retain_until, kind, rows, }) } - fn bounds(&self, scope: &ProjectionScope) -> Result<(i64, i64), ProjectionError> { - let expires_at = self + /// Expiry is also the purge deadline. + fn expiry(&self, scope: &ProjectionScope) -> Result { + Ok(self .observed_at .checked_add(scope.cache_ttl_seconds) .ok_or(ProjectionError::InvalidSnapshot)? - .min(self.source_expires_at); - let purge_at = self - .observed_at - .checked_add(scope.retention_seconds) - .ok_or(ProjectionError::InvalidSnapshot)? - .min(self.retain_until); - Ok((expires_at, purge_at)) + .min(self.source_expires_at)) } } @@ -623,20 +614,19 @@ pub(crate) async fn configure_scope( ) -> Result<(), ProjectionError> { let changed = migration.execute( "INSERT INTO registry_internal.registry_external_projection_scopes - (projection_id,epoch,binding_digest,grant_digest,cache_ttl_seconds,retention_seconds) - VALUES ($1,$2,$3,$4,$5,$6) + (projection_id,epoch,binding_digest,grant_digest,cache_ttl_seconds) + VALUES ($1,$2,$3,$4,$5) ON CONFLICT (projection_id) DO UPDATE SET epoch=EXCLUDED.epoch,binding_digest=EXCLUDED.binding_digest,grant_digest=EXCLUDED.grant_digest, - cache_ttl_seconds=EXCLUDED.cache_ttl_seconds,retention_seconds=EXCLUDED.retention_seconds, + cache_ttl_seconds=EXCLUDED.cache_ttl_seconds, active=true,generation=registry_internal.registry_external_projection_scopes.generation + CASE WHEN registry_internal.registry_external_projection_scopes.epoch=EXCLUDED.epoch THEN 0 ELSE 1 END WHERE registry_internal.registry_external_projection_scopes.epoch <> EXCLUDED.epoch OR (registry_internal.registry_external_projection_scopes.active AND registry_internal.registry_external_projection_scopes.binding_digest=EXCLUDED.binding_digest AND registry_internal.registry_external_projection_scopes.grant_digest=EXCLUDED.grant_digest - AND registry_internal.registry_external_projection_scopes.cache_ttl_seconds=EXCLUDED.cache_ttl_seconds - AND registry_internal.registry_external_projection_scopes.retention_seconds=EXCLUDED.retention_seconds)", - &[&scope.projection_id,&scope.epoch,&scope.binding_digest,&scope.grant_digest,&scope.cache_ttl_seconds,&scope.retention_seconds], + AND registry_internal.registry_external_projection_scopes.cache_ttl_seconds=EXCLUDED.cache_ttl_seconds)", + &[&scope.projection_id,&scope.epoch,&scope.binding_digest,&scope.grant_digest,&scope.cache_ttl_seconds], ).await.map_err(|_| ProjectionError::Unavailable)?; if changed != 1 { return Err(ProjectionError::ScopeChanged); @@ -663,7 +653,7 @@ pub(crate) async fn load_active_scope( let row = transaction .transaction() .query_opt( - "SELECT epoch,cache_ttl_seconds,retention_seconds + "SELECT epoch,cache_ttl_seconds FROM registry_internal.registry_external_projection_scopes WHERE projection_id=$1 AND binding_digest=$2 AND grant_digest=$3 AND active", &[&projection_id, &binding_digest, &grant_digest], @@ -677,7 +667,6 @@ pub(crate) async fn load_active_scope( binding_digest, grant_digest, row.get(1), - row.get(2), ) } @@ -786,7 +775,7 @@ async fn allocate_inner( } /// Evaluate only a trusted compiled root-read query in the publication statement. -/// Its parameters are $13::uuid[] for local roots and $6::text for the source key. +/// Its parameters are $12::uuid[] for local roots and $6::text for the source key. /// Ineligibility does not invalidate values shared by other authorized roots. pub(crate) async fn publish_for_roots( transaction: &GuardedTransaction<'_>, @@ -799,7 +788,7 @@ pub(crate) async fn publish_for_roots( return Err(ProjectionError::InvalidSnapshot); } let scope = &ticket.scope; - let (expires_at, purge_at) = snapshot.bounds(scope)?; + let expires_at = snapshot.expiry(scope)?; let rows = serde_json::to_value(&snapshot.rows).map_err(|_| ProjectionError::InvalidSnapshot)?; let kind = snapshot.kind.as_str(); @@ -818,7 +807,6 @@ pub(crate) async fn publish_for_roots( &snapshot.snapshot_id, &snapshot.observed_at, &expires_at, - &purge_at, &kind, &rows, &root_ids, @@ -836,7 +824,7 @@ pub(crate) async fn publish_for_roots( } pub(super) fn publication_statement(eligibility_sql: &str) -> String { - format!("SELECT CASE WHEN EXISTS ({eligibility_sql}) THEN registry_internal.publish_external_projection($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12) ELSE 'ineligible' END") + format!("SELECT CASE WHEN EXISTS ({eligibility_sql}) THEN registry_internal.publish_external_projection($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) ELSE 'ineligible' END") } #[cfg(all(test, feature = "postgres-test"))] @@ -849,11 +837,11 @@ async fn publish_inner( return Err(ProjectionError::InvalidSnapshot); } let scope = &ticket.scope; - let (expires_at, purge_at) = snapshot.bounds(scope)?; + let expires_at = snapshot.expiry(scope)?; let rows = serde_json::to_value(&snapshot.rows).map_err(|_| ProjectionError::InvalidSnapshot)?; let kind = snapshot.kind.as_str(); - let row = client.query_one("SELECT registry_internal.publish_external_projection($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)", &[&scope.projection_id,&scope.epoch,&scope.binding_digest,&scope.grant_digest,&ticket.sequence,&ticket.subject_key,&snapshot.snapshot_id,&snapshot.observed_at,&expires_at,&purge_at,&kind,&rows]).await.map_err(|_| ProjectionError::Unavailable)?; + let row = client.query_one("SELECT registry_internal.publish_external_projection($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)", &[&scope.projection_id,&scope.epoch,&scope.binding_digest,&scope.grant_digest,&ticket.sequence,&ticket.subject_key,&snapshot.snapshot_id,&snapshot.observed_at,&expires_at,&kind,&rows]).await.map_err(|_| ProjectionError::Unavailable)?; match row.get::<_, String>(0).as_str() { "published" => Ok(Publication::Published), "replay" => Ok(Publication::Replay), @@ -975,7 +963,6 @@ CREATE TABLE IF NOT EXISTS registry_internal.registry_external_projection_scopes binding_digest text NOT NULL, grant_digest text NOT NULL, cache_ttl_seconds bigint NOT NULL CHECK (cache_ttl_seconds BETWEEN 1 AND 31536000), - retention_seconds bigint NOT NULL CHECK (retention_seconds BETWEEN cache_ttl_seconds AND 31536000), active boolean NOT NULL DEFAULT true, next_sequence bigint NOT NULL DEFAULT 0 CHECK (next_sequence >= 0), generation bigint NOT NULL DEFAULT 0 CHECK (generation >= 0) @@ -997,7 +984,6 @@ CREATE TABLE IF NOT EXISTS registry_internal.registry_external_projection_snapsh snapshot_id text NOT NULL CHECK (octet_length(snapshot_id) BETWEEN 1 AND 1024), observed_at bigint NOT NULL CHECK (observed_at >= 0), expires_at bigint NOT NULL CHECK (expires_at > observed_at), - purge_at bigint NOT NULL CHECK (purge_at >= expires_at), refresh_sequence bigint NOT NULL CHECK (refresh_sequence > 0), kind text NOT NULL CHECK (kind IN ('one','absent','complete-collection')), PRIMARY KEY (projection_id,epoch,subject_key) @@ -1047,7 +1033,7 @@ END $body$; CREATE OR REPLACE FUNCTION registry_internal.publish_external_projection( p_projection text,p_epoch uuid,p_binding text,p_grant text,p_sequence bigint,p_subject text, - p_snapshot text,p_observed bigint,p_expires bigint,p_purge bigint,p_kind text,p_rows jsonb + p_snapshot text,p_observed bigint,p_expires bigint,p_kind text,p_rows jsonb ) RETURNS text LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, registry_internal AS $body$ DECLARE scope registry_internal.registry_external_projection_scopes; @@ -1072,7 +1058,6 @@ BEGIN OR p_observed IS NULL OR p_observed<0 OR p_observed>projection_now OR p_expires IS NULL OR p_expires<=projection_now OR p_expires<=p_observed OR p_expires-p_observed>scope.cache_ttl_seconds - OR p_purge IS NULL OR p_purgescope.retention_seconds OR p_kind IS NULL OR p_kind NOT IN ('one','absent','complete-collection') OR p_rows IS NULL OR jsonb_typeof(p_rows)<>'array' OR octet_length(p_rows::text)>2097152 THEN RETURN 'invalid'; END IF; @@ -1106,11 +1091,11 @@ BEGIN IF attempt.state<>'allocated' OR (has_previous AND (previous.refresh_sequence>=p_sequence OR previous.observed_at>p_observed)) THEN RETURN 'stale'; END IF; INSERT INTO registry_internal.registry_external_projection_snapshots - (projection_id,epoch,subject_key,snapshot_id,observed_at,expires_at,purge_at,refresh_sequence,kind) - VALUES (p_projection,p_epoch,p_subject,p_snapshot,p_observed,p_expires,p_purge,p_sequence,p_kind) + (projection_id,epoch,subject_key,snapshot_id,observed_at,expires_at,refresh_sequence,kind) + VALUES (p_projection,p_epoch,p_subject,p_snapshot,p_observed,p_expires,p_sequence,p_kind) ON CONFLICT (projection_id,epoch,subject_key) DO UPDATE SET snapshot_id=EXCLUDED.snapshot_id,observed_at=EXCLUDED.observed_at, - expires_at=EXCLUDED.expires_at,purge_at=EXCLUDED.purge_at, + expires_at=EXCLUDED.expires_at, refresh_sequence=EXCLUDED.refresh_sequence,kind=EXCLUDED.kind; DELETE FROM registry_internal.registry_external_projection_values WHERE projection_id=p_projection AND epoch=p_epoch AND subject_key=p_subject; @@ -1219,7 +1204,7 @@ END $body$; const MAINTENANCE_SQL: &str = r#" CREATE OR REPLACE FUNCTION registry_internal.maintain_external_projection_scope( p_action text,p_projection text,p_epoch uuid,p_package text,p_activation uuid, - p_binding text,p_grant text,p_ttl bigint,p_retention bigint + p_binding text,p_grant text,p_ttl bigint ) RETURNS jsonb LANGUAGE plpgsql SECURITY INVOKER SET search_path = pg_catalog, registry_internal AS $body$ DECLARE allowed jsonb := __COMPILED_EXTERNAL_ALLOWLIST__; @@ -1259,16 +1244,15 @@ BEGIN ELSE IF p_binding IS NULL OR octet_length(p_binding) NOT BETWEEN 1 AND 1024 OR p_grant IS NULL OR octet_length(p_grant) NOT BETWEEN 1 AND 1024 - OR p_ttl IS NULL OR p_ttl NOT BETWEEN 1 AND (expected->>'ttl')::bigint - OR p_retention IS NULL OR p_retention NOT BETWEEN p_ttl AND 31536000 THEN + OR p_ttl IS NULL OR p_ttl NOT BETWEEN 1 AND (expected->>'ttl')::bigint THEN RAISE EXCEPTION USING ERRCODE='22023', MESSAGE='external projection maintenance binding is invalid'; END IF; INSERT INTO registry_internal.registry_external_projection_scopes AS current - (projection_id,contract_digest,epoch,binding_digest,grant_digest,cache_ttl_seconds,retention_seconds) - VALUES (p_projection,expected->>'contract',gen_random_uuid(),p_binding,p_grant,p_ttl,p_retention) + (projection_id,contract_digest,epoch,binding_digest,grant_digest,cache_ttl_seconds) + VALUES (p_projection,expected->>'contract',gen_random_uuid(),p_binding,p_grant,p_ttl) ON CONFLICT (projection_id) DO UPDATE SET contract_digest=EXCLUDED.contract_digest, epoch=EXCLUDED.epoch,binding_digest=EXCLUDED.binding_digest,grant_digest=EXCLUDED.grant_digest, - cache_ttl_seconds=EXCLUDED.cache_ttl_seconds,retention_seconds=EXCLUDED.retention_seconds, + cache_ttl_seconds=EXCLUDED.cache_ttl_seconds, active=true,generation=current.generation+1 RETURNING * INTO scope; END IF; @@ -1291,13 +1275,13 @@ BEGIN END IF; RETURN jsonb_build_object('projection',p_projection,'epoch',scope.epoch,'active',COALESCE(scope.active,false), 'bindingDigest',scope.binding_digest,'grantDigest',scope.grant_digest,'cacheTtlSeconds',scope.cache_ttl_seconds, - 'retentionSeconds',scope.retention_seconds,'packageDigest',package_digest,'activationId',activation_id) || counters; + 'packageDigest',package_digest,'activationId',activation_id) || counters; END $body$; "#; const BIND_ONCE_SQL: &str = r#" CREATE OR REPLACE FUNCTION registry_internal.bind_external_projection_once( - p_projection text,p_epoch uuid,p_binding text,p_grant text,p_ttl bigint,p_retention bigint, + p_projection text,p_epoch uuid,p_binding text,p_grant text,p_ttl bigint, p_contract text,p_package text,p_activation uuid ) RETURNS uuid LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, registry_internal AS $body$ @@ -1310,21 +1294,20 @@ BEGIN OR p_binding IS NULL OR octet_length(p_binding) NOT BETWEEN 1 AND 1024 OR p_grant IS NULL OR octet_length(p_grant) NOT BETWEEN 1 AND 1024 OR p_ttl IS NULL OR p_ttl NOT BETWEEN 1 AND (expected->>'ttl')::bigint - OR p_retention IS NULL OR p_retention NOT BETWEEN p_ttl AND 31536000 OR p_package IS NULL OR p_activation IS NULL THEN RETURN NULL; END IF; IF NOT EXISTS (SELECT 1 FROM registry_internal.registry_state WHERE singleton AND active_package_digest=p_package AND active_activation_id=p_activation AND maintenance_status='ready') THEN RETURN NULL; END IF; INSERT INTO registry_internal.registry_external_projection_scopes - (projection_id,contract_digest,epoch,binding_digest,grant_digest,cache_ttl_seconds,retention_seconds) - VALUES (p_projection,p_contract,p_epoch,p_binding,p_grant,p_ttl,p_retention) + (projection_id,contract_digest,epoch,binding_digest,grant_digest,cache_ttl_seconds) + VALUES (p_projection,p_contract,p_epoch,p_binding,p_grant,p_ttl) ON CONFLICT (projection_id) DO NOTHING; SELECT * INTO scope FROM registry_internal.registry_external_projection_scopes WHERE projection_id=p_projection FOR UPDATE; IF scope.active AND scope.contract_digest=p_contract AND scope.binding_digest=p_binding AND scope.grant_digest=p_grant - AND scope.cache_ttl_seconds=p_ttl AND scope.retention_seconds=p_retention THEN RETURN scope.epoch; END IF; + AND scope.cache_ttl_seconds=p_ttl THEN RETURN scope.epoch; END IF; IF NOT scope.active AND scope.epoch='00000000-0000-0000-0000-000000000000'::uuid AND scope.binding_digest='' AND scope.grant_digest='' AND scope.contract_digest=p_contract THEN UPDATE registry_internal.registry_external_projection_scopes - SET epoch=p_epoch,binding_digest=p_binding,grant_digest=p_grant,cache_ttl_seconds=p_ttl,retention_seconds=p_retention,active=true,generation=generation+1 + SET epoch=p_epoch,binding_digest=p_binding,grant_digest=p_grant,cache_ttl_seconds=p_ttl,active=true,generation=generation+1 WHERE projection_id=p_projection; RETURN p_epoch; END IF; @@ -1338,15 +1321,7 @@ mod tests { use serde_json::json; fn scope() -> ProjectionScope { - ProjectionScope::new( - "people", - Uuid::nil(), - "sha256:binding", - "sha256:grant", - 60, - 120, - ) - .unwrap() + ProjectionScope::new("people", Uuid::nil(), "sha256:binding", "sha256:grant", 60).unwrap() } fn row(key: &str) -> ProjectionRow { @@ -1367,7 +1342,10 @@ mod tests { .unwrap_err(), ProjectionError::RetentionRefused ); - assert!(ProjectionScope::new("people", Uuid::nil(), "binding", "grant", 60, 59).is_err()); + assert!(ProjectionScope::new("people", Uuid::nil(), "binding", "grant", 0).is_err()); + assert!( + ProjectionScope::new("people", Uuid::nil(), "binding", "grant", 31_536_001).is_err() + ); let snapshot = AcceptedSnapshot::one( "key", "snapshot", @@ -1424,7 +1402,7 @@ mod tests { } #[test] - fn original_snapshot_bounds_expiry_and_purge() { + fn one_cache_lifetime_bounds_expiry_and_purge() { let accepted = AcceptedSnapshot::one( "key", "snapshot", @@ -1434,7 +1412,7 @@ mod tests { row("key"), ) .unwrap(); - assert_eq!(accepted.bounds(&scope()).unwrap(), (160, 220)); + assert_eq!(accepted.expiry(&scope()).unwrap(), 160); let accepted = AcceptedSnapshot::one( "key", "snapshot", @@ -1444,7 +1422,7 @@ mod tests { row("key"), ) .unwrap(); - assert_eq!(accepted.bounds(&scope()).unwrap(), (150, 180)); + assert_eq!(accepted.expiry(&scope()).unwrap(), 150); } #[test] @@ -1497,7 +1475,7 @@ mod postgres_tests { } fn scope(epoch: Uuid) -> ProjectionScope { - ProjectionScope::new("projection", epoch, "binding", "grant", 60, 120).unwrap() + ProjectionScope::new("projection", epoch, "binding", "grant", 60).unwrap() } fn record(key: &str, value: &str) -> ProjectionRow { @@ -1625,7 +1603,7 @@ mod postgres_tests { record("subject", "fresh"), ) .unwrap(); - let eligibility = "SELECT 1 WHERE $6::text='subject' AND cardinality($13::uuid[])>0"; + let eligibility = "SELECT 1 WHERE $6::text='subject' AND cardinality($12::uuid[])>0"; assert_eq!( publish_for_roots(&transaction, &ticket, &accepted, eligibility, &[]) .await @@ -1778,7 +1756,8 @@ mod postgres_tests { ) .await .unwrap(); - let maintenance="SELECT registry_internal.maintain_external_projection_scope($1,$2,$3,$4,$5,$6,$7,$8,$9)"; + let maintenance = + "SELECT registry_internal.maintain_external_projection_scope($1,$2,$3,$4,$5,$6,$7,$8)"; let activation = Uuid::parse_str(&expected.activation_id).unwrap(); let package = expected.package_digest.as_str(); let runtime = pool.get().await.unwrap(); @@ -1794,7 +1773,6 @@ mod postgres_tests { &"renewed-binding", &"renewed-grant", &20_i64, - &120_i64 ] ) .await @@ -1812,7 +1790,6 @@ mod postgres_tests { &"renewed-binding", &"renewed-grant", &20_i64, - &120_i64, ], ) .await @@ -1821,6 +1798,7 @@ mod postgres_tests { let new_epoch = Uuid::parse_str(rotated["epoch"].as_str().unwrap()).unwrap(); assert_ne!(bound.epoch, new_epoch); assert_eq!(rotated["active"], true); + assert!(rotated.get("retentionSeconds").is_none()); let inspected: Value = migration .query_one( maintenance, @@ -1833,7 +1811,6 @@ mod postgres_tests { &"", &"", &20_i64, - &120_i64, ], ) .await @@ -1861,7 +1838,6 @@ mod postgres_tests { &"", &"", &20_i64, - &120_i64 ] ) .await @@ -1878,7 +1854,6 @@ mod postgres_tests { &"", &"", &20_i64, - &120_i64, ], ) .await @@ -2216,7 +2191,7 @@ mod postgres_tests { let visible: i64 = migration.query_one("SELECT count(*) FROM registry_internal.registry_external_projection_snapshots WHERE expires_at>1050", &[]).await.unwrap().get(0); assert_eq!(visible, 0); assert!(publish_inner(&migration, &ticket, &observed).await.is_err()); - // Purge at freshness expiry leaves retention headroom. + // Purge removes a value once its one cache lifetime expires. let pruned = migration .query_one( "SELECT * FROM registry_internal.purge_external_projections(100)", diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index 9b520cf000..75422d6479 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -196,16 +196,12 @@ impl PostgresExternalViewService { .await .map_err(|_| ExternalViewError::Unavailable)?; for (projection, bound, ttl) in configured { - let retention = bound - .maximum_retention_seconds() - .ok_or(ExternalViewError::Unavailable)?; let scope = ProjectionScope::new( projection.id.as_str(), Uuid::new_v4(), bound.binding_fingerprint(), bound.grant_fingerprint(), i64::from(ttl), - i64::from(retention), ) .map_err(|_| ExternalViewError::Unavailable)?; store::bind_initial_scope(&transaction, &self.registry, &self.expected, &scope) @@ -1106,7 +1102,7 @@ impl RootKeys { relations.id_expression, key.sql, relations.from_sql ), publication_eligibility: format!( - "SELECT 1 FROM {} WHERE {} = ANY($13::uuid[]) AND ({})::text = $6::text", + "SELECT 1 FROM {} WHERE {} = ANY($12::uuid[]) AND ({})::text = $6::text", relations.from_sql, relations.id_expression, key.sql, ), id: relations.id_expression, @@ -1174,9 +1170,7 @@ async fn active_scope( ) .await .map_err(|_| ExternalViewError::Unavailable)?; - if Some(scope.cache_ttl_seconds) != source.effective_cache_ttl_seconds().map(i64::from) - || Some(scope.retention_seconds) != source.maximum_retention_seconds().map(i64::from) - { + if Some(scope.cache_ttl_seconds) != source.effective_cache_ttl_seconds().map(i64::from) { return Err(ExternalViewError::Unavailable); } Ok(scope) diff --git a/crates/registry-breg/src/postgres/external_view_query.rs b/crates/registry-breg/src/postgres/external_view_query.rs index ff03b48bb9..20acc5f424 100644 --- a/crates/registry-breg/src/postgres/external_view_query.rs +++ b/crates/registry-breg/src/postgres/external_view_query.rs @@ -700,8 +700,8 @@ pub(crate) mod tests { #[test] fn query_kernel_refuses_hidden_fields_in_selection_filter_and_order() { let registry = registry(); - let scope = ProjectionScope::new("projection", Uuid::new_v4(), "binding", "grant", 60, 120) - .unwrap(); + let scope = + ProjectionScope::new("projection", Uuid::new_v4(), "binding", "grant", 60).unwrap(); let mut query = request(&scope, 0); query.fields = BTreeSet::from(["hidden".to_owned()]); assert!(matches!( @@ -853,7 +853,7 @@ mod postgres_tests { let rows = label .map(|label| serde_json::json!([{"source_key":key,"fields":{"label":label}}])) .unwrap_or_else(|| serde_json::json!([])); - let result:String=client.query_one("SELECT registry_internal.publish_external_projection($1,$2,$3,$4,$5,$6,$7,999,1059,1119,$8,$9)",&[&scope.projection_id,&scope.epoch,&scope.binding_digest,&scope.grant_digest,&sequence,&key,&id,&kind,&rows]).await.unwrap().get(0); + let result:String=client.query_one("SELECT registry_internal.publish_external_projection($1,$2,$3,$4,$5,$6,$7,999,1059,$8,$9)",&[&scope.projection_id,&scope.epoch,&scope.binding_digest,&scope.grant_digest,&sequence,&key,&id,&kind,&rows]).await.unwrap().get(0); assert_eq!(result, "published"); } @@ -873,8 +873,8 @@ mod postgres_tests { )) .await .unwrap(); - let scope = ProjectionScope::new("projection", Uuid::new_v4(), "binding", "grant", 60, 120) - .unwrap(); + let scope = + ProjectionScope::new("projection", Uuid::new_v4(), "binding", "grant", 60).unwrap(); configure_scope(&migration, &scope).await.unwrap(); let entity = ®istry.entities()["local-record"]; let key = &entity @@ -1034,7 +1034,7 @@ mod postgres_tests { // real generated source relation, including its forced root RLS. let relations = ReadRelations::statistics(entity, &["subject-key".to_owned()]).unwrap(); let subject = relations.field_expression(entity, "subject-key").unwrap(); - let eligibility=format!("SELECT 1 FROM {} WHERE {}=ANY($13::uuid[]) AND ({})::text=$6::text AND current_setting('registry.access_profile',true)='reader'",relations.from_sql,relations.id_expression,subject.sql); + let eligibility=format!("SELECT 1 FROM {} WHERE {}=ANY($12::uuid[]) AND ({})::text=$6::text AND current_setting('registry.access_profile',true)='reader'",relations.from_sql,relations.id_expression,subject.sql); let sql = super::super::external_projection::publication_statement(&eligibility); let ticket:i64=runtime.query_one("SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,'first')",&[&scope.projection_id,&scope.epoch,&scope.binding_digest,&scope.grant_digest]).await.unwrap().get(0); let fields = serde_json::json!([{"source_key":"first","fields":{"label":"retained"}}]); @@ -1052,7 +1052,6 @@ mod postgres_tests { &"association", &1000_i64, &1060_i64, - &1120_i64, &"one", &fields, &roots, @@ -1082,7 +1081,6 @@ mod postgres_tests { &"rejected", &1000_i64, &1060_i64, - &1120_i64, &"one", &fields, &roots, diff --git a/crates/registry-breg/src/runtime_config.rs b/crates/registry-breg/src/runtime_config.rs index d6aed48495..a7cb220b96 100644 --- a/crates/registry-breg/src/runtime_config.rs +++ b/crates/registry-breg/src/runtime_config.rs @@ -497,7 +497,7 @@ impl RuntimeConfigError { "Configure at most 32 distinct taskGrantStatus source issuers, safe service/token URLs, a client and assertion audience, absolute resource identifiers, one statusScope token, and a statusPath that starts with / and holds exactly one {grantId} segment." } Self::InvalidExternalSources => { - "Use distinct projection IDs, a bounded absolute packFile, approved fields and an explicit grant cutoff. Set a refresh interval shorter than the permitted retention window." + "Use distinct projection IDs, a bounded absolute packFile, approved fields and an explicit grant cutoff. A deployment cacheTtlSeconds may only shorten an indexed projection's cache lifetime." } Self::InvalidBinding => { "Give every identity value as trimmed text without control characters, and declare in the runtime file, with a valid endpoint and credential, every task grant status source, Evidence provider, and review authority the package relies on." diff --git a/crates/registry-bregctl/src/lib.rs b/crates/registry-bregctl/src/lib.rs index 92b163cf4a..839316f9bd 100644 --- a/crates/registry-bregctl/src/lib.rs +++ b/crates/registry-bregctl/src/lib.rs @@ -8627,8 +8627,6 @@ fn generate_external_source_facts(args: &GenerateArgs) -> Result&2 exit 2 } @@ -16,13 +16,13 @@ projection=$3 case "$action" in inspect) [[ $# -eq 3 ]] || usage ;; withdraw) [[ $# -eq 6 ]] || usage ;; -rotate) [[ $# -eq 10 ]] || usage ;; +rotate) [[ $# -eq 9 ]] || usage ;; *) usage ;; esac psql --no-psqlrc --dbname="service=$service" --set=ON_ERROR_STOP=1 \ --set=action="$action" --set=projection="$projection" --set=epoch="${4:-unbound}" \ --set=package="${5:-}" --set=activation="${6:-}" --set=binding="${7:-}" \ - --set=grant="${8:-}" --set=ttl="${9:-}" --set=retention="${10:-}" <<'SQL' + --set=grant="${8:-}" --set=ttl="${9:-}" <<'SQL' BEGIN; SET LOCAL lock_timeout='10s'; SET LOCAL statement_timeout='30s'; @@ -30,6 +30,6 @@ SELECT jsonb_pretty(registry_internal.maintain_external_projection_scope( :'action', :'projection', NULLIF(:'epoch','unbound')::uuid, NULLIF(:'package',''), NULLIF(:'activation','')::uuid, NULLIF(:'binding',''), NULLIF(:'grant',''), - NULLIF(:'ttl','')::bigint, NULLIF(:'retention','')::bigint)); + NULLIF(:'ttl','')::bigint)); COMMIT; SQL From f9fa7cd6ceaaa7e18131287de31b75f47d1c01cc Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 11:41:35 +0700 Subject: [PATCH 05/64] fix(breg): rebind external projections on authority changes only Scope fingerprints now cover only the reviewed source contract, its operation and the allowed fields, so a renewed grant cutoff, a CA change or a connection edit no longer discards the cache. Each projection binds on its own: a contract mismatch rotates to a fresh epoch, and a projection that still cannot bind disables only its views with an unavailable Problem that names it. Startup and doctor report unavailable projections with the reason. Deployment configuration is the only acquisition cutoff: withdrawal purges data and a binding still configured binds again at the next start. Signed-off-by: Jeremi Joslin --- .../registry-breg/src/api/external_views.rs | 70 ++- crates/registry-breg/src/external_source.rs | 424 ++++++++++++++---- .../src/postgres/external_projection.rs | 415 +++++++++++++++-- .../src/postgres/external_runtime.rs | 243 ++++++++-- .../src/postgres/external_view_query.rs | 41 +- crates/registry-breg/src/startup.rs | 39 +- crates/registry-breg/tests/startup_http.rs | 2 +- crates/registry-bregctl/src/doctor.rs | 2 +- crates/registry-bregctl/src/lib.rs | 72 ++- products/breg/external-projection-storage.md | 22 +- 10 files changed, 1135 insertions(+), 195 deletions(-) diff --git a/crates/registry-breg/src/api/external_views.rs b/crates/registry-breg/src/api/external_views.rs index 3b1073543b..35cdab042e 100644 --- a/crates/registry-breg/src/api/external_views.rs +++ b/crates/registry-breg/src/api/external_views.rs @@ -45,12 +45,15 @@ pub(crate) struct ExternalOrder { pub(crate) descending: bool, } -#[derive(Clone, Copy)] +#[derive(Clone)] pub(crate) enum ExternalViewError { Concealed, InvalidCursor, Incomplete, Unavailable, + /// This projection cannot serve: its binding was refused, its grant has + /// expired or its scope moved to another binding. Other views still serve. + ProjectionUnavailable(String), } #[derive(Clone, Copy, Eq, PartialEq)] @@ -339,14 +342,7 @@ async fn handle( }; let mut response = match result { Ok(value) => Json(value).into_response(), - Err(ExternalViewError::Concealed) => concealed(), - Err(ExternalViewError::InvalidCursor) => cursor_invalid(), - Err(ExternalViewError::Incomplete) => ( - StatusCode::SERVICE_UNAVAILABLE, - Json(json!({"status": "incomplete"})), - ) - .into_response(), - Err(ExternalViewError::Unavailable) => unavailable(), + Err(error) => external_view_problem(error), }; response .headers_mut() @@ -354,6 +350,36 @@ async fn handle( response } +fn external_view_problem(error: ExternalViewError) -> Response { + match error { + ExternalViewError::Concealed => concealed(), + ExternalViewError::InvalidCursor => cursor_invalid(), + ExternalViewError::Incomplete => ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({"status": "incomplete"})), + ) + .into_response(), + ExternalViewError::Unavailable => unavailable(), + ExternalViewError::ProjectionUnavailable(projection) => projection_unavailable(&projection), + } +} + +/// The backend reports this only after the caller's view and link grant was +/// admitted, so naming the compiled projection discloses no source value. +fn projection_unavailable(projection: &str) -> Response { + let code = ProblemCode::SourceUnavailable; + crate::correlation::problem_response_with_field_path( + StatusCode::from_u16(code.status()).expect("problem catalogue status is valid"), + code.title(), + code.description(), + code.code(), + format!( + "/externalProjections/{}", + projection.replace('~', "~0").replace('/', "~1") + ), + ) +} + fn admission_profile(raw: Option<&str>) -> Option { raw.filter(|value| value.len() <= MAX_RAW_QUERY_BYTES)? .split('&') @@ -477,3 +503,29 @@ fn literal_value(field_type: ExternalFieldType, literal: Literal) -> Option None, } } + +#[cfg(test)] +mod tests { + use super::*; + + async fn body(response: Response) -> Value { + serde_json::from_slice(&to_bytes(response.into_body(), 4096).await.unwrap()).unwrap() + } + + #[tokio::test] + async fn an_unavailable_projection_names_itself_and_other_failures_stay_generic() { + let named = external_view_problem(ExternalViewError::ProjectionUnavailable( + "person-indexed".to_owned(), + )); + assert_eq!(named.status(), StatusCode::SERVICE_UNAVAILABLE); + let named = body(named).await; + assert_eq!(named["code"], "source.unavailable"); + assert_eq!(named["fieldPath"], "/externalProjections/person-indexed"); + + let generic = external_view_problem(ExternalViewError::Unavailable); + assert_eq!(generic.status(), StatusCode::SERVICE_UNAVAILABLE); + let generic = body(generic).await; + assert_eq!(generic["code"], "source.unavailable"); + assert!(generic.get("fieldPath").is_none()); + } +} diff --git a/crates/registry-breg/src/external_source.rs b/crates/registry-breg/src/external_source.rs index 12b848882d..cb4488bdb7 100644 --- a/crates/registry-breg/src/external_source.rs +++ b/crates/registry-breg/src/external_source.rs @@ -99,6 +99,10 @@ pub enum ExternalSourceError { Contract, #[error("no external source is bound to this projection")] Unbound, + #[error("the active package declares no projection with this ID")] + UnknownProjection, + #[error("another binding describes the same connection differently")] + ConnectionConflict, #[error("the external source grant does not permit this acquisition")] Grant, #[error("the source result is incompatible with the approved BReg field types")] @@ -110,6 +114,7 @@ pub enum ExternalSourceError { #[derive(Default)] pub struct ExternalSourceService { bindings: BTreeMap, + unavailable: BTreeMap, } pub struct BoundExternalSource { @@ -156,106 +161,50 @@ impl BoundExternalSource { impl ExternalSourceService { /// Capture the pack and public TLS material without source calls or /// resolving service credentials. Missing bindings never get a fallback. + /// Only a malformed binding list refuses the whole load. A binding that + /// fails its own checks leaves that projection unavailable, with the + /// reason, while every other projection keeps its binding. pub fn load( compiled: &CompiledRegistry, bindings: &[ExternalSourceBinding], resolver: Arc, ) -> Result { validate_bindings(bindings)?; - let catalog = compiled.external_views(); let mut result = Self::default(); let mut connections = BTreeMap::new(); for binding in bindings { let id = binding.id.as_str(); - let projection = catalog - .projections - .get(id) - .ok_or(ExternalSourceError::Contract)?; - let record = catalog - .record_types - .get(projection.record_type.as_str()) - .ok_or(ExternalSourceError::Contract)?; - let (pack, artifacts) = load_pack(Path::new(&binding.pack_file), &binding.pack_digest)?; - let limits = EvidenceLimits::try_new( - binding.acquisition.maximum_data_operations.get(), - binding.acquisition.timeout_milliseconds.get(), - binding.acquisition.maximum_response_bytes.get() as usize, - ) - .map_err(|_| ExternalSourceError::Configuration)?; - let program = - compile_projection_pack(&pack, projection.operation.as_str(), &artifacts, limits) - .map_err(|_| ExternalSourceError::Contract)?; - check_pack_contract(projection, record, &program)?; - let (fields, field_types) = project_fields(projection, record, binding)?; - let cache_ttl_seconds = - effective_ttl(projection, binding.cache_ttl_seconds.map(|ttl| ttl.get()))?; - let grant_valid_until = parse_grant_cutoff(&binding.grant_valid_until)?; - - let ca_bundles = capture_ca_bundles(&binding.connection)?; - let ca_digests: BTreeMap<_, _> = ca_bundles - .iter() - .map(|(name, bytes)| (name, sha256_uri(bytes))) - .collect(); - let connection_fingerprint = fingerprint(&json!({ - "connection":binding.connection, - "endpointPolicies":binding.endpoint_policies, - "caDigests":ca_digests - }))?; - let connection_id = binding.connection.id.as_str(); - let connection = match connections.entry(connection_id) { - std::collections::btree_map::Entry::Vacant(entry) => { - &entry - .insert(( - connection_fingerprint.clone(), - EvidenceRuntimeBindings::from_shared_http( - binding.connection.clone(), - Arc::clone(&resolver), - ca_bundles, - ), - )) - .1 + match bind_one(compiled, binding, &mut connections, &resolver) { + Ok(bound) => { + result.bindings.insert(id.to_owned(), bound); } - std::collections::btree_map::Entry::Occupied(entry) => { - let (previous, runtime) = entry.into_mut(); - if previous.as_str() != connection_fingerprint.as_str() { - return Err(ExternalSourceError::Configuration); - } - runtime + Err(error) => { + result.unavailable.insert(id.to_owned(), error.to_string()); } - }; - let program = program - .bind_live(connection, connection_id, &binding.endpoint_policies) - .map_err(|_| ExternalSourceError::Contract)?; - let binding_fingerprint = fingerprint(&json!({ - "packDigest":binding.pack_digest, - "operation":projection.operation, - "connectionFingerprint":connection_fingerprint, - "acquisition":binding.acquisition - }))?; - let grant_fingerprint = fingerprint(&json!({ - "projection":projection, - "externalType":record, - "sourceGrant":program.contract(), - "allowedFields":fields, - "grantValidUntil":grant_valid_until.to_rfc3339() - }))?; - result.bindings.insert( - id.to_owned(), - BoundExternalSource { - program, - fields, - mode: projection.mode, - grant_valid_until, - binding_fingerprint, - grant_fingerprint, - cache_ttl_seconds, - field_types, - }, - ); + } + } + for id in compiled.external_views().projections.keys() { + if !result.bindings.contains_key(id) && !result.unavailable.contains_key(id) { + result + .unavailable + .insert(id.clone(), ExternalSourceError::Unbound.to_string()); + } } Ok(result) } + /// Every declared or configured projection without a usable binding, + /// with the reason. Views over these projections answer unavailable. + pub fn unavailable_projections(&self) -> &BTreeMap { + &self.unavailable + } + + /// Withdraw one projection's binding for this process, recording why. + pub(crate) fn refuse(&mut self, projection: &str, reason: String) { + self.bindings.remove(projection); + self.unavailable.insert(projection.to_owned(), reason); + } + pub fn binding(&self, projection: &str) -> Option<&BoundExternalSource> { self.bindings.get(projection) } @@ -301,6 +250,101 @@ impl ExternalSourceService { } } +type SharedConnections<'a> = BTreeMap<&'a str, (String, EvidenceRuntimeBindings)>; + +fn bind_one<'a>( + compiled: &CompiledRegistry, + binding: &'a ExternalSourceBinding, + connections: &mut SharedConnections<'a>, + resolver: &Arc, +) -> Result { + let catalog = compiled.external_views(); + let projection = catalog + .projections + .get(binding.id.as_str()) + .ok_or(ExternalSourceError::UnknownProjection)?; + let record = catalog + .record_types + .get(projection.record_type.as_str()) + .ok_or(ExternalSourceError::Contract)?; + let (pack, artifacts) = load_pack(Path::new(&binding.pack_file), &binding.pack_digest)?; + let limits = EvidenceLimits::try_new( + binding.acquisition.maximum_data_operations.get(), + binding.acquisition.timeout_milliseconds.get(), + binding.acquisition.maximum_response_bytes.get() as usize, + ) + .map_err(|_| ExternalSourceError::Configuration)?; + let program = compile_projection_pack(&pack, projection.operation.as_str(), &artifacts, limits) + .map_err(|_| ExternalSourceError::Contract)?; + check_pack_contract(projection, record, &program)?; + let (fields, field_types) = project_fields(projection, record, binding)?; + let cache_ttl_seconds = + effective_ttl(projection, binding.cache_ttl_seconds.map(|ttl| ttl.get()))?; + let grant_valid_until = parse_grant_cutoff(&binding.grant_valid_until)?; + + let ca_bundles = capture_ca_bundles(&binding.connection)?; + let ca_digests: BTreeMap<_, _> = ca_bundles + .iter() + .map(|(name, bytes)| (name, sha256_uri(bytes))) + .collect(); + // Bindings that share a connection must describe it identically. This + // fingerprint only enforces that; it grants no authority. + let connection_fingerprint = fingerprint(&json!({ + "connection":binding.connection, + "endpointPolicies":binding.endpoint_policies, + "caDigests":ca_digests + }))?; + let connection_id = binding.connection.id.as_str(); + let connection = match connections.entry(connection_id) { + std::collections::btree_map::Entry::Vacant(entry) => { + &entry + .insert(( + connection_fingerprint, + EvidenceRuntimeBindings::from_shared_http( + binding.connection.clone(), + Arc::clone(resolver), + ca_bundles, + ), + )) + .1 + } + std::collections::btree_map::Entry::Occupied(entry) => { + let (previous, runtime) = entry.into_mut(); + if previous.as_str() != connection_fingerprint.as_str() { + return Err(ExternalSourceError::ConnectionConflict); + } + runtime + } + }; + let program = program + .bind_live(connection, connection_id, &binding.endpoint_policies) + .map_err(|_| ExternalSourceError::Contract)?; + // The scope fingerprints cover only what grants authority over cached + // rows: the reviewed source contract, its operation and the allowed + // fields. Connection details, CA bundles, acquisition limits, the grant + // cutoff and the cache lifetime can change without discarding the cache. + let source_contract = program.contract(); + let binding_fingerprint = fingerprint(&json!({ + "operation":projection.operation, + "sourceContract":source_contract + }))?; + let grant_fingerprint = fingerprint(&json!({ + "operation":projection.operation, + "sourceContract":source_contract, + "allowedFields":fields + }))?; + Ok(BoundExternalSource { + program, + fields, + mode: projection.mode, + grant_valid_until, + binding_fingerprint, + grant_fingerprint, + cache_ttl_seconds, + field_types, + }) +} + pub(crate) fn validate_bindings( bindings: &[ExternalSourceBinding], ) -> Result<(), ExternalSourceError> { @@ -664,11 +708,54 @@ fn read_relative(_: &File, _: &Path, _: usize) -> Result, ExternalSource } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; use registry_adapter_engine::evidence::{EvidenceAcquisitionIntent, EvidenceHookFuture}; use registry_platform_yaml::ExternalId; + /// Indexed bindings over the example pack for store and startup tests, + /// as (projection, binding fingerprint, grant fingerprint, cache + /// lifetime). The replay transcript is empty, so no source call succeeds. + #[cfg(feature = "postgres-test")] + pub(crate) fn indexed_service(bindings: &[(&str, &str, &str, u32)]) -> ExternalSourceService { + let (pack, artifacts) = example(); + let mut service = ExternalSourceService::default(); + for (id, binding, grant, ttl) in bindings { + let transcript = Arc::new( + registry_adapter_engine::replay::ReplayTranscript::from_exchanges( + serde_json::from_value( + json!([{"type":"endpoint","endpoint":"synthetic-registry"}]), + ) + .unwrap(), + serde_json::from_value(json!([])).unwrap(), + ), + ); + let program = compile_projection_pack( + &pack, + "person-batch", + &artifacts, + EvidenceLimits::default(), + ) + .unwrap() + .bind_replay(transcript) + .unwrap(); + service.bindings.insert( + (*id).to_owned(), + BoundExternalSource { + program, + fields: BTreeSet::from(["label".to_owned()]), + mode: ExternalProjectionMode::Indexed, + grant_valid_until: Utc::now() + chrono::Duration::days(1), + binding_fingerprint: (*binding).to_owned(), + grant_fingerprint: (*grant).to_owned(), + cache_ttl_seconds: Some(*ttl), + field_types: BTreeMap::new(), + }, + ); + } + service + } + struct Hooks; impl EvidenceHostHooks for Hooks { fn before_acquisition<'a>( @@ -811,6 +898,7 @@ mod tests { )]), }, )]), + unavailable: BTreeMap::new(), }; let samples = std::sync::atomic::AtomicUsize::new(0); let result = service @@ -850,6 +938,172 @@ mod tests { assert!(matches!(effective_ttl(&indexed, Some(900)), Ok(Some(300)))); } + fn example_registry() -> CompiledRegistry { + let source = std::fs::read( + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../products/breg/examples/external-views/registry.yaml"), + ) + .unwrap(); + let project = crate::contract::parse_project_yaml(&source).unwrap(); + crate::compiler::compile_project(&project, &[], crate::compiler::CompileProfile::Authoring) + .unwrap() + } + + fn example_binding(id: &str, fields: &[&str], ca_file: &Path) -> Value { + let pack = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../products/adapter/examples/external-projections/pack.yaml") + .canonicalize() + .unwrap(); + let digest = sha256_uri(&std::fs::read(&pack).unwrap()); + json!({ + "id":id,"packFile":pack.to_str().unwrap(),"packDigest":digest, + "connection":{"id":"synthetic-licensing","endpoints":[{"endpoint":"synthetic-registry", + "baseUrl":"https://source.example.test","tlsTrustProfile":"source-ca", + "credential":{"type":"static-api-key","valueRef":"secret:file/source-key"}}], + "outboundTls":{"trustProfiles":{"source-ca":{"caBundleFile":ca_file.to_str().unwrap()}}}}, + "endpointPolicies":[{"endpoint":"synthetic-registry","baseUrl":"https://source.example.test", + "tlsTrustProfile":"source-ca","clientIdentity":"forbidden", + "authentication":{"type":"static-api-key"}}], + "allowedFields":fields,"grantValidUntil":"2030-01-01T00:00:00Z", + "acquisition":{"maximumDataOperations":8,"timeoutMilliseconds":8000,"maximumResponseBytes":262144} + }) + } + + /// Copy the example pack into `dir` with an edited pack.yaml and point + /// the binding at the copy. + fn use_pack_copy(binding: &mut Value, dir: &Path, edit: impl Fn(&str) -> String) { + let source = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../products/adapter/examples/external-projections"); + let target = dir.canonicalize().unwrap(); + for folder in ["scripts", "schemas"] { + std::fs::create_dir_all(target.join(folder)).unwrap(); + for entry in std::fs::read_dir(source.join(folder)).unwrap() { + let entry = entry.unwrap(); + std::fs::copy(entry.path(), target.join(folder).join(entry.file_name())).unwrap(); + } + } + let pack = edit(&std::fs::read_to_string(source.join("pack.yaml")).unwrap()); + let pack_file = target.join("pack.yaml"); + std::fs::write(&pack_file, &pack).unwrap(); + binding["packFile"] = json!(pack_file.to_str().unwrap()); + binding["packDigest"] = json!(sha256_uri(pack.as_bytes())); + } + + fn load_values(registry: &CompiledRegistry, bindings: &[Value]) -> ExternalSourceService { + let bindings = bindings + .iter() + .map(|binding| serde_json::from_value(binding.clone()).unwrap()) + .collect::>(); + ExternalSourceService::load(registry, &bindings, Arc::new(SecretResolver::disabled())) + .unwrap() + } + + fn fingerprints(service: &ExternalSourceService, id: &str) -> (String, String) { + let bound = service.binding(id).unwrap_or_else(|| { + panic!( + "{id} is unavailable: {:?}", + service.unavailable_projections().get(id) + ) + }); + ( + bound.binding_fingerprint().to_owned(), + bound.grant_fingerprint().to_owned(), + ) + } + + #[test] + fn only_the_pack_contract_operation_and_fields_enter_the_authority_fingerprints() { + let registry = example_registry(); + let trust = tempfile::tempdir().unwrap(); + let ca_file = trust.path().canonicalize().unwrap().join("source-ca.pem"); + std::fs::write(&ca_file, b"synthetic certificate authority one").unwrap(); + let original = example_binding("person-indexed", &["family-name", "given-name"], &ca_file); + let first = fingerprints( + &load_values(®istry, std::slice::from_ref(&original)), + "person-indexed", + ); + + // A renewed grant, a replaced CA bundle, a moved endpoint and new + // acquisition limits grant no new authority. + std::fs::write(&ca_file, b"synthetic certificate authority two").unwrap(); + let mut renewed = original.clone(); + renewed["grantValidUntil"] = json!("2031-06-01T00:00:00Z"); + renewed["connection"]["endpoints"][0]["baseUrl"] = json!("https://renewed.example.test"); + renewed["endpointPolicies"][0]["baseUrl"] = json!("https://renewed.example.test"); + renewed["acquisition"] = json!({"maximumDataOperations":2,"timeoutMilliseconds":3000,"maximumResponseBytes":65536}); + renewed["cacheTtlSeconds"] = json!(120); + assert_eq!( + fingerprints(&load_values(®istry, &[renewed]), "person-indexed"), + first + ); + + // A rebuilt pack with the same reviewed contract keeps the authority, + // and fields approved beyond the projection's own grant nothing. + let packs = tempfile::tempdir().unwrap(); + let mut rebuilt = original.clone(); + use_pack_copy(&mut rebuilt, packs.path(), |pack| { + format!("{pack}# rebuilt without a contract change\n") + }); + rebuilt["allowedFields"] = json!(["family-name", "given-name", "unprojected"]); + assert_eq!( + fingerprints(&load_values(®istry, &[rebuilt]), "person-indexed"), + first + ); + + // A changed source contract is a different authority. + let changed = tempfile::tempdir().unwrap(); + let mut recontracted = original.clone(); + use_pack_copy(&mut recontracted, changed.path(), |pack| { + pack.replacen("maximumBatchKeys: 64", "maximumBatchKeys: 32", 1) + }); + let (binding, grant) = + fingerprints(&load_values(®istry, &[recontracted]), "person-indexed"); + assert_ne!(binding, first.0); + assert_ne!(grant, first.1); + + // Fields approved below the projection's own leave it unbound. + let mut narrowed = original; + narrowed["allowedFields"] = json!(["family-name"]); + let service = load_values(®istry, &[narrowed]); + assert_eq!( + service.unavailable_projections().get("person-indexed"), + Some(&ExternalSourceError::Grant.to_string()) + ); + } + + #[test] + fn a_refused_binding_leaves_the_other_projections_bound() { + let registry = example_registry(); + let trust = tempfile::tempdir().unwrap(); + let ca_file = trust.path().canonicalize().unwrap().join("source-ca.pem"); + std::fs::write(&ca_file, b"synthetic certificate authority").unwrap(); + let good = example_binding("person-indexed", &["family-name", "given-name"], &ca_file); + let mut pinned_elsewhere = example_binding("person-live", &["family-name"], &ca_file); + pinned_elsewhere["packDigest"] = + json!("sha256:0000000000000000000000000000000000000000000000000000000000000000"); + let retired = example_binding("retired-projection", &["family-name"], &ca_file); + let service = load_values(®istry, &[good, pinned_elsewhere, retired]); + + assert!(service.binding("person-indexed").is_some()); + assert!(service.binding("person-live").is_none()); + let unavailable = service.unavailable_projections(); + assert_eq!( + unavailable.get("person-live").map(String::as_str), + Some(ExternalSourceError::Artifact.to_string().as_str()) + ); + assert_eq!( + unavailable.get("retired-projection").map(String::as_str), + Some(ExternalSourceError::UnknownProjection.to_string().as_str()) + ); + for unbound in ["holdings-live", "holdings-indexed"] { + assert_eq!( + unavailable.get(unbound).map(String::as_str), + Some(ExternalSourceError::Unbound.to_string().as_str()) + ); + } + assert!(!unavailable.contains_key("person-indexed")); + } + #[test] fn retired_retention_and_selector_members_are_refused() { let binding = json!({ diff --git a/crates/registry-breg/src/postgres/external_projection.rs b/crates/registry-breg/src/postgres/external_projection.rs index babfd37bf4..9d1cabef18 100644 --- a/crates/registry-breg/src/postgres/external_projection.rs +++ b/crates/registry-breg/src/postgres/external_projection.rs @@ -16,7 +16,9 @@ pub(crate) const BIND_ONCE_FUNCTION: &str = "registry_internal.bind_external_projection_once(text,uuid,text,text,bigint,text,text,uuid)"; pub(crate) const MAINTENANCE_FUNCTION:&str="registry_internal.maintain_external_projection_scope(text,text,uuid,text,uuid,text,text,bigint)"; /// Withdrawal stays with migration authority: the runtime role holds no -/// EXECUTE grant on it, so a compromised runtime cannot purge a scope. +/// EXECUTE grant on it. The runtime bind can still move a configured +/// projection to a fresh epoch, which discards rebuildable cache but never +/// makes a withdrawn or replaced epoch readable again. pub(crate) const WITHDRAW_FUNCTION: &str = "registry_internal.withdraw_external_projection(text,uuid,text,text)"; /// Exact catalog identities include PostgreSQL argument names and spacing. @@ -48,19 +50,29 @@ pub(crate) fn enabled(registry: &crate::model::CompiledRegistry) -> bool { .any(|projection| projection.mode == crate::external_views::ExternalProjectionMode::Indexed) } -/// Scope identity includes the reviewed external inventory, so changing a hop, -/// field, grant or eligible acquisition path invalidates retained associations. +/// Scope identity covers what defines one projection's cached rows: its +/// declaration (record type, fields, operation, selector scheme, cache +/// lifetime ceiling) and the external record type it reads. Editing a view, +/// an access profile or another projection leaves this cache in place; views +/// and grants are rechecked on every read. pub(crate) fn scope_contract_digest( registry: &crate::model::CompiledRegistry, projection_id: &str, ) -> Result { - let projection = registry - .external_views() + let catalog = registry.external_views(); + let projection = catalog .projections .get(projection_id) .ok_or(ProjectionError::InvalidScope)?; - let value = serde_json::to_value((registry.external_views(), projection)) - .map_err(|_| ProjectionError::InvalidScope)?; + let record_type = catalog + .record_types + .get(projection.record_type.as_str()) + .ok_or(ProjectionError::InvalidScope)?; + let value = serde_json::to_value(serde_json::json!({ + "projection": projection, + "recordType": record_type, + })) + .map_err(|_| ProjectionError::InvalidScope)?; let canonical = registry_platform_canonical_json::canonicalize_json(&value) .map_err(|_| ProjectionError::InvalidScope)?; Ok(registry_platform_config::sha256_uri(&canonical)) @@ -173,9 +185,12 @@ pub(crate) async fn reconcile( Ok(()) } -/// Initialize only a migration-allowlisted empty slot under the exact active -/// package. Same bindings reuse their stored epoch; withdrawal or mismatched -/// bindings require explicit migration authority and cannot reactivate here. +/// Bind a migration-allowlisted projection under the exact active package. +/// The same binding fingerprints and cache lifetime reuse the stored epoch, so +/// a restart keeps the cache. Changed fingerprints, a changed cache lifetime, a +/// withdrawn scope or an empty slot start the requested fresh epoch, which +/// leaves every older row for purge. A requested epoch equal to the stored one +/// is refused, so a withdrawn or replaced epoch never becomes readable again. pub(crate) async fn bind_initial_scope( transaction: &GuardedTransaction<'_>, registry: &crate::model::CompiledRegistry, @@ -1302,16 +1317,16 @@ BEGIN VALUES (p_projection,p_contract,p_epoch,p_binding,p_grant,p_ttl) ON CONFLICT (projection_id) DO NOTHING; SELECT * INTO scope FROM registry_internal.registry_external_projection_scopes WHERE projection_id=p_projection FOR UPDATE; - IF scope.active AND scope.contract_digest=p_contract AND scope.binding_digest=p_binding AND scope.grant_digest=p_grant + IF scope.contract_digest<>p_contract THEN RETURN NULL; END IF; + IF scope.active AND scope.binding_digest=p_binding AND scope.grant_digest=p_grant AND scope.cache_ttl_seconds=p_ttl THEN RETURN scope.epoch; END IF; - IF NOT scope.active AND scope.epoch='00000000-0000-0000-0000-000000000000'::uuid - AND scope.binding_digest='' AND scope.grant_digest='' AND scope.contract_digest=p_contract THEN - UPDATE registry_internal.registry_external_projection_scopes - SET epoch=p_epoch,binding_digest=p_binding,grant_digest=p_grant,cache_ttl_seconds=p_ttl,active=true,generation=generation+1 - WHERE projection_id=p_projection; - RETURN p_epoch; - END IF; - RETURN NULL; + IF scope.epoch=p_epoch THEN RETURN NULL; END IF; + -- Changed authority or cache lifetime, a withdrawn scope or an empty slot + -- start a fresh epoch. Rows under any older epoch become purgeable. + UPDATE registry_internal.registry_external_projection_scopes + SET epoch=p_epoch,binding_digest=p_binding,grant_digest=p_grant,cache_ttl_seconds=p_ttl,active=true,generation=generation+1 + WHERE projection_id=p_projection; + RETURN p_epoch; END $body$; "#; @@ -1460,13 +1475,13 @@ mod tests { #[cfg(test)] #[cfg(feature = "postgres-test")] -mod postgres_tests { +pub(crate) mod postgres_tests { use super::*; use serde_json::json; use tokio_postgres::{Client, NoTls}; #[allow(dead_code)] - mod harness { + pub(crate) mod harness { use crate as registry_breg; include!(concat!( env!("CARGO_MANIFEST_DIR"), @@ -1562,14 +1577,6 @@ mod postgres_tests { .await .unwrap(); assert_eq!(bound.epoch, restarted.epoch); - let mut substituted = scope(Uuid::new_v4()); - substituted.binding_digest = "another-binding".to_owned(); - assert_eq!( - bind_initial_scope(&transaction, ®istry, &expected, &substituted) - .await - .unwrap_err(), - ProjectionError::ScopeChanged - ); assert!( load_active_scope(&transaction, "projection", "binding", "grant") .await @@ -1711,6 +1718,7 @@ mod postgres_tests { .unwrap_err(), ProjectionError::ScopeChanged ); + // A withdrawn epoch is never reused, so its rows stay purgeable. assert_eq!( bind_initial_scope(&transaction, ®istry, &expected, &requested) .await @@ -1718,12 +1726,40 @@ mod postgres_tests { ProjectionError::ScopeChanged ); transaction.commit().await.unwrap(); - drop(runtime); - // Reapplying an unchanged compiled inventory cannot undo withdrawal. + // Reapplying an unchanged compiled inventory does not undo withdrawal. reconcile(&migration, ®istry, &database.runtime_role) .await .unwrap(); - assert!(!migration.query_one("SELECT active FROM registry_internal.registry_external_projection_scopes WHERE projection_id='projection'",&[]).await.unwrap().get::<_,bool>(0)); + let active = "SELECT active FROM registry_internal.registry_external_projection_scopes WHERE projection_id='projection'"; + assert!(!migration + .query_one(active, &[]) + .await + .unwrap() + .get::<_, bool>(0)); + // Deployment configuration is the cutoff: a binding still configured + // reacquires at the next start, under a fresh epoch and an empty cache. + let transaction = super::super::begin_registry_transaction( + &mut runtime, + RegistryLockKey::derive(registry.registry_id()).unwrap(), + std::time::Duration::from_secs(2), + &expected, + ) + .await + .unwrap(); + let mut restarted = scope(Uuid::new_v4()); + restarted.cache_ttl_seconds = 30; + let reacquired = bind_initial_scope(&transaction, ®istry, &expected, &restarted) + .await + .unwrap(); + transaction.commit().await.unwrap(); + drop(runtime); + assert_ne!(reacquired.epoch, bound.epoch); + assert!(migration + .query_one(active, &[]) + .await + .unwrap() + .get::<_, bool>(0)); + let bound = reacquired; migration .batch_execute(&format!( "GRANT UPDATE ON registry_internal.registry_external_projection_values TO {}", @@ -2238,4 +2274,319 @@ mod postgres_tests { restored.cleanup().await; database.cleanup().await; } + + /// The fixture registry with a second indexed projection and its own + /// granted view link. + pub(crate) fn two_projection_project() -> Value { + let mut project = super::super::external_view_query::tests::project(); + project["externalProjections"].as_array_mut().unwrap().push(json!({ + "id":"other","recordType":"external-record","operation":"batch","operationVersion":"1", + "purpose":"consultation","scope":"eligible","selectorScheme":"urn:test:keys", + "mode":"indexed","cardinality":"one","cacheTtlSeconds":120,"fields":["label"] + })); + project["externalViews"][0]["links"] + .as_array_mut() + .unwrap() + .push( + json!({"id":"other-link","projection":"other","fromField":"subject-key", + "identifierScheme":"urn:test:keys","fields":["label"]}), + ); + project["accessProfiles"][0]["permissions"]["externalViews"][0]["links"] + .as_array_mut() + .unwrap() + .push(json!({"link":"other-link","readableFields":["label"]})); + project + } + + pub(crate) struct Activated { + pub(crate) database: harness::TestDatabase, + pub(crate) migration: Client, + migration_task: tokio::task::JoinHandle<()>, + pub(crate) expected: ExpectedRegistryIdentity, + pub(crate) pool: super::super::RuntimePool, + } + + impl Activated { + pub(crate) async fn new(registry: &crate::model::CompiledRegistry) -> Self { + let database = harness::TestDatabase::create(2).await; + let (migration, migration_task) = database.connect_migration().await; + super::super::install_compiled_schema(&migration, registry, &database.runtime_role) + .await + .unwrap(); + let expected = super::super::initialize_compiled_registry_state_for_test( + &migration, + &database.runtime_role, + registry, + super::super::RegistryStateTestIdentity { + package_id: "projection-package", + database_id: "projection-database", + label: "projection-catalog", + }, + ) + .await + .unwrap(); + let pool = database.runtime_config.build_pool().unwrap(); + Self { + database, + migration, + migration_task, + expected, + pool, + } + } + + /// One startup bind attempt: a fresh epoch in its own transaction. + async fn bind( + &self, + registry: &crate::model::CompiledRegistry, + projection: &str, + binding: &str, + grant: &str, + ttl: i64, + ) -> Result { + let requested = + ProjectionScope::new(projection, Uuid::new_v4(), binding, grant, ttl).unwrap(); + self.bind_requested(registry, &requested).await + } + + async fn bind_requested( + &self, + registry: &crate::model::CompiledRegistry, + requested: &ProjectionScope, + ) -> Result { + let mut runtime = self.pool.get().await.unwrap(); + let transaction = super::super::begin_registry_transaction( + &mut runtime, + super::super::RegistryLockKey::derive(registry.registry_id()).unwrap(), + std::time::Duration::from_secs(2), + &self.expected, + ) + .await + .unwrap(); + let bound = bind_initial_scope(&transaction, registry, &self.expected, requested).await; + transaction.commit().await.unwrap(); + bound + } + + /// Whether a process holding these fingerprints can still serve. + async fn serves( + &self, + registry: &crate::model::CompiledRegistry, + projection: &str, + binding: &str, + grant: &str, + ) -> bool { + let mut runtime = self.pool.get().await.unwrap(); + let transaction = super::super::begin_registry_transaction( + &mut runtime, + super::super::RegistryLockKey::derive(registry.registry_id()).unwrap(), + std::time::Duration::from_secs(2), + &self.expected, + ) + .await + .unwrap(); + let loaded = load_active_scope(&transaction, projection, binding, grant).await; + transaction.rollback().await.unwrap(); + loaded.is_ok() + } + + async fn cache(&self, scope: &ProjectionScope) { + let now: i64 = self + .migration + .query_one( + "SELECT floor(extract(epoch FROM clock_timestamp()))::bigint", + &[], + ) + .await + .unwrap() + .get(0); + let ticket = allocate_inner(&self.migration, scope, "subject") + .await + .unwrap(); + assert_eq!( + publish_inner(&self.migration, &ticket, &snapshot("cached", now, "cached")) + .await + .unwrap(), + Publication::Published + ); + } + + async fn cached(&self, projection: &str, epoch: Uuid) -> i64 { + self.migration + .query_one( + "SELECT count(*) FROM registry_internal.registry_external_projection_snapshots + WHERE projection_id=$1 AND epoch=$2", + &[&projection, &epoch], + ) + .await + .unwrap() + .get(0) + } + + pub(crate) async fn cleanup(self) { + drop(self.pool); + self.migration_task.abort(); + drop(self.migration); + self.database.cleanup().await; + } + } + + #[tokio::test] + async fn authority_changes_rotate_only_their_projection_and_restarts_keep_the_cache() { + let registry = + super::super::external_view_query::tests::compile_project(&two_projection_project()); + let fixture = Activated::new(®istry).await; + let first = fixture + .bind(®istry, "projection", "binding", "grant", 60) + .await + .unwrap(); + let other = fixture + .bind(®istry, "other", "other-binding", "other-grant", 120) + .await + .unwrap(); + fixture.cache(&first).await; + fixture.cache(&other).await; + + // A restart with the same authority keeps the epoch and its cache. + let restarted = fixture + .bind(®istry, "projection", "binding", "grant", 60) + .await + .unwrap(); + assert_eq!(restarted.epoch, first.epoch); + assert_eq!(fixture.cached("projection", first.epoch).await, 1); + + // Changed authority rotates only that projection to a fresh epoch. A + // process still holding the old fingerprints stops serving it. + let rotated = fixture + .bind(®istry, "projection", "binding", "narrowed-grant", 60) + .await + .unwrap(); + assert_ne!(rotated.epoch, first.epoch); + assert!( + !fixture + .serves(®istry, "projection", "binding", "grant") + .await + ); + assert!( + fixture + .serves(®istry, "projection", "binding", "narrowed-grant") + .await + ); + let unchanged = fixture + .bind(®istry, "other", "other-binding", "other-grant", 120) + .await + .unwrap(); + assert_eq!(unchanged.epoch, other.epoch); + assert!( + fixture + .serves(®istry, "other", "other-binding", "other-grant") + .await + ); + + // A shorter deployment cache lifetime also starts a fresh epoch, and + // a bind cannot reuse the epoch it would replace. + let shortened = fixture + .bind(®istry, "projection", "binding", "narrowed-grant", 30) + .await + .unwrap(); + assert_ne!(shortened.epoch, rotated.epoch); + let mut reused = ProjectionScope::new( + "projection", + shortened.epoch, + "another", + "narrowed-grant", + 30, + ) + .unwrap(); + assert_eq!( + fixture + .bind_requested(®istry, &reused) + .await + .unwrap_err(), + ProjectionError::ScopeChanged + ); + reused.cache_ttl_seconds = 61; + reused.epoch = Uuid::new_v4(); + assert_eq!( + fixture + .bind_requested(®istry, &reused) + .await + .unwrap_err(), + ProjectionError::ScopeChanged + ); + + // Purge removes the replaced epoch's cache; the other projection's + // cache stays warm. + fixture + .migration + .query_one( + "SELECT * FROM registry_internal.purge_external_projections(100)", + &[], + ) + .await + .unwrap(); + assert_eq!(fixture.cached("projection", first.epoch).await, 0); + assert_eq!(fixture.cached("other", other.epoch).await, 1); + fixture.cleanup().await; + } + + #[tokio::test] + async fn an_unrelated_edit_leaves_other_projection_caches_warm() { + let project = two_projection_project(); + let registry = super::super::external_view_query::tests::compile_project(&project); + let fixture = Activated::new(®istry).await; + let first = fixture + .bind(®istry, "projection", "binding", "grant", 60) + .await + .unwrap(); + let other = fixture + .bind(®istry, "other", "other-binding", "other-grant", 120) + .await + .unwrap(); + fixture.cache(&first).await; + + // A profile grant edit and another projection's cache lifetime are + // outside this projection's contract. + let mut edited = project; + edited["accessProfiles"][0]["permissions"]["externalViews"][0]["links"][0]["allowCount"] = + json!(false); + edited["externalProjections"][1]["cacheTtlSeconds"] = json!(90); + let edited = super::super::external_view_query::tests::compile_project(&edited); + assert_eq!( + scope_contract_digest(®istry, "projection").unwrap(), + scope_contract_digest(&edited, "projection").unwrap() + ); + assert_ne!( + scope_contract_digest(®istry, "other").unwrap(), + scope_contract_digest(&edited, "other").unwrap() + ); + reconcile(&fixture.migration, &edited, &fixture.database.runtime_role) + .await + .unwrap(); + + assert!( + fixture + .serves(&edited, "projection", "binding", "grant") + .await + ); + assert_eq!(fixture.cached("projection", first.epoch).await, 1); + let restarted = fixture + .bind(&edited, "projection", "binding", "grant", 60) + .await + .unwrap(); + assert_eq!(restarted.epoch, first.epoch); + + // The changed projection reacquires under a fresh epoch. + assert!( + !fixture + .serves(&edited, "other", "other-binding", "other-grant") + .await + ); + let rebound = fixture + .bind(&edited, "other", "other-binding", "other-grant", 90) + .await + .unwrap(); + assert_ne!(rebound.epoch, other.epoch); + fixture.cleanup().await; + } } diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index 75422d6479..1c39046579 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -168,7 +168,13 @@ impl PostgresExternalViewService { } } - pub(crate) async fn bind_configured_scopes(&self) -> Result<()> { + /// Bind each configured indexed projection in its own transaction. A + /// projection the database refuses becomes unavailable for this process, + /// with the reason, while the others bind and serve. A database that + /// cannot be reached still fails the whole start. With `persist` false + /// (doctor) every bind rolls back, so a check never rotates a serving + /// scope or writes one. + pub(crate) async fn bind_configured_scopes(&mut self, persist: bool) -> Result<()> { let configured = self .registry .external_views() @@ -176,46 +182,77 @@ impl PostgresExternalViewService { .values() .filter_map(|projection| { let bound = self.sources.binding(projection.id.as_str())?; - Some((projection, bound, bound.effective_cache_ttl_seconds()?)) + let ttl = bound.effective_cache_ttl_seconds()?; + Some(( + projection.id.to_string(), + ProjectionScope::new( + projection.id.as_str(), + Uuid::new_v4(), + bound.binding_fingerprint(), + bound.grant_fingerprint(), + i64::from(ttl), + ), + )) }) .collect::>(); if configured.is_empty() { return Ok(()); } let mut session = self.session().await?; - let transaction = begin_registry_transaction( - session.client(), - self.lock_key, - self.lock_timeout, - &self.expected, - ) - .await - .map_err(|_| ExternalViewError::Unavailable)?; - transaction - .set_statement_budget(Duration::from_secs(10)) - .await - .map_err(|_| ExternalViewError::Unavailable)?; - for (projection, bound, ttl) in configured { - let scope = ProjectionScope::new( - projection.id.as_str(), - Uuid::new_v4(), - bound.binding_fingerprint(), - bound.grant_fingerprint(), - i64::from(ttl), + for (id, scope) in configured { + let transaction = begin_registry_transaction( + session.client(), + self.lock_key, + self.lock_timeout, + &self.expected, ) + .await .map_err(|_| ExternalViewError::Unavailable)?; - store::bind_initial_scope(&transaction, &self.registry, &self.expected, &scope) + transaction + .set_statement_budget(Duration::from_secs(10)) .await .map_err(|_| ExternalViewError::Unavailable)?; + let bound = match scope { + Ok(scope) => { + store::bind_initial_scope(&transaction, &self.registry, &self.expected, &scope) + .await + } + Err(error) => Err(error), + }; + let refusal = match bound { + Ok(_) => None, + Err(store::ProjectionError::ScopeChanged) => Some( + "the database refused its cache scope; run migrate for this package with this release, then restart", + ), + Err(store::ProjectionError::InvalidScope) => { + Some("its cache scope is invalid for this package") + } + Err(_) => return Err(ExternalViewError::Unavailable), + }; + if let Some(reason) = refusal { + tracing::error!( + projection = %id, + reason, + "external projection is unavailable; its views answer source.unavailable" + ); + self.sources.refuse(&id, reason.to_owned()); + } + let finished = if persist && refusal.is_none() { + transaction.commit().await + } else { + transaction.rollback().await + }; + finished.map_err(|_| ExternalViewError::Unavailable)?; } - transaction - .commit() - .await - .map_err(|_| ExternalViewError::Unavailable)?; session.disarm(); Ok(()) } + /// Projections without a usable binding in this process, with the reason. + pub(crate) fn unavailable_projections(&self) -> &BTreeMap { + self.sources.unavailable_projections() + } + pub(crate) async fn run_retention( self: Arc, last_success: Arc, @@ -269,6 +306,13 @@ impl PostgresExternalViewService { Ok(()) } + /// Called only after the caller's view, link and purpose were admitted. + fn bound_source(&self, projection: &ExternalProjectionSource) -> Result<&BoundExternalSource> { + self.sources + .binding(projection.id.as_str()) + .ok_or_else(|| ExternalViewError::ProjectionUnavailable(projection.id.to_string())) + } + fn plan<'a>( &'a self, request: &ExternalViewRequest, @@ -321,10 +365,7 @@ impl PostgresExternalViewService { } label_parent = Some(LabelParent { projection, - source: self - .sources - .binding(projection.id.as_str()) - .ok_or(ExternalViewError::Unavailable)?, + source: self.bound_source(projection)?, }); projection = label; } @@ -348,10 +389,7 @@ impl PostgresExternalViewService { .record_types .get(projection.record_type.as_str()) .ok_or(ExternalViewError::Unavailable)?; - let source = self - .sources - .binding(projection.id.as_str()) - .ok_or(ExternalViewError::Unavailable)?; + let source = self.bound_source(projection)?; Ok(Plan { root, link, @@ -1159,9 +1197,12 @@ async fn active_scope( source: &BoundExternalSource, projection: &str, ) -> Result { + let unavailable = || ExternalViewError::ProjectionUnavailable(projection.to_owned()); if Utc::now() >= source.grant_valid_until() { - return Err(ExternalViewError::Unavailable); + return Err(unavailable()); } + // Another process bound this projection with different authority or a + // different cache lifetime, or an operator withdrew it. let scope = store::load_active_scope( transaction, projection, @@ -1169,9 +1210,14 @@ async fn active_scope( source.grant_fingerprint(), ) .await - .map_err(|_| ExternalViewError::Unavailable)?; + .map_err(|error| match error { + store::ProjectionError::ScopeChanged | store::ProjectionError::InvalidScope => { + unavailable() + } + _ => ExternalViewError::Unavailable, + })?; if Some(scope.cache_ttl_seconds) != source.effective_cache_ttl_seconds().map(i64::from) { - return Err(ExternalViewError::Unavailable); + return Err(unavailable()); } Ok(scope) } @@ -1757,3 +1803,126 @@ mod tests { assert!(Uuid::parse_str(&snapshot_identity(&codec, "binding-a", None).unwrap()).is_ok()); } } + +#[cfg(test)] +#[cfg(feature = "postgres-test")] +mod postgres_tests { + use super::*; + use crate::correlation::RequestCorrelation; + use crate::postgres::external_projection::postgres_tests::{two_projection_project, Activated}; + + fn service(registry: &CompiledRegistry, fixture: &Activated) -> PostgresExternalViewService { + PostgresExternalViewService::new( + fixture.pool.clone(), + Arc::new(registry.clone()), + fixture.expected.clone(), + RegistryLockKey::derive(registry.registry_id()).unwrap(), + Duration::from_secs(2), + crate::audit::test_support::capturing( + registry_platform_audit::AuditProfile::unkeyed_dev_only(), + ) + .0, + Arc::new( + CursorCodec::new( + zeroize::Zeroizing::new(vec![7; 32]), + Duration::from_secs(60), + ) + .unwrap(), + ), + crate::external_source::tests::indexed_service(&[ + ("projection", "binding", "grant", 60), + ("other", "other-binding", "other-grant", 120), + ]), + ) + } + + fn request(link: &str) -> ExternalViewRequest { + ExternalViewRequest { + context: crate::api::AuthorizedRequestContext::new( + Some("reader-principal".to_owned()), + Some("consultation".to_owned()), + "reader".to_owned(), + Vec::new(), + ), + correlation: RequestCorrelation::breg_created(), + view: "view".to_owned(), + link: link.to_owned(), + root: None, + query: ExternalQueryMode::First(ExternalQuery::default()), + deadline: Instant::now() + Duration::from_secs(5), + } + } + + async fn scopes(fixture: &Activated) -> BTreeMap { + fixture + .migration + .query( + "SELECT projection_id,contract_digest,active + FROM registry_internal.registry_external_projection_scopes", + &[], + ) + .await + .unwrap() + .into_iter() + .map(|row| (row.get(0), (row.get(1), row.get(2)))) + .collect() + } + + #[tokio::test] + async fn a_refused_projection_scope_leaves_the_other_projections_serving() { + let registry = + super::super::external_view_query::tests::compile_project(&two_projection_project()); + let fixture = Activated::new(®istry).await; + + let mut checked = service(®istry, &fixture); + checked.bind_configured_scopes(false).await.ok().unwrap(); + assert!(checked.unavailable_projections().is_empty()); + assert!( + scopes(&fixture).await.is_empty(), + "a doctor check binds and rolls back" + ); + + // A scope this release did not reconcile: its stored contract differs. + fixture + .migration + .execute( + "INSERT INTO registry_internal.registry_external_projection_scopes + (projection_id,contract_digest,epoch,binding_digest,grant_digest,cache_ttl_seconds) + VALUES ('projection','sha256:another',gen_random_uuid(),'binding','grant',60)", + &[], + ) + .await + .unwrap(); + + let mut serving = service(®istry, &fixture); + serving.bind_configured_scopes(true).await.ok().unwrap(); + let unavailable = serving.unavailable_projections(); + assert_eq!( + unavailable.keys().collect::>(), + ["projection"], + "only the refused projection is unavailable" + ); + assert!(unavailable["projection"].contains("run migrate")); + + match serving.plan(&request("link"), None) { + Err(ExternalViewError::ProjectionUnavailable(id)) => assert_eq!(id, "projection"), + Err(_) => panic!("the refused projection must name itself"), + Ok(_) => panic!("the refused projection must not plan"), + } + assert!( + serving.plan(&request("other-link"), None).is_ok(), + "the other projection still serves" + ); + + let stored = scopes(&fixture).await; + assert_eq!(stored["projection"].0, "sha256:another"); + assert_eq!( + stored["other"], + ( + store::scope_contract_digest(®istry, "other").unwrap(), + true + ) + ); + fixture.cleanup().await; + } +} diff --git a/crates/registry-breg/src/postgres/external_view_query.rs b/crates/registry-breg/src/postgres/external_view_query.rs index 20acc5f424..c9794a87d9 100644 --- a/crates/registry-breg/src/postgres/external_view_query.rs +++ b/crates/registry-breg/src/postgres/external_view_query.rs @@ -637,7 +637,26 @@ pub(crate) mod tests { } pub(crate) fn registry_with_external(include_external: bool) -> CompiledRegistry { - let mut project = json!({ + let mut project = project(); + if !include_external { + let root = project.as_object_mut().unwrap(); + root.remove("externalRecordTypes"); + root.remove("externalProjections"); + root.remove("externalViews"); + root.get_mut("accessProfiles") + .unwrap() + .as_array_mut() + .unwrap()[0]["permissions"] + .as_object_mut() + .unwrap() + .remove("externalViews"); + } + compile_project(&project) + } + + /// The authoring project behind [`registry`], for tests that edit it. + pub(crate) fn project() -> Value { + json!({ "apiVersion":crate::compiler::AUTHORING_API_VERSION,"kind":"BRegProject", "project":{"id":"external-query-test","version":"1","defaultLanguage":"en","canonicalBaseIri":"https://fixture.example.test"}, "datasets":[{"id":"records"}], @@ -655,22 +674,12 @@ pub(crate) mod tests { "records":[{"recordType":"local-record","operations":["get","list"],"allowCount":true,"rowBoundaries":[{"field":"tenant","claim":"tenant","operator":"equals"}]}], "externalViews":[{"view":"view","links":[{"link":"link","readableFields":["label"],"filterableFields":["label"],"sortableFields":["label"],"allowCount":true}]}] }}] - }); - if !include_external { - let root = project.as_object_mut().unwrap(); - root.remove("externalRecordTypes"); - root.remove("externalProjections"); - root.remove("externalViews"); - root.get_mut("accessProfiles") - .unwrap() - .as_array_mut() - .unwrap()[0]["permissions"] - .as_object_mut() - .unwrap() - .remove("externalViews"); - } + }) + } + + pub(crate) fn compile_project(project: &Value) -> CompiledRegistry { let project = - crate::contract::parse_project_json(&serde_json::to_vec(&project).unwrap()).unwrap(); + crate::contract::parse_project_json(&serde_json::to_vec(project).unwrap()).unwrap(); crate::compiler::compile_project(&project, &[], crate::compiler::CompileProfile::Authoring) .unwrap() } diff --git a/crates/registry-breg/src/startup.rs b/crates/registry-breg/src/startup.rs index b8f2ea6a44..5519398e01 100644 --- a/crates/registry-breg/src/startup.rs +++ b/crates/registry-breg/src/startup.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 //! Startup ordering gate for one verified Registry package. +use std::collections::BTreeMap; use std::future::Future; use std::net::SocketAddr; use std::path::Path; @@ -141,7 +142,7 @@ pub enum StartupError { AuditDestination(String), #[error("the Registry cursor profile was refused")] Cursor, - #[error("an external source binding or its reviewed Adapter pack was refused")] + #[error("the external source binding list or the external projection store was refused")] ExternalSources, #[error( "indexed external projections require separate migration and runtime PostgreSQL roles" @@ -765,7 +766,7 @@ impl StartupError { Self::Audit => "the Registry audit profile or destination was refused", Self::AuditDestination(_) => "the Registry audit destination was refused", Self::Cursor => "the Registry cursor profile was refused", - Self::ExternalSources => "an external source binding or its reviewed Adapter pack was refused", + Self::ExternalSources => "the external source binding list or the external projection store was refused", Self::ExternalProjectionRoleMode => "indexed external projections require separate migration and runtime PostgreSQL roles", Self::Oidc => "the Registry OIDC key source was refused", Self::Authentication => "the Registry authentication profile was refused", @@ -851,6 +852,7 @@ pub struct PreparedServer { #[cfg(feature = "wasm")] wasm_runtime: Option, postgres_advisories: Vec, + unavailable_external_projections: BTreeMap, role_mode: RoleMode, #[cfg(all(feature = "postgres-test", feature = "tooling"))] fixture_pool: Option, @@ -880,6 +882,13 @@ impl PreparedServer { &self.postgres_advisories } + /// External projections this server cannot serve, with the reason. Their + /// views answer unavailable while every other view serves. + #[must_use] + pub fn unavailable_external_projections(&self) -> &BTreeMap { + &self.unavailable_external_projections + } + /// The role mode the runtime file selects and startup verified. #[must_use] pub fn role_mode(&self) -> RoleMode { @@ -932,6 +941,7 @@ impl PreparedServer { external_projection_retention: None, metrics: None, postgres_advisories: Vec::new(), + unavailable_external_projections: BTreeMap::new(), role_mode: RoleMode::Split, #[cfg(feature = "wasm")] wasm_runtime: None, @@ -973,6 +983,7 @@ impl PreparedServer { external_projection_retention: None, metrics: None, postgres_advisories: Vec::new(), + unavailable_external_projections: BTreeMap::new(), role_mode: RoleMode::Split, #[cfg(feature = "wasm")] wasm_runtime: None, @@ -1044,16 +1055,19 @@ pub async fn check_loaded(config: RuntimeConfig) -> Result { postgres_advisories: prepared.postgres_advisories().to_vec(), role_mode: prepared.role_mode(), unbound_review_authorities, + unavailable_external_projections: prepared.unavailable_external_projections().clone(), }) } /// What a startup check that passed decided without refusing: the -/// PostgreSQL baseline advisories and the role mode it verified. +/// PostgreSQL baseline advisories, the role mode it verified, and the external +/// projections a server would start without, with the reason. #[derive(Debug, Clone)] pub struct CheckedStartup { pub postgres_advisories: Vec, pub role_mode: RoleMode, pub unbound_review_authorities: Vec, + pub unavailable_external_projections: BTreeMap, } /// Prepare the clean database capability consumed by the production pre-sign @@ -1324,6 +1338,7 @@ async fn prepare_verified_package_with_connection( key_source, audit, cursor_codec, + opening, ) .await } @@ -1353,6 +1368,7 @@ async fn prepare_verified_package_with_key_source( key_source, audit, cursor_codec, + AuditOpening::Serve, ) .await } @@ -1466,6 +1482,7 @@ async fn prepare_database_startup( Ok((pool, startup, advisories)) } +#[allow(clippy::too_many_arguments)] async fn finish_prepared_server( config: RuntimeConfig, startup: VerifiedStartup, @@ -1474,6 +1491,7 @@ async fn finish_prepared_server( key_source: Arc, audit: RegistryAudit, cursor_codec: Arc, + opening: AuditOpening, ) -> Result { let oidc = config.authentication().oidc(); key_source @@ -1611,6 +1629,7 @@ async fn finish_prepared_server( } else { None }; + let mut unavailable_external_projections = BTreeMap::new(); let external_views = if registry.external_views().views.is_empty() && config.external_sources().is_empty() { None @@ -1625,7 +1644,14 @@ async fn finish_prepared_server( ), ) .map_err(|_| StartupError::ExternalSources)?; - let external = crate::postgres::external_runtime::PostgresExternalViewService::new( + for (projection, reason) in sources.unavailable_projections() { + tracing::warn!( + projection = %projection, + reason = %reason, + "external projection is unavailable; its views answer source.unavailable" + ); + } + let mut external = crate::postgres::external_runtime::PostgresExternalViewService::new( pool.clone(), Arc::clone(®istry), expected.clone(), @@ -1635,10 +1661,12 @@ async fn finish_prepared_server( Arc::clone(&cursor_codec), sources, ); + // A check binds and rolls back, so it never rotates a serving scope. external - .bind_configured_scopes() + .bind_configured_scopes(opening == AuditOpening::Serve) .await .map_err(|_| StartupError::ExternalSources)?; + unavailable_external_projections = external.unavailable_projections().clone(); Some(Arc::new(external)) }; let records = PostgresRecordReadService::new( @@ -1944,6 +1972,7 @@ async fn finish_prepared_server( external_projection_retention, metrics, postgres_advisories, + unavailable_external_projections, role_mode: RoleMode::from_roles( config.database().roles().migration(), config.database().roles().runtime(), diff --git a/crates/registry-breg/tests/startup_http.rs b/crates/registry-breg/tests/startup_http.rs index 9e2d35bba8..15e67dd64b 100644 --- a/crates/registry-breg/tests/startup_http.rs +++ b/crates/registry-breg/tests/startup_http.rs @@ -806,7 +806,7 @@ fn expected_startup_error(error: StartupError) -> &'static str { StartupError::AuditDestination(_) => "the Registry audit destination was refused", StartupError::Cursor => "the Registry cursor profile was refused", StartupError::ExternalSources => { - "an external source binding or its reviewed Adapter pack was refused" + "the external source binding list or the external projection store was refused" } StartupError::ExternalProjectionRoleMode => { "indexed external projections require separate migration and runtime PostgreSQL roles" diff --git a/crates/registry-bregctl/src/doctor.rs b/crates/registry-bregctl/src/doctor.rs index ec1d2f5345..076b8f98d9 100644 --- a/crates/registry-bregctl/src/doctor.rs +++ b/crates/registry-bregctl/src/doctor.rs @@ -282,7 +282,7 @@ fn startup_diagnostic(error: StartupError) -> Diagnostic { StartupError::ExternalSources => ( "startup.external-sources.refused", "externalSources", - "review the indexed projection scope and the configured source grant and pinned Adapter pack", + "check each binding for a unique ID, an absolute pack file, allowed fields, endpoint policies and an RFC 3339 grant cutoff, and that the database is reachable; a binding that fails its own checks only leaves its views unavailable", ), StartupError::ExternalProjectionRoleMode => ( "startup.external-projections.role-mode", diff --git a/crates/registry-bregctl/src/lib.rs b/crates/registry-bregctl/src/lib.rs index 839316f9bd..4ecc52e2fa 100644 --- a/crates/registry-bregctl/src/lib.rs +++ b/crates/registry-bregctl/src/lib.rs @@ -3022,6 +3022,7 @@ where success.checked.role_mode, success.insecure_http_warning, &success.checked.unbound_review_authorities, + &success.checked.unavailable_external_projections, format, stdout, stderr, @@ -12792,11 +12793,13 @@ fn write_dev_success( write_result(result, stderr) } +#[allow(clippy::too_many_arguments)] fn write_doctor_success( advisories: &[registry_breg::postgres::BaselineAdvisory], role_mode: registry_breg::postgres::RoleMode, insecure_http_warning: Option<&'static str>, unbound_review_authorities: &[String], + unavailable_external_projections: &BTreeMap, format: OutputFormat, stdout: &mut dyn Write, stderr: &mut dyn Write, @@ -12829,6 +12832,20 @@ fn write_doctor_success( observed: ObservedNumbers(&[]), }); } + let projection_messages = unavailable_external_projections + .iter() + .map(|(projection, reason)| format!( + "External projection `{projection}` is unavailable: {reason}. Its views answer 503 source.unavailable; other views serve." + )) + .collect::>(); + for message in &projection_messages { + doctor_advisories.push(DoctorAdvisory { + code: "external-projection.unavailable".to_owned(), + severity: "warning", + message, + observed: ObservedNumbers(&[]), + }); + } let report = DoctorSuccessReport { ok: true, command: "doctor", @@ -16872,7 +16889,7 @@ mod tests { let mut stderr = Vec::new(); assert_eq!( - write_doctor_success(&[], RoleMode::Split, None, &[], format, &mut stdout, &mut stderr), + write_doctor_success(&[], RoleMode::Split, None, &[], &BTreeMap::new(), format, &mut stdout, &mut stderr), ExitCode::SUCCESS ); assert_eq!( @@ -16893,6 +16910,7 @@ mod tests { RoleMode::Single, None, &[], + &BTreeMap::new(), OutputFormat::Human, &mut stdout, &mut stderr @@ -16917,6 +16935,7 @@ mod tests { RoleMode::Single, None, &[], + &BTreeMap::new(), OutputFormat::Json, &mut stdout, &mut stderr @@ -16950,6 +16969,7 @@ mod tests { RoleMode::Split, None, &[], + &BTreeMap::new(), OutputFormat::Human, &mut stdout, &mut stderr @@ -16989,6 +17009,7 @@ mod tests { RoleMode::Split, None, &[], + &BTreeMap::new(), OutputFormat::Json, &mut stdout, &mut stderr @@ -17046,6 +17067,7 @@ mod tests { RoleMode::Split, None, &["local-review".into()], + &BTreeMap::new(), format, &mut stdout, &mut stderr @@ -17070,6 +17092,53 @@ mod tests { } } + #[test] + fn doctor_names_each_unavailable_external_projection_and_its_reason() { + let unavailable = BTreeMap::from([( + "person-indexed".to_owned(), + "no external source is bound to this projection".to_owned(), + )]); + for format in [OutputFormat::Human, OutputFormat::Json] { + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + assert_eq!( + write_doctor_success( + &[], + RoleMode::Split, + None, + &[], + &unavailable, + format, + &mut stdout, + &mut stderr + ), + ExitCode::SUCCESS + ); + assert!(stderr.is_empty()); + let text = plain(&stdout) + .split_whitespace() + .collect::>() + .join(" "); + for expected in [ + "external-projection.unavailable", + "person-indexed", + "no external source is bound to this projection", + "other views serve", + ] { + assert!(text.contains(expected), "{text}"); + } + if format == OutputFormat::Json { + let report: Value = serde_json::from_slice(&stdout).unwrap(); + assert_eq!(report["ok"], true); + assert_eq!(report["advisories"][0]["severity"], "warning"); + assert_eq!( + report["advisories"][0]["code"], + "external-projection.unavailable" + ); + } + } + } + #[test] fn doctor_reports_the_plain_http_issuer_opt_in_as_a_value_free_advisory() { const WARNING: &str = @@ -17083,6 +17152,7 @@ mod tests { RoleMode::Split, Some(WARNING), &[], + &BTreeMap::new(), format, &mut stdout, &mut stderr, diff --git a/products/breg/external-projection-storage.md b/products/breg/external-projection-storage.md index 4acb8850a0..476eddf8cd 100644 --- a/products/breg/external-projection-storage.md +++ b/products/breg/external-projection-storage.md @@ -6,9 +6,13 @@ When you operate indexed external views, keep the projection cache separate from Indexed projections require separate migration and runtime PostgreSQL roles. Activation and startup refuse a shared role; packages without indexed projections can keep their existing role layout. The migration role installs the projection store when the compiled project contains an indexed projection. A project with live projections only, or without external projections, does not install this store. The runtime role can read the four tables and execute their bounded functions. It cannot insert, update, or delete their rows directly. -The migration-installed binding function contains the compiled projection allowlist. Its definition is part of the managed catalog fingerprint. At startup, a current reviewed deployment binding can fill an empty allowlisted slot under the exact active package and activation. A restart with the same binding and source grant reuses the stored epoch. +The migration-installed binding function contains the compiled projection allowlist. Its definition is part of the managed catalog fingerprint. At startup, each configured deployment binding binds its allowlisted slot under the exact active package and activation, one projection at a time. A restart with the same binding, source grant, and cache lifetime reuses the stored epoch, so the cache stays warm. -A different binding or source grant cannot replace an already bound slot through runtime startup. A withdrawn slot also stays withdrawn. These changes require operator maintenance under migration authority. Changing the compiled external contract invalidates its previous associations and requires a fresh binding and acquisition. Removing the final indexed projection removes its rebuildable store during package activation. +The binding and source grant fingerprints cover only what grants authority over cached rows: the reviewed pack contract, the operation, and the allowed fields. Renewing the grant cutoff, replacing a CA bundle, moving an endpoint, or changing acquisition limits keeps the stored epoch. A changed contract, operation, allowed field list, or effective cache lifetime starts a fresh epoch at the next start. A withdrawn slot whose binding is still configured also starts a fresh epoch. Rows under every older epoch are left for purge, and a withdrawn or replaced epoch never becomes readable again. + +Each projection's scope covers its own declaration and its external record type. Editing a view, an access profile, or another projection keeps its cache. Changing the projection's own declaration resets its scope during package activation and requires fresh acquisition. Removing the final indexed projection removes its rebuildable store during package activation. + +A projection that cannot bind leaves only its own views unavailable. Its binding may fail its own checks, its source grant may have expired, or the database may refuse its scope. Those views answer `503` with the `source.unavailable` Problem, whose `fieldPath` is `/externalProjections/{id}`. Other views keep serving. Startup logs each unavailable projection with the reason, and `bregctl doctor` reports each one as an `external-projection.unavailable` warning. A doctor check binds and rolls back, so it never moves a serving scope to a new epoch. The current link requires the local reference scheme to match the source selector scheme. The named link reserves `identifierBinding` for a future versioned resolver contract. @@ -33,15 +37,17 @@ Run the named view’s `POST /v1/external-views/{view}/{link}/refresh` with the The refresh report counts processed roots or related rows, published values, reused keys and unavailable results. The migration role’s scope inspection reports receipt counts by outcome and fresh, expired and invalidated snapshot counts. Use an authorised indexed query to test readiness for its local root scope. -## Operator withdrawal and grant renewal +## Stop acquisition, withdraw, and renew + +Deployment configuration is the only cutoff for acquisition. To stop acquiring a projection, remove its binding from the deployment configuration and restart. Its views then answer `source.unavailable`. A binding that stays in the configuration binds again at the next start and acquires fresh data, even after a withdrawal. Record a source withdrawal in deployment configuration or at the source authority so that it also holds after a backup restore. Use [external-projection-scope.sh](../../scripts/external-projection-scope.sh) with a PostgreSQL service profile for the migration role. The script never requests source values. Runtime cannot execute its maintenance function. It takes the package advisory lock and checks the exact inspected package, activation, and scope epoch before mutation. -1. Stop dependent acquisition and update or disable the deployment binding first. Record the source withdrawal in deployment configuration or the source authority so it remains effective after a backup restore. +1. Remove or update the deployment binding first, then restart. 2. Run `bash scripts/external-projection-scope.sh inspect breg-migration PROJECTION`. Preserve the returned epoch, packageDigest, and activationId for the next command. A null epoch is passed as `unbound`. -3. Withdraw with `bash scripts/external-projection-scope.sh withdraw breg-migration PROJECTION EPOCH PACKAGE_DIGEST ACTIVATION_UUID`. This disables the scope and erases its retained values and refresh receipts. Runtime startup cannot reactivate it. -4. After reviewing a renewed binding and grant, obtain its validated fingerprints and effective limits with `bregctl generate external-source-facts --runtime-config ABSOLUTE_FILE --output NEW_FILE PROJECT`. This reads deployment configuration without acquiring source data. Use the chosen projection entry's `bindingDigest`, `grantDigest`, and `cacheTtlSeconds`; do not calculate the digests by hand. Inspect again, then run `bash scripts/external-projection-scope.sh rotate breg-migration PROJECTION EPOCH PACKAGE_DIGEST ACTIVATION_UUID BINDING_DIGEST GRANT_DIGEST TTL_SECONDS`. Use the exact digests and effective bounds from the reviewed deployment configuration. Rotation allocates a fresh epoch and removes previous cache contents. A changed package or epoch refuses the operation; inspect and review again instead of retrying old values. -5. Start with the matching reviewed configuration and acquire fresh authorized data. Rotation does not assert coverage. +3. Withdraw with `bash scripts/external-projection-scope.sh withdraw breg-migration PROJECTION EPOCH PACKAGE_DIGEST ACTIVATION_UUID`. This disables the scope and erases its retained values and refresh receipts. Withdrawal purges data; it does not stop acquisition. If the binding is still configured, the next start binds a fresh epoch and acquires again. +4. A renewed binding or grant needs no maintenance. When its contract, operation, allowed fields, or cache lifetime changed, the next start moves the projection to a fresh epoch and leaves the old rows for purge. Otherwise the stored epoch and its cache stay in place. To discard the cache under an unchanged binding, obtain its validated fingerprints and effective limits with `bregctl generate external-source-facts --runtime-config ABSOLUTE_FILE --output NEW_FILE PROJECT`. This reads deployment configuration without acquiring source data. Use the chosen projection entry's `bindingDigest`, `grantDigest`, and `cacheTtlSeconds`; do not calculate the digests by hand. Inspect again, then run `bash scripts/external-projection-scope.sh rotate breg-migration PROJECTION EPOCH PACKAGE_DIGEST ACTIVATION_UUID BINDING_DIGEST GRANT_DIGEST TTL_SECONDS`. Rotation allocates a fresh epoch and removes previous cache contents. A changed package or epoch refuses the operation; inspect and review again instead of retrying old values. +5. Acquire fresh authorized data. Neither a fresh epoch nor rotation asserts coverage. These operations do not change the governed schema or package digest. They mutate only the rebuildable projection cache. A database-only withdrawal cannot be durable across a backup that excludes those rows; the current deployment/source authority is the durable cutoff. @@ -68,7 +74,7 @@ Restore with the registry's ordinary PostgreSQL and instance-claim procedure. Fe Before startup, check the current source grant and deployment binding. A withdrawal stored only in excluded database rows does not survive the restore. Reflect withdrawn source authority in the current deployment configuration or at the source before allowing reacquisition. The absence of cached values prevents old bytes from being served, but it is not permission for a new acquisition. -Startup can bind empty slots using the current reviewed deployment configuration. Indexed queries remain incomplete until authorized refresh covers their eligible roots. Restore does not mark a projection ready or extend any source lifetime. Live reads continue to use their own current source contract and retain no projection copy. +Startup binds each configured projection using the current reviewed deployment configuration. Indexed queries remain incomplete until authorized refresh covers their eligible roots. Restore does not mark a projection ready or extend any source lifetime. Live reads continue to use their own current source contract and retain no projection copy. ## Implementation references From 69f60253331a1fab716fb3e70293b014ac7f17d8 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 11:49:57 +0700 Subject: [PATCH 06/64] test(cli-docs): expect the external-source-facts generate target The bregctl generate target list gained external-source-facts with external views; the published reference test still expected the earlier list. Signed-off-by: Jeremi Joslin --- crates/registry-cli-docs/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/registry-cli-docs/src/lib.rs b/crates/registry-cli-docs/src/lib.rs index 44baf62258..2f14a0e8aa 100644 --- a/crates/registry-cli-docs/src/lib.rs +++ b/crates/registry-cli-docs/src/lib.rs @@ -454,6 +454,7 @@ mod tests { "evidence-source", "sdk", "identity-facts", + "external-source-facts", ] })); for invocation in [ From b2ef3b11ecb71353046bbf3751ec77cd542f4aa7 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 11:53:17 +0700 Subject: [PATCH 07/64] feat(bregctl): add external-projection inspect, withdraw and rotate Operators maintained a projection cache scope through a psql script that took digests, epochs and activation IDs copied by hand. bregctl now reads the runtime configuration, computes the binding and grant digests and the effective cache lifetime as startup does, and acts on one indexed projection under the configured migration role. Withdraw and rotate take the registry lock, check the active package, activation and managed catalog in the changing transaction, and are audited under breg-external-projection-audit/v1. Inspect reads the scope and fixed counters only. The scope script is removed; the recovery drill and the operator guide use the bregctl commands. Signed-off-by: Jeremi Joslin --- .../src/external_projection_maintenance.rs | 765 ++++++++++++++++++ crates/registry-breg/src/lib.rs | 2 + .../src/postgres/external_projection.rs | 8 +- crates/registry-bregctl/src/lib.rs | 172 ++++ crates/registry-bregctl/tests/cli.rs | 11 + .../tests/external_projection.rs | 85 ++ crates/registry-cli-docs/catalog.json | 195 +++++ crates/registry-cli-docs/src/lib.rs | 3 + .../breg/examples/external-views/README.md | 9 +- products/breg/external-projection-storage.md | 10 +- .../breg/loadtest/external-views/recovery.py | 53 +- .../loadtest/external-views/run_candidate.py | 3 +- .../loadtest/external-views/test_runners.py | 29 +- scripts/external-projection-scope.sh | 35 - 14 files changed, 1277 insertions(+), 103 deletions(-) create mode 100644 crates/registry-breg/src/external_projection_maintenance.rs create mode 100644 crates/registry-bregctl/tests/external_projection.rs delete mode 100755 scripts/external-projection-scope.sh diff --git a/crates/registry-breg/src/external_projection_maintenance.rs b/crates/registry-breg/src/external_projection_maintenance.rs new file mode 100644 index 0000000000..e4f6f3638f --- /dev/null +++ b/crates/registry-breg/src/external_projection_maintenance.rs @@ -0,0 +1,765 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Operator-only inspection, withdrawal and rotation of one indexed external +//! projection's cache scope. +//! +//! Deployment configuration is the only cutoff for acquisition: a binding +//! still configured binds again at the next start. Withdrawal deactivates the +//! scope and purges its cached rows; rotation starts a fresh epoch under the +//! currently configured binding, grant and cache lifetime, and purges the old +//! one. Both run under the configured migration authority, the exclusive +//! registry lock and a catalog identity check in one transaction, and both are +//! audited. Inspection reads the scope and fixed counters only. + +use std::collections::BTreeSet; +use std::{path::Path, sync::Arc, time::Duration}; + +use registry_platform_audit::AuditEntry; +use serde_json::{json, Value}; +use thiserror::Error; +use uuid::Uuid; + +use crate::audit::RegistryAudit; +use crate::external_source::ExternalSourceService; +use crate::external_views::ExternalProjectionMode; +use crate::history_maintenance::{lock_registry, HistoryMaintenanceError}; +use crate::model::CompiledRegistry; +use crate::package::PackageError; +use crate::postgres::{ + lock_wait_ended, verify_catalog_identity_for_catalog, verify_migration_role, ConnectionConfig, + ExpectedManagedCatalog, ExpectedRegistryIdentity, RegistryLockKey, SqlIdentifier, +}; +use crate::runtime_config::{load_runtime_config, RuntimeConfig}; + +/// Schema of the request and response entries of one external projection +/// withdrawal or rotation. +pub const EXTERNAL_PROJECTION_AUDIT_SCHEMA: &str = "breg-external-projection-audit/v1"; + +const MAINTENANCE: &str = + "SELECT registry_internal.maintain_external_projection_scope($1,$2,$3,$4,$5,$6,$7,$8)"; + +/// Value-free failures. Projection IDs come from the reviewed package and a +/// binding reason names the failed check, never a configured value. +#[derive(Debug, Error)] +pub enum ExternalProjectionMaintenanceError { + #[error("the external projection maintenance request is invalid")] + InvalidRequest, + #[error("external projection `{0}` is not an indexed projection of the active package")] + UnknownProjection(String), + #[error("external projection `{projection}` has no usable binding: {reason}")] + BindingUnavailable { projection: String, reason: String }, + #[error("external projection `{0}` has no bound scope to withdraw")] + NotBound(String), + #[error("another session held the exclusive migration lock past the lock timeout")] + MigrationLockHeld, + #[error("the active package does not match its configured pin")] + PackagePinMismatch(registry_platform_config::blocks::PackageDigestMismatch), + #[error("external projection maintenance is unavailable")] + Unavailable, +} + +type Result = std::result::Result; +use ExternalProjectionMaintenanceError as Error; + +/// The cache scope a configured binding asks for, as startup computes it. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ConfiguredScope { + pub binding_digest: String, + pub grant_digest: String, + pub cache_ttl_seconds: i64, +} + +/// The scope the loaded bindings ask for `projection`. A projection whose +/// binding failed its own checks, or that has none, is refused with the reason. +pub(crate) fn scope_from_sources( + sources: &ExternalSourceService, + projection: &str, +) -> Result { + let refused = || Error::BindingUnavailable { + projection: projection.to_owned(), + reason: sources + .unavailable_projections() + .get(projection) + .cloned() + .unwrap_or_else(|| "no external source is bound to this projection".to_owned()), + }; + let bound = sources.binding(projection).ok_or_else(refused)?; + let ttl = bound.effective_cache_ttl_seconds().ok_or_else(refused)?; + Ok(ConfiguredScope { + binding_digest: bound.binding_fingerprint().to_owned(), + grant_digest: bound.grant_fingerprint().to_owned(), + cache_ttl_seconds: i64::from(ttl), + }) +} + +/// The scope the runtime configuration's binding asks for `projection`. Pack +/// and public TLS material are read; no credential is resolved and no source +/// is called. +pub fn configured_scope( + registry: &CompiledRegistry, + config: &RuntimeConfig, + projection: &str, +) -> Result { + let resolver = config.secret_resolver().map_err(|_| Error::Unavailable)?; + let sources = + ExternalSourceService::load(registry, config.external_sources(), Arc::new(resolver)) + .map_err(|error| Error::BindingUnavailable { + projection: projection.to_owned(), + reason: error.to_string(), + })?; + scope_from_sources(&sources, projection) +} + +fn indexed_projections(registry: &CompiledRegistry) -> BTreeSet { + registry + .external_views() + .projections + .values() + .filter(|projection| projection.mode == ExternalProjectionMode::Indexed) + .map(|projection| projection.id.to_string()) + .collect() +} + +/// The stored scope row, read to tell a committed change from a rolled-back +/// one. Every withdrawal and rotation moves the generation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct ScopeMark { + epoch: Uuid, + generation: i64, +} + +async fn scope_mark( + client: &impl tokio_postgres::GenericClient, + projection: &str, +) -> Result> { + Ok(client + .query_opt( + "SELECT epoch, generation FROM registry_internal.registry_external_projection_scopes + WHERE projection_id=$1", + &[&projection], + ) + .await + .map_err(|_| Error::Unavailable)? + .map(|row| ScopeMark { + epoch: row.get(0), + generation: row.get(1), + })) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Action { + Withdraw, + Rotate, +} + +impl Action { + fn name(self) -> &'static str { + match self { + Self::Withdraw => "withdraw", + Self::Rotate => "rotate", + } + } + + fn outcome(self) -> &'static str { + match self { + Self::Withdraw => "withdrawn", + Self::Rotate => "rotated", + } + } +} + +/// How a maintenance transaction ended once every statement in it succeeded. +enum Change { + Committed(Value), + Unacknowledged { + scope: Value, + before: Option, + after: Option, + }, +} + +/// Package-bound authority over the indexed external projection scopes. The +/// migration identity, actual target catalog and registry interlock are checked +/// again in the same transaction that changes a scope. +pub struct ExternalProjectionOperatorService { + indexed: BTreeSet, + expected: ExpectedRegistryIdentity, + expected_catalog: ExpectedManagedCatalog, + lock_key: RegistryLockKey, + migration_connection: ConnectionConfig, + migration_role: SqlIdentifier, + runtime_role: SqlIdentifier, + lock_timeout: Duration, + statement_timeout: Duration, + audit: RegistryAudit, +} + +impl ExternalProjectionOperatorService { + pub async fn from_runtime_config(path: &Path) -> Result { + Ok(Self::open(path).await?.0) + } + + /// The service, its configuration and the active package's registry. + async fn open(path: &Path) -> Result<(Self, RuntimeConfig, CompiledRegistry)> { + if !path.is_absolute() { + return Err(Error::InvalidRequest); + } + let config = load_runtime_config(path).map_err(|_| Error::Unavailable)?; + let package = config.load_active_package().map_err(|error| match error { + PackageError::ExpectedDigestMismatch(mismatch) => Error::PackagePinMismatch(mismatch), + _ => Error::Unavailable, + })?; + let pool = config + .runtime_database_connection_config() + .map_err(|_| Error::Unavailable)? + .build_pool() + .map_err(|_| Error::Unavailable)?; + let mut client = pool.get().await.map_err(|_| Error::Unavailable)?; + let startup = crate::startup::prepare_loaded_startup( + package, + config.identity().database_id(), + &mut client, + config.database().roles().migration(), + config.database().roles().runtime(), + ) + .await + .map_err(|_| Error::Unavailable)?; + let registry = startup.package().registry().clone(); + let service = Self { + indexed: indexed_projections(®istry), + expected: startup.expected_identity().clone(), + expected_catalog: startup.expected_catalog().clone(), + lock_key: startup.lock_key(), + migration_connection: config + .migration_database_connection_config() + .map_err(|_| Error::Unavailable)?, + migration_role: config.database().roles().migration().clone(), + runtime_role: config.database().roles().runtime().clone(), + lock_timeout: config.operational_timeouts().migration_lock, + statement_timeout: config.operational_timeouts().migration_statement, + audit: RegistryAudit::open_companion(&config) + .await + .map_err(|_| Error::Unavailable)?, + }; + Ok((service, config, registry)) + } + + #[cfg(feature = "postgres-test")] + #[doc(hidden)] + pub fn new_for_test( + registry: &CompiledRegistry, + expected: ExpectedRegistryIdentity, + migration_connection: ConnectionConfig, + migration_role: SqlIdentifier, + runtime_role: SqlIdentifier, + audit: RegistryAudit, + ) -> Self { + Self { + indexed: indexed_projections(registry), + expected, + expected_catalog: ExpectedManagedCatalog::compiled(registry), + lock_key: RegistryLockKey::derive(registry.registry_id()) + .expect("the registry lock key derives"), + migration_connection, + migration_role, + runtime_role, + lock_timeout: Duration::from_secs(5), + statement_timeout: Duration::from_secs(10), + audit, + } + } + + fn require_indexed(&self, projection: &str) -> Result<()> { + if self.indexed.contains(projection) { + Ok(()) + } else { + Err(Error::UnknownProjection(projection.to_owned())) + } + } + + /// The projection's scope, its fixed refresh outcome counts and its + /// snapshot counts. Nothing is changed and no source value is read. + pub async fn inspect(&self, projection: &str) -> Result { + self.require_indexed(projection)?; + let pool = self + .migration_connection + .build_pool() + .map_err(|_| Error::Unavailable)?; + let mut client = pool.get().await.map_err(|_| Error::Unavailable)?; + let client: &mut tokio_postgres::Client = &mut client; + let transaction = self.open_transaction(client).await?; + transaction + .execute( + "SELECT pg_catalog.pg_advisory_xact_lock_shared($1)", + &[&self.lock_key.get()], + ) + .await + .map_err(lock_failure)?; + self.verify_catalog(&transaction).await?; + let scope = self + .call(&transaction, "inspect", projection, None, None) + .await?; + transaction + .rollback() + .await + .map_err(|_| Error::Unavailable)?; + Ok(scope) + } + + /// Deactivate the projection's scope under its current epoch and purge its + /// cached rows. Acquisition stops only when the binding leaves the + /// deployment configuration; a binding still configured binds again at + /// the next start. + pub async fn withdraw(&self, projection: &str) -> Result { + self.require_indexed(projection)?; + self.audited(Action::Withdraw, projection, None).await + } + + /// Start a fresh epoch under `configured` and purge the projection's + /// cached rows. Earlier epochs are never readable again. + pub async fn rotate(&self, projection: &str, configured: &ConfiguredScope) -> Result { + self.require_indexed(projection)?; + self.audited(Action::Rotate, projection, Some(configured)) + .await + } + + /// The request entry is accepted before the transaction opens, so an + /// audit outage changes nothing. Its response records the outcome the + /// database holds. + async fn audited( + &self, + action: Action, + projection: &str, + configured: Option<&ConfiguredScope>, + ) -> Result { + let correlation = Uuid::new_v4().to_string(); + let record = |phase: &str, outcome: &str| -> Value { + json!({ + "kind": "externalProjectionMaintenance", + "action": action.name(), + "projection": projection, + "phase": phase, + "outcome": outcome, + "activationId": self.expected.activation_id, + "actor": "breg:external-projection-operator", + "correlation": correlation, + }) + }; + let mut attempt = self + .audit + .begin( + AuditEntry::request( + EXTERNAL_PROJECTION_AUDIT_SCHEMA, + correlation.clone(), + record("attempt", "started"), + ), + record("terminal", "unfinished"), + ) + .await + .map_err(|_| Error::Unavailable)?; + let changed = match self + .change_in_transaction(action, projection, configured) + .await + { + Ok(Change::Committed(scope)) => Ok(scope), + // A commit that returned an error may still have committed, so the + // outcome recorded is the one the database holds, read on a fresh + // connection. + Ok(Change::Unacknowledged { + scope, + before, + after, + }) => match self.stored_mark(projection).await { + Some(stored) if stored == after => Ok(scope), + resolved => { + let outcome = if resolved == Some(before) { + "failed" + } else { + "unfinished" + }; + if attempt.respond(record("terminal", outcome)).await.is_err() { + tracing::error!( + "the unacknowledged external projection maintenance's response audit entry was not recorded" + ); + } + return Err(Error::Unavailable); + } + }, + Err(error) => Err(error), + }; + match changed { + Ok(scope) => { + let mut response = record("terminal", action.outcome()); + response["epoch"] = scope["epoch"].clone(); + // The change committed; a refused entry reports the command + // unavailable, and the writer then refuses every later entry. + attempt + .respond(response) + .await + .map_err(|_| Error::Unavailable)?; + Ok(scope) + } + Err(error) => { + if attempt.respond(record("terminal", "failed")).await.is_err() { + tracing::error!( + "the failed external projection maintenance's response audit entry was not recorded" + ); + } + Err(error) + } + } + } + + /// The stored scope row read on a fresh connection, `None` inside when no + /// row exists, and `None` outside when it cannot be read. + async fn stored_mark(&self, projection: &str) -> Option> { + let pool = self.migration_connection.build_pool().ok()?; + let client = pool.get().await.ok()?; + let client: &tokio_postgres::Client = &client; + scope_mark(client, projection).await.ok() + } + + async fn change_in_transaction( + &self, + action: Action, + projection: &str, + configured: Option<&ConfiguredScope>, + ) -> Result { + let pool = self + .migration_connection + .build_pool() + .map_err(|_| Error::Unavailable)?; + let mut client = pool.get().await.map_err(|_| Error::Unavailable)?; + let client: &mut tokio_postgres::Client = &mut client; + let transaction = self.open_transaction(client).await?; + lock_registry(&transaction, self.lock_key) + .await + .map_err(|error| match error { + HistoryMaintenanceError::MigrationLockHeld => Error::MigrationLockHeld, + _ => Error::Unavailable, + })?; + self.verify_catalog(&transaction).await?; + let before = scope_mark(&transaction, projection).await?; + if action == Action::Withdraw && before.is_none() { + return Err(Error::NotBound(projection.to_owned())); + } + let scope = self + .call( + &transaction, + action.name(), + projection, + before.map(|mark| mark.epoch), + configured, + ) + .await?; + let after = scope_mark(&transaction, projection).await?; + if transaction.commit().await.is_err() { + return Ok(Change::Unacknowledged { + scope, + before, + after, + }); + } + Ok(Change::Committed(scope)) + } + + async fn open_transaction<'a>( + &self, + client: &'a mut tokio_postgres::Client, + ) -> Result> { + verify_migration_role(client, &self.migration_role) + .await + .map_err(|_| Error::Unavailable)?; + let transaction = client.transaction().await.map_err(|_| Error::Unavailable)?; + transaction + .query_one( + "SELECT set_config('lock_timeout', $1, true), set_config('statement_timeout', $2, true)", + &[ + &format!("{}ms", self.lock_timeout.as_millis()), + &format!("{}ms", self.statement_timeout.as_millis()), + ], + ) + .await + .map_err(|_| Error::Unavailable)?; + Ok(transaction) + } + + async fn verify_catalog(&self, transaction: &tokio_postgres::Transaction<'_>) -> Result<()> { + verify_catalog_identity_for_catalog( + transaction, + &self.expected, + &self.expected_catalog, + &self.migration_role, + &self.runtime_role, + ) + .await + .map(|_| ()) + .map_err(|_| Error::Unavailable) + } + + /// One call to the migration-only maintenance function, bound to the + /// expected package and activation. The function refuses a change when + /// either, or the epoch, moved. + async fn call( + &self, + transaction: &tokio_postgres::Transaction<'_>, + action: &str, + projection: &str, + epoch: Option, + configured: Option<&ConfiguredScope>, + ) -> Result { + let activation = + Uuid::parse_str(&self.expected.activation_id).map_err(|_| Error::Unavailable)?; + Ok(transaction + .query_one( + MAINTENANCE, + &[ + &action, + &projection, + &epoch, + &self.expected.package_digest, + &activation, + &configured.map(|scope| scope.binding_digest.as_str()), + &configured.map(|scope| scope.grant_digest.as_str()), + &configured.map(|scope| scope.cache_ttl_seconds), + ], + ) + .await + .map_err(lock_failure)? + .get(0)) + } +} + +fn lock_failure(error: tokio_postgres::Error) -> Error { + if lock_wait_ended(&error) { + Error::MigrationLockHeld + } else { + Error::Unavailable + } +} + +/// Inspect one indexed projection's scope. Diagnostics contain no connection +/// or configured values. +pub async fn inspect(path: &Path, projection: &str) -> Result { + ExternalProjectionOperatorService::from_runtime_config(path) + .await? + .inspect(projection) + .await +} + +/// Withdraw one indexed projection's scope and purge its cached rows. +pub async fn withdraw(path: &Path, projection: &str) -> Result { + ExternalProjectionOperatorService::from_runtime_config(path) + .await? + .withdraw(projection) + .await +} + +/// Rotate one indexed projection to a fresh epoch under the binding, grant and +/// cache lifetime its runtime configuration now asks for. +pub async fn rotate(path: &Path, projection: &str) -> Result { + let (service, config, registry) = ExternalProjectionOperatorService::open(path).await?; + service.require_indexed(projection)?; + let configured = configured_scope(®istry, &config, projection)?; + service.rotate(projection, &configured).await +} + +#[cfg(test)] +#[cfg(feature = "postgres-test")] +mod postgres_tests { + use super::*; + use crate::postgres::external_projection::postgres_tests::{two_projection_project, Activated}; + use crate::postgres::external_view_query::tests::compile_project; + use registry_platform_audit::AuditProfile; + + fn audit(fixture: &Activated) -> RegistryAudit { + fixture.database.audit( + AuditProfile::production_from_secret_bytes(vec![0x4e; 32].into()) + .expect("test owns a strong keyed audit profile"), + ) + } + + fn operator( + registry: &CompiledRegistry, + fixture: &Activated, + ) -> ExternalProjectionOperatorService { + ExternalProjectionOperatorService::new_for_test( + registry, + fixture.expected.clone(), + fixture.database.migration_config.clone(), + fixture.database.migration_role.clone(), + fixture.database.runtime_role.clone(), + audit(fixture), + ) + } + + #[test] + fn rotation_takes_the_configured_binding_and_names_a_missing_one() { + let sources = crate::external_source::tests::indexed_service(&[( + "projection", + "binding", + "grant", + 45, + )]); + assert_eq!( + scope_from_sources(&sources, "projection").unwrap(), + ConfiguredScope { + binding_digest: "binding".to_owned(), + grant_digest: "grant".to_owned(), + cache_ttl_seconds: 45, + } + ); + let Err(Error::BindingUnavailable { projection, reason }) = + scope_from_sources(&sources, "other") + else { + panic!("a projection without a binding is refused"); + }; + assert_eq!(projection, "other"); + assert_eq!(reason, "no external source is bound to this projection"); + } + + #[tokio::test] + async fn operators_inspect_withdraw_and_rotate_one_projection_under_audit() { + let registry = compile_project(&two_projection_project()); + let fixture = Activated::new(®istry).await; + let operator = operator(®istry, &fixture); + + let unbound = operator.inspect("projection").await.unwrap(); + assert!(unbound["epoch"].is_null()); + assert_eq!(unbound["active"], false); + assert!(matches!( + operator.inspect("missing").await, + Err(Error::UnknownProjection(id)) if id == "missing" + )); + assert!(matches!( + operator.withdraw("projection").await, + Err(Error::NotBound(id)) if id == "projection" + )); + + let first = fixture + .bind(®istry, "projection", "binding", "grant", 60) + .await + .unwrap(); + let other = fixture + .bind(®istry, "other", "other-binding", "other-grant", 120) + .await + .unwrap(); + fixture.cache(&first).await; + fixture.cache(&other).await; + let inspected = operator.inspect("projection").await.unwrap(); + assert_eq!(inspected["epoch"], first.epoch.to_string()); + assert_eq!(inspected["active"], true); + assert_eq!(inspected["bindingDigest"], "binding"); + assert_eq!(inspected["snapshotCounts"]["fresh"], 1); + assert_eq!(inspected["packageDigest"], fixture.expected.package_digest); + + let withdrawn = operator.withdraw("projection").await.unwrap(); + assert_eq!(withdrawn["epoch"], first.epoch.to_string()); + assert_eq!(withdrawn["active"], false); + assert_eq!(fixture.cached("projection", first.epoch).await, 0); + assert!( + !fixture + .serves(®istry, "projection", "binding", "grant") + .await + ); + assert_eq!(fixture.cached("other", other.epoch).await, 1); + assert!( + fixture + .serves(®istry, "other", "other-binding", "other-grant") + .await + ); + + let configured = ConfiguredScope { + binding_digest: "renewed-binding".to_owned(), + grant_digest: "renewed-grant".to_owned(), + cache_ttl_seconds: 30, + }; + let rotated = operator.rotate("projection", &configured).await.unwrap(); + let epoch = Uuid::parse_str(rotated["epoch"].as_str().unwrap()).unwrap(); + assert_ne!(epoch, first.epoch); + assert_eq!(rotated["active"], true); + assert_eq!(rotated["bindingDigest"], "renewed-binding"); + assert_eq!(rotated["grantDigest"], "renewed-grant"); + assert_eq!(rotated["cacheTtlSeconds"], 30); + assert!( + fixture + .serves(®istry, "projection", "renewed-binding", "renewed-grant") + .await + ); + let restarted = fixture + .bind( + ®istry, + "projection", + "renewed-binding", + "renewed-grant", + 30, + ) + .await + .unwrap(); + assert_eq!(restarted.epoch, epoch, "a restart keeps the rotated epoch"); + assert_eq!(fixture.cached("other", other.epoch).await, 1); + + let responses = fixture + .database + .audit_entries() + .into_iter() + .filter(|entry| { + entry["schema"] == EXTERNAL_PROJECTION_AUDIT_SCHEMA && entry["phase"] == "response" + }) + .map(|entry| { + ( + entry["record"]["action"].as_str().unwrap().to_owned(), + entry["record"]["outcome"].as_str().unwrap().to_owned(), + ) + }) + .collect::>(); + assert_eq!( + responses, + [ + ("withdraw".to_owned(), "failed".to_owned()), + ("withdraw".to_owned(), "withdrawn".to_owned()), + ("rotate".to_owned(), "rotated".to_owned()), + ] + ); + fixture.database.assert_every_audit_request_answered_once(); + fixture.cleanup().await; + } + + #[tokio::test] + async fn the_runtime_role_cannot_operate_a_projection_scope() { + let registry = compile_project(&two_projection_project()); + let fixture = Activated::new(®istry).await; + let bound = fixture + .bind(®istry, "projection", "binding", "grant", 60) + .await + .unwrap(); + let runtime = ExternalProjectionOperatorService::new_for_test( + ®istry, + fixture.expected.clone(), + fixture.database.runtime_config.clone(), + fixture.database.migration_role.clone(), + fixture.database.runtime_role.clone(), + audit(&fixture), + ); + assert!(matches!( + runtime.withdraw("projection").await, + Err(Error::Unavailable) + )); + assert!(matches!( + runtime.inspect("projection").await, + Err(Error::Unavailable) + )); + assert!( + fixture + .serves(®istry, "projection", "binding", "grant") + .await + ); + assert_eq!( + operator(®istry, &fixture) + .inspect("projection") + .await + .unwrap()["epoch"], + bound.epoch.to_string() + ); + fixture.cleanup().await; + } +} diff --git a/crates/registry-breg/src/lib.rs b/crates/registry-breg/src/lib.rs index c3e0894a68..deeee3dec8 100644 --- a/crates/registry-breg/src/lib.rs +++ b/crates/registry-breg/src/lib.rs @@ -70,6 +70,8 @@ pub(crate) mod event_record; pub mod evidence_source; #[cfg(feature = "tooling")] pub mod example_references; +#[cfg(all(feature = "runtime", feature = "tooling"))] +pub mod external_projection_maintenance; #[cfg(feature = "runtime")] pub mod external_source; pub(crate) mod external_view_artifacts; diff --git a/crates/registry-breg/src/postgres/external_projection.rs b/crates/registry-breg/src/postgres/external_projection.rs index 9d1cabef18..632ca47350 100644 --- a/crates/registry-breg/src/postgres/external_projection.rs +++ b/crates/registry-breg/src/postgres/external_projection.rs @@ -2336,7 +2336,7 @@ pub(crate) mod postgres_tests { } /// One startup bind attempt: a fresh epoch in its own transaction. - async fn bind( + pub(crate) async fn bind( &self, registry: &crate::model::CompiledRegistry, projection: &str, @@ -2369,7 +2369,7 @@ pub(crate) mod postgres_tests { } /// Whether a process holding these fingerprints can still serve. - async fn serves( + pub(crate) async fn serves( &self, registry: &crate::model::CompiledRegistry, projection: &str, @@ -2390,7 +2390,7 @@ pub(crate) mod postgres_tests { loaded.is_ok() } - async fn cache(&self, scope: &ProjectionScope) { + pub(crate) async fn cache(&self, scope: &ProjectionScope) { let now: i64 = self .migration .query_one( @@ -2411,7 +2411,7 @@ pub(crate) mod postgres_tests { ); } - async fn cached(&self, projection: &str, epoch: Uuid) -> i64 { + pub(crate) async fn cached(&self, projection: &str, epoch: Uuid) -> i64 { self.migration .query_one( "SELECT count(*) FROM registry_internal.registry_external_projection_snapshots diff --git a/crates/registry-bregctl/src/lib.rs b/crates/registry-bregctl/src/lib.rs index 4ecc52e2fa..95f1dbdbdf 100644 --- a/crates/registry-bregctl/src/lib.rs +++ b/crates/registry-bregctl/src/lib.rs @@ -274,6 +274,8 @@ enum Command { EvidenceRetention(EvidenceRetentionArgs), /// Drop held idempotency responses past their receipt horizon using configured migration authority. IdempotencyRetention(IdempotencyRetentionArgs), + /// Inspect, withdraw, or rotate an indexed external projection's cache scope using configured migration authority. + ExternalProjection(ExternalProjectionArgs), /// Open, close, and list the authorities that bound `import` runs. ImportAuthority(ImportAuthorityArgs), /// Inspect and adopt the claim naming the database the Registry serves from. @@ -444,6 +446,32 @@ struct IdempotencyRetentionEraseArgs { before: String, } +#[derive(Debug, Args)] +struct ExternalProjectionArgs { + #[command(subcommand)] + command: ExternalProjectionCommand, +} + +#[derive(Debug, Subcommand)] +enum ExternalProjectionCommand { + /// Report one projection's scope, refresh outcome counts and snapshot counts; reads no source values. + Inspect(ExternalProjectionTargetArgs), + /// Deactivate the scope and purge its cached rows; a binding still configured binds again at the next start. + Withdraw(ExternalProjectionTargetArgs), + /// Start a fresh epoch under the configured binding, grant and cache lifetime, and purge cached rows. + Rotate(ExternalProjectionTargetArgs), +} + +#[derive(Debug, Args)] +struct ExternalProjectionTargetArgs { + /// Runtime configuration file. + #[arg(long, value_name = "ABSOLUTE_FILE")] + runtime_config: PathBuf, + /// Indexed external projection ID declared by the active package. + #[arg(long, value_name = "ID")] + projection: String, +} + #[derive(Debug, Args)] struct InitArgs { /// New directory that will receive the project closure. @@ -1974,6 +2002,7 @@ enum DiagnosticArtifact { ReviewRecoveryOperation, EvidenceRetentionOperation, IdempotencyRetentionOperation, + ExternalProjectionOperation, ImportAuthority, InstanceClaim, HistoryErasure, @@ -2029,6 +2058,7 @@ enum SuggestedAction { VerifyReviewRecoveryOperation, VerifyEvidenceRetentionOperation, VerifyIdempotencyRetentionOperation, + VerifyExternalProjectionOperation, CorrectImportAuthorityRequest, VerifyImportAuthority, VerifyInstanceClaim, @@ -3231,6 +3261,68 @@ where ), }; } + Command::ExternalProjection(args) => { + use registry_breg::external_projection_maintenance as maintenance; + let (command, lead, target) = match &args.command { + ExternalProjectionCommand::Inspect(target) => ( + "external-projection inspect", + "External projection scope.", + target, + ), + ExternalProjectionCommand::Withdraw(target) => ( + "external-projection withdraw", + "Withdrew the external projection scope and purged its cached rows.", + target, + ), + ExternalProjectionCommand::Rotate(target) => ( + "external-projection rotate", + "Rotated the external projection scope to a fresh epoch.", + target, + ), + }; + let path = target.runtime_config.as_path(); + let projection = target.projection.as_str(); + let outcome = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|_| maintenance::ExternalProjectionMaintenanceError::Unavailable) + .and_then(|runtime| { + runtime.block_on(async { + match &args.command { + ExternalProjectionCommand::Inspect(_) => { + maintenance::inspect(path, projection).await + } + ExternalProjectionCommand::Withdraw(_) => { + maintenance::withdraw(path, projection).await + } + ExternalProjectionCommand::Rotate(_) => { + maintenance::rotate(path, projection).await + } + } + }) + }); + return match outcome { + Ok(scope) => { + let result = if format == OutputFormat::Json { + write_ctl_report( + &mut *stdout, + &json!({"ok":true,"command":command,"scope":scope}), + ) + .map_err(io::Error::other) + .and_then(|()| writeln!(stdout)) + } else { + render_report(lead, &external_projection_pairs(&scope), stdout) + }; + write_result(result, stderr) + } + Err(error) => write_failure( + &external_projection_failure(command, error), + format, + stdout, + stderr, + ), + }; + } Command::ReviewRecovery(args) => { let (command, operation, args) = match args.command { ReviewRecoveryCommand::Resubmit(args) => ( @@ -3724,6 +3816,85 @@ fn idempotency_retention_failure(error: registry_breg::mutation::MutationError) } } +/// The scope fields an operator reads in human output. Counts appear only +/// when inspection returned them. +fn external_projection_pairs(scope: &Value) -> Vec<(&'static str, String)> { + let text = |value: &Value| match value { + Value::String(text) => text.clone(), + Value::Null => "none".to_owned(), + other => other.to_string(), + }; + let mut pairs = vec![ + ("projection", text(&scope["projection"])), + ("epoch", text(&scope["epoch"])), + ("active", text(&scope["active"])), + ("cache TTL seconds", text(&scope["cacheTtlSeconds"])), + ]; + if let Some(counts) = scope.get("snapshotCounts") { + pairs.push(("fresh snapshots", text(&counts["fresh"]))); + pairs.push(("expired snapshots", text(&counts["expired"]))); + pairs.push(("invalidated snapshots", text(&counts["invalidated"]))); + } + pairs +} + +fn external_projection_failure( + command: &'static str, + error: registry_breg::external_projection_maintenance::ExternalProjectionMaintenanceError, +) -> FailureReport { + use registry_breg::external_projection_maintenance::ExternalProjectionMaintenanceError as Error; + let operation = |code: &str, message: &str| { + source_failure( + command, + diagnostic(code, "externalProjection", message), + DiagnosticArtifact::ExternalProjectionOperation, + SuggestedAction::VerifyExternalProjectionOperation, + ) + }; + match error { + Error::MigrationLockHeld => source_failure( + command, + diagnostic( + "external-projection.in-progress", + "database", + "another session held the exclusive migration lock past the lock timeout, so an apply, an adoption, a migration reconcile, or other registry maintenance is in progress. Nothing was changed. Retry once it releases", + ), + DiagnosticArtifact::DatabaseMigration, + SuggestedAction::RetryAfterMigrationLockReleases, + ), + Error::PackagePinMismatch(mismatch) => package_pin_failure( + command, + "external-projection.package.refused", + "package", + &mismatch, + ), + Error::UnknownProjection(_) => operation( + "external-projection.unknown", + "the projection is not an indexed external projection of the active package; name one the package declares", + ), + Error::BindingUnavailable { .. } => source_failure( + command, + diagnostic( + "external-projection.binding.unavailable", + "externalSources", + &format!( + "{error}. Rotation uses the configured binding: correct it, or remove it from the deployment configuration to stop acquisition" + ), + ), + DiagnosticArtifact::RuntimeConfiguration, + SuggestedAction::CorrectRuntimeConfiguration, + ), + Error::NotBound(_) => operation( + "external-projection.not-bound", + &format!("{error}; nothing was changed"), + ), + Error::InvalidRequest | Error::Unavailable => operation( + "external-projection.unavailable", + "Verify the absolute runtime configuration, migration authority and that the active package is ready.", + ), + } +} + fn proposal_retention_failure( command: &'static str, error: ProposalRetentionCliError, @@ -16525,6 +16696,7 @@ mod tests { "review-recovery", "evidence-retention", "idempotency-retention", + "external-projection", "import-authority", "instance-claim", "field-encryption" diff --git a/crates/registry-bregctl/tests/cli.rs b/crates/registry-bregctl/tests/cli.rs index 5c09d886cb..6bd8df78c7 100644 --- a/crates/registry-bregctl/tests/cli.rs +++ b/crates/registry-bregctl/tests/cli.rs @@ -9334,6 +9334,17 @@ fn idempotency_retention_names_both_digests_when_the_active_package_misses_its_p ); } +#[test] +fn external_projection_operations_name_both_digests_when_the_active_package_misses_its_pin() { + for command in ["inspect", "withdraw", "rotate"] { + assert_operator_command_names_both_digests_of_a_package_pin_mismatch( + &["external-projection", command], + &["--projection", "holdings-indexed"], + "external-projection.package.refused", + ); + } +} + #[test] fn canonical_package_tampering_is_refused_without_rendering_package_values() { let fixture = RuntimePackageFixture::production("127.0.0.1:1".parse().unwrap()); diff --git a/crates/registry-bregctl/tests/external_projection.rs b/crates/registry-bregctl/tests/external_projection.rs new file mode 100644 index 0000000000..d2ae8c4697 --- /dev/null +++ b/crates/registry-bregctl/tests/external_projection.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::ffi::OsStr; +use std::io::Write; + +use serde_json::Value; + +#[test] +fn invalid_runtime_configuration_reports_external_projection_recovery_without_values() { + const PATH_CANARY: &str = "external-projection-path-canary"; + const CONFIG_CANARY: &str = "external-projection-config-canary"; + for command in ["inspect", "withdraw", "rotate"] { + let mut config = tempfile::Builder::new() + .prefix(PATH_CANARY) + .tempfile() + .unwrap(); + writeln!(config, "invalid: [{CONFIG_CANARY}").unwrap(); + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let status = registry_bregctl::run_from( + [ + OsStr::new("bregctl"), + OsStr::new("--format"), + OsStr::new("json"), + OsStr::new("external-projection"), + OsStr::new(command), + OsStr::new("--runtime-config"), + config.path().as_os_str(), + OsStr::new("--projection"), + OsStr::new("holdings-indexed"), + ], + &mut stdout, + &mut stderr, + ); + assert_ne!(status, std::process::ExitCode::SUCCESS); + assert!(stderr.is_empty()); + let output = String::from_utf8(stdout).unwrap(); + for value in [PATH_CANARY, CONFIG_CANARY] { + assert!(!output.contains(value)); + } + let report: Value = serde_json::from_str(&output).unwrap(); + assert_eq!(report["ok"], false); + assert_eq!(report["command"], format!("external-projection {command}")); + assert_eq!(report["diagnostics"].as_array().unwrap().len(), 1); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["code"], "external-projection.unavailable"); + assert_eq!(diagnostic["path"], "externalProjection"); + assert_eq!(diagnostic["artifact"], "external-projection-operation"); + assert_eq!( + diagnostic["suggestedAction"], + "verify-external-projection-operation" + ); + } +} + +#[test] +fn a_relative_runtime_configuration_is_refused_before_it_is_read() { + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let status = registry_bregctl::run_from( + [ + "bregctl", + "--format", + "json", + "external-projection", + "inspect", + "--runtime-config", + "relative/runtime.yaml", + "--projection", + "holdings-indexed", + ], + &mut stdout, + &mut stderr, + ); + assert_ne!(status, std::process::ExitCode::SUCCESS); + let output = String::from_utf8(stdout).unwrap(); + assert!(!output.contains("relative/runtime.yaml")); + let report: Value = serde_json::from_str(&output).unwrap(); + assert_eq!(report["ok"], false); + assert_eq!(report["command"], "external-projection inspect"); + assert_eq!( + report["diagnostics"][0]["code"], + "external-projection.unavailable" + ); +} diff --git a/crates/registry-cli-docs/catalog.json b/crates/registry-cli-docs/catalog.json index 074dbf1312..34fbd4c741 100644 --- a/crates/registry-cli-docs/catalog.json +++ b/crates/registry-cli-docs/catalog.json @@ -8038,6 +8038,201 @@ } ] }, + { + "name": "external-projection", + "invocation": "bregctl external-projection", + "about": "Inspect, withdraw, or rotate an indexed external projection's cache scope using configured migration authority", + "long_about": null, + "usage": "bregctl external-projection [OPTIONS] ", + "arguments": [], + "options": [ + { + "display": "--format ", + "description": "Emit the selected command's report in this format", + "always_required": false, + "repeatable": false, + "default_values": [ + "human" + ], + "possible_values": [ + "human", + "json" + ], + "environment": null + }, + { + "display": "-h, --help", + "description": "Print help", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "constraints": [], + "subcommands": [ + { + "name": "inspect", + "invocation": "bregctl external-projection inspect", + "about": "Report one projection's scope, refresh outcome counts and snapshot counts; reads no source values", + "long_about": null, + "usage": "bregctl external-projection inspect [OPTIONS] --runtime-config --projection ", + "arguments": [], + "options": [ + { + "display": "--runtime-config ", + "description": "Runtime configuration file. A symbolic link at any component of this path is refused.", + "always_required": true, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--projection ", + "description": "Indexed external projection ID declared by the active package", + "always_required": true, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--format ", + "description": "Emit the selected command's report in this format", + "always_required": false, + "repeatable": false, + "default_values": [ + "human" + ], + "possible_values": [ + "human", + "json" + ], + "environment": null + }, + { + "display": "-h, --help", + "description": "Print help", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "constraints": [], + "subcommands": [] + }, + { + "name": "withdraw", + "invocation": "bregctl external-projection withdraw", + "about": "Deactivate the scope and purge its cached rows; a binding still configured binds again at the next start", + "long_about": null, + "usage": "bregctl external-projection withdraw [OPTIONS] --runtime-config --projection ", + "arguments": [], + "options": [ + { + "display": "--runtime-config ", + "description": "Runtime configuration file. A symbolic link at any component of this path is refused.", + "always_required": true, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--projection ", + "description": "Indexed external projection ID declared by the active package", + "always_required": true, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--format ", + "description": "Emit the selected command's report in this format", + "always_required": false, + "repeatable": false, + "default_values": [ + "human" + ], + "possible_values": [ + "human", + "json" + ], + "environment": null + }, + { + "display": "-h, --help", + "description": "Print help", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "constraints": [], + "subcommands": [] + }, + { + "name": "rotate", + "invocation": "bregctl external-projection rotate", + "about": "Start a fresh epoch under the configured binding, grant and cache lifetime, and purge cached rows", + "long_about": null, + "usage": "bregctl external-projection rotate [OPTIONS] --runtime-config --projection ", + "arguments": [], + "options": [ + { + "display": "--runtime-config ", + "description": "Runtime configuration file. A symbolic link at any component of this path is refused.", + "always_required": true, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--projection ", + "description": "Indexed external projection ID declared by the active package", + "always_required": true, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + }, + { + "display": "--format ", + "description": "Emit the selected command's report in this format", + "always_required": false, + "repeatable": false, + "default_values": [ + "human" + ], + "possible_values": [ + "human", + "json" + ], + "environment": null + }, + { + "display": "-h, --help", + "description": "Print help", + "always_required": false, + "repeatable": false, + "default_values": [], + "possible_values": [], + "environment": null + } + ], + "constraints": [], + "subcommands": [] + } + ] + }, { "name": "import-authority", "invocation": "bregctl import-authority", diff --git a/crates/registry-cli-docs/src/lib.rs b/crates/registry-cli-docs/src/lib.rs index 2f14a0e8aa..0d59ae65f8 100644 --- a/crates/registry-cli-docs/src/lib.rs +++ b/crates/registry-cli-docs/src/lib.rs @@ -464,6 +464,9 @@ mod tests { "bregctl hook replay", "bregctl hook discard", "bregctl review-recovery retry-application", + "bregctl external-projection inspect", + "bregctl external-projection withdraw", + "bregctl external-projection rotate", ] { assert!(!find_command(&catalog.binaries, invocation).usage.is_empty()); } diff --git a/products/breg/examples/external-views/README.md b/products/breg/examples/external-views/README.md index 2e2f250d25..c9c66bbef4 100644 --- a/products/breg/examples/external-views/README.md +++ b/products/breg/examples/external-views/README.md @@ -123,11 +123,12 @@ stages those three files unchanged and applies the same binding and HTTP checks. Add `--recovery` to the small candidate runner to exercise migration-role -scope maintenance. It uses the staged migration connection and requires -`psql`. The runner pauses the first source response of a paged holdings +scope maintenance with `bregctl external-projection` and the staged runtime +configuration. The runner pauses the first source response of a paged holdings refresh, withdraws the inspected scope, and checks that the in-flight refresh sends no later source requests or restores indexed data. It -rotates the scope using the checked synthetic binding, restarts BReg, +rotates the scope under the configured synthetic binding, checks the +rotated digests against the generated source facts, restarts BReg, and verifies that indexed coverage remains incomplete until an explicit fresh refresh. Owner-only `scope-*.json` and `recovery-*.json` reports record the transition and operation counts. @@ -158,7 +159,7 @@ authorized absence, label reuse, company shareholders, retention refusal, complete snapshot replacement, and interrupted acquisition. It verifies that a retained Person becomes unavailable after source denial, ambiguity or disconnection and that recovery restores coverage. Source call counts check -that indexed and denied reads skip acquisition. Database races, retention +that indexed and denied reads skip acquisition. Database races, expiry purges and scale have separate acceptance checks. diff --git a/products/breg/external-projection-storage.md b/products/breg/external-projection-storage.md index 476eddf8cd..2627dae4d8 100644 --- a/products/breg/external-projection-storage.md +++ b/products/breg/external-projection-storage.md @@ -35,18 +35,18 @@ The bounded purge worker runs every few seconds and deletes snapshots once they Run the named view’s `POST /v1/external-views/{view}/{link}/refresh` with the approved machine profile. Continue with the returned `nextCursor` until the sweep ends. For independent related labels, run `POST /v1/external-views/{view}/{link}/refresh-labels` after the related collection refresh. That machine profile needs explicit refresh grants on both projections under the same purpose. The label sweep selects keys only from authorised roots with fresh complete collections. A changed collection generation requires a fresh sweep. -The refresh report counts processed roots or related rows, published values, reused keys and unavailable results. The migration role’s scope inspection reports receipt counts by outcome and fresh, expired and invalidated snapshot counts. Use an authorised indexed query to test readiness for its local root scope. +The refresh report counts processed roots or related rows, published values, reused keys and unavailable results. `bregctl external-projection inspect` reports receipt counts by outcome and fresh, expired and invalidated snapshot counts. Use an authorised indexed query to test readiness for its local root scope. ## Stop acquisition, withdraw, and renew Deployment configuration is the only cutoff for acquisition. To stop acquiring a projection, remove its binding from the deployment configuration and restart. Its views then answer `source.unavailable`. A binding that stays in the configuration binds again at the next start and acquires fresh data, even after a withdrawal. Record a source withdrawal in deployment configuration or at the source authority so that it also holds after a backup restore. -Use [external-projection-scope.sh](../../scripts/external-projection-scope.sh) with a PostgreSQL service profile for the migration role. The script never requests source values. Runtime cannot execute its maintenance function. It takes the package advisory lock and checks the exact inspected package, activation, and scope epoch before mutation. +Use `bregctl external-projection` with the runtime configuration. It connects with the configured migration role, reads no source values, and takes the registry lock. Runtime cannot execute its maintenance function. Withdraw and rotate check the active package, activation, and managed catalog in the same transaction that changes the scope, and record an audit request and response under `breg-external-projection-audit/v1`. While another session holds the migration lock, they answer `external-projection.in-progress` and change nothing. 1. Remove or update the deployment binding first, then restart. -2. Run `bash scripts/external-projection-scope.sh inspect breg-migration PROJECTION`. Preserve the returned epoch, packageDigest, and activationId for the next command. A null epoch is passed as `unbound`. -3. Withdraw with `bash scripts/external-projection-scope.sh withdraw breg-migration PROJECTION EPOCH PACKAGE_DIGEST ACTIVATION_UUID`. This disables the scope and erases its retained values and refresh receipts. Withdrawal purges data; it does not stop acquisition. If the binding is still configured, the next start binds a fresh epoch and acquires again. -4. A renewed binding or grant needs no maintenance. When its contract, operation, allowed fields, or cache lifetime changed, the next start moves the projection to a fresh epoch and leaves the old rows for purge. Otherwise the stored epoch and its cache stay in place. To discard the cache under an unchanged binding, obtain its validated fingerprints and effective limits with `bregctl generate external-source-facts --runtime-config ABSOLUTE_FILE --output NEW_FILE PROJECT`. This reads deployment configuration without acquiring source data. Use the chosen projection entry's `bindingDigest`, `grantDigest`, and `cacheTtlSeconds`; do not calculate the digests by hand. Inspect again, then run `bash scripts/external-projection-scope.sh rotate breg-migration PROJECTION EPOCH PACKAGE_DIGEST ACTIVATION_UUID BINDING_DIGEST GRANT_DIGEST TTL_SECONDS`. Rotation allocates a fresh epoch and removes previous cache contents. A changed package or epoch refuses the operation; inspect and review again instead of retrying old values. +2. Run `bregctl external-projection inspect --runtime-config ABSOLUTE_FILE --projection PROJECTION`. It reports the scope epoch, whether it is active, its binding and grant digests, its cache lifetime, the active package and activation, and the counts above. A null epoch means the projection has not bound yet. +3. Withdraw with `bregctl external-projection withdraw --runtime-config ABSOLUTE_FILE --projection PROJECTION`. This disables the scope and erases its retained values and refresh receipts. Withdrawal purges data; it does not stop acquisition. If the binding is still configured, the next start binds a fresh epoch and acquires again. +4. A renewed binding or grant needs no maintenance. When its contract, operation, allowed fields, or cache lifetime changed, the next start moves the projection to a fresh epoch and leaves the old rows for purge. Otherwise the stored epoch and its cache stay in place. To discard the cache under an unchanged binding, run `bregctl external-projection rotate --runtime-config ABSOLUTE_FILE --projection PROJECTION`. It computes the binding and grant digests and the effective cache lifetime from the configured binding, as startup does, without acquiring source data. It then allocates a fresh epoch and removes previous cache contents. A projection whose binding is missing or fails its own checks is refused with the reason. 5. Acquire fresh authorized data. Neither a fresh epoch nor rotation asserts coverage. These operations do not change the governed schema or package digest. They mutate only the rebuildable projection cache. A database-only withdrawal cannot be durable across a backup that excludes those rows; the current deployment/source authority is the durable cutoff. diff --git a/products/breg/loadtest/external-views/recovery.py b/products/breg/loadtest/external-views/recovery.py index a3ccdfa007..d96ec3d212 100644 --- a/products/breg/loadtest/external-views/recovery.py +++ b/products/breg/loadtest/external-views/recovery.py @@ -4,11 +4,8 @@ from concurrent.futures import ThreadPoolExecutor import json -import os -from pathlib import Path import re import subprocess -from urllib.parse import unquote, urlsplit import fixture_path # noqa: F401 from journey import JourneyError, expect, records, request @@ -17,19 +14,18 @@ VIEW = "business-holdings-indexed" LINK = "holdings" OPERATION = "holdings-by-business" -OPERATOR = Path(__file__).resolve().parents[4] / "scripts/external-projection-scope.sh" -def operator_report(output): - lines = [line.strip().removesuffix("+").rstrip() for line in output.decode().splitlines()] - expect(lines.count("{") == lines.count("}") == lines.count("COMMIT") == 1 - and "(1 row)" in lines, "the operator did not return one committed scope report") +def operator_report(output, action): try: - report = json.loads("\n".join(lines[lines.index("{"):lines.index("}") + 1])) + report = json.loads(output) except ValueError as error: raise JourneyError("the operator returned an invalid scope report") from error - expect(isinstance(report, dict), "the operator returned an invalid scope report") - return report + expect(isinstance(report, dict) and report.get("ok") is True + and report.get("command") == f"external-projection {action}" + and isinstance(report.get("scope"), dict), + "the operator did not return one successful scope report") + return report["scope"] def verify_hidden(journey): @@ -51,36 +47,24 @@ def verify_holdings(journey): class ScopeRecovery: - def __init__(self, journey, fixture, work, facts, private_write, write_json): + def __init__(self, journey, fixture, work, facts, ctl, runtime, write_json): self.journey = journey self.fixture = fixture self.work = work + self.ctl = ctl + self.runtime = runtime self.write_json = write_json self.fact = next(fact for fact in facts["projections"] if fact["id"] == PROJECTION) - keys = work / "secrets" - # The launcher validates these as loopback, split-role fixture URLs before staging them. - target = urlsplit((keys / "migration-url").read_text(encoding="ascii")) - service = keys / "recovery-pg-service.conf" - private_write(service, ( - f"[external-fixture-migration]\nhost={target.hostname}\nport={target.port}\n" - f"dbname={target.path[1:]}\nuser={unquote(target.username)}\n" - ).encode("ascii")) - self.env = {key: value for key, value in os.environ.items() if not key.startswith("PG")} - self.env.update(PGSERVICEFILE=str(service), PGSSLMODE="verify-full", - PGSSLROOTCERT=str(keys / "postgres-ca"), PGCONNECT_TIMEOUT="3", - PGPASSFILE="/dev/null", LC_ALL="C") - if target.password is not None: - self.env["PGPASSWORD"] = unquote(target.password) - - def operator(self, action, step, current=None, arguments=()): - identity = () if current is None else (current["epoch"], current["packageDigest"], current["activationId"]) + + def operator(self, action, step, current=None): with (self.work / f"scope-{step}.stderr").open("xb") as error: result = subprocess.run( - ["bash", OPERATOR, action, "external-fixture-migration", PROJECTION, *identity, *arguments], - cwd=self.work, env=self.env, stdout=subprocess.PIPE, stderr=error, timeout=20, check=False, + [*self.ctl, "external-projection", action, "--runtime-config", self.runtime, + "--projection", PROJECTION], + cwd=self.work, stdout=subprocess.PIPE, stderr=error, timeout=60, check=False, ) expect(result.returncode == 0, "the migration-only scope operator failed; see the owner-only fixture log") - report = operator_report(result.stdout) + report = operator_report(result.stdout, action) expect(report.get("projection") == PROJECTION and type(report.get("active")) is bool and all(isinstance(report.get(key), str) for key in ("epoch", "packageDigest", "activationId")) and all(re.fullmatch(r"[0-9a-f-]{36}", report[key]) for key in ("epoch", "activationId")) @@ -134,10 +118,7 @@ def withdraw_and_rotate(self): self.rotate(inactive) def rotate(self, inactive): - rotated = self.operator("rotate", "rotation", inactive, ( - self.fact["bindingDigest"], self.fact["grantDigest"], - str(self.fact["cacheTtlSeconds"]), - )) + rotated = self.operator("rotate", "rotation", inactive) expect(rotated["active"] and rotated["epoch"] != inactive["epoch"] and rotated["bindingDigest"] == self.fact["bindingDigest"] and rotated["grantDigest"] == self.fact["grantDigest"] diff --git a/products/breg/loadtest/external-views/run_candidate.py b/products/breg/loadtest/external-views/run_candidate.py index 744478ca51..0f00d8bbe2 100644 --- a/products/breg/loadtest/external-views/run_candidate.py +++ b/products/breg/loadtest/external-views/run_candidate.py @@ -289,7 +289,6 @@ def run(options): else: scale = None if options.recovery: - expect(shutil.which("psql") is not None, "psql is required for the recovery operator") from recovery import ScopeRecovery work = options.work_dir.resolve() work.mkdir(mode=0o700) @@ -372,7 +371,7 @@ def run(options): print("Basic external view HTTP checks passed", flush=True) if options.recovery: print("Withdrawing holdings scope during a paged refresh", flush=True) - recovery = ScopeRecovery(journey, fixture, work, facts, private_write, write_json) + recovery = ScopeRecovery(journey, fixture, work, facts, ctl, runtime, write_json) recovery.withdraw_and_rotate() if options.recovery: print("Restarting and rebuilding the rotated holdings scope", flush=True) diff --git a/products/breg/loadtest/external-views/test_runners.py b/products/breg/loadtest/external-views/test_runners.py index 28652a6b0c..90399d3db3 100644 --- a/products/breg/loadtest/external-views/test_runners.py +++ b/products/breg/loadtest/external-views/test_runners.py @@ -15,24 +15,19 @@ class RecoveryTests(unittest.TestCase): - def test_operator_report_reads_the_psql_json_cell(self): - output = b"""BEGIN -SET -SET - jsonb_pretty ----------------------------------------------------- - { + - "projection": "holdings-indexed", + - "active": false + - } -(1 row) - -COMMIT -""" - self.assertEqual(operator_report(output), {"projection": "holdings-indexed", "active": False}) - for invalid in (output.replace(b"COMMIT", b"ROLLBACK"), b"COMMIT", output + output): + def test_operator_report_reads_the_bregctl_scope(self): + output = (b'{"ok":true,"command":"external-projection withdraw",' + b'"scope":{"projection":"holdings-indexed","active":false}}\n') + self.assertEqual(operator_report(output, "withdraw"), {"projection": "holdings-indexed", "active": False}) + for invalid, action in ( + (output, "rotate"), + (output.replace(b'"ok":true', b'"ok":false'), "withdraw"), + (b'{"ok":true,"command":"external-projection withdraw"}', "withdraw"), + (output + output, "withdraw"), + (b"COMMIT", "withdraw"), + ): with self.assertRaisesRegex(JourneyError, "scope report"): - operator_report(invalid) + operator_report(invalid, action) class CandidateFixtureTests(unittest.TestCase): diff --git a/scripts/external-projection-scope.sh b/scripts/external-projection-scope.sh deleted file mode 100755 index aa4420fcd5..0000000000 --- a/scripts/external-projection-scope.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -usage() { - printf '%s\n' 'Usage: external-projection-scope.sh inspect SERVICE PROJECTION' \ - ' external-projection-scope.sh withdraw SERVICE PROJECTION EPOCH PACKAGE_DIGEST ACTIVATION_UUID' \ - ' external-projection-scope.sh rotate SERVICE PROJECTION EPOCH PACKAGE_DIGEST ACTIVATION_UUID BINDING_DIGEST GRANT_DIGEST TTL_SECONDS' \ - 'Use unbound for EPOCH only when inspect reports a null epoch. SERVICE must use the migration role.' >&2 - exit 2 -} -[[ $# -ge 3 ]] || usage -action=$1 -service=$2 -projection=$3 -[[ "$service" =~ ^[A-Za-z0-9_-]+$ ]] || usage -case "$action" in -inspect) [[ $# -eq 3 ]] || usage ;; -withdraw) [[ $# -eq 6 ]] || usage ;; -rotate) [[ $# -eq 9 ]] || usage ;; -*) usage ;; -esac -psql --no-psqlrc --dbname="service=$service" --set=ON_ERROR_STOP=1 \ - --set=action="$action" --set=projection="$projection" --set=epoch="${4:-unbound}" \ - --set=package="${5:-}" --set=activation="${6:-}" --set=binding="${7:-}" \ - --set=grant="${8:-}" --set=ttl="${9:-}" <<'SQL' -BEGIN; -SET LOCAL lock_timeout='10s'; -SET LOCAL statement_timeout='30s'; -SELECT jsonb_pretty(registry_internal.maintain_external_projection_scope( - :'action', :'projection', NULLIF(:'epoch','unbound')::uuid, - NULLIF(:'package',''), NULLIF(:'activation','')::uuid, - NULLIF(:'binding',''), NULLIF(:'grant',''), - NULLIF(:'ttl','')::bigint)); -COMMIT; -SQL From 5eb4ba9c052fc5a9cd073a3054c40530d85254b4 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 12:43:32 +0700 Subject: [PATCH 08/64] fix(breg): answer incomplete external view coverage with its own problem Incomplete indexed coverage now returns an external_view.incomplete Problem instead of a bare status body. Scope lock and database failures on the read path answer source.unavailable rather than incomplete coverage or an invalid cursor; only a withdrawn or rebound projection keeps the detail fallback and cursor refusal. Signed-off-by: Jeremi Joslin --- crates/registry-breg-client/src/error.rs | 11 +- .../registry-breg/src/api/external_views.rs | 39 ++- .../src/postgres/external_projection.rs | 30 ++ .../src/postgres/external_runtime.rs | 281 +++++++++++++++++- crates/registry-breg/src/problem.rs | 19 +- docs/site/src/data/breg-api.yaml | 1 + .../identifiers/generated/catalog.v1.json | 213 +++++++------ 7 files changed, 475 insertions(+), 119 deletions(-) diff --git a/crates/registry-breg-client/src/error.rs b/crates/registry-breg-client/src/error.rs index d1453fe47d..c5b07cd742 100644 --- a/crates/registry-breg-client/src/error.rs +++ b/crates/registry-breg-client/src/error.rs @@ -143,6 +143,7 @@ pub enum BRegProblemCode { ActionHandlerFailed, ActionRefused, AuthenticationRefused, + ExternalViewIncomplete, IdempotencyConflict, IdempotencyExpired, IngestionChunkMismatch, @@ -175,12 +176,13 @@ pub enum BRegProblemCode { } impl BRegProblemCode { - pub const ALL: [Self; 34] = [ + pub const ALL: [Self; 35] = [ Self::ActionEvidenceFailed, Self::ActionGuardFailed, Self::ActionHandlerFailed, Self::ActionRefused, Self::AuthenticationRefused, + Self::ExternalViewIncomplete, Self::IdempotencyConflict, Self::IdempotencyExpired, Self::IngestionChunkMismatch, @@ -220,6 +222,7 @@ impl BRegProblemCode { Self::ActionHandlerFailed => "action.handler_failed", Self::ActionRefused => "action.refused", Self::AuthenticationRefused => "authentication.refused", + Self::ExternalViewIncomplete => "external_view.incomplete", Self::IdempotencyConflict => "idempotency.key-reused", Self::IdempotencyExpired => "idempotency.expired", Self::IngestionChunkMismatch => "ingestion.chunk_mismatch", @@ -279,7 +282,8 @@ impl BRegProblemCode { Self::ActionRefused | Self::StatisticalDatasetReleaseRefused => 422, Self::PreconditionRequired => 428, Self::StatisticalDatasetDomainViolation => 500, - Self::RuntimeFieldEncryptionUnavailable + Self::ExternalViewIncomplete + | Self::RuntimeFieldEncryptionUnavailable | Self::RuntimeNotReady | Self::ServiceUnavailable | Self::SourceUnavailable => 503, @@ -330,6 +334,9 @@ impl BRegProblemCode { // package-declared label on the wire instead. Self::ActionRefused => "The action was refused by a declared business rule.", Self::AuthenticationRefused => "The bearer credential is missing or refused.", + Self::ExternalViewIncomplete => { + "The external view's indexed coverage is incomplete for this request." + } Self::IdempotencyConflict => "The idempotency key is bound to another request.", Self::IdempotencyExpired => { "The held response of the idempotency key expired; the key stays spent." diff --git a/crates/registry-breg/src/api/external_views.rs b/crates/registry-breg/src/api/external_views.rs index 35cdab042e..90c6c3b3e8 100644 --- a/crates/registry-breg/src/api/external_views.rs +++ b/crates/registry-breg/src/api/external_views.rs @@ -354,11 +354,7 @@ fn external_view_problem(error: ExternalViewError) -> Response { match error { ExternalViewError::Concealed => concealed(), ExternalViewError::InvalidCursor => cursor_invalid(), - ExternalViewError::Incomplete => ( - StatusCode::SERVICE_UNAVAILABLE, - Json(json!({"status": "incomplete"})), - ) - .into_response(), + ExternalViewError::Incomplete => fixed_problem(ProblemCode::ExternalViewIncomplete), ExternalViewError::Unavailable => unavailable(), ExternalViewError::ProjectionUnavailable(projection) => projection_unavailable(&projection), } @@ -528,4 +524,37 @@ mod tests { assert_eq!(generic["code"], "source.unavailable"); assert!(generic.get("fieldPath").is_none()); } + + #[tokio::test] + async fn incomplete_coverage_is_a_fixed_problem_with_its_own_code() { + let response = external_view_problem(ExternalViewError::Incomplete); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + response.headers()[axum::http::header::CONTENT_TYPE], + "application/problem+json" + ); + let problem = body(response).await; + assert_eq!(problem["code"], "external_view.incomplete"); + assert_eq!( + problem["type"], + "https://id.registrystack.org/problems/registry-breg/external_view/incomplete" + ); + assert_eq!(problem["title"], "Service Unavailable"); + assert_eq!(problem["status"], 503); + assert_eq!( + problem["detail"], + "The external view's indexed coverage is incomplete for this request." + ); + let members = problem + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect::>(); + // The correlation layer adds traceId on the served route. + assert_eq!( + members, + ["code", "detail", "status", "title", "type"].into() + ); + } } diff --git a/crates/registry-breg/src/postgres/external_projection.rs b/crates/registry-breg/src/postgres/external_projection.rs index 632ca47350..1d4cc7c390 100644 --- a/crates/registry-breg/src/postgres/external_projection.rs +++ b/crates/registry-breg/src/postgres/external_projection.rs @@ -2411,6 +2411,36 @@ pub(crate) mod postgres_tests { ); } + /// Publish one exact-key label under migration authority at the + /// database clock, without a root eligibility recheck. + pub(crate) async fn publish_label(&self, scope: &ProjectionScope, key: &str, label: &str) { + let now: i64 = self + .migration + .query_one( + "SELECT floor(extract(epoch FROM clock_timestamp()))::bigint", + &[], + ) + .await + .unwrap() + .get(0); + let ticket = allocate_inner(&self.migration, scope, key).await.unwrap(); + let snapshot = AcceptedSnapshot::one( + key, + &format!("{key}-{label}"), + now, + now + scope.cache_ttl_seconds, + RetentionPermission::PermittedUntil(now + scope.cache_ttl_seconds), + record(key, label), + ) + .unwrap(); + assert_eq!( + publish_inner(&self.migration, &ticket, &snapshot) + .await + .unwrap(), + Publication::Published + ); + } + pub(crate) async fn cached(&self, projection: &str, epoch: Uuid) -> i64 { self.migration .query_one( diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index 1c39046579..122ef22317 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -587,7 +587,10 @@ impl PostgresExternalViewService { .await { Ok(scope) => scope, - Err(_) + // A withdrawn or rebound projection still answers a plain detail + // read with its root, as unavailable. Database and lock failures + // are never reported as incomplete coverage or a bad cursor. + Err(ExternalViewError::ProjectionUnavailable(_)) if request.root.is_some() && matches!(&request.query, ExternalQueryMode::First(query) if query.equality.is_none() @@ -612,10 +615,12 @@ impl PostgresExternalViewService { expires_at: i64::MAX, }); } - Err(_) if matches!(&request.query, ExternalQueryMode::Continue(_)) => { + Err(ExternalViewError::ProjectionUnavailable(_)) + if matches!(&request.query, ExternalQueryMode::Continue(_)) => + { return Err(ExternalViewError::InvalidCursor); } - Err(_) => return Err(ExternalViewError::Incomplete), + Err(error) => return Err(error), }; let grant = plan.read_grant()?; let label = if grant.label_fields.is_empty() { @@ -627,13 +632,19 @@ impl PostgresExternalViewService { .as_ref() .ok_or(ExternalViewError::Unavailable)?; match self.sources.binding(label.projection.as_str()) { - Some(source) => self - .lock_scope(&transaction, source, label.projection.as_str()) - .await - .ok() - .map(|(scope, generation)| { - (scope, generation, source.grant_valid_until().timestamp()) - }), + Some(source) => { + match self + .lock_scope(&transaction, source, label.projection.as_str()) + .await + { + Ok((scope, generation)) => { + Some((scope, generation, source.grant_valid_until().timestamp())) + } + // Labels from an unavailable projection read as unavailable. + Err(ExternalViewError::ProjectionUnavailable(_)) => None, + Err(error) => return Err(error), + } + } None => None, } }; @@ -776,7 +787,12 @@ impl PostgresExternalViewService { let scope = active_scope(transaction, source, projection).await?; let generation = store::lock_generation(transaction, &scope) .await - .map_err(|_| ExternalViewError::Unavailable)?; + .map_err(|error| match error { + store::ProjectionError::ScopeChanged => { + ExternalViewError::ProjectionUnavailable(projection.to_owned()) + } + _ => ExternalViewError::Unavailable, + })?; Ok((scope, generation)) } @@ -1812,14 +1828,25 @@ mod postgres_tests { use crate::postgres::external_projection::postgres_tests::{two_projection_project, Activated}; fn service(registry: &CompiledRegistry, fixture: &Activated) -> PostgresExternalViewService { + service_with(registry, fixture, Duration::from_secs(2)) + } + + fn service_with( + registry: &CompiledRegistry, + fixture: &Activated, + lock_timeout: Duration, + ) -> PostgresExternalViewService { PostgresExternalViewService::new( fixture.pool.clone(), Arc::new(registry.clone()), fixture.expected.clone(), RegistryLockKey::derive(registry.registry_id()).unwrap(), - Duration::from_secs(2), + lock_timeout, crate::audit::test_support::capturing( - registry_platform_audit::AuditProfile::unkeyed_dev_only(), + registry_platform_audit::AuditProfile::production_from_secret_bytes( + vec![0x81; 32].into(), + ) + .unwrap(), ) .0, Arc::new( @@ -1925,4 +1952,232 @@ mod postgres_tests { ); fixture.cleanup().await; } + + const ROOTS: [(&str, &str, &str); 4] = [ + ("00000000-0000-4000-8000-000000000001", "first", "allowed"), + ("00000000-0000-4000-8000-000000000002", "second", "allowed"), + ("00000000-0000-4000-8000-000000000003", "third", "allowed"), + ("00000000-0000-4000-8000-000000000004", "hidden", "other"), + ]; + + /// A bound, keyed runtime behind the in-process HTTP router. Three roots + /// sit in the `allowed` tenant and one outside it. + struct Served { + fixture: Activated, + router: axum::Router, + scope: ProjectionScope, + } + + impl Served { + async fn new(lock_timeout: Duration) -> Self { + let registry = Arc::new(super::super::external_view_query::tests::compile_project( + &two_projection_project(), + )); + let fixture = Activated::new(®istry).await; + let mut service = service_with(®istry, &fixture, lock_timeout); + service.bind_configured_scopes(true).await.ok().unwrap(); + let row = fixture + .migration + .query_one( + "SELECT epoch,binding_digest,grant_digest,cache_ttl_seconds + FROM registry_internal.registry_external_projection_scopes + WHERE projection_id='projection'", + &[], + ) + .await + .unwrap(); + let scope = ProjectionScope::new( + "projection", + row.get(0), + row.get::<_, String>(1).as_str(), + row.get::<_, String>(2).as_str(), + row.get(3), + ) + .unwrap(); + let entity = ®istry.entities()["local-record"]; + let column = |id: &str| { + entity + .stored_fields + .iter() + .find(|field| field.logical.id == id) + .unwrap() + .physical_name + .clone() + }; + let values = ROOTS + .iter() + .map(|(id, key, tenant)| format!("('{id}','fixture','{key}','{tenant}')")) + .collect::>() + .join(","); + fixture + .database + .admin + .batch_execute(&format!( + "INSERT INTO registry_data.\"{}\" (record_id,active_package_revision,\"{}\",\"{}\") VALUES {values}", + entity.physical_table, + column("subject-key"), + column("tenant"), + )) + .await + .unwrap(); + let http = crate::api::HttpService::new( + Arc::clone(®istry), + crate::api::ReadRuntimeIdentity { + package_revision: "package-revision".to_owned(), + schema_fingerprint: "schema-fingerprint".to_owned(), + }, + Arc::new(NoRecords), + Arc::new(Ready), + Arc::new( + CursorCodec::new( + zeroize::Zeroizing::new(vec![9; 32]), + Duration::from_secs(60), + ) + .unwrap(), + ), + ) + .with_external_views(Arc::new(service)); + Self { + router: crate::api::router(Arc::new(http)), + fixture, + scope, + } + } + + async fn publish_all(&self) { + for (_, key, _) in ROOTS { + self.fixture.publish_label(&self.scope, key, key).await; + } + } + + async fn get( + &self, + query: &str, + claims: crate::api::VerifiedRequestClaims, + ) -> (axum::http::StatusCode, Value) { + use tower::ServiceExt; + let mut request = axum::http::Request::builder() + .uri(format!( + "/v1/external-views/view/link/records?accessProfile=reader{query}" + )) + .body(axum::body::Body::empty()) + .unwrap(); + request.extensions_mut().insert(claims); + let response = self.router.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap(); + (status, serde_json::from_slice(&bytes).unwrap()) + } + } + + fn reader(principal: &str, tenants: &[&str]) -> crate::api::VerifiedRequestClaims { + crate::api::VerifiedRequestClaims::authenticated( + "registry_principal", + principal, + BTreeSet::from(["registry.read".to_owned()]), + Some("consultation".to_owned()), + BTreeMap::from([( + "tenant".to_owned(), + crate::api::VerifiedClaimValue::direct_string_set(tenants.iter().copied()).unwrap(), + )]), + ) + .unwrap() + } + + fn labels(body: &Value) -> Vec { + body["records"] + .as_array() + .unwrap() + .iter() + .map(|record| record["fields"]["label"].as_str().unwrap().to_owned()) + .collect() + } + + struct Ready; + impl crate::api::ReadinessProbe for Ready { + fn is_ready(&self) -> ServiceFuture<'_, bool> { + Box::pin(async { true }) + } + } + + struct NoRecords; + impl crate::api::RecordReadService for NoRecords { + fn get( + &self, + _: crate::api::RecordReadRequest, + ) -> ServiceFuture< + '_, + std::result::Result, crate::api::ReadServiceError>, + > { + Box::pin(async { Ok(None) }) + } + fn list( + &self, + _: crate::api::RecordReadRequest, + ) -> ServiceFuture< + '_, + std::result::Result, + > { + Box::pin(async { Err(crate::api::ReadServiceError::Unavailable) }) + } + fn lookup( + &self, + _: crate::api::RecordReadRequest, + ) -> ServiceFuture< + '_, + std::result::Result, crate::api::ReadServiceError>, + > { + Box::pin(async { Ok(None) }) + } + fn refusal( + &self, + _: crate::api::RecordReadRefusal, + ) -> ServiceFuture<'_, std::result::Result<(), crate::api::ReadServiceError>> { + Box::pin(async { Ok(()) }) + } + } + + #[tokio::test] + async fn a_scope_lock_timeout_is_unavailable_never_incomplete_or_an_invalid_cursor() { + let served = Served::new(Duration::from_millis(300)).await; + served.publish_all().await; + let caller = reader("reader-principal", &["allowed"]); + let (status, first) = served.get("&$top=1", caller.clone()).await; + assert_eq!(status, axum::http::StatusCode::OK, "{first}"); + let cursor = first["nextCursor"].as_str().unwrap().to_owned(); + + // Another session holds the scope row, as a long publication would. + served + .fixture + .database + .admin + .batch_execute( + "BEGIN; SELECT 1 FROM registry_internal.registry_external_projection_scopes + WHERE projection_id='projection' FOR UPDATE", + ) + .await + .unwrap(); + let (status, problem) = served.get("&$top=1", caller.clone()).await; + assert_eq!(status, axum::http::StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(problem["code"], "source.unavailable", "{problem}"); + let (status, problem) = served + .get(&format!("&$skiptoken={cursor}"), caller.clone()) + .await; + assert_eq!(status, axum::http::StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(problem["code"], "source.unavailable", "{problem}"); + served + .fixture + .database + .admin + .batch_execute("COMMIT") + .await + .unwrap(); + + let (status, next) = served.get(&format!("&$skiptoken={cursor}"), caller).await; + assert_eq!(status, axum::http::StatusCode::OK, "{next}"); + assert_eq!(labels(&next).len(), 1); + served.fixture.cleanup().await; + } } diff --git a/crates/registry-breg/src/problem.rs b/crates/registry-breg/src/problem.rs index 32911505fa..8e06850d76 100644 --- a/crates/registry-breg/src/problem.rs +++ b/crates/registry-breg/src/problem.rs @@ -27,6 +27,7 @@ pub enum ProblemCode { ActionHandlerFailed, ActionRefused, AuthenticationRefused, + ExternalViewIncomplete, IdempotencyConflict, IdempotencyExpired, IngestionChunkMismatch, @@ -66,6 +67,7 @@ impl ProblemCode { Self::ActionHandlerFailed, Self::ActionRefused, Self::AuthenticationRefused, + Self::ExternalViewIncomplete, Self::IdempotencyExpired, Self::IdempotencyConflict, Self::IngestionChunkMismatch, @@ -100,6 +102,8 @@ impl ProblemCode { /// The codes a documented HTTP operation can answer with. The readiness /// probe is an operational route with no documented operation, so its code /// is registered and published but never listed in a generated document. + /// External view routes are not package operations either, so their + /// incomplete-coverage code is published in the catalogue only. pub const DOCUMENTED: &'static [Self] = &[ Self::ActionEvidenceFailed, Self::ActionGuardFailed, @@ -144,6 +148,7 @@ impl ProblemCode { Self::ActionHandlerFailed => "action.handler_failed", Self::ActionRefused => "action.refused", Self::AuthenticationRefused => "authentication.refused", + Self::ExternalViewIncomplete => "external_view.incomplete", Self::IdempotencyConflict => "idempotency.key-reused", Self::IdempotencyExpired => "idempotency.expired", Self::IngestionChunkMismatch => "ingestion.chunk_mismatch", @@ -203,6 +208,7 @@ impl ProblemCode { Self::PreconditionRequired => 428, Self::ActionHandlerFailed | Self::StatisticalDatasetDomainViolation => 500, Self::ActionEvidenceFailed + | Self::ExternalViewIncomplete | Self::RuntimeFieldEncryptionUnavailable | Self::RuntimeNotReady | Self::ServiceUnavailable @@ -244,6 +250,9 @@ impl ProblemCode { Self::ActionHandlerFailed => "The action handler could not produce an accepted result.", Self::ActionRefused => "The action was refused by a declared business rule.", Self::AuthenticationRefused => "The bearer credential is missing or refused.", + Self::ExternalViewIncomplete => { + "The external view's indexed coverage is incomplete for this request." + } Self::IdempotencyConflict => "The idempotency key is bound to another request.", Self::IdempotencyExpired => { "The held response of the idempotency key expired; the key stays spent." @@ -367,13 +376,19 @@ mod tests { } #[test] - fn the_readiness_probe_is_the_only_undocumented_code() { + fn only_routes_outside_package_operations_have_undocumented_codes() { let undocumented: Vec<_> = ProblemCode::ALL .iter() .filter(|code| !ProblemCode::DOCUMENTED.contains(code)) .copied() .collect(); - assert_eq!(undocumented, vec![ProblemCode::RuntimeNotReady]); + assert_eq!( + undocumented, + vec![ + ProblemCode::ExternalViewIncomplete, + ProblemCode::RuntimeNotReady + ] + ); } #[test] diff --git a/docs/site/src/data/breg-api.yaml b/docs/site/src/data/breg-api.yaml index 01fb5ec771..4f5e40573c 100644 --- a/docs/site/src/data/breg-api.yaml +++ b/docs/site/src/data/breg-api.yaml @@ -98,6 +98,7 @@ - ['`source.unavailable`', '503', 'The database refused or timed out while the request held it.'] - ['`service.unavailable`', '503', 'The runtime could not complete the mutation for a reason other than the request itself.'] - ['`runtime.not_ready`', '503', 'The readiness probe before the package and database are usable.'] + - ['`external_view.incomplete`', '503', 'An indexed external view does not hold a fresh snapshot for every eligible root the request covers, so it cannot answer a filter, an order, a count, a collection, or a continuation. A projection refresh restores coverage.'] - ['`runtime.field_encryption.unavailable`', '503', 'The route reaches a Record type with encrypted fields while the field-encryption key state is not usable. Record types without encrypted fields keep serving.'] - ['`statistical_dataset.domain_violation`', '500', 'A non-null source value falls outside a declared dimension domain; the value is not disclosed.'] - ['`statistical_dataset.release_refused`', '422', 'The period or lifecycle state is ineligible; `refusalCode` names a closed reason.'] diff --git a/products/identifiers/generated/catalog.v1.json b/products/identifiers/generated/catalog.v1.json index b148c71780..598c91119c 100644 --- a/products/identifiers/generated/catalog.v1.json +++ b/products/identifiers/generated/catalog.v1.json @@ -61,7 +61,7 @@ "description": "The declared Evidence dependency could not be accepted.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "action.evidence_failed", @@ -80,7 +80,7 @@ "description": "The action guard was not satisfied.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "action.guard_failed", @@ -99,7 +99,7 @@ "description": "The action handler could not produce an accepted result.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "action.handler_failed", @@ -118,7 +118,7 @@ "description": "The action was refused by a declared business rule.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "action.refused", @@ -137,7 +137,7 @@ "description": "The bearer credential is missing or refused.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "authentication.refused", @@ -146,6 +146,25 @@ ] } }, + { + "uri": "https://id.registrystack.org/problems/registry-breg/external_view/incomplete", + "kind": "problem", + "status": "active", + "compatibilityLine": "v1alpha1", + "owner": "breg", + "title": "Service Unavailable", + "description": "The external view's indexed coverage is incomplete for this request.", + "source": { + "path": "crates/registry-breg/src/problem.rs", + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" + }, + "problem": { + "code": "external_view.incomplete", + "httpStatuses": [ + 503 + ] + } + }, { "uri": "https://id.registrystack.org/problems/registry-breg/idempotency/expired", "kind": "problem", @@ -156,7 +175,7 @@ "description": "The held response of the idempotency key expired; the key stays spent.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "idempotency.expired", @@ -175,7 +194,7 @@ "description": "The idempotency key is bound to another request.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "idempotency.key-reused", @@ -194,7 +213,7 @@ "description": "The chunk does not match the expected next chunk.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "ingestion.chunk_mismatch", @@ -213,7 +232,7 @@ "description": "The selected access profile does not match the run's bound profile.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "ingestion.profile_mismatch", @@ -232,7 +251,7 @@ "description": "The stored receipt of the chunk was erased.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "ingestion.receipt_erased", @@ -251,7 +270,7 @@ "description": "The ingestion run is blocked and refuses further chunks.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "ingestion.run_blocked", @@ -270,7 +289,7 @@ "description": "The ingestion run is not open for this transition.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "ingestion.run_not_open", @@ -289,7 +308,7 @@ "description": "The lookup did not resolve exactly one record.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "lookup.unresolved", @@ -308,7 +327,7 @@ "description": "The mutation conflicts with current state.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "mutation.conflict", @@ -327,7 +346,7 @@ "description": "The mutation precondition failed.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "precondition.failed", @@ -346,7 +365,7 @@ "description": "The mutation precondition is required.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "precondition.required", @@ -365,7 +384,7 @@ "description": "The review rejected this Proposal, and a rejection is final. A review result of changes-requested is what lets a Proposal be revised for another attempt.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "proposal.rejected", @@ -384,7 +403,7 @@ "description": "The current review result does not match the retained approval.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "proposal.review_result_mismatch", @@ -403,7 +422,7 @@ "description": "The review result and application receipt of the submission are no longer retained.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "proposal.submission_expired", @@ -422,7 +441,7 @@ "description": "The query cursor is invalid.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "query.cursor_invalid", @@ -441,7 +460,7 @@ "description": "The query request is invalid.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "query.invalid", @@ -460,7 +479,7 @@ "description": "The request is invalid.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "request.invalid", @@ -479,7 +498,7 @@ "description": "The request timed out.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "request.timeout", @@ -498,7 +517,7 @@ "description": "The requested resource was not found.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "resource.not_found", @@ -517,7 +536,7 @@ "description": "The Registry field-encryption service is unavailable.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "runtime.field_encryption.unavailable", @@ -536,7 +555,7 @@ "description": "Registry runtime is not ready.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "runtime.not_ready", @@ -555,7 +574,7 @@ "description": "The Registry mutation service is unavailable.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "service.unavailable", @@ -574,7 +593,7 @@ "description": "The Registry data service is unavailable.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "source.unavailable", @@ -593,7 +612,7 @@ "description": "A statistical dataset contains a code outside its declared domain.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "statistical_dataset.domain_violation", @@ -612,7 +631,7 @@ "description": "The statistical dataset release operation is not eligible.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "statistical_dataset.release_refused", @@ -631,7 +650,7 @@ "description": "The statistical dataset computation was superseded or its package changed.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "statistical_dataset.version_conflict", @@ -650,7 +669,7 @@ "description": "The statistical dataset version was withdrawn.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "statistical_dataset.version_withdrawn", @@ -669,7 +688,7 @@ "description": "The request media type is not supported.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "unsupported.media_type", @@ -688,7 +707,7 @@ "description": "The resulting record falls outside the selected access profile's row boundary.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "ea3600e7289468a622df039523b4a9c6c52829ed8bec6bf3cc0ab8026ce91a20" + "sha256": "0d852a01e6a4f71e84e4766721859af45e4aa9c114012b27a7cd4faefd5e8052" }, "problem": { "code": "write.outside-boundary", @@ -1410,7 +1429,7 @@ "description": "Starts stay held after a restore until an operator releases the admission hold.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "admission.held", @@ -1429,7 +1448,7 @@ "description": "This instance runs in recovery-only mode and admits no runs. Send starts to the ordinary service.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "admission.recovery-only", @@ -1448,7 +1467,7 @@ "description": "The operation may have completed, but its audited result could not be released. Inspect the run before repeating the operation.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "audit.response-unavailable", @@ -1467,7 +1486,7 @@ "description": "The bearer credential is missing, invalid, or expired.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "authentication.refused", @@ -1486,7 +1505,7 @@ "description": "The run's original connection settings differ from this service's runtime configuration.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "binding.conflict", @@ -1505,7 +1524,7 @@ "description": "Connection settings that runs in progress use differ from this service's runtime configuration.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "binding.live-conflict", @@ -1524,7 +1543,7 @@ "description": "That step is not waiting.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "check.not-waiting", @@ -1543,7 +1562,7 @@ "description": "The workflow's workflow definition, one of its mappings or one of its functions refused the request.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "definition.refused", @@ -1562,7 +1581,7 @@ "description": "An apply changed the active packages after this service started. Restart it with the runtime file that apply used.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "deployment.activation-changed", @@ -1581,7 +1600,7 @@ "description": "The deployment's activation, runtime configuration or state key custody check failed.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "deployment.refused", @@ -1600,7 +1619,7 @@ "description": "An evaluation may have completed at its provider, so the run is never evaluated again.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "evaluation.uncertain", @@ -1619,7 +1638,7 @@ "description": "The signed event must satisfy the accepted canonical envelope and typed event contract.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "event.invalid", @@ -1638,7 +1657,7 @@ "description": "Restore the configured event source signing key and its enabled secret provider before retrying delivery.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "event.source-unavailable", @@ -1657,7 +1676,7 @@ "description": "The signed event exceeds the event value profile: 131072 canonical bytes, 32 levels of nesting, 4096 values, 256 items per array, 256 members per object, and 16384 bytes per string. Have the producer project fewer or smaller fields into the event.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "event.too-large", @@ -1676,7 +1695,7 @@ "description": "Send a fresh signed delivery from the exact configured source to its accepted trigger.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "event.unauthenticated", @@ -1695,7 +1714,7 @@ "description": "Execution is under restore hold until an operator releases it.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "execution.held", @@ -1714,7 +1733,7 @@ "description": "Supply an Idempotency-Key header of 1 to 256 bytes.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "idempotency.key-invalid", @@ -1733,7 +1752,7 @@ "description": "This idempotency key already started a run with different input.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "idempotency.key-reused", @@ -1752,7 +1771,7 @@ "description": "The start input does not satisfy the workflow's declared input schema and value bounds.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "input.invalid", @@ -1771,7 +1790,7 @@ "description": "An unkeyed mutation may have completed at its provider, so the run never sends it again.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "mutation.uncertain", @@ -1790,7 +1809,7 @@ "description": "The caller's policy does not allow this operation.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "operation.not-authorized", @@ -1809,7 +1828,7 @@ "description": "The original command's receipt can no longer be recovered. This is not evidence that the command was not sent.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "receipt.expired", @@ -1828,7 +1847,7 @@ "description": "The run's progress changed while it was being reconciled. Inspect it again.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "reconcile.raced", @@ -1847,7 +1866,7 @@ "description": "Reconcile applies only to a held mutation with a prepared command, its original binding and no active lease.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "reconcile.refused", @@ -1866,7 +1885,7 @@ "description": "The held operation has no original-result lookup.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "reconcile.unsupported", @@ -1885,7 +1904,7 @@ "description": "The request body exceeds the accepted size: 65536 bytes for a JSON request, or 1048576 bytes for a signed event delivery.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "request.body-too-large", @@ -1904,7 +1923,7 @@ "description": "The request could not be read as a Coordinator request.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "request.invalid", @@ -1923,7 +1942,7 @@ "description": "The list limit must be between 1 and 100.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "request.limit-out-of-range", @@ -1942,7 +1961,7 @@ "description": "The route exists but not for this method.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "request.method-not-allowed", @@ -1961,7 +1980,7 @@ "description": "The requested route does not exist.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "request.not-found", @@ -1980,7 +1999,7 @@ "description": "The reason or recovery reference must be a bounded reference without personal data or control characters.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "request.reason-invalid", @@ -1999,7 +2018,7 @@ "description": "The retention cutoff must be in the past and the limit between 1 and 100.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "request.retention-invalid", @@ -2018,7 +2037,7 @@ "description": "The request body could not be processed.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "request.unprocessable", @@ -2037,7 +2056,7 @@ "description": "The request body is not JSON.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "request.unsupported-media-type", @@ -2056,7 +2075,7 @@ "description": "This recovery action requires execution to be under restore hold.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "restore.hold-required", @@ -2075,7 +2094,7 @@ "description": "Held work no longer matches its protected definition or prepared command.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "restore.lease-incompatible", @@ -2094,7 +2113,7 @@ "description": "Execution history must be complete and the prior deployment fenced before recovery is reviewed.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "restore.review-required", @@ -2113,7 +2132,7 @@ "description": "Held work remains unresolved, so the restore hold cannot be released.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "restore.unresolved", @@ -2132,7 +2151,7 @@ "description": "Retry-same applies only to held work before its original deadline.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "retry.refused", @@ -2151,7 +2170,7 @@ "description": "This service's Coordinator version cannot interpret the run's pinned definition.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "run.definition-incompatible", @@ -2170,7 +2189,7 @@ "description": "No run with this identifier is visible to the caller.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "run.not-visible", @@ -2189,7 +2208,7 @@ "description": "Retention erased this run's payloads, so it can no longer be retried or reconciled.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "run.payload-erased", @@ -2208,7 +2227,7 @@ "description": "Coordinator could not complete the request.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "runtime.failure", @@ -2227,7 +2246,7 @@ "description": "A dependency of the Coordinator, such as its database, audit stream or token verifier, is unavailable. Try again after the service recovers.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "service.unavailable", @@ -2246,7 +2265,7 @@ "description": "This deployment does not serve the named workflow.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "workflow.not-served", @@ -2265,7 +2284,7 @@ "description": "The expected workflow revision differs from the active revision or the original admission.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" + "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" }, "problem": { "code": "workflow.revision-mismatch", @@ -4599,7 +4618,7 @@ }, "artifact": { "path": "products/adapter/schemas/pack.v1alpha1.schema.json", - "sha256": "a8d908e534dffcbeee5b1e3352007a4205e0a4614c64a24fd22130f59f0ff264", + "sha256": "413643debf89c216d445df79d8d5c74f585ce84fc25650c11e8e28355c1ba373", "mediaType": "application/schema+json" } }, @@ -4721,7 +4740,7 @@ "description": "Base Registry Engine portable Action contract export schema, reproduced from the canonical compiled project.", "source": { "path": "crates/registry-breg/src/artifacts.rs", - "sha256": "8e7e1c82cc1ddb3bc1d41866c812c2ff7d0007d0b57b83422260681ff549e795" + "sha256": "1e95863f83e08914cb08a8b382ecf3c1b0b9a55fe25e00cb96fbd3b863e40cd1" }, "artifact": { "path": "products/breg/generated/facility/generated/action-contracts.schema.json", @@ -4905,7 +4924,7 @@ }, "artifact": { "path": "products/breg/generated/tools/migration-descriptor.v1alpha1.schema.json", - "sha256": "a21e02c4ca6980e0eb4dc733b99b1fdb465aec8fa1c661ec70e86d71d4146e06", + "sha256": "1f9fb0ab8696b817e755116bcd629c06018729ecf64c538ab60ed04c54a9726c", "mediaType": "application/schema+json" } }, @@ -4919,11 +4938,11 @@ "description": "Base Registry Engine bregctl explain output JSON Schemas, written by hand and held to bregctl's output by its contract test.", "source": { "path": "products/breg/contracts/explain/ModelExplanation.schema.json", - "sha256": "c5c5461590e22ba0896a312e9235ad707c066b803c14759f34d402c186823797" + "sha256": "d559294995dfc1049aad49574d9ca9b185971f5c6201054dc143cee96a3395e7" }, "artifact": { "path": "products/breg/contracts/explain/ModelExplanation.schema.json", - "sha256": "c5c5461590e22ba0896a312e9235ad707c066b803c14759f34d402c186823797", + "sha256": "d559294995dfc1049aad49574d9ca9b185971f5c6201054dc143cee96a3395e7", "mediaType": "application/schema+json" } }, @@ -4959,7 +4978,7 @@ }, "artifact": { "path": "products/breg/generated/authoring/registry-module.schema.json", - "sha256": "7beb51728884e600dccd3b47fd1c46e47936423883079f7cffb9dba147f0339a", + "sha256": "2948465f69e8dd8856e245826adb47addbb0b0b3ae5a4a3eece660dfa77ba060", "mediaType": "application/schema+json" } }, @@ -4977,7 +4996,7 @@ }, "artifact": { "path": "products/breg/generated/authoring/registry-project.schema.json", - "sha256": "85dd5a7829de9151091e6674c0cf9883d80f1bd6410a9a6585516e919d2cb67b", + "sha256": "9d432a58cfcb304ae9116be9f96b2a00f35c87da1e543d3598b2bf9207a8bd36", "mediaType": "application/schema+json" } }, @@ -5067,7 +5086,7 @@ }, "artifact": { "path": "products/breg/generated/runtime/runtime.schema.json", - "sha256": "785867b4f14c78862010b53406f62a420e53c4b17d5c8b9f7f95636130b75bd4", + "sha256": "29266d2d65e256cd54d27641dca372a18c4380771e09740f56a81fa59289d8bf", "mediaType": "application/schema+json" } }, @@ -5117,7 +5136,7 @@ "description": "Registry Casework fixture JSON Schema.", "source": { "path": "crates/registry-casework-core/src/schema.rs", - "sha256": "aae3e2891d651ad99989ae4720c08991d954c0f3232e194ec68d18e30642a711" + "sha256": "1604cf79433172ea4553b2f56365ea9e999a515aa3f16d60959cc652972ae185" }, "artifact": { "path": "products/casework/generated/fixture/fixture.schema.json", @@ -5135,7 +5154,7 @@ "description": "Registry Casework holiday set JSON Schema.", "source": { "path": "crates/registry-casework-core/src/schema.rs", - "sha256": "aae3e2891d651ad99989ae4720c08991d954c0f3232e194ec68d18e30642a711" + "sha256": "1604cf79433172ea4553b2f56365ea9e999a515aa3f16d60959cc652972ae185" }, "artifact": { "path": "products/casework/generated/holiday-set/holiday-set.schema.json", @@ -5153,11 +5172,11 @@ "description": "Registry Casework project JSON Schema.", "source": { "path": "crates/registry-casework-core/src/schema.rs", - "sha256": "aae3e2891d651ad99989ae4720c08991d954c0f3232e194ec68d18e30642a711" + "sha256": "1604cf79433172ea4553b2f56365ea9e999a515aa3f16d60959cc652972ae185" }, "artifact": { "path": "products/casework/generated/project/project.schema.json", - "sha256": "3973b27d8d378de8f11877013c36c0bacc0aec6db003f70f1375c52b21857505", + "sha256": "87a1a61127de6eb20c130f17b2d804c8b209c7343e7e4275f480f37833b83fcf", "mediaType": "application/schema+json" } }, @@ -5175,7 +5194,7 @@ }, "artifact": { "path": "products/casework/contracts/cli/report.schema.json", - "sha256": "442c29fe6db3bd93c2a35a5303e341d8aaa70c95d58dcd589fee9fff529fdff4", + "sha256": "2cbf5e3d2fd943abfa3777b63f204feac0dc0f5c02241a121aa3a135f3d420c7", "mediaType": "application/schema+json" } }, @@ -5207,11 +5226,11 @@ "description": "Registry Coordinator portable project authoring JSON Schema.", "source": { "path": "crates/registry-coordinator/src/authoring.rs", - "sha256": "8aff778324c1f05ca2615121980be6e3212d3e80f1bc99104b7256dfe18445a5" + "sha256": "032aa502fa63b5e7fe1cfb8f400be84d0802637409ac82ddf9f40152c482b91f" }, "artifact": { "path": "products/coordinator/generated/project/project.schema.json", - "sha256": "c4bc1e7a160804ec03dc958eed753e6cc59fb3132b702bf6bd43638f0c6829a1", + "sha256": "b22c06678f15e4a5aa028df61819cde8d37ce2311e1249a3a8fcc18d00aa3eab", "mediaType": "application/schema+json" } }, @@ -5243,11 +5262,11 @@ "description": "Registry Coordinator deterministic scenario authoring JSON Schema.", "source": { "path": "crates/registry-coordinator/src/scenarios.rs", - "sha256": "5e5de3c3a4ca9d36a2286b0abd1d3f551acb28691ae4a16439ee3fafcb83d4ef" + "sha256": "e20158e5382f690d46879161f32b5adda80385767b2c99b6efb11f166d43a394" }, "artifact": { "path": "products/coordinator/generated/scenarios/scenarios.schema.json", - "sha256": "7a91cc37393fabc0bd05d3ea22708a382a7ef11f4b7c63cdeedba5b939cf12fa", + "sha256": "e42e044eb828ff7e12f0a199fc95065c04baeadf91773c2d28ba9a1b0252090e", "mediaType": "application/schema+json" } }, From cd4e6506b6c57b604800aeff59c1909a778cf2ef Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 12:49:43 +0700 Subject: [PATCH 09/64] fix(breg): keep external snapshots through transient refresh failures An unavailable, incomplete or unbindable refresh for one key no longer removes the previous snapshot or bumps the scope generation. The snapshot stays readable until its own expiry; only explicit denial or no-store removes it, so a transient source failure no longer turns covered indexed reads into incomplete coverage. Signed-off-by: Jeremi Joslin --- .../src/postgres/external_projection.rs | 42 +++++++++++----- .../src/postgres/external_runtime.rs | 48 +++++++++++++++++++ products/breg/external-projection-storage.md | 2 +- 3 files changed, 80 insertions(+), 12 deletions(-) diff --git a/crates/registry-breg/src/postgres/external_projection.rs b/crates/registry-breg/src/postgres/external_projection.rs index 1d4cc7c390..81577da954 100644 --- a/crates/registry-breg/src/postgres/external_projection.rs +++ b/crates/registry-breg/src/postgres/external_projection.rs @@ -919,8 +919,9 @@ pub(crate) async fn withdraw_scope( Ok(()) } -/// Exact-key failures invalidate coverage. Generic collection interruption -/// preserves its prior complete snapshot; denial and no-store invalidate both. +/// A transient failure keeps the previous snapshot of either kind until its +/// own expiry. Explicit denial and no-store remove it, fenced against newer +/// publications. pub(crate) async fn fail_refresh( transaction: &GuardedTransaction<'_>, ticket: &RefreshTicket, @@ -1127,7 +1128,7 @@ CREATE OR REPLACE FUNCTION registry_internal.fail_external_projection_refresh( p_projection text,p_epoch uuid,p_sequence bigint,p_subject text,p_reason text ) RETURNS boolean LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, registry_internal AS $body$ -DECLARE current_epoch uuid; allocated boolean; removed bigint; +DECLARE current_epoch uuid; allocated boolean; BEGIN IF p_reason IS NULL OR p_reason NOT IN ('abandoned','unavailable','denied','incomplete','invalid-binding','retention-refused') THEN RETURN false; END IF; SELECT epoch INTO current_epoch FROM registry_internal.registry_external_projection_scopes @@ -1142,14 +1143,11 @@ BEGIN WHERE projection_id=p_projection AND epoch=p_epoch AND sequence=p_sequence AND subject_key=p_subject AND lease_expires_at>floor(extract(epoch FROM clock_timestamp()))::bigint FOR UPDATE; IF NOT FOUND OR NOT allocated THEN RETURN false; END IF; - IF p_reason IN ('retention-refused','denied','unavailable','incomplete','invalid-binding') THEN + -- Only the source's explicit denial or no-store removes a snapshot. A + -- transient failure leaves the previous one until its own expiry. + IF p_reason IN ('retention-refused','denied') THEN DELETE FROM registry_internal.registry_external_projection_snapshots - WHERE projection_id=p_projection AND epoch=p_epoch AND subject_key=p_subject AND refresh_sequence<=p_sequence - AND (p_reason IN ('retention-refused','denied') OR kind IN ('one','absent')); - GET DIAGNOSTICS removed=ROW_COUNT; - IF removed>0 THEN - UPDATE registry_internal.registry_external_projection_scopes SET generation=generation+1 WHERE projection_id=p_projection; - END IF; + WHERE projection_id=p_projection AND epoch=p_epoch AND subject_key=p_subject AND refresh_sequence<=p_sequence; END IF; UPDATE registry_internal.registry_external_projection_refreshes SET state=p_reason WHERE projection_id=p_projection AND epoch=p_epoch AND sequence=p_sequence AND subject_key=p_subject AND state='allocated'; @@ -1978,7 +1976,8 @@ pub(crate) mod postgres_tests { assert_eq!(runtime.query_one("SELECT count(*) FROM registry_internal.registry_external_projection_snapshots WHERE subject_key='subject'",&[]).await.unwrap().get::<_,i64>(0),1); let failed = allocate_inner(&runtime, &scope, "subject").await.unwrap(); assert!(runtime.query_one("SELECT registry_internal.fail_external_projection_refresh($1,$2,$3,$4,'unavailable')",&[&scope.projection_id,&scope.epoch,&failed.sequence,&failed.subject_key]).await.unwrap().get::<_,bool>(0)); - assert_eq!(runtime.query_one("SELECT count(*) FROM registry_internal.registry_external_projection_snapshots WHERE subject_key='subject'",&[]).await.unwrap().get::<_,i64>(0),0); + // A transient failure keeps the snapshot until its own expiry. + assert_eq!(runtime.query_one("SELECT count(*) FROM registry_internal.registry_external_projection_snapshots WHERE subject_key='subject'",&[]).await.unwrap().get::<_,i64>(0),1); let restore = allocate_inner(&runtime, &scope, "subject").await.unwrap(); publish_inner(&runtime, &restore, &fresh).await.unwrap(); let no_store = allocate_inner(&runtime, &scope, "subject").await.unwrap(); @@ -2441,6 +2440,27 @@ pub(crate) mod postgres_tests { ); } + /// Record a refresh failure for one key, as a refresh worker would. + pub(crate) async fn fail(&self, scope: &ProjectionScope, key: &str, reason: &str) { + let ticket = allocate_inner(&self.migration, scope, key).await.unwrap(); + let failed: bool = self + .migration + .query_one( + "SELECT registry_internal.fail_external_projection_refresh($1,$2,$3,$4,$5)", + &[ + &scope.projection_id, + &scope.epoch, + &ticket.sequence, + &ticket.subject_key, + &reason, + ], + ) + .await + .unwrap() + .get(0); + assert!(failed, "{reason} is recorded"); + } + pub(crate) async fn cached(&self, projection: &str, epoch: Uuid) -> i64 { self.migration .query_one( diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index 122ef22317..0f5808888c 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -2044,6 +2044,18 @@ mod postgres_tests { } } + async fn generation(&self) -> i64 { + self.fixture + .migration + .query_one( + "SELECT generation FROM registry_internal.registry_external_projection_scopes WHERE projection_id=$1", + &[&self.scope.projection_id], + ) + .await + .unwrap() + .get(0) + } + async fn publish_all(&self) { for (_, key, _) in ROOTS { self.fixture.publish_label(&self.scope, key, key).await; @@ -2180,4 +2192,40 @@ mod postgres_tests { assert_eq!(labels(&next).len(), 1); served.fixture.cleanup().await; } + + #[tokio::test] + async fn a_transient_one_key_failure_keeps_coverage_and_denial_removes_it() { + let served = Served::new(Duration::from_secs(2)).await; + served.publish_all().await; + let caller = reader("reader-principal", &["allowed"]); + let generation = served.generation().await; + for reason in ["unavailable", "incomplete"] { + served.fixture.fail(&served.scope, "second", reason).await; + let (status, page) = served.get("&$count=true", caller.clone()).await; + assert_eq!(status, axum::http::StatusCode::OK, "{reason}: {page}"); + assert_eq!( + page["count"], 3, + "{reason} keeps the snapshot until it expires" + ); + assert_eq!(labels(&page), ["first", "second", "third"]); + } + served.fixture.fail(&served.scope, "second", "denied").await; + let (status, problem) = served.get("&$count=true", caller.clone()).await; + assert_eq!(status, axum::http::StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(problem["code"], "external_view.incomplete", "{problem}"); + // A per-key outcome never invalidates cursors over the rest of the scope. + assert_eq!(served.generation().await, generation); + served + .fixture + .publish_label(&served.scope, "second", "second") + .await; + served + .fixture + .fail(&served.scope, "second", "retention-refused") + .await; + let (status, problem) = served.get("&$count=true", caller).await; + assert_eq!(status, axum::http::StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(problem["code"], "external_view.incomplete", "{problem}"); + served.fixture.cleanup().await; + } } diff --git a/products/breg/external-projection-storage.md b/products/breg/external-projection-storage.md index 2627dae4d8..2d86a247bf 100644 --- a/products/breg/external-projection-storage.md +++ b/products/breg/external-projection-storage.md @@ -25,7 +25,7 @@ before Adapter acquisition. The current checker refuses unknown binding configur A captured snapshot stores the observation time and a keyed revision reference when the source supplies a revision. The reference is scoped to the bound source. -Replaying an accepted result keeps its original expiry. A failed exact-key refresh removes the previous one/absent snapshot, making coverage unproven. An interrupted related-collection refresh preserves its previous verified complete snapshot without extending its lifetime. Explicit grant denial or no-store removes either kind, fenced against newer publications. A source response marked `no-store` cannot enter the indexed store. If it supersedes an earlier retained result for the same subject, the guarded retention-refusal path removes that result; an older refresh cannot erase a newer publication. +Replaying an accepted result keeps its original expiry. A refresh that fails because the source is unavailable, returns an incomplete result, or cannot be acquired under the current binding keeps the previous snapshot of either kind until its own expiry, without extending its lifetime. Explicit grant denial or no-store removes the previous snapshot, making coverage unproven, fenced against newer publications. A source response marked `no-store` cannot enter the indexed store. If it supersedes an earlier retained result for the same subject, the guarded retention-refusal path removes that result; an older refresh cannot erase a newer publication. Reads check expiry independently of the refresh job. Indexed predicates and counts refuse incomplete coverage before selecting matching rows. A complete empty related collection is covered; a partial collection, source failure, or protected-existence response is not evidence of emptiness or absence. From 1ed6bcf12abae046421d5aa551066fd76376216c Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 12:58:40 +0700 Subject: [PATCH 10/64] fix(breg): keep external view cursors and label sweeps across per-key refreshes The scope generation now moves only on rotation, withdrawal and contract reset, so a reader's continuation survives concurrent publication. A label sweep confirms that the exact parent snapshots supplying its selectors are still current before each source call, instead of stopping whenever any parent root refreshed. Publication still rechecks current membership in the same statement. A database failure while rechecking the parent is reported as unavailable instead of abandoning the page. Signed-off-by: Jeremi Joslin --- .../postgres/external_label_acquisition.rs | 57 ++++++- .../src/postgres/external_projection.rs | 37 ++++- .../src/postgres/external_runtime.rs | 151 +++++++++++++++--- .../src/postgres/external_view_query.rs | 4 +- products/breg/external-projection-storage.md | 4 +- 5 files changed, 222 insertions(+), 31 deletions(-) diff --git a/crates/registry-breg/src/postgres/external_label_acquisition.rs b/crates/registry-breg/src/postgres/external_label_acquisition.rs index a019042114..602a2a09fa 100644 --- a/crates/registry-breg/src/postgres/external_label_acquisition.rs +++ b/crates/registry-breg/src/postgres/external_label_acquisition.rs @@ -35,9 +35,25 @@ pub(crate) struct LabelSelector { pub(crate) root_id: Uuid, pub(crate) member_key: String, pub(crate) selector: String, + parent_key: String, + parent_snapshot: String, parent_expires_at: i64, } +/// One parent snapshot that supplied label selectors. Acquisition goes ahead +/// only while that exact snapshot is still current and unexpired. +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct SupportingSnapshot { + pub(crate) subject_key: String, + pub(crate) snapshot_id: String, +} + +impl fmt::Debug for SupportingSnapshot { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("SupportingSnapshot()") + } +} + impl fmt::Debug for LabelSelector { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str("LabelSelector()") @@ -48,6 +64,7 @@ pub(crate) struct LabelSelectorPage { pub(crate) rows: Vec, pub(crate) continuation: Option, pub(crate) minimum_parent_expires_at: Option, + pub(crate) supporting: Vec, } pub(crate) struct LabelAcquisition { @@ -222,10 +239,20 @@ fn bounded_page(mut rows: Vec, limit: u16) -> LabelSelectorPage { None }; let minimum_parent_expires_at = rows.iter().map(|row| row.parent_expires_at).min(); + let supporting = rows + .iter() + .map(|row| SupportingSnapshot { + subject_key: row.parent_key.clone(), + snapshot_id: row.parent_snapshot.clone(), + }) + .collect::>() + .into_iter() + .collect(); LabelSelectorPage { rows, continuation, minimum_parent_expires_at, + supporting, } } @@ -234,7 +261,8 @@ fn literal(value: &str) -> String { } const ELIGIBLE_SQL: &str = r#" -SELECT roots.root_id,members.source_key AS member_key,members.fields->>__SELECTOR_FIELD__ AS selector,snapshot.expires_at AS parent_expires_at +SELECT roots.root_id,members.source_key AS member_key,members.fields->>__SELECTOR_FIELD__ AS selector, + snapshot.subject_key AS parent_key,snapshot.snapshot_id AS parent_snapshot,snapshot.expires_at AS parent_expires_at FROM (__ROOT_SQL__) AS roots JOIN parent_state AS scope ON true JOIN registry_internal.registry_external_projection_snapshots AS snapshot @@ -252,12 +280,13 @@ const PAGE_SQL: &str = r#" WITH parent_state AS MATERIALIZED (__STATE_SQL__), selected AS ( __ELIGIBLE_SQL__ ), page AS MATERIALIZED ( - SELECT root_id,member_key,selector,parent_expires_at FROM selected + SELECT root_id,member_key,selector,parent_key,parent_snapshot,parent_expires_at FROM selected WHERE $1::uuid IS NULL OR root_id>$1::uuid OR (root_id=$1::uuid AND member_key COLLATE "C">$2::text COLLATE "C") ORDER BY root_id,member_key COLLATE "C" LIMIT $3::bigint ) SELECT EXISTS(SELECT 1 FROM parent_state),COALESCE((SELECT jsonb_agg(jsonb_build_object( - 'root_id',root_id,'member_key',member_key,'selector',selector,'parent_expires_at',parent_expires_at) ORDER BY root_id,member_key COLLATE "C") FROM page),'[]'::jsonb) + 'root_id',root_id,'member_key',member_key,'selector',selector,'parent_key',parent_key, + 'parent_snapshot',parent_snapshot,'parent_expires_at',parent_expires_at) ORDER BY root_id,member_key COLLATE "C") FROM page),'[]'::jsonb) "#; const PUBLICATION_SQL: &str = r#" @@ -363,11 +392,20 @@ mod tests { root_id: Uuid::nil(), member_key: index.to_string(), selector: "internal".to_owned(), + parent_key: format!("parent-{}", index / 2), + parent_snapshot: format!("snapshot-{}", index / 2), parent_expires_at: expiry, }) .collect(); let page = bounded_page(rows, 2); assert_eq!(page.minimum_parent_expires_at, Some(200)); + assert_eq!( + page.supporting, + [SupportingSnapshot { + subject_key: "parent-0".to_owned(), + snapshot_id: "snapshot-0".to_owned(), + }] + ); assert_eq!(page.continuation.unwrap().member_key, "1"); assert_eq!(bounded_page(vec![], 2).minimum_parent_expires_at, None); } @@ -382,9 +420,15 @@ mod tests { root_id: Uuid::nil(), member_key: "PRIVATE_MEMBER_CANARY".to_owned(), selector: "PRIVATE_SELECTOR_CANARY".to_owned(), + parent_key: "PRIVATE_PARENT_CANARY".to_owned(), + parent_snapshot: "PRIVATE_SNAPSHOT_CANARY".to_owned(), parent_expires_at: 1000, }; - assert!(!format!("{position:?} {selector:?}").contains("CANARY")); + let supporting = SupportingSnapshot { + subject_key: "PRIVATE_PARENT_CANARY".to_owned(), + snapshot_id: "PRIVATE_SNAPSHOT_CANARY".to_owned(), + }; + assert!(!format!("{position:?} {selector:?} {supporting:?}").contains("CANARY")); } } @@ -478,9 +522,10 @@ mod postgres_tests { "view", "members", &parent, + // Publication leaves the configured scope generation unchanged. ProjectionGeneration { epoch: parent.epoch, - generation: 3, + generation: 0, }, ) .unwrap(); @@ -496,6 +541,8 @@ mod postgres_tests { let rows: Value = page.get(1); assert_eq!(rows.as_array().unwrap().len(), 2); assert_eq!(rows[0]["selector"], "indirect-person"); + assert_eq!(rows[0]["parent_key"], "allowed-parent"); + assert_eq!(rows[0]["parent_snapshot"], "members"); assert_eq!(rows[1]["member_key"], "d"); let allowed = Uuid::parse_str("00000000-0000-4000-8000-000000000001").unwrap(); let next = runtime diff --git a/crates/registry-breg/src/postgres/external_projection.rs b/crates/registry-breg/src/postgres/external_projection.rs index 81577da954..d0dfb5268d 100644 --- a/crates/registry-breg/src/postgres/external_projection.rs +++ b/crates/registry-breg/src/postgres/external_projection.rs @@ -894,6 +894,40 @@ pub(crate) async fn lock_generation( }) } +/// Whether each named snapshot is still the current, unexpired one for its +/// subject in this scope epoch. A label sweep uses this to confirm that the +/// parent snapshots supplying its selectors were not replaced or removed. +pub(crate) async fn snapshots_current( + transaction: &GuardedTransaction<'_>, + scope: &ProjectionScope, + subject_keys: &[&str], + snapshot_ids: &[&str], +) -> Result { + if subject_keys.len() != snapshot_ids.len() { + return Err(ProjectionError::InvalidScope); + } + let row = transaction + .transaction() + .query_one( + "SELECT count(*)=cardinality($3::text[]) + FROM unnest($3::text[],$4::text[]) AS expected(subject_key,snapshot_id) + JOIN registry_internal.registry_external_projection_snapshots AS snapshot + ON snapshot.projection_id=$1 AND snapshot.epoch=$2 + AND snapshot.subject_key=expected.subject_key + AND snapshot.snapshot_id=expected.snapshot_id + AND snapshot.expires_at>floor(extract(epoch FROM clock_timestamp()))::bigint", + &[ + &scope.projection_id, + &scope.epoch, + &subject_keys, + &snapshot_ids, + ], + ) + .await + .map_err(|_| ProjectionError::Unavailable)?; + Ok(row.get(0)) +} + /// Tests withdraw under migration authority; operators use the maintenance /// function, and the runtime role holds no EXECUTE grant on either. #[cfg(all(test, feature = "postgres-test"))] @@ -1120,7 +1154,8 @@ BEGIN SELECT p_projection,p_epoch,p_subject,r->>'source_key',r->'fields' FROM jsonb_array_elements(p_rows) AS r; UPDATE registry_internal.registry_external_projection_refreshes SET state='published' WHERE projection_id=p_projection AND sequence=p_sequence; - UPDATE registry_internal.registry_external_projection_scopes SET generation=generation+1 WHERE projection_id=p_projection; + -- Generation tracks scope-wide changes only, so cursors and label sweeps + -- survive per-key publication. RETURN 'published'; END $body$; diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index 0f5808888c..5aa75b4424 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -35,7 +35,7 @@ use crate::external_views::{ use crate::model::{CompiledEntity, CompiledRegistry, HttpMethod}; use super::cancellation::QueryCancellationGuard; -use super::external_label_acquisition::{LabelAcquisition, LabelAfter}; +use super::external_label_acquisition::{LabelAcquisition, LabelAfter, SupportingSnapshot}; use super::external_projection::{ self as store, AcceptedSnapshot, ProjectionGeneration, ProjectionRow, ProjectionScope, RefreshFailure, RefreshTicket, RetentionPermission, @@ -1035,6 +1035,8 @@ struct RefreshAuthority { lock_timeout: Duration, scope: ProjectionScope, parent: Option<(ProjectionScope, ProjectionGeneration)>, + /// The parent snapshots that supplied this page's label selectors. + supporting: Vec, } impl RefreshAuthority { @@ -1081,6 +1083,24 @@ impl RefreshAuthority { if ¤t != generation { return Err(EvidenceHookError::Authority); } + // Only the parent snapshots that supplied this page's selectors + // matter; refreshes of other parents leave the sweep running. + let keys = self + .supporting + .iter() + .map(|parent| parent.subject_key.as_str()) + .collect::>(); + let ids = self + .supporting + .iter() + .map(|parent| parent.snapshot_id.as_str()) + .collect::>(); + if !store::snapshots_current(&transaction, parent, &keys, &ids) + .await + .map_err(|_| EvidenceHookError::Authority)? + { + return Err(EvidenceHookError::Authority); + } } transaction .commit() @@ -1310,6 +1330,7 @@ struct RefreshRows { next: Option, eligibility: String, supporting_until: Option, + supporting: Vec, } struct RefreshWork { @@ -1374,6 +1395,7 @@ impl PostgresExternalViewService { }), eligibility: selectors.publication_eligibility().to_owned(), supporting_until: page.minimum_parent_expires_at, + supporting: page.supporting, }); } let after = match after { @@ -1400,6 +1422,7 @@ impl PostgresExternalViewService { next, eligibility: keys.publication_eligibility, supporting_until: None, + supporting: Vec::new(), }) } @@ -1498,6 +1521,7 @@ impl PostgresExternalViewService { lock_timeout: self.lock_timeout, scope: scope.clone(), parent: parent.clone(), + supporting: page.supporting.clone(), }); let outcomes = self .fetch_refresh(request, plan, &work, page.supporting_until, authority) @@ -1512,10 +1536,17 @@ impl PostgresExternalViewService { return Err(ExternalViewError::Unavailable); } if let (Some(parent), Some((_, selected_generation))) = (&plan.label_parent, &parent) { - let parent_is_current = self + // A rotated or withdrawn parent abandons the page. A database + // or lock failure is reported as such and leaves the receipts + // to expire with their leases. + let parent_is_current = match self .lock_scope(&transaction, parent.source, parent.projection.id.as_str()) .await - .is_ok_and(|(_, generation)| &generation == selected_generation); + { + Ok((_, generation)) => &generation == selected_generation, + Err(ExternalViewError::ProjectionUnavailable(_)) => false, + Err(error) => return Err(error), + }; if !parent_is_current || page .supporting_until @@ -1960,6 +1991,27 @@ mod postgres_tests { ("00000000-0000-4000-8000-000000000004", "hidden", "other"), ]; + async fn bound_scope(fixture: &Activated, projection: &str) -> ProjectionScope { + let row = fixture + .migration + .query_one( + "SELECT epoch,binding_digest,grant_digest,cache_ttl_seconds + FROM registry_internal.registry_external_projection_scopes + WHERE projection_id=$1", + &[&projection], + ) + .await + .unwrap(); + ProjectionScope::new( + projection, + row.get(0), + row.get::<_, String>(1).as_str(), + row.get::<_, String>(2).as_str(), + row.get(3), + ) + .unwrap() + } + /// A bound, keyed runtime behind the in-process HTTP router. Three roots /// sit in the `allowed` tenant and one outside it. struct Served { @@ -1976,24 +2028,7 @@ mod postgres_tests { let fixture = Activated::new(®istry).await; let mut service = service_with(®istry, &fixture, lock_timeout); service.bind_configured_scopes(true).await.ok().unwrap(); - let row = fixture - .migration - .query_one( - "SELECT epoch,binding_digest,grant_digest,cache_ttl_seconds - FROM registry_internal.registry_external_projection_scopes - WHERE projection_id='projection'", - &[], - ) - .await - .unwrap(); - let scope = ProjectionScope::new( - "projection", - row.get(0), - row.get::<_, String>(1).as_str(), - row.get::<_, String>(2).as_str(), - row.get(3), - ) - .unwrap(); + let scope = bound_scope(&fixture, "projection").await; let entity = ®istry.entities()["local-record"]; let column = |id: &str| { entity @@ -2193,6 +2228,80 @@ mod postgres_tests { served.fixture.cleanup().await; } + #[tokio::test] + async fn a_continuation_reads_through_a_concurrent_publish() { + let served = Served::new(Duration::from_secs(2)).await; + served.publish_all().await; + let caller = reader("reader-principal", &["allowed"]); + let ordered = "&$top=1&$orderby=label%20asc&$count=true"; + let (status, first) = served.get(ordered, caller.clone()).await; + assert_eq!(status, axum::http::StatusCode::OK, "{first}"); + assert_eq!(labels(&first), ["first"]); + assert_eq!(first["count"], 3); + let cursor = first["nextCursor"].as_str().unwrap().to_owned(); + + // A refresh publishes a later key between pages. The continuation + // keeps its position and reads the value current for its own page. + served + .fixture + .publish_label(&served.scope, "third", "thirdly") + .await; + let (status, second) = served + .get(&format!("&$skiptoken={cursor}"), caller.clone()) + .await; + assert_eq!(status, axum::http::StatusCode::OK, "{second}"); + assert_eq!(labels(&second), ["second"]); + assert_eq!(second["count"], 3); + let cursor = second["nextCursor"].as_str().unwrap().to_owned(); + let (status, third) = served.get(&format!("&$skiptoken={cursor}"), caller).await; + assert_eq!(status, axum::http::StatusCode::OK, "{third}"); + assert_eq!(labels(&third), ["thirdly"]); + assert!(third.get("nextCursor").is_none()); + served.fixture.cleanup().await; + } + + #[tokio::test] + async fn a_label_sweep_survives_a_holdings_refresh_for_another_parent() { + let served = Served::new(Duration::from_secs(2)).await; + let registry = + super::super::external_view_query::tests::compile_project(&two_projection_project()); + served.publish_all().await; + let parent = served.scope.clone(); + let generation = ProjectionGeneration { + epoch: parent.epoch, + generation: served.generation().await, + }; + let authority = RefreshAuthority { + pool: served.fixture.pool.clone(), + expected: served.fixture.expected.clone(), + lock_key: RegistryLockKey::derive(registry.registry_id()).unwrap(), + lock_timeout: Duration::from_secs(2), + scope: bound_scope(&served.fixture, "other").await, + parent: Some((parent.clone(), generation)), + supporting: vec![SupportingSnapshot { + subject_key: "first".to_owned(), + snapshot_id: "first-first".to_owned(), + }], + }; + let deadline = Instant::now() + Duration::from_secs(5); + authority.check(deadline).await.unwrap(); + // A holdings refresh for another root leaves this sweep's parent as + // selected, so acquisition for its selectors goes ahead. + served + .fixture + .publish_label(&parent, "second", "renewed") + .await; + authority.check(deadline).await.unwrap(); + // A refresh of the sweep's own parent replaces the snapshot its + // selectors came from, so acquisition stops before the source call. + served + .fixture + .publish_label(&parent, "first", "renewed") + .await; + assert!(authority.check(deadline).await.is_err()); + served.fixture.cleanup().await; + } + #[tokio::test] async fn a_transient_one_key_failure_keeps_coverage_and_denial_removes_it() { let served = Served::new(Duration::from_secs(2)).await; diff --git a/crates/registry-breg/src/postgres/external_view_query.rs b/crates/registry-breg/src/postgres/external_view_query.rs index c9794a87d9..165bd7efdd 100644 --- a/crates/registry-breg/src/postgres/external_view_query.rs +++ b/crates/registry-breg/src/postgres/external_view_query.rs @@ -911,7 +911,8 @@ mod postgres_tests { connection.await.expect("query fixture connection"); }); runtime.batch_execute("SET registry.access_profile='reader'; SET registry.principal='staff'; SET registry.purpose='consultation'; SET registry.row_boundaries='[{\"field\":\"tenant\",\"operator\":\"equals\",\"values\":[\"allowed\"]}]';").await.unwrap(); - let mut request = super::tests::request(&scope, 2); + // Publication leaves the configured scope generation unchanged. + let mut request = super::tests::request(&scope, 0); request.include_count = true; request.filter = Some(ExactFilter { field: "label".to_owned(), @@ -922,7 +923,6 @@ mod postgres_tests { assert_eq!(incomplete.get::<_, Option>(4), None); assert_eq!(incomplete.get::<_, Value>(5), serde_json::json!([])); publish(&migration, &scope, "uncovered", Some("B"), "third").await; - request.generation.generation = 3; let complete = run(&runtime, ®istry, &request).await; assert!(complete.get::<_, bool>(1)); assert_eq!(complete.get::<_, Option>(4), Some(1)); diff --git a/products/breg/external-projection-storage.md b/products/breg/external-projection-storage.md index 2d86a247bf..eb631c269e 100644 --- a/products/breg/external-projection-storage.md +++ b/products/breg/external-projection-storage.md @@ -27,13 +27,13 @@ A captured snapshot stores the observation time and a keyed revision reference w Replaying an accepted result keeps its original expiry. A refresh that fails because the source is unavailable, returns an incomplete result, or cannot be acquired under the current binding keeps the previous snapshot of either kind until its own expiry, without extending its lifetime. Explicit grant denial or no-store removes the previous snapshot, making coverage unproven, fenced against newer publications. A source response marked `no-store` cannot enter the indexed store. If it supersedes an earlier retained result for the same subject, the guarded retention-refusal path removes that result; an older refresh cannot erase a newer publication. -Reads check expiry independently of the refresh job. Indexed predicates and counts refuse incomplete coverage before selecting matching rows. A complete empty related collection is covered; a partial collection, source failure, or protected-existence response is not evidence of emptiness or absence. +Reads check expiry independently of the refresh job. Indexed predicates and counts refuse incomplete coverage before selecting matching rows. Each page reads coverage, rows and any count from one statement snapshot, so a count always matches complete coverage at that moment. A `nextCursor` stays valid while refreshes publish or fail individual keys; the next page reads the values current when it runs. A row whose ordering value changed between pages can appear twice or not at all. Rotating, withdrawing or resetting the projection invalidates outstanding cursors. A complete empty related collection is covered; a partial collection, source failure, or protected-existence response is not evidence of emptiness or absence. The bounded purge worker runs every few seconds and deletes snapshots once they expire. It also removes data from withdrawn or replaced epochs. Size database capacity for the backlog. Where a storage obligation is strict, choose a cache lifetime shorter than that obligation by more than the worker interval and the expected recovery delay. Freshness enforcement keeps expired values out of responses even when refresh or purge is delayed, but a delayed purge can still breach a storage obligation. Monitor purge failures, expiry backlog, and database downtime separately. No physical erasure guarantee holds during a database outage or when backlog exceeds that margin. Database WAL and physical snapshots need their own retention policy. ## Schedule refresh and inspect coverage -Run the named view’s `POST /v1/external-views/{view}/{link}/refresh` with the approved machine profile. Continue with the returned `nextCursor` until the sweep ends. For independent related labels, run `POST /v1/external-views/{view}/{link}/refresh-labels` after the related collection refresh. That machine profile needs explicit refresh grants on both projections under the same purpose. The label sweep selects keys only from authorised roots with fresh complete collections. A changed collection generation requires a fresh sweep. +Run the named view’s `POST /v1/external-views/{view}/{link}/refresh` with the approved machine profile. Continue with the returned `nextCursor` until the sweep ends. For independent related labels, run `POST /v1/external-views/{view}/{link}/refresh-labels` after the related collection refresh. That machine profile needs explicit refresh grants on both projections under the same purpose. The label sweep selects keys only from authorised roots with fresh complete collections. Before each source call it confirms that the collections which supplied that page are still current and unexpired. A refresh of other collections leaves the sweep running; a refresh of one of its own collections stops acquisition for that page, and the next sweep picks up the new members. A rotated, withdrawn or reset collection projection ends the sweep, so start a fresh one. The refresh report counts processed roots or related rows, published values, reused keys and unavailable results. `bregctl external-projection inspect` reports receipt counts by outcome and fresh, expired and invalidated snapshot counts. Use an authorised indexed query to test readiness for its local root scope. From 702b81cb84843eb70880c90112811d24caf8c6c9 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 13:01:11 +0700 Subject: [PATCH 11/64] test(breg): name the store publication helper that skips eligibility The test-only helper publishes without the root eligibility statement that runtime publication always wraps around the store function. Its name now says so, so a reader cannot mistake it for the guarded path. Signed-off-by: Jeremi Joslin --- .../src/postgres/external_projection.rs | 79 +++++++++++++------ 1 file changed, 54 insertions(+), 25 deletions(-) diff --git a/crates/registry-breg/src/postgres/external_projection.rs b/crates/registry-breg/src/postgres/external_projection.rs index d0dfb5268d..4d28b03a1b 100644 --- a/crates/registry-breg/src/postgres/external_projection.rs +++ b/crates/registry-breg/src/postgres/external_projection.rs @@ -842,8 +842,11 @@ pub(super) fn publication_statement(eligibility_sql: &str) -> String { format!("SELECT CASE WHEN EXISTS ({eligibility_sql}) THEN registry_internal.publish_external_projection($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) ELSE 'ineligible' END") } +/// Publishes straight through the store function, skipping the root +/// eligibility statement that `publish_for_roots` wraps around it. Tests use +/// it to exercise the store's own fences in isolation. #[cfg(all(test, feature = "postgres-test"))] -async fn publish_inner( +async fn publish_without_eligibility( client: &impl GenericClient, ticket: &RefreshTicket, snapshot: &AcceptedSnapshot, @@ -1987,24 +1990,28 @@ pub(crate) mod postgres_tests { let newer = allocate_inner(&runtime, &scope, "subject").await.unwrap(); let fresh = snapshot("newer", 999, "fresh"); assert_eq!( - publish_inner(&runtime, &newer, &fresh).await.unwrap(), + publish_without_eligibility(&runtime, &newer, &fresh) + .await + .unwrap(), Publication::Published ); assert_eq!( - publish_inner(&runtime, &older, &snapshot("older", 998, "stale")) + publish_without_eligibility(&runtime, &older, &snapshot("older", 998, "stale")) .await .unwrap_err(), ProjectionError::StaleRefresh ); let expiry_before: i64 = runtime.query_one("SELECT expires_at FROM registry_internal.registry_external_projection_snapshots WHERE subject_key='subject'", &[]).await.unwrap().get(0); assert_eq!( - publish_inner(&runtime, &newer, &fresh).await.unwrap(), + publish_without_eligibility(&runtime, &newer, &fresh) + .await + .unwrap(), Publication::Replay ); let expiry_after: i64 = runtime.query_one("SELECT expires_at FROM registry_internal.registry_external_projection_snapshots WHERE subject_key='subject'", &[]).await.unwrap().get(0); assert_eq!(expiry_before, expiry_after); let substitution = snapshot("newer", 999, "substitution"); - assert!(publish_inner(&runtime, &newer, &substitution) + assert!(publish_without_eligibility(&runtime, &newer, &substitution) .await .is_err()); assert!(runtime.query_one("SELECT registry_internal.fail_external_projection_refresh($1,$2,$3,$4,'retention-refused')",&[&scope.projection_id,&scope.epoch,&older.sequence,&older.subject_key]).await.unwrap().get::<_,bool>(0)); @@ -2014,7 +2021,9 @@ pub(crate) mod postgres_tests { // A transient failure keeps the snapshot until its own expiry. assert_eq!(runtime.query_one("SELECT count(*) FROM registry_internal.registry_external_projection_snapshots WHERE subject_key='subject'",&[]).await.unwrap().get::<_,i64>(0),1); let restore = allocate_inner(&runtime, &scope, "subject").await.unwrap(); - publish_inner(&runtime, &restore, &fresh).await.unwrap(); + publish_without_eligibility(&runtime, &restore, &fresh) + .await + .unwrap(); let no_store = allocate_inner(&runtime, &scope, "subject").await.unwrap(); assert!(runtime.query_one("SELECT registry_internal.fail_external_projection_refresh($1,$2,$3,$4,'retention-refused')",&[&scope.projection_id,&scope.epoch,&no_store.sequence,&no_store.subject_key]).await.unwrap().get::<_,bool>(0)); assert_eq!(runtime.query_one("SELECT count(*) FROM registry_internal.registry_external_projection_snapshots WHERE subject_key='subject'",&[]).await.unwrap().get::<_,i64>(0),0); @@ -2029,7 +2038,9 @@ pub(crate) mod postgres_tests { vec![record("a", "first"), record("b", "second")], ) .unwrap(); - publish_inner(&runtime, &first, &collection).await.unwrap(); + publish_without_eligibility(&runtime, &first, &collection) + .await + .unwrap(); let count = || async { runtime.query_one("SELECT count(*) FROM registry_internal.registry_external_projection_values WHERE subject_key='parent'", &[]).await.unwrap().get::<_,i64>(0) }; @@ -2047,7 +2058,9 @@ pub(crate) mod postgres_tests { vec![], ) .unwrap(); - publish_inner(&runtime, &next, &empty).await.unwrap(); + publish_without_eligibility(&runtime, &next, &empty) + .await + .unwrap(); assert_eq!(count().await, 0); assert_eq!(runtime.query_one("SELECT kind FROM registry_internal.registry_external_projection_snapshots WHERE subject_key='parent'", &[]).await.unwrap().get::<_,String>(0), "complete-collection"); @@ -2073,7 +2086,7 @@ pub(crate) mod postgres_tests { assert!(migration.query_one("SELECT active FROM registry_internal.registry_external_projection_scopes WHERE projection_id=$1",&[&scope.projection_id]).await.unwrap().get::<_,bool>(0)); withdraw_scope(&migration, &scope).await.unwrap(); assert_eq!( - publish_inner( + publish_without_eligibility( &runtime, &pending, &snapshot("after-withdrawal", 1000, "forbidden") @@ -2086,7 +2099,9 @@ pub(crate) mod postgres_tests { let replacement = super::postgres_tests::scope(Uuid::new_v4()); configure_scope(&migration, &replacement).await.unwrap(); assert_eq!( - publish_inner(&runtime, &pending, &fresh).await.unwrap_err(), + publish_without_eligibility(&runtime, &pending, &fresh) + .await + .unwrap_err(), ProjectionError::ScopeChanged ); @@ -2106,7 +2121,7 @@ pub(crate) mod postgres_tests { let scope = scope(Uuid::new_v4()); configure_scope(&migration, &scope).await.unwrap(); let initial = allocate_inner(&migration, &scope, "subject").await.unwrap(); - publish_inner(&migration, &initial, &snapshot("initial", 999, "retained")) + publish_without_eligibility(&migration, &initial, &snapshot("initial", 999, "retained")) .await .unwrap(); let older = allocate_inner(&migration, &scope, "subject").await.unwrap(); @@ -2142,7 +2157,7 @@ pub(crate) mod postgres_tests { .get::<_, bool>(0)); assert_eq!(migration.query_one("SELECT count(*) FROM registry_internal.registry_external_projection_refreshes WHERE projection_id=$1 AND sequence=$2",&[&scope.projection_id,&abandoned.sequence]).await.unwrap().get::<_,i64>(0),0); assert_eq!(migration.query_one("SELECT snapshot_id,expires_at FROM registry_internal.registry_external_projection_snapshots WHERE projection_id=$1 AND subject_key='subject'",&[&scope.projection_id]).await.unwrap().get::<_,String>(0),"initial"); - publish_inner(&migration, &older, &snapshot("older", 999, "accepted")) + publish_without_eligibility(&migration, &older, &snapshot("older", 999, "accepted")) .await .unwrap(); assert!(migration @@ -2182,7 +2197,7 @@ pub(crate) mod postgres_tests { ] { let initial = allocate_inner(&migration, &scope, "subject").await.unwrap(); let accepted = snapshot(reason, 999, "retained"); - publish_inner(&migration, &initial, &accepted) + publish_without_eligibility(&migration, &initial, &accepted) .await .unwrap(); let older = allocate_inner(&migration, &scope, "subject").await.unwrap(); @@ -2203,13 +2218,17 @@ pub(crate) mod postgres_tests { .get::<_, bool>(0)); assert!(migration.query_one("SELECT registry_internal.fail_external_projection_refresh($1,$2,$3,$4,'abandoned')",&[&scope.projection_id,&scope.epoch,&newer.sequence,&newer.subject_key]).await.unwrap().get::<_,bool>(0)); assert_eq!( - publish_inner(&migration, &older, &snapshot("older", 999, "forbidden")) - .await - .unwrap_err(), + publish_without_eligibility( + &migration, + &older, + &snapshot("older", 999, "forbidden") + ) + .await + .unwrap_err(), ProjectionError::StaleRefresh ); assert_eq!( - publish_inner(&migration, &initial, &accepted) + publish_without_eligibility(&migration, &initial, &accepted) .await .unwrap_err(), ProjectionError::StaleRefresh @@ -2225,13 +2244,15 @@ pub(crate) mod postgres_tests { .await .unwrap(); assert_eq!( - publish_inner(&migration, &older, &accepted) + publish_without_eligibility(&migration, &older, &accepted) .await .unwrap_err(), ProjectionError::StaleRefresh ); let later = allocate_inner(&migration, &scope, "subject").await.unwrap(); - publish_inner(&migration, &later, &accepted).await.unwrap(); + publish_without_eligibility(&migration, &later, &accepted) + .await + .unwrap(); migration.query_one("SELECT registry_internal.fail_external_projection_refresh($1,$2,$3,$4,'retention-refused')",&[&scope.projection_id,&scope.epoch,&older.sequence,&older.subject_key]).await.unwrap(); assert_eq!(migration.query_one("SELECT refresh_sequence FROM registry_internal.registry_external_projection_snapshots WHERE projection_id=$1 AND subject_key='subject'",&[&scope.projection_id]).await.unwrap().get::<_,i64>(0),later.sequence); } @@ -2256,11 +2277,15 @@ pub(crate) mod postgres_tests { configure_scope(&migration, &scope).await.unwrap(); let ticket = allocate_inner(&migration, &scope, "subject").await.unwrap(); let observed = snapshot("original", 990, "retained"); - publish_inner(&migration, &ticket, &observed).await.unwrap(); + publish_without_eligibility(&migration, &ticket, &observed) + .await + .unwrap(); clock(&migration, 1050).await; let visible: i64 = migration.query_one("SELECT count(*) FROM registry_internal.registry_external_projection_snapshots WHERE expires_at>1050", &[]).await.unwrap().get(0); assert_eq!(visible, 0); - assert!(publish_inner(&migration, &ticket, &observed).await.is_err()); + assert!(publish_without_eligibility(&migration, &ticket, &observed) + .await + .is_err()); // Purge removes a value once its one cache lifetime expires. let pruned = migration .query_one( @@ -2438,9 +2463,13 @@ pub(crate) mod postgres_tests { .await .unwrap(); assert_eq!( - publish_inner(&self.migration, &ticket, &snapshot("cached", now, "cached")) - .await - .unwrap(), + publish_without_eligibility( + &self.migration, + &ticket, + &snapshot("cached", now, "cached") + ) + .await + .unwrap(), Publication::Published ); } @@ -2468,7 +2497,7 @@ pub(crate) mod postgres_tests { ) .unwrap(); assert_eq!( - publish_inner(&self.migration, &ticket, &snapshot) + publish_without_eligibility(&self.migration, &ticket, &snapshot) .await .unwrap(), Publication::Published From 29b4197c4c6d533c4effeeaa33f7a54d2a575cc9 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 13:04:53 +0700 Subject: [PATCH 12/64] fix(breg): derive external refresh leases from the page deadline A refresh receipt now stays live for the time left before the page deadline plus a ten second margin, instead of a fixed two minutes. A page that runs longer than the old lease but inside its own deadline still publishes, and a short page no longer holds receipts live longer than it can use them. The store refuses leases outside one second to one hour. Signed-off-by: Jeremi Joslin --- .../postgres/external_label_acquisition.rs | 4 +- .../src/postgres/external_projection.rs | 95 ++++++++++++++++--- .../src/postgres/external_runtime.rs | 31 +++++- .../src/postgres/external_view_query.rs | 6 +- 4 files changed, 117 insertions(+), 19 deletions(-) diff --git a/crates/registry-breg/src/postgres/external_label_acquisition.rs b/crates/registry-breg/src/postgres/external_label_acquisition.rs index 602a2a09fa..e3b4a9828f 100644 --- a/crates/registry-breg/src/postgres/external_label_acquisition.rs +++ b/crates/registry-breg/src/postgres/external_label_acquisition.rs @@ -488,7 +488,7 @@ mod postgres_tests { for subject in ["allowed-parent", "denied-parent", "expired-parent"] { let ticket: i64 = migration .query_one( - "SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,$5)", + "SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,$5,120)", &[ &parent.projection_id, &parent.epoch, @@ -554,7 +554,7 @@ mod postgres_tests { "floor(extract(epoch FROM clock_timestamp()))::bigint", "1000", ); - let ticket:i64=runtime.query_one("SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,'indirect-person')",&[&label.projection_id,&label.epoch,&label.binding_digest,&label.grant_digest]).await.unwrap().get(0); + let ticket:i64=runtime.query_one("SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,'indirect-person',120)",&[&label.projection_id,&label.epoch,&label.binding_digest,&label.grant_digest]).await.unwrap().get(0); let rows = json!([{"source_key":"indirect-person","fields":{"name":"retained"}}]); let roots = vec![allowed]; let outcome: String = runtime diff --git a/crates/registry-breg/src/postgres/external_projection.rs b/crates/registry-breg/src/postgres/external_projection.rs index 4d28b03a1b..3ab6b28d89 100644 --- a/crates/registry-breg/src/postgres/external_projection.rs +++ b/crates/registry-breg/src/postgres/external_projection.rs @@ -4,6 +4,7 @@ use std::collections::BTreeSet; use std::fmt; +use std::time::Duration; use serde::Serialize; use serde_json::Value; @@ -25,7 +26,7 @@ pub(crate) const WITHDRAW_FUNCTION: &str = /// ACL/install signatures above remain type-only SQL call signatures. /// Entries are (identity, result type, returns set, security definer, runtime execute). pub(crate) const MANAGED_FUNCTIONS: &[(&str,&str,bool,bool,bool)] = &[ - ("registry_internal.allocate_external_projection_refresh(p_projection text, p_epoch uuid, p_binding text, p_grant text, p_subject text)","bigint",false,true,true), + ("registry_internal.allocate_external_projection_refresh(p_projection text, p_epoch uuid, p_binding text, p_grant text, p_subject text, p_lease bigint)","bigint",false,true,true), ("registry_internal.publish_external_projection(p_projection text, p_epoch uuid, p_binding text, p_grant text, p_sequence bigint, p_subject text, p_snapshot text, p_observed bigint, p_expires bigint, p_kind text, p_rows jsonb)","text",false,true,true), ("registry_internal.fail_external_projection_refresh(p_projection text, p_epoch uuid, p_sequence bigint, p_subject text, p_reason text)","boolean",false,true,true), ("registry_internal.withdraw_external_projection(p_projection text, p_epoch uuid, p_binding text, p_grant text)","boolean",false,true,false), @@ -246,7 +247,7 @@ pub(crate) const SNAPSHOTS_TABLE: &str = BACKUP_EXCLUDED_TABLES[2]; pub(crate) const VALUES_TABLE: &str = BACKUP_EXCLUDED_TABLES[3]; pub(crate) const RUNTIME_FUNCTIONS: &[&str] = &[ - "registry_internal.allocate_external_projection_refresh(text,uuid,text,text,text)", + "registry_internal.allocate_external_projection_refresh(text,uuid,text,text,text,bigint)", "registry_internal.publish_external_projection(text,uuid,text,text,bigint,text,text,bigint,bigint,text,jsonb)", "registry_internal.fail_external_projection_refresh(text,uuid,bigint,text,text)", "registry_internal.purge_external_projections(integer)", @@ -649,12 +650,15 @@ pub(crate) async fn configure_scope( Ok(()) } +/// The lease must outlive the whole refresh page: acquisition and publication +/// both finish by the caller's deadline, so pass the time left plus a margin. pub(crate) async fn allocate_refresh( transaction: &GuardedTransaction<'_>, scope: &ProjectionScope, subject_key: &str, + lease: Duration, ) -> Result { - allocate_inner(transaction.transaction(), scope, subject_key).await + allocate_with_lease(transaction.transaction(), scope, subject_key, lease).await } /// Read the operator-provisioned binding after the ordinary exact-package @@ -760,23 +764,36 @@ WITH active_scope AS MATERIALIZED ( SELECT EXISTS(SELECT 1 FROM active_scope),COALESCE((SELECT array_agg(subject_key ORDER BY subject_key COLLATE "C") FROM due),ARRAY[]::text[]) "#; +/// Tests allocate with the two-minute lease a short refresh page needs. +#[cfg(all(test, feature = "postgres-test"))] async fn allocate_inner( client: &impl GenericClient, scope: &ProjectionScope, subject_key: &str, +) -> Result { + allocate_with_lease(client, scope, subject_key, Duration::from_secs(120)).await +} + +async fn allocate_with_lease( + client: &impl GenericClient, + scope: &ProjectionScope, + subject_key: &str, + lease: Duration, ) -> Result { if !valid_key(subject_key) { return Err(ProjectionError::InvalidSnapshot); } + let lease = i64::try_from(lease.as_secs()).map_err(|_| ProjectionError::StaleRefresh)?; let row = client .query_one( - "SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,$5)", + "SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,$5,$6)", &[ &scope.projection_id, &scope.epoch, &scope.binding_digest, &scope.grant_digest, &subject_key, + &lease, ], ) .await @@ -1063,7 +1080,7 @@ CREATE INDEX IF NOT EXISTS registry_external_projection_allocated_idx WHERE state='allocated'; CREATE OR REPLACE FUNCTION registry_internal.allocate_external_projection_refresh( - p_projection text,p_epoch uuid,p_binding text,p_grant text,p_subject text + p_projection text,p_epoch uuid,p_binding text,p_grant text,p_subject text,p_lease bigint ) RETURNS bigint LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, registry_internal AS $body$ DECLARE scope registry_internal.registry_external_projection_scopes; projection_now bigint; allocated bigint; @@ -1073,14 +1090,15 @@ BEGIN WHERE projection_id=p_projection FOR UPDATE; IF NOT FOUND OR p_epoch IS NULL OR p_binding IS NULL OR p_grant IS NULL OR NOT scope.active OR scope.epoch<>p_epoch OR scope.binding_digest<>p_binding OR scope.grant_digest<>p_grant - OR p_subject IS NULL OR octet_length(p_subject) NOT BETWEEN 1 AND 1024 THEN RETURN NULL; END IF; + OR p_subject IS NULL OR octet_length(p_subject) NOT BETWEEN 1 AND 1024 + OR p_lease IS NULL OR p_lease NOT BETWEEN 1 AND 3600 THEN RETURN NULL; END IF; IF (SELECT count(*) FROM registry_internal.registry_external_projection_refreshes WHERE projection_id=p_projection AND state='allocated' AND lease_expires_at>projection_now) >= 1024 THEN RETURN NULL; END IF; UPDATE registry_internal.registry_external_projection_scopes SET next_sequence=next_sequence+1 WHERE projection_id=p_projection RETURNING next_sequence INTO allocated; INSERT INTO registry_internal.registry_external_projection_refreshes (projection_id,sequence,epoch,subject_key,started_at,lease_expires_at,state) - VALUES (p_projection,allocated,p_epoch,p_subject,projection_now,projection_now+120,'allocated'); + VALUES (p_projection,allocated,p_epoch,p_subject,projection_now,projection_now+p_lease,'allocated'); RETURN allocated; END $body$; @@ -1599,7 +1617,7 @@ pub(crate) mod postgres_tests { let transaction = super::super::begin_registry_transaction( &mut runtime, RegistryLockKey::derive(registry.registry_id()).unwrap(), - std::time::Duration::from_secs(2), + Duration::from_secs(2), &expected, ) .await @@ -1634,7 +1652,7 @@ pub(crate) mod postgres_tests { .unwrap() .contains("subject") ); - let ticket = allocate_refresh(&transaction, &bound, "subject") + let ticket = allocate_refresh(&transaction, &bound, "subject", Duration::from_secs(120)) .await .unwrap(); let accepted = AcceptedSnapshot::one( @@ -1743,7 +1761,7 @@ pub(crate) mod postgres_tests { let transaction = super::super::begin_registry_transaction( &mut runtime, RegistryLockKey::derive(registry.registry_id()).unwrap(), - std::time::Duration::from_secs(2), + Duration::from_secs(2), &expected, ) .await @@ -1777,7 +1795,7 @@ pub(crate) mod postgres_tests { let transaction = super::super::begin_registry_transaction( &mut runtime, RegistryLockKey::derive(registry.registry_id()).unwrap(), - std::time::Duration::from_secs(2), + Duration::from_secs(2), &expected, ) .await @@ -2180,6 +2198,57 @@ pub(crate) mod postgres_tests { database.cleanup().await; } + #[tokio::test] + async fn a_refresh_lease_covers_the_acquisition_deadline() { + let database = harness::TestDatabase::create(2).await; + let (migration, migration_task) = database.connect_migration().await; + install(&migration, &database.runtime_role).await.unwrap(); + clock(&migration, 1000).await; + let scope = scope(Uuid::new_v4()); + configure_scope(&migration, &scope).await.unwrap(); + let allocate = |lease: i64| { + let migration = &migration; + let scope = &scope; + async move { + migration + .query_one( + "SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,'subject',$5)", + &[ + &scope.projection_id, + &scope.epoch, + &scope.binding_digest, + &scope.grant_digest, + &lease, + ], + ) + .await + .unwrap() + .get::<_, Option>(0) + } + }; + for refused in [0, -1, 3601] { + assert_eq!(allocate(refused).await, None, "lease {refused}"); + } + let sequence = allocate(300).await.unwrap(); + // The page outlives the previous fixed two-minute lease but stays + // inside its own deadline, so its result is still accepted. + clock(&migration, 1150).await; + let ticket = RefreshTicket { + scope: scope.clone(), + subject_key: "subject".to_owned(), + sequence, + }; + assert_eq!( + publish_without_eligibility(&migration, &ticket, &snapshot("slow", 1149, "accepted")) + .await + .unwrap(), + Publication::Published + ); + migration_task.abort(); + drop(migration); + database.cleanup().await; + } + #[tokio::test] async fn terminal_receipts_fence_older_results_after_snapshot_removal() { let database = harness::TestDatabase::create(2).await; @@ -2417,7 +2486,7 @@ pub(crate) mod postgres_tests { let transaction = super::super::begin_registry_transaction( &mut runtime, super::super::RegistryLockKey::derive(registry.registry_id()).unwrap(), - std::time::Duration::from_secs(2), + Duration::from_secs(2), &self.expected, ) .await @@ -2439,7 +2508,7 @@ pub(crate) mod postgres_tests { let transaction = super::super::begin_registry_transaction( &mut runtime, super::super::RegistryLockKey::derive(registry.registry_id()).unwrap(), - std::time::Duration::from_secs(2), + Duration::from_secs(2), &self.expected, ) .await diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index 5aa75b4424..a7a9999fba 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -1495,12 +1495,13 @@ impl PostgresExternalViewService { .await .map_err(|_| ExternalViewError::Unavailable)?; let reused = keys.len() - due.len(); + let lease = refresh_lease(request.deadline); let mut work = Vec::with_capacity(due.len()); for (key, roots) in by_key { if !due.contains(&key) { continue; } - let ticket = store::allocate_refresh(&transaction, &scope, &key) + let ticket = store::allocate_refresh(&transaction, &scope, &key, lease) .await .map_err(|_| ExternalViewError::Unavailable)?; work.push(RefreshWork { key, roots, ticket }); @@ -1785,6 +1786,19 @@ fn accepted_record( ) } +/// Time a refresh receipt stays live beyond the page deadline. It absorbs +/// scheduling delay and a publication committing as the deadline passes. +const REFRESH_LEASE_MARGIN: Duration = Duration::from_secs(10); + +/// A refresh page acquires and publishes before its deadline, so its receipts +/// stay live for the time left plus a margin. A slow page is never treated +/// as abandoned while it can still publish. +fn refresh_lease(deadline: Instant) -> Duration { + let remaining = deadline.saturating_duration_since(Instant::now()); + Duration::from_secs(remaining.as_secs() + u64::from(remaining.subsec_nanos() > 0)) + + REFRESH_LEASE_MARGIN +} + fn acquisition_failure(error: Option) -> RefreshFailure { match error { Some(ExternalSourceError::Grant) => RefreshFailure::Denied, @@ -1849,6 +1863,21 @@ mod tests { ); assert!(Uuid::parse_str(&snapshot_identity(&codec, "binding-a", None).unwrap()).is_ok()); } + + #[test] + fn a_refresh_lease_outlives_the_page_deadline() { + // The longest configurable request deadline is a minute. + let lease = refresh_lease(Instant::now() + Duration::from_secs(60)); + assert!( + lease > Duration::from_secs(60) + Duration::from_secs(5), + "{lease:?}" + ); + assert!( + lease <= Duration::from_secs(60) + REFRESH_LEASE_MARGIN, + "{lease:?}" + ); + assert_eq!(refresh_lease(Instant::now()), REFRESH_LEASE_MARGIN); + } } #[cfg(test)] diff --git a/crates/registry-breg/src/postgres/external_view_query.rs b/crates/registry-breg/src/postgres/external_view_query.rs index 165bd7efdd..0274a6dc14 100644 --- a/crates/registry-breg/src/postgres/external_view_query.rs +++ b/crates/registry-breg/src/postgres/external_view_query.rs @@ -846,7 +846,7 @@ mod postgres_tests { ) { let sequence: i64 = client .query_one( - "SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,$5)", + "SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,$5,120)", &[ &scope.projection_id, &scope.epoch, @@ -1045,7 +1045,7 @@ mod postgres_tests { let subject = relations.field_expression(entity, "subject-key").unwrap(); let eligibility=format!("SELECT 1 FROM {} WHERE {}=ANY($12::uuid[]) AND ({})::text=$6::text AND current_setting('registry.access_profile',true)='reader'",relations.from_sql,relations.id_expression,subject.sql); let sql = super::super::external_projection::publication_statement(&eligibility); - let ticket:i64=runtime.query_one("SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,'first')",&[&scope.projection_id,&scope.epoch,&scope.binding_digest,&scope.grant_digest]).await.unwrap().get(0); + let ticket:i64=runtime.query_one("SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,'first',120)",&[&scope.projection_id,&scope.epoch,&scope.binding_digest,&scope.grant_digest]).await.unwrap().get(0); let fields = serde_json::json!([{"source_key":"first","fields":{"label":"retained"}}]); let roots = vec![Uuid::parse_str("00000000-0000-4000-8000-000000000001").unwrap()]; let result: String = runtime @@ -1070,7 +1070,7 @@ mod postgres_tests { .unwrap() .get(0); assert_eq!(result, "published"); - let ticket:i64=runtime.query_one("SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,'first')",&[&scope.projection_id,&scope.epoch,&scope.binding_digest,&scope.grant_digest]).await.unwrap().get(0); + let ticket:i64=runtime.query_one("SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,'first',120)",&[&scope.projection_id,&scope.epoch,&scope.binding_digest,&scope.grant_digest]).await.unwrap().get(0); database.admin.batch_execute(&format!("INSERT INTO registry_data.\"{}\" (record_id,active_package_revision,\"{key}\",\"{tenant}\") VALUES ('00000000-0000-4000-8000-000000000005','fixture','first','allowed'); UPDATE registry_data.\"{}\" SET \"{key}\"='detached' WHERE record_id='00000000-0000-4000-8000-000000000001'; UPDATE registry_data.\"{}\" SET \"{key}\"='first' WHERE record_id='00000000-0000-4000-8000-000000000004';",entity.physical_table,entity.physical_table,entity.physical_table)).await.unwrap(); for id in [ "00000000-0000-4000-8000-000000000001", From 10cd0813634bfe14db3fdd6845092f5b4764e296 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 13:18:41 +0700 Subject: [PATCH 13/64] fix(breg): record external refresh outcomes one key per transaction A refresh page published every key in one transaction, holding the projection scope row from its first publication to commit. Indexed reads share that row, so they waited for the whole page and fell into their lock timeout. Each outcome now commits on its own, after rechecking the projection epoch and, for label sweeps, the parent generation. Signed-off-by: Jeremi Joslin --- .../src/postgres/external_projection.rs | 24 +- .../src/postgres/external_runtime.rs | 287 ++++++++++++++---- products/breg/external-projection-storage.md | 2 +- 3 files changed, 243 insertions(+), 70 deletions(-) diff --git a/crates/registry-breg/src/postgres/external_projection.rs b/crates/registry-breg/src/postgres/external_projection.rs index 3ab6b28d89..e01206de73 100644 --- a/crates/registry-breg/src/postgres/external_projection.rs +++ b/crates/registry-breg/src/postgres/external_projection.rs @@ -2546,6 +2546,23 @@ pub(crate) mod postgres_tests { /// Publish one exact-key label under migration authority at the /// database clock, without a root eligibility recheck. pub(crate) async fn publish_label(&self, scope: &ProjectionScope, key: &str, label: &str) { + let (ticket, snapshot) = self.label_refresh(scope, key, label).await; + assert_eq!( + publish_without_eligibility(&self.migration, &ticket, &snapshot) + .await + .unwrap(), + Publication::Published + ); + } + + /// Allocate a refresh for one key and the accepted label snapshot a + /// source would return for it, stamped at the database clock. + pub(crate) async fn label_refresh( + &self, + scope: &ProjectionScope, + key: &str, + label: &str, + ) -> (RefreshTicket, AcceptedSnapshot) { let now: i64 = self .migration .query_one( @@ -2565,12 +2582,7 @@ pub(crate) mod postgres_tests { record(key, label), ) .unwrap(); - assert_eq!( - publish_without_eligibility(&self.migration, &ticket, &snapshot) - .await - .unwrap(), - Publication::Published - ); + (ticket, snapshot) } /// Record a refresh failure for one key, as a refresh worker would. diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index a7a9999fba..b0f1a70e9e 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -1527,7 +1527,55 @@ impl PostgresExternalViewService { let outcomes = self .fetch_refresh(request, plan, &work, page.supporting_until, authority) .await; - let mut session = self.session().await?; + let attempted = work.len(); + published = self + .record_refresh( + request, + plan, + claims, + &scope, + parent.as_ref().map(|(_, generation)| generation), + &page, + work.into_iter().zip(outcomes).collect(), + ) + .await?; + unavailable += attempted - published; + } + let expires_at = plan.grant_valid_until().timestamp(); + let mut body = json!({"processed": page.rows.len(), "published": published, "reused": reused, "unavailable": unavailable}); + if let Some(after) = page.next { + body["nextCursor"] = json!(self.next_cursor( + binding, + query, + Position::Refresh { + after, + minimum_sequence + }, + expires_at + )?); + } + Ok(HeldResponse { body, expires_at }) + } + + /// Records each acquisition outcome in its own short transaction and + /// returns how many were published. A publication holds the projection + /// scope row until it commits, and indexed reads share that row, so one + /// transaction for the whole page would hold readers back for every key. + #[allow(clippy::too_many_arguments)] // The page's scope, parent and rows come from its selection transaction. + async fn record_refresh( + &self, + request: &ExternalViewRequest, + plan: &Plan<'_>, + claims: &ClaimContext, + scope: &ProjectionScope, + parent: Option<&ProjectionGeneration>, + page: &RefreshRows, + results: Vec<(RefreshWork, RefreshResult)>, + ) -> Result { + let mut published = 0usize; + let mut session = self.session().await?; + let mut results = results.into_iter(); + while let Some((work, result)) = results.next() { let transaction = self .transaction(&mut session, claims, request.deadline) .await?; @@ -1536,15 +1584,15 @@ impl PostgresExternalViewService { if current.epoch != scope.epoch { return Err(ExternalViewError::Unavailable); } - if let (Some(parent), Some((_, selected_generation))) = (&plan.label_parent, &parent) { - // A rotated or withdrawn parent abandons the page. A database - // or lock failure is reported as such and leaves the receipts - // to expire with their leases. + if let (Some(parent), Some(selected_generation)) = (&plan.label_parent, parent) { + // A rotated or withdrawn parent abandons the rest of the + // page. A database or lock failure is reported as such and + // leaves the receipts to expire with their leases. let parent_is_current = match self .lock_scope(&transaction, parent.source, parent.projection.id.as_str()) .await { - Ok((_, generation)) => &generation == selected_generation, + Ok((_, generation)) => generation == *selected_generation, Err(ExternalViewError::ProjectionUnavailable(_)) => false, Err(error) => return Err(error), }; @@ -1553,7 +1601,8 @@ impl PostgresExternalViewService { .supporting_until .is_some_and(|cutoff| Utc::now().timestamp() >= cutoff) { - for work in &work { + let abandoned = std::iter::once(work).chain(results.map(|(work, _)| work)); + for work in abandoned { store::fail_refresh(&transaction, &work.ticket, RefreshFailure::Abandoned) .await .map_err(|_| ExternalViewError::Unavailable)?; @@ -1566,70 +1615,53 @@ impl PostgresExternalViewService { return Err(ExternalViewError::Unavailable); } } - for (work, result) in work.into_iter().zip(outcomes) { - let accepted = match result { - RefreshResult::Accepted(snapshot) => { - let root_ids = work.roots.into_iter().collect::>(); - // The RLS recheck and publication share a statement - // snapshot. A detached association does not erase a - // shared fact still used by another root. - match store::publish_for_roots( - &transaction, - &work.ticket, - &snapshot, - &page.eligibility, - &root_ids, - ) - .await - { - Ok(store::Publication::Published | store::Publication::Replay) => true, - Ok(store::Publication::Ineligible) - | Err(store::ProjectionError::StaleRefresh) => { - store::fail_refresh( - &transaction, - &work.ticket, - RefreshFailure::Abandoned, - ) - .await - .map_err(|_| ExternalViewError::Unavailable)?; - false - } - Err(_) => return Err(ExternalViewError::Unavailable), - } - } - RefreshResult::Failed(failure) => { - store::fail_refresh(&transaction, &work.ticket, failure) + let accepted = match result { + RefreshResult::Accepted(snapshot) => { + let root_ids = work.roots.into_iter().collect::>(); + // The RLS recheck and publication share a statement + // snapshot. A detached association does not erase a + // shared fact still used by another root. + match store::publish_for_roots( + &transaction, + &work.ticket, + &snapshot, + &page.eligibility, + &root_ids, + ) + .await + { + Ok(store::Publication::Published | store::Publication::Replay) => true, + Ok(store::Publication::Ineligible) + | Err(store::ProjectionError::StaleRefresh) => { + store::fail_refresh( + &transaction, + &work.ticket, + RefreshFailure::Abandoned, + ) .await .map_err(|_| ExternalViewError::Unavailable)?; - false + false + } + Err(_) => return Err(ExternalViewError::Unavailable), } - }; - if accepted { - published += 1; - } else { - unavailable += 1; } - } + RefreshResult::Failed(failure) => { + store::fail_refresh(&transaction, &work.ticket, failure) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + false + } + }; transaction .commit() .await .map_err(|_| ExternalViewError::Unavailable)?; - session.disarm(); - } - let expires_at = plan.grant_valid_until().timestamp(); - let mut body = json!({"processed": page.rows.len(), "published": published, "reused": reused, "unavailable": unavailable}); - if let Some(after) = page.next { - body["nextCursor"] = json!(self.next_cursor( - binding, - query, - Position::Refresh { - after, - minimum_sequence - }, - expires_at - )?); + if accepted { + published += 1; + } } - Ok(HeldResponse { body, expires_at }) + session.disarm(); + Ok(published) } async fn fetch_refresh( @@ -2046,6 +2078,7 @@ mod postgres_tests { struct Served { fixture: Activated, router: axum::Router, + service: Arc, scope: ProjectionScope, } @@ -2057,6 +2090,7 @@ mod postgres_tests { let fixture = Activated::new(®istry).await; let mut service = service_with(®istry, &fixture, lock_timeout); service.bind_configured_scopes(true).await.ok().unwrap(); + let service = Arc::new(service); let scope = bound_scope(&fixture, "projection").await; let entity = ®istry.entities()["local-record"]; let column = |id: &str| { @@ -2100,10 +2134,11 @@ mod postgres_tests { .unwrap(), ), ) - .with_external_views(Arc::new(service)); + .with_external_views(service.clone()); Self { router: crate::api::router(Arc::new(http)), fixture, + service, scope, } } @@ -2366,4 +2401,130 @@ mod postgres_tests { assert_eq!(problem["code"], "external_view.incomplete", "{problem}"); served.fixture.cleanup().await; } + + #[tokio::test] + async fn indexed_reads_go_on_while_a_refresh_records_its_page() { + const BARRIER: i64 = 732_995_011; + let served = Served::new(Duration::from_millis(300)).await; + served.publish_all().await; + let caller = reader("reader-principal", &["allowed"]); + // The refresh worker waits out the barrier below; readers keep the + // short lock timeout. + let mut worker = service_with( + &served.service.registry, + &served.fixture, + Duration::from_secs(5), + ); + worker.bind_configured_scopes(true).await.ok().unwrap(); + let refresh = request("link"); + let plan = worker.plan(&refresh, None).ok().unwrap(); + let claims = ClaimContext::for_compiled( + &served.service.registry, + &plan.root.id, + Some("reader-principal".to_owned()), + "reader", + Some("consultation".to_owned()), + vec![crate::postgres::context::RowBoundaryContext::Equals { + field: "tenant".to_owned(), + value: "allowed".to_owned(), + }], + ) + .unwrap(); + // The second key's publication pauses in its eligibility recheck, + // after the first key's publication. + let eligibility = RootKeys::new(&plan).ok().unwrap().publication_eligibility; + let page = RefreshRows { + rows: Vec::new(), + next: None, + eligibility: format!( + "SELECT 1 FROM ({eligibility}) AS eligible + WHERE CASE WHEN $6::text='second' + THEN pg_catalog.pg_advisory_xact_lock({BARRIER}) IS NOT NULL + ELSE true END" + ), + supporting_until: None, + supporting: Vec::new(), + }; + let mut results = Vec::new(); + for (id, key, _) in &ROOTS[..3] { + let (ticket, snapshot) = served + .fixture + .label_refresh(&served.scope, key, "renewed") + .await; + results.push(( + RefreshWork { + key: (*key).to_owned(), + roots: BTreeSet::from([Uuid::parse_str(id).unwrap()]), + ticket, + }, + RefreshResult::Accepted(snapshot), + )); + } + served + .fixture + .migration + .execute(&format!("SELECT pg_advisory_lock({BARRIER})"), &[]) + .await + .unwrap(); + let read_during_refresh = async { + let waited = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let waiting: bool = served + .fixture + .database + .admin + .query_one( + &format!( + "SELECT EXISTS (SELECT 1 FROM pg_locks + WHERE locktype='advisory' AND NOT granted AND objid={BARRIER})" + ), + &[], + ) + .await + .unwrap() + .get(0); + if waiting { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .is_ok(); + let read = if waited { + Some(served.get("&$count=true", caller.clone()).await) + } else { + None + }; + served + .fixture + .migration + .execute(&format!("SELECT pg_advisory_unlock({BARRIER})"), &[]) + .await + .unwrap(); + read.expect("the second publication reached the barrier") + }; + let (recorded, (status, during)) = tokio::join!( + worker.record_refresh( + &refresh, + &plan, + &claims, + &served.scope, + None, + &page, + results + ), + read_during_refresh, + ); + // The reader sees the first key's new value and the previous values + // of the keys still in flight, all under one complete coverage. + assert_eq!(status, axum::http::StatusCode::OK, "{during}"); + assert_eq!(labels(&during), ["renewed", "second", "third"]); + assert_eq!(during["count"], 3); + assert_eq!(recorded.ok(), Some(3)); + let (status, after) = served.get("&$count=true", caller).await; + assert_eq!(status, axum::http::StatusCode::OK, "{after}"); + assert_eq!(labels(&after), ["renewed", "renewed", "renewed"]); + served.fixture.cleanup().await; + } } diff --git a/products/breg/external-projection-storage.md b/products/breg/external-projection-storage.md index eb631c269e..de6968c681 100644 --- a/products/breg/external-projection-storage.md +++ b/products/breg/external-projection-storage.md @@ -27,7 +27,7 @@ A captured snapshot stores the observation time and a keyed revision reference w Replaying an accepted result keeps its original expiry. A refresh that fails because the source is unavailable, returns an incomplete result, or cannot be acquired under the current binding keeps the previous snapshot of either kind until its own expiry, without extending its lifetime. Explicit grant denial or no-store removes the previous snapshot, making coverage unproven, fenced against newer publications. A source response marked `no-store` cannot enter the indexed store. If it supersedes an earlier retained result for the same subject, the guarded retention-refusal path removes that result; an older refresh cannot erase a newer publication. -Reads check expiry independently of the refresh job. Indexed predicates and counts refuse incomplete coverage before selecting matching rows. Each page reads coverage, rows and any count from one statement snapshot, so a count always matches complete coverage at that moment. A `nextCursor` stays valid while refreshes publish or fail individual keys; the next page reads the values current when it runs. A row whose ordering value changed between pages can appear twice or not at all. Rotating, withdrawing or resetting the projection invalidates outstanding cursors. A complete empty related collection is covered; a partial collection, source failure, or protected-existence response is not evidence of emptiness or absence. +Reads check expiry independently of the refresh job. Indexed predicates and counts refuse incomplete coverage before selecting matching rows. Each page reads coverage, rows and any count from one statement snapshot, so a count always matches complete coverage at that moment. A refresh records each key in its own short transaction, so a read waits for at most one key's publication and can show new values for some keys of a page in flight and previous values for the rest. A `nextCursor` stays valid while refreshes publish or fail individual keys; the next page reads the values current when it runs. A row whose ordering value changed between pages can appear twice or not at all. Rotating, withdrawing or resetting the projection invalidates outstanding cursors. A complete empty related collection is covered; a partial collection, source failure, or protected-existence response is not evidence of emptiness or absence. The bounded purge worker runs every few seconds and deletes snapshots once they expire. It also removes data from withdrawn or replaced epochs. Size database capacity for the backlog. Where a storage obligation is strict, choose a cache lifetime shorter than that obligation by more than the worker interval and the expected recovery delay. Freshness enforcement keeps expired values out of responses even when refresh or purge is delayed, but a delayed purge can still breach a storage obligation. Monitor purge failures, expiry backlog, and database downtime separately. No physical erasure guarantee holds during a database outage or when backlog exceeds that margin. Database WAL and physical snapshots need their own retention policy. From b50067b6d20c0d59b51b5ba7f8cee4337194a1a8 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 13:21:34 +0700 Subject: [PATCH 14/64] test(breg): refuse external view cursors replayed under other authority An HTTP continuation under another principal or another row boundary claim answers query.cursor_invalid, and the original caller continues. Signed-off-by: Jeremi Joslin --- .../src/postgres/external_runtime.rs | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index b0f1a70e9e..7357f2b755 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -2527,4 +2527,45 @@ mod postgres_tests { assert_eq!(labels(&after), ["renewed", "renewed", "renewed"]); served.fixture.cleanup().await; } + + #[tokio::test] + async fn a_cursor_replayed_under_other_authority_is_an_invalid_cursor() { + let served = Served::new(Duration::from_secs(2)).await; + served.publish_all().await; + let (status, first) = served + .get("&$top=1", reader("reader-principal", &["allowed"])) + .await; + assert_eq!(status, axum::http::StatusCode::OK, "{first}"); + let cursor = first["nextCursor"].as_str().unwrap().to_owned(); + for (caller, authority) in [ + ( + reader("another-principal", &["allowed"]), + "another principal", + ), + ( + reader("reader-principal", &["other"]), + "another row boundary", + ), + ] { + let (status, problem) = served.get(&format!("&$skiptoken={cursor}"), caller).await; + assert_eq!( + status, + axum::http::StatusCode::BAD_REQUEST, + "{authority}: {problem}" + ); + assert_eq!( + problem["code"], "query.cursor_invalid", + "{authority}: {problem}" + ); + } + let (status, next) = served + .get( + &format!("&$skiptoken={cursor}"), + reader("reader-principal", &["allowed"]), + ) + .await; + assert_eq!(status, axum::http::StatusCode::OK, "{next}"); + assert_eq!(labels(&next), ["second"]); + served.fixture.cleanup().await; + } } From 05497f2d08d4f8fc90f13675c817243244943a8f Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 13:26:54 +0700 Subject: [PATCH 15/64] fix(breg): render external view date cursors in ISO 8601 A date's text output follows the session DateStyle, so a cursor issued under one style failed to continue under another. The cursor now carries the ISO 8601 form, which the continuation parses under any style. Signed-off-by: Jeremi Joslin --- .../src/postgres/external_view_query.rs | 117 +++++++++++++++++- 1 file changed, 113 insertions(+), 4 deletions(-) diff --git a/crates/registry-breg/src/postgres/external_view_query.rs b/crates/registry-breg/src/postgres/external_view_query.rs index 0274a6dc14..da1a3c81cd 100644 --- a/crates/registry-breg/src/postgres/external_view_query.rs +++ b/crates/registry-breg/src/postgres/external_view_query.rs @@ -356,6 +356,13 @@ fn compile( (Some(order), Some(kind)) => field_expression("data", &order.field, kind), _ => "NULL::text".to_owned(), }; + // A date's text output follows the session's DateStyle. The cursor + // carries ISO 8601, which the continuation parses under any style. + let sort_value = if order_type == Some(ExternalFieldType::Date) { + "to_char(matched.sort_key,'YYYY-MM-DD')" + } else { + "matched.sort_key::text" + }; let direction = request .order .as_ref() @@ -475,7 +482,7 @@ fn compile( ), displayed AS ( SELECT matched.ordinal,{label_expiry} AS label_expires_at, jsonb_build_object('root_id',matched.root_id,'source_key',matched.source_key, - 'availability',matched.availability,'fields',{selected},'label',{label_value},'sort_value',matched.sort_key::text) AS payload + 'availability',matched.availability,'fields',{selected},'label',{label_value},'sort_value',{sort_value}) AS payload FROM page AS matched {label_join} ) SELECT EXISTS (SELECT 1 FROM projection_state){label_valid},coverage.complete,coverage.root_found, LEAST(coverage.expires_at,(SELECT min(label_expires_at) FROM displayed)),{count}, @@ -843,6 +850,17 @@ mod postgres_tests { key: &str, label: Option<&str>, id: &str, + ) { + let fields = label.map(|label| serde_json::json!({"label": label})); + publish_fields(client, scope, key, fields, id).await; + } + + async fn publish_fields( + client: &Client, + scope: &ProjectionScope, + key: &str, + fields: Option, + id: &str, ) { let sequence: i64 = client .query_one( @@ -858,9 +876,9 @@ mod postgres_tests { .await .unwrap() .get(0); - let kind = if label.is_some() { "one" } else { "absent" }; - let rows = label - .map(|label| serde_json::json!([{"source_key":key,"fields":{"label":label}}])) + let kind = if fields.is_some() { "one" } else { "absent" }; + let rows = fields + .map(|fields| serde_json::json!([{"source_key":key,"fields":fields}])) .unwrap_or_else(|| serde_json::json!([])); let result:String=client.query_one("SELECT registry_internal.publish_external_projection($1,$2,$3,$4,$5,$6,$7,999,1059,$8,$9)",&[&scope.projection_id,&scope.epoch,&scope.binding_digest,&scope.grant_digest,&sequence,&key,&id,&kind,&rows]).await.unwrap().get(0); assert_eq!(result, "published"); @@ -1107,4 +1125,95 @@ mod postgres_tests { drop(migration); database.cleanup().await; } + + #[tokio::test] + async fn a_date_cursor_reads_the_same_under_any_date_style() { + let database = harness::TestDatabase::create(2).await; + let (migration, migration_task) = database.connect_migration().await; + let mut project = super::tests::project(); + project["externalRecordTypes"][0]["fields"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({"id":"issued","type":"date","classification":"internal"})); + for path in [ + "/externalProjections/0/fields", + "/externalViews/0/links/0/fields", + "/accessProfiles/0/permissions/externalViews/0/links/0/readableFields", + "/accessProfiles/0/permissions/externalViews/0/links/0/sortableFields", + ] { + project + .pointer_mut(path) + .and_then(Value::as_array_mut) + .unwrap() + .push(Value::from("issued")); + } + let registry = super::tests::compile_project(&project); + super::super::install_compiled_schema(&migration, ®istry, &database.runtime_role) + .await + .unwrap(); + install(&migration, &database.runtime_role).await.unwrap(); + migration + .batch_execute(&INSTALL_SQL.replace( + "floor(extract(epoch FROM clock_timestamp()))::bigint", + "1000", + )) + .await + .unwrap(); + let scope = + ProjectionScope::new("projection", Uuid::new_v4(), "binding", "grant", 60).unwrap(); + configure_scope(&migration, &scope).await.unwrap(); + let entity = ®istry.entities()["local-record"]; + let column = |id: &str| { + entity + .stored_fields + .iter() + .find(|field| field.logical.id == id) + .unwrap() + .physical_name + .clone() + }; + database.admin.batch_execute(&format!("INSERT INTO registry_data.\"{}\" (record_id,active_package_revision,\"{}\",\"{}\") VALUES ('00000000-0000-4000-8000-000000000001','fixture','first','allowed'),('00000000-0000-4000-8000-000000000002','fixture','second','allowed')",entity.physical_table,column("subject-key"),column("tenant"))).await.unwrap(); + for (key, label, issued) in [("first", "A", "2026-01-02"), ("second", "B", "2026-02-01")] { + let fields = serde_json::json!({"label": label, "issued": issued}); + publish_fields(&migration, &scope, key, Some(fields), key).await; + } + let (runtime, connection) = database + .runtime_config + .postgres() + .connect(NoTls) + .await + .unwrap(); + let runtime_task = tokio::spawn(async move { + connection.await.expect("query fixture connection"); + }); + runtime.batch_execute("SET registry.access_profile='reader'; SET registry.principal='staff'; SET registry.purpose='consultation'; SET registry.row_boundaries='[{\"field\":\"tenant\",\"operator\":\"equals\",\"values\":[\"allowed\"]}]';").await.unwrap(); + // A database default can choose any date output style. + runtime + .batch_execute("SET DateStyle='SQL, DMY'") + .await + .unwrap(); + let mut request = super::tests::request(&scope, 0); + request.order = Some(ExternalOrder { + field: "issued".to_owned(), + direction: SortDirection::Ascending, + }); + request.page_size = 1; + let first = run(&runtime, ®istry, &request).await; + let rows: Vec = serde_json::from_value(first.get(5)).unwrap(); + assert_eq!(rows[0].fields["label"], "A"); + assert_eq!(rows[0].sort_value.as_deref(), Some("2026-01-02")); + request.after = Some(PageAfter { + root_id: rows[0].root_id, + source_key: rows[0].source_key.clone(), + sort_value: rows[0].sort_value.clone(), + }); + let second = run(&runtime, ®istry, &request).await; + let rows: Vec = serde_json::from_value(second.get(5)).unwrap(); + assert_eq!(rows[0].fields["label"], "B"); + runtime_task.abort(); + migration_task.abort(); + drop(runtime); + drop(migration); + database.cleanup().await; + } } From e881c4e77a28aba1a890faddd46d6e0902e8bc6f Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 13:26:54 +0700 Subject: [PATCH 16/64] fix(breg): bind external view cursors to the row boundary operator name The cursor binding used the operator's Rust Debug form, which changes with a rename. It now uses the contract names equals and in, as query bindings do. Signed-off-by: Jeremi Joslin --- .../registry-breg/src/postgres/external_runtime.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index 7357f2b755..1bda7ea486 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -21,7 +21,7 @@ use crate::api::external_views::{ ExternalQuery, ExternalQueryMode, ExternalViewError, ExternalViewRequest, ExternalViewService, RefreshMode, }; -use crate::api::ServiceFuture; +use crate::api::{RowBoundaryOperator, ServiceFuture}; use crate::audit::{ begin_pre_io_audit, profile_is_keyed, read_terminal_entry, PreIoAudit, PreIoAuditKind, ReadTerminalAudit, RegistryAudit, TerminalAudit, TerminalAuditOutcome, @@ -931,11 +931,11 @@ impl PostgresExternalViewService { .row_boundaries() .iter() .map(|boundary| { - json!([ - boundary.field(), - format!("{:?}", boundary.operator()), - boundary.values() - ]) + let operator = match boundary.operator() { + RowBoundaryOperator::Equals => "equals", + RowBoundaryOperator::In => "in", + }; + json!([boundary.field(), operator, boundary.values()]) }) .collect::>(); self.cursors From 0c5c6e8cd4e7a9fb85c26b400f687d9e393d5568 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 13:32:32 +0700 Subject: [PATCH 17/64] fix(breg): let the planner choose the external refresh page plan A dense eligible scope already pages in primary-key order without a sort, and a selective row boundary pages faster with a bounded sort over its few rows than with a forced primary-key walk. The page query no longer turns sorting off; a test pins the dense-scope plan. Signed-off-by: Jeremi Joslin --- .../src/postgres/external_runtime.rs | 102 +++++++++++++++--- 1 file changed, 87 insertions(+), 15 deletions(-) diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index 1bda7ea486..da0d3f832d 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -1196,29 +1196,22 @@ impl RootKeys { Ok(row.map(|row| row.get(1))) } + fn page_sql(&self) -> String { + format!( + "{} WHERE ($1::uuid IS NULL OR {} > $1) ORDER BY {} LIMIT $2", + self.select, self.id, self.id + ) + } + async fn page( &self, transaction: &GuardedTransaction<'_>, after: Option, limit: u16, ) -> Result)>> { - // The row policy can make PostgreSQL underestimate the eligible - // population. Prefer the existing primary-key order for this - // bounded refresh transaction to avoid sorting the scope per page. - transaction - .transaction() - .batch_execute("SET LOCAL enable_sort = off") - .await - .map_err(|_| ExternalViewError::Unavailable)?; let rows = transaction .transaction() - .query( - &format!( - "{} WHERE ($1::uuid IS NULL OR {} > $1) ORDER BY {} LIMIT $2", - self.select, self.id, self.id - ), - &[&after, &i64::from(limit)], - ) + .query(&self.page_sql(), &[&after, &i64::from(limit)]) .await .map_err(|_| ExternalViewError::Unavailable)?; Ok(rows @@ -2568,4 +2561,83 @@ mod postgres_tests { assert_eq!(labels(&next), ["second"]); served.fixture.cleanup().await; } + + #[tokio::test] + async fn a_refresh_pages_a_dense_scope_in_key_order_without_sorting_it() { + let served = Served::new(Duration::from_secs(2)).await; + let entity = &served.service.registry.entities()["local-record"]; + let column = |id: &str| { + entity + .stored_fields + .iter() + .find(|field| field.logical.id == id) + .unwrap() + .physical_name + .clone() + }; + // Most of a large scope is eligible, so sorting it for each page + // would read the whole scope. + served + .fixture + .database + .admin + .batch_execute(&format!( + "INSERT INTO registry_data.\"{table}\" + (record_id,active_package_revision,\"{key}\",\"{tenant}\") + SELECT gen_random_uuid(),'fixture','bulk-'||g, + CASE WHEN g % 20 = 0 THEN 'other' ELSE 'allowed' END + FROM generate_series(1,20000) AS g; + ANALYZE registry_data.\"{table}\";", + table = entity.physical_table, + key = column("subject-key"), + tenant = column("tenant"), + )) + .await + .unwrap(); + let refresh = request("link"); + let plan = served.service.plan(&refresh, None).ok().unwrap(); + let claims = ClaimContext::for_compiled( + &served.service.registry, + &plan.root.id, + Some("reader-principal".to_owned()), + "reader", + Some("consultation".to_owned()), + vec![crate::postgres::context::RowBoundaryContext::Equals { + field: "tenant".to_owned(), + value: "allowed".to_owned(), + }], + ) + .unwrap(); + let keys = RootKeys::new(&plan).ok().unwrap(); + let middle = Uuid::parse_str("80000000-0000-4000-8000-000000000000").unwrap(); + for after in [None, Some(middle)] { + let mut session = served.service.session().await.ok().unwrap(); + let transaction = served + .service + .transaction( + &mut session, + &claims, + Instant::now() + Duration::from_secs(5), + ) + .await + .ok() + .unwrap(); + let page = keys.page(&transaction, after, 101).await.ok().unwrap(); + assert_eq!(page.len(), 101); + let explained = transaction + .transaction() + .query( + &format!("EXPLAIN (COSTS OFF) {}", keys.page_sql()), + &[&after, &101i64], + ) + .await + .unwrap() + .into_iter() + .map(|row| row.get::<_, String>(0)) + .collect::>() + .join("\n"); + assert!(!explained.contains("Sort"), "after {after:?}: {explained}"); + } + served.fixture.cleanup().await; + } } From bc451d45be2528190e586df8ef1f82537863879d Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 13:37:53 +0700 Subject: [PATCH 18/64] test(breg): prove the indexed no-store refusal through the source binding A source reply marked no-store refuses an indexed acquisition through the bound source, and the refresh records it as a retention refusal, which removes the previous snapshot. Signed-off-by: Jeremi Joslin --- crates/registry-breg/src/external_source.rs | 84 +++++++++++++++++++ .../src/postgres/external_runtime.rs | 10 +++ 2 files changed, 94 insertions(+) diff --git a/crates/registry-breg/src/external_source.rs b/crates/registry-breg/src/external_source.rs index cb4488bdb7..d0e5807e0a 100644 --- a/crates/registry-breg/src/external_source.rs +++ b/crates/registry-breg/src/external_source.rs @@ -922,6 +922,90 @@ pub(crate) mod tests { transcript.finish().unwrap(); } + #[tokio::test] + async fn a_no_store_source_response_is_refused_for_the_indexed_store() { + async fn indexed_batch( + no_store: bool, + ) -> ( + Result, + Arc, + ) { + let (pack, artifacts) = example(); + let mut response = json!({"status":200,"body":{"records":[ + {"subject":"person-7","state":"found","properties":{"family-name":"Lee"}} + ]}}); + if no_store { + response["headers"] = json!({"Cache-Control":"private, no-store"}); + } + let transcript = Arc::new( + registry_adapter_engine::replay::ReplayTranscript::from_exchanges( + serde_json::from_value( + json!([{"type":"endpoint","endpoint":"synthetic-registry"}]), + ) + .unwrap(), + serde_json::from_value(json!([{ + "request":{"endpoint":"synthetic-registry","method":"POST","path":"/people/batch","body":{ + "type":"batch","keys":["person-7"],"fields":["family-name"] + }}, + "response":response + }])) + .unwrap(), + ), + ); + let program = compile_projection_pack( + &pack, + "person-batch", + &artifacts, + EvidenceLimits::default(), + ) + .unwrap() + .bind_replay(Arc::clone(&transcript)) + .unwrap(); + let fields = BTreeSet::from(["family-name".to_owned()]); + let service = ExternalSourceService { + bindings: BTreeMap::from([( + "holder".to_owned(), + BoundExternalSource { + program, + fields: fields.clone(), + mode: ExternalProjectionMode::Indexed, + grant_valid_until: Utc::now() + chrono::Duration::days(1), + binding_fingerprint: "binding".to_owned(), + grant_fingerprint: "grant".to_owned(), + cache_ttl_seconds: Some(60), + field_types: BTreeMap::from([( + "family-name".to_owned(), + (ExternalFieldType::String, false), + )]), + }, + )]), + unavailable: BTreeMap::new(), + }; + let result = service + .acquire( + "holder", + Arc::new(Hooks), + ProjectionRequest::Batch { + keys: vec!["person-7".to_owned()], + }, + &fields, + Utc::now, + ) + .await; + (result, transcript) + } + + let (stored, transcript) = indexed_batch(false).await; + assert!(stored.is_ok()); + transcript.finish().unwrap(); + let (refused, transcript) = indexed_batch(true).await; + assert!(matches!( + refused, + Err(ExternalSourceError::Acquisition(ProjectionError::NoStore)) + )); + transcript.finish().unwrap(); + } + #[test] fn a_deployment_cache_lifetime_only_shortens_the_authored_one() { let (live, _) = project_contracts(); diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index da0d3f832d..13a00920d1 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -1889,6 +1889,16 @@ mod tests { assert!(Uuid::parse_str(&snapshot_identity(&codec, "binding-a", None).unwrap()).is_ok()); } + #[test] + fn a_no_store_acquisition_is_recorded_as_a_retention_refusal() { + assert_eq!( + acquisition_failure(Some(ExternalSourceError::Acquisition( + AdapterProjectionError::NoStore + ))), + RefreshFailure::RetentionRefused + ); + } + #[test] fn a_refresh_lease_outlives_the_page_deadline() { // The longest configurable request deadline is a minute. From 8cbbe26bc6338ddab009f739dc9c592c6d3bdf92 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 13:37:53 +0700 Subject: [PATCH 19/64] refactor(breg): drop the unreachable no-store retention permission No caller ever built the no-store variant; the Adapter refuses a no-store reply before a snapshot exists. The store keeps its check that retention covers the source expiry. Signed-off-by: Jeremi Joslin --- .../src/postgres/external_projection.rs | 24 ++++--------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/crates/registry-breg/src/postgres/external_projection.rs b/crates/registry-breg/src/postgres/external_projection.rs index e01206de73..51e778cceb 100644 --- a/crates/registry-breg/src/postgres/external_projection.rs +++ b/crates/registry-breg/src/postgres/external_projection.rs @@ -326,11 +326,11 @@ impl ProjectionScope { } } +/// A source response marked no-store never reaches the store: the Adapter +/// refuses it for an indexed acquisition, and the refresh records a +/// retention refusal that removes the previous snapshot. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum RetentionPermission { - // Retain an explicit refusal variant even though the source adapter also rejects it. - #[allow(dead_code)] - NoStore, /// Source-approved absolute last retention time, as Unix seconds. PermittedUntil(i64), } @@ -471,9 +471,7 @@ impl AcceptedSnapshot { ) -> Result { // The source must permit storage until its own expiry. The cache // never keeps a value past that expiry, so no later deadline is held. - let RetentionPermission::PermittedUntil(retain_until) = retention else { - return Err(ProjectionError::RetentionRefused); - }; + let RetentionPermission::PermittedUntil(retain_until) = retention; if retain_until < source_expires_at { return Err(ProjectionError::RetentionRefused); } @@ -1398,19 +1396,7 @@ mod tests { } #[test] - fn no_store_and_unbounded_retention_are_refused() { - assert_eq!( - AcceptedSnapshot::one( - "key", - "snapshot", - 100, - 160, - RetentionPermission::NoStore, - row("key") - ) - .unwrap_err(), - ProjectionError::RetentionRefused - ); + fn short_and_unbounded_retention_are_refused() { assert!(ProjectionScope::new("people", Uuid::nil(), "binding", "grant", 0).is_err()); assert!( ProjectionScope::new("people", Uuid::nil(), "binding", "grant", 31_536_001).is_err() From aaa52d04a5327a1df808774b15bab20721cd23a9 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 14:17:13 +0700 Subject: [PATCH 20/64] fix(breg): log external projection refresh and purge outcomes by reason Operators could not tell why external projection data went stale or stayed stored: a failed purge pass logged no cause, refresh outcomes were not logged, and a failed live acquisition was discarded. Each purge pass now logs the rows it removed per projection and reason (expired, replaced, withdrawn), a failed pass logs its failure category, and a pass that reaches its batch bound warns. Each refresh page logs its published count and its failed keys per reason, a refresh that fails as a whole logs its reason, and a failed live acquisition logs its reason instead of being dropped. No line carries a source key, selector, value or secret. A startup check also reports, for each projection a server would serve, the source grant expiry and the observation time of the newest snapshot its active epoch holds. Signed-off-by: Jeremi Joslin --- crates/registry-breg/src/hooks.rs | 8 +- .../src/postgres/external_projection.rs | 173 ++++-- .../src/postgres/external_runtime.rs | 556 +++++++++++++++++- crates/registry-breg/src/startup.rs | 31 +- 4 files changed, 697 insertions(+), 71 deletions(-) diff --git a/crates/registry-breg/src/hooks.rs b/crates/registry-breg/src/hooks.rs index 07a372808d..91ff6235e9 100644 --- a/crates/registry-breg/src/hooks.rs +++ b/crates/registry-breg/src/hooks.rs @@ -368,7 +368,7 @@ struct ProposalCompletionBarrier { const MAX_RETENTION_BATCHES_PER_TICK: usize = 100; #[derive(Clone, Copy)] -enum RetentionFailureReason { +pub(crate) enum RetentionFailureReason { RegistryNotReady, LockTimeout, QueryCancelled, @@ -376,7 +376,7 @@ enum RetentionFailureReason { } impl RetentionFailureReason { - fn as_str(self) -> &'static str { + pub(crate) fn as_str(self) -> &'static str { match self { Self::RegistryNotReady => "registry-not-ready", Self::LockTimeout => "lock-timeout", @@ -385,7 +385,7 @@ impl RetentionFailureReason { } } - fn sqlstate(code: Option<&str>) -> Self { + pub(crate) fn sqlstate(code: Option<&str>) -> Self { match code { Some("55P03") => Self::LockTimeout, Some("57014") => Self::QueryCancelled, @@ -393,7 +393,7 @@ impl RetentionFailureReason { } } - fn admission(error: crate::postgres::RegistryTransactionError) -> Self { + pub(crate) fn admission(error: crate::postgres::RegistryTransactionError) -> Self { use crate::postgres::{PostgresKernelError, RegistryLockFailure, RegistryTransactionError}; match error { RegistryTransactionError::Kernel(PostgresKernelError::RegistryUnavailable) => { diff --git a/crates/registry-breg/src/postgres/external_projection.rs b/crates/registry-breg/src/postgres/external_projection.rs index 51e778cceb..9b176c1e9b 100644 --- a/crates/registry-breg/src/postgres/external_projection.rs +++ b/crates/registry-breg/src/postgres/external_projection.rs @@ -12,6 +12,7 @@ use tokio_postgres::GenericClient; use uuid::Uuid; use super::{ExpectedRegistryIdentity, GuardedTransaction, RuntimeRevoke, SqlIdentifier}; +use crate::hooks::RetentionFailureReason; pub(crate) const BIND_ONCE_FUNCTION: &str = "registry_internal.bind_external_projection_once(text,uuid,text,text,bigint,text,text,uuid)"; @@ -553,7 +554,7 @@ pub(crate) enum RefreshFailure { } impl RefreshFailure { - fn as_str(self) -> &'static str { + pub(crate) fn as_str(self) -> &'static str { match self { Self::Abandoned => "abandoned", Self::Unavailable => "unavailable", @@ -999,28 +1000,110 @@ pub(crate) async fn fail_refresh( Ok(()) } +/// When the newest snapshot the projection's active epoch holds was observed, +/// if it holds any. Withdrawn and replaced epochs do not count. +pub(crate) async fn newest_publication( + transaction: &GuardedTransaction<'_>, + projection: &str, +) -> Result, ProjectionError> { + let row = transaction + .transaction() + .query_one( + "SELECT max(o.observed_at) + FROM registry_internal.registry_external_projection_snapshots AS o + JOIN registry_internal.registry_external_projection_scopes AS s USING (projection_id) + WHERE o.projection_id=$1 AND o.epoch=s.epoch AND s.active", + &[&projection], + ) + .await + .map_err(|_| ProjectionError::Unavailable)?; + Ok(row.get(0)) +} + +/// Why the purge removed rows. Only these fixed categories leave the store. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum PurgeReason { + /// The cache lifetime or refresh lease ended. + Expired, + /// A rotation moved the projection to a fresh epoch. + Replaced, + /// The projection was withdrawn. + Withdrawn, +} + +impl PurgeReason { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Expired => "expired", + Self::Replaced => "replaced", + Self::Withdrawn => "withdrawn", + } + } + + fn parse(value: &str) -> Option { + match value { + "expired" => Some(Self::Expired), + "replaced" => Some(Self::Replaced), + "withdrawn" => Some(Self::Withdrawn), + _ => None, + } + } +} + +/// Rows one purge batch removed for one projection and reason. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Purged { + pub(crate) projection: String, + pub(crate) reason: PurgeReason, + pub(crate) snapshots: u64, + pub(crate) receipts: u64, +} + /// Each lane removes at most `limit` rows, with values cascading only from the -/// selected bounded snapshot headers. No expired value is made readable. +/// selected bounded snapshot headers. No expired value is made readable. A +/// failure carries only its category, for the retention worker's log. pub(crate) async fn purge( transaction: &GuardedTransaction<'_>, limit: u32, -) -> Result<(u64, u64), ProjectionError> { +) -> Result, RetentionFailureReason> { if !(1..=1000).contains(&limit) { - return Err(ProjectionError::InvalidScope); + return Err(RetentionFailureReason::Database); } - let limit = i32::try_from(limit).map_err(|_| ProjectionError::InvalidScope)?; - let row = transaction + let limit = i32::try_from(limit).map_err(|_| RetentionFailureReason::Database)?; + let rows = transaction .transaction() - .query_one( + .query( "SELECT * FROM registry_internal.purge_external_projections($1)", &[&limit], ) .await - .map_err(|_| ProjectionError::Unavailable)?; - Ok(( - u64::try_from(row.get::<_, i64>(0)).map_err(|_| ProjectionError::Unavailable)?, - u64::try_from(row.get::<_, i64>(1)).map_err(|_| ProjectionError::Unavailable)?, - )) + .map_err(|error| RetentionFailureReason::sqlstate(error.code().map(|code| code.code())))?; + let mut purged: Vec = Vec::new(); + for row in rows { + let projection: String = row.get(0); + let reason = + PurgeReason::parse(row.get::<_, &str>(1)).ok_or(RetentionFailureReason::Database)?; + let count = |column| { + u64::try_from(row.get::<_, i64>(column)).map_err(|_| RetentionFailureReason::Database) + }; + let (snapshots, receipts) = (count(2)?, count(3)?); + match purged + .iter_mut() + .find(|entry| entry.projection == projection && entry.reason == reason) + { + Some(entry) => { + entry.snapshots += snapshots; + entry.receipts += receipts; + } + None => purged.push(Purged { + projection, + reason, + snapshots, + receipts, + }), + } + } + Ok(purged) } pub(crate) const INSTALL_SQL: &str = r#" @@ -1218,8 +1301,12 @@ BEGIN RETURN FOUND; END $body$; +-- One row per projection, reason and lane: the snapshot lane fills +-- snapshots_pruned, the receipt lane fills refreshes_pruned. The reason is a +-- fixed category (withdrawn, replaced or expired) and never a stored value. CREATE OR REPLACE FUNCTION registry_internal.purge_external_projections(p_limit integer) -RETURNS TABLE (snapshots_pruned bigint,refreshes_pruned bigint) LANGUAGE plpgsql SECURITY DEFINER +RETURNS TABLE (pruned_projection text,pruned_reason text,snapshots_pruned bigint,refreshes_pruned bigint) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, registry_internal AS $body$ DECLARE projection_now bigint; BEGIN @@ -1227,18 +1314,22 @@ BEGIN RAISE EXCEPTION USING ERRCODE='22023', MESSAGE='external projection purge limit is invalid'; END IF; projection_now := floor(extract(epoch FROM clock_timestamp()))::bigint; - WITH expired AS MATERIALIZED ( - SELECT o.projection_id,o.epoch,o.subject_key + RETURN QUERY WITH expired AS MATERIALIZED ( + SELECT o.projection_id,o.epoch,o.subject_key, + CASE WHEN NOT s.active THEN 'withdrawn' WHEN o.epoch<>s.epoch THEN 'replaced' ELSE 'expired' END AS reason FROM registry_internal.registry_external_projection_snapshots AS o JOIN registry_internal.registry_external_projection_scopes AS s USING (projection_id) WHERE o.expires_at<=projection_now OR o.epoch<>s.epoch OR NOT s.active ORDER BY o.expires_at,o.projection_id,o.epoch,o.subject_key LIMIT p_limit FOR UPDATE OF o SKIP LOCKED ), removed AS ( DELETE FROM registry_internal.registry_external_projection_snapshots AS o USING expired AS e - WHERE o.projection_id=e.projection_id AND o.epoch=e.epoch AND o.subject_key=e.subject_key RETURNING 1 - ) SELECT count(*) INTO snapshots_pruned FROM removed; - WITH expired AS MATERIALIZED ( - SELECT r.projection_id,r.sequence + WHERE o.projection_id=e.projection_id AND o.epoch=e.epoch AND o.subject_key=e.subject_key + RETURNING e.projection_id,e.reason + ) SELECT removed.projection_id,removed.reason,count(*),0::bigint FROM removed + GROUP BY removed.projection_id,removed.reason; + RETURN QUERY WITH expired AS MATERIALIZED ( + SELECT r.projection_id,r.sequence, + CASE WHEN NOT s.active THEN 'withdrawn' WHEN r.epoch<>s.epoch THEN 'replaced' ELSE 'expired' END AS reason FROM registry_internal.registry_external_projection_refreshes AS r JOIN registry_internal.registry_external_projection_scopes AS s USING (projection_id) WHERE r.epoch<>s.epoch OR NOT s.active OR (r.lease_expires_at<=projection_now @@ -1250,9 +1341,10 @@ BEGIN ORDER BY r.lease_expires_at,r.projection_id,r.sequence LIMIT p_limit FOR UPDATE OF r SKIP LOCKED ), removed AS ( DELETE FROM registry_internal.registry_external_projection_refreshes AS r USING expired AS e - WHERE r.projection_id=e.projection_id AND r.sequence=e.sequence RETURNING 1 - ) SELECT count(*) INTO refreshes_pruned FROM removed; - RETURN NEXT; + WHERE r.projection_id=e.projection_id AND r.sequence=e.sequence + RETURNING e.projection_id,e.reason + ) SELECT removed.projection_id,removed.reason,0::bigint,count(*) FROM removed + GROUP BY removed.projection_id,removed.reason; END $body$; CREATE OR REPLACE FUNCTION registry_internal.lock_external_projection_generation( @@ -2292,7 +2384,7 @@ pub(crate) mod postgres_tests { // remains live, independently of wall-clock lease ordering. migration.execute("UPDATE registry_internal.registry_external_projection_refreshes SET lease_expires_at=999 WHERE projection_id=$1 AND sequence=$2",&[&scope.projection_id,&newer.sequence]).await.unwrap(); migration - .query_one( + .query( "SELECT * FROM registry_internal.purge_external_projections(1000)", &[], ) @@ -2341,15 +2433,19 @@ pub(crate) mod postgres_tests { assert!(publish_without_eligibility(&migration, &ticket, &observed) .await .is_err()); - // Purge removes a value once its one cache lifetime expires. - let pruned = migration - .query_one( - "SELECT * FROM registry_internal.purge_external_projections(100)", + // Purge removes a value once its one cache lifetime expires and names + // the projection and reason, never the value. + let pruned: Vec<(String, String, i64)> = migration + .query( + "SELECT pruned_projection,pruned_reason,snapshots_pruned FROM registry_internal.purge_external_projections(100) WHERE snapshots_pruned>0", &[], ) .await - .unwrap(); - assert_eq!(pruned.get::<_, i64>(0), 1); + .unwrap() + .iter() + .map(|row| (row.get(0), row.get(1), row.get(2))) + .collect(); + assert_eq!(pruned, [("projection".to_owned(), "expired".to_owned(), 1)]); assert_eq!( migration .query_one( @@ -2696,16 +2792,23 @@ pub(crate) mod postgres_tests { ProjectionError::ScopeChanged ); - // Purge removes the replaced epoch's cache; the other projection's - // cache stays warm. - fixture + // Purge removes the replaced epoch's cache and says so; the other + // projection's cache stays warm. + let pruned: Vec<(String, String, i64)> = fixture .migration - .query_one( - "SELECT * FROM registry_internal.purge_external_projections(100)", + .query( + "SELECT pruned_projection,pruned_reason,snapshots_pruned FROM registry_internal.purge_external_projections(100) WHERE snapshots_pruned>0", &[], ) .await - .unwrap(); + .unwrap() + .iter() + .map(|row| (row.get(0), row.get(1), row.get(2))) + .collect(); + assert_eq!( + pruned, + [("projection".to_owned(), "replaced".to_owned(), 1)] + ); assert_eq!(fixture.cached("projection", first.epoch).await, 0); assert_eq!(fixture.cached("other", other.epoch).await, 1); fixture.cleanup().await; diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index 13a00920d1..b96f46b4aa 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -45,11 +45,15 @@ use super::external_view_query::{ }; use super::read::{strict_claim_context, ReadRelations}; use super::{ - begin_record_transaction, begin_registry_transaction, ClaimContext, ExpectedRegistryIdentity, - GuardedTransaction, RegistryLockKey, RuntimePool, + begin_record_transaction, begin_registry_transaction, begin_registry_transaction_with_reason, + ClaimContext, ExpectedRegistryIdentity, GuardedTransaction, RegistryLockKey, RuntimePool, }; +use crate::hooks::RetentionFailureReason; type Result = std::result::Result; +/// Rows one retention tick committed, as (snapshots, receipts) by projection +/// and reason. +type PurgeTally = BTreeMap<(String, store::PurgeReason), (u64, u64)>; pub(crate) struct PostgresExternalViewService { pool: RuntimePool, @@ -253,6 +257,53 @@ impl PostgresExternalViewService { self.sources.unavailable_projections() } + /// Each bound projection's grant horizon and, for an indexed projection, + /// when the newest snapshot its active epoch holds was observed. Reads + /// only, in one transaction that is rolled back. + pub(crate) async fn projection_status( + &self, + ) -> Result> { + let mut session = self.session().await?; + let transaction = begin_registry_transaction( + session.client(), + self.lock_key, + self.lock_timeout, + &self.expected, + ) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + transaction + .set_statement_budget(Duration::from_secs(10)) + .await + .map_err(|_| ExternalViewError::Unavailable)?; + let mut status = Vec::new(); + for (id, projection) in &self.registry.external_views().projections { + let Some(source) = self.sources.binding(id.as_str()) else { + continue; + }; + let indexed = projection.mode == ExternalProjectionMode::Indexed; + let newest_publication = if indexed { + store::newest_publication(&transaction, id.as_str()) + .await + .map_err(|_| ExternalViewError::Unavailable)? + } else { + None + }; + status.push(crate::startup::ExternalProjectionStatus { + projection: id.to_string(), + indexed, + grant_valid_until: source.grant_valid_until().timestamp(), + newest_publication, + }); + } + transaction + .rollback() + .await + .map_err(|_| ExternalViewError::Unavailable)?; + session.disarm(); + Ok(status) + } + pub(crate) async fn run_retention( self: Arc, last_success: Arc, @@ -266,43 +317,91 @@ impl PostgresExternalViewService { if change.is_err() || *shutdown.borrow() { return; } } _ = interval.tick() => { - match timeout_at(Instant::now() + Duration::from_secs(5), self.purge_tick()).await { - Ok(Ok(())) => last_success.record(), - _ => tracing::error!("external projection retention tick failed"), + if self.retention_tick().await { + last_success.record(); } } } } } - async fn purge_tick(&self) -> Result<()> { + /// One bounded purge pass. Logs the committed removals by projection and + /// reason, and a failure by category; returns whether the pass completed. + async fn retention_tick(&self) -> bool { + let mut purged = PurgeTally::new(); + let outcome = timeout_at( + Instant::now() + Duration::from_secs(5), + self.purge_tick(&mut purged), + ) + .await; + for ((projection, reason), (snapshots, receipts)) in &purged { + tracing::info!( + projection = projection.as_str(), + reason = reason.as_str(), + snapshots, + receipts, + "external projection retention removed cache rows" + ); + } + let reason = match outcome { + Ok(Ok(())) => return true, + Ok(Err(reason)) => reason.as_str(), + Err(_) => "deadline", + }; + tracing::error!(reason, "external projection retention tick failed"); + false + } + + async fn purge_tick( + &self, + purged: &mut PurgeTally, + ) -> std::result::Result<(), RetentionFailureReason> { const BATCH: u32 = 1000; - let mut session = self.session().await?; - for _ in 0..8 { - let transaction = begin_registry_transaction( + const BATCHES: usize = 8; + let client = self + .pool + .get() + .await + .map_err(|_| RetentionFailureReason::Database)?; + let mut session = QueryCancellationGuard::new(self.pool.clone(), client); + for _ in 0..BATCHES { + let transaction = begin_registry_transaction_with_reason( session.client(), self.lock_key, self.lock_timeout, &self.expected, ) .await - .map_err(|_| ExternalViewError::Unavailable)?; + .map_err(RetentionFailureReason::admission)?; transaction .set_statement_budget(Duration::from_secs(2)) .await - .map_err(|_| ExternalViewError::Unavailable)?; - let (snapshots, receipts) = store::purge(&transaction, BATCH) - .await - .map_err(|_| ExternalViewError::Unavailable)?; + .map_err(|_| RetentionFailureReason::Database)?; + let batch = store::purge(&transaction, BATCH).await?; transaction .commit() .await - .map_err(|_| ExternalViewError::Unavailable)?; + .map_err(|_| RetentionFailureReason::Database)?; + let (mut snapshots, mut receipts) = (0, 0); + for removed in batch { + snapshots += removed.snapshots; + receipts += removed.receipts; + let entry = purged + .entry((removed.projection, removed.reason)) + .or_default(); + entry.0 += removed.snapshots; + entry.1 += removed.receipts; + } if snapshots < u64::from(BATCH) && receipts < u64::from(BATCH) { - break; + session.disarm(); + return Ok(()); } } session.disarm(); + tracing::warn!( + batches = BATCHES, + "external projection retention reached its per-tick batch bound" + ); Ok(()) } @@ -532,9 +631,11 @@ impl PostgresExternalViewService { parent_key: key.clone(), }, }; - self.acquire(request, plan, source_request, &fields, None, None) - .await - .ok() + live_acquisition( + plan.projection.id.as_str(), + self.acquire(request, plan, source_request, &fields, None, None) + .await, + ) } else { None }; @@ -1021,13 +1122,59 @@ impl ExternalViewService for PostgresExternalViewService { mode: RefreshMode, ) -> ServiceFuture<'_, Result> { Box::pin(async move { - timeout_at(request.deadline, self.run(request, Some(mode))) - .await - .map_err(|_| ExternalViewError::Unavailable)? + let projection = self.refresh_projection(&request, mode); + let outcome = timeout_at(request.deadline, self.run(request, Some(mode))).await; + if let Some(projection) = projection { + log_refresh(&projection, &outcome); + } + outcome.map_err(|_| ExternalViewError::Unavailable)? }) } } +impl PostgresExternalViewService { + /// The projection a refresh request writes, when its view and link exist. + fn refresh_projection( + &self, + request: &ExternalViewRequest, + mode: RefreshMode, + ) -> Option { + let link = self + .registry + .external_views() + .views + .get(&request.view)? + .links + .iter() + .find(|link| link.id.as_str() == request.link)?; + match mode { + RefreshMode::Projection => Some(link.projection.to_string()), + RefreshMode::Labels => link + .label + .as_ref() + .map(|label| label.projection.to_string()), + } + } +} + +/// A refresh page that fails as a whole is logged by projection and reason +/// category. Per-key outcomes of a completed page are logged when recorded. +fn log_refresh( + projection: &str, + outcome: &std::result::Result, tokio::time::error::Elapsed>, +) { + let reason = match outcome { + Ok(Ok(_)) => return, + Ok(Err(ExternalViewError::Concealed)) => "concealed", + Ok(Err(ExternalViewError::InvalidCursor)) => "invalid-cursor", + Ok(Err(ExternalViewError::Incomplete)) => "incomplete", + Ok(Err(ExternalViewError::Unavailable)) => "unavailable", + Ok(Err(ExternalViewError::ProjectionUnavailable(_))) => "projection-unavailable", + Err(_) => "deadline", + }; + tracing::warn!(projection, reason, "external projection refresh failed"); +} + struct RefreshAuthority { pool: RuntimePool, expected: ExpectedRegistryIdentity, @@ -1554,6 +1701,7 @@ impl PostgresExternalViewService { /// returns how many were published. A publication holds the projection /// scope row until it commits, and indexed reads share that row, so one /// transaction for the whole page would hold readers back for every key. + /// The committed outcomes are logged by reason category, without keys. #[allow(clippy::too_many_arguments)] // The page's scope, parent and rows come from its selection transaction. async fn record_refresh( &self, @@ -1565,7 +1713,28 @@ impl PostgresExternalViewService { page: &RefreshRows, results: Vec<(RefreshWork, RefreshResult)>, ) -> Result { - let mut published = 0usize; + let mut tally = RefreshTally::default(); + let recorded = self + .record_outcomes( + request, plan, claims, scope, parent, page, results, &mut tally, + ) + .await; + tally.log(plan.projection.id.as_str()); + recorded.map(|()| tally.published) + } + + #[allow(clippy::too_many_arguments)] // See `record_refresh`. + async fn record_outcomes( + &self, + request: &ExternalViewRequest, + plan: &Plan<'_>, + claims: &ClaimContext, + scope: &ProjectionScope, + parent: Option<&ProjectionGeneration>, + page: &RefreshRows, + results: Vec<(RefreshWork, RefreshResult)>, + tally: &mut RefreshTally, + ) -> Result<()> { let mut session = self.session().await?; let mut results = results.into_iter(); while let Some((work, result)) = results.next() { @@ -1595,20 +1764,23 @@ impl PostgresExternalViewService { .is_some_and(|cutoff| Utc::now().timestamp() >= cutoff) { let abandoned = std::iter::once(work).chain(results.map(|(work, _)| work)); + let mut keys = 0usize; for work in abandoned { store::fail_refresh(&transaction, &work.ticket, RefreshFailure::Abandoned) .await .map_err(|_| ExternalViewError::Unavailable)?; + keys += 1; } transaction .commit() .await .map_err(|_| ExternalViewError::Unavailable)?; session.disarm(); + tally.failed(RefreshFailure::Abandoned, keys); return Err(ExternalViewError::Unavailable); } } - let accepted = match result { + let failure = match result { RefreshResult::Accepted(snapshot) => { let root_ids = work.roots.into_iter().collect::>(); // The RLS recheck and publication share a statement @@ -1623,7 +1795,7 @@ impl PostgresExternalViewService { ) .await { - Ok(store::Publication::Published | store::Publication::Replay) => true, + Ok(store::Publication::Published | store::Publication::Replay) => None, Ok(store::Publication::Ineligible) | Err(store::ProjectionError::StaleRefresh) => { store::fail_refresh( @@ -1633,7 +1805,7 @@ impl PostgresExternalViewService { ) .await .map_err(|_| ExternalViewError::Unavailable)?; - false + Some(RefreshFailure::Abandoned) } Err(_) => return Err(ExternalViewError::Unavailable), } @@ -1642,19 +1814,20 @@ impl PostgresExternalViewService { store::fail_refresh(&transaction, &work.ticket, failure) .await .map_err(|_| ExternalViewError::Unavailable)?; - false + Some(failure) } }; transaction .commit() .await .map_err(|_| ExternalViewError::Unavailable)?; - if accepted { - published += 1; + match failure { + None => tally.published += 1, + Some(failure) => tally.failed(failure, 1), } } session.disarm(); - Ok(published) + Ok(()) } async fn fetch_refresh( @@ -1824,6 +1997,55 @@ fn refresh_lease(deadline: Instant) -> Duration { + REFRESH_LEASE_MARGIN } +/// Committed outcomes of one refresh page: published keys and failed keys by +/// reason category. +#[derive(Default)] +struct RefreshTally { + published: usize, + failed: BTreeMap<&'static str, usize>, +} + +impl RefreshTally { + fn failed(&mut self, failure: RefreshFailure, keys: usize) { + *self.failed.entry(failure.as_str()).or_default() += keys; + } + + fn log(&self, projection: &str) { + tracing::info!( + projection, + published = self.published, + "external projection refresh recorded" + ); + for (reason, keys) in &self.failed { + tracing::warn!( + projection, + reason, + keys, + "external projection refresh failed for some keys" + ); + } + } +} + +/// A live read that cannot reach its source answers unavailable. The failure +/// is logged by projection and reason category, never with the key. +fn live_acquisition( + projection: &str, + result: std::result::Result, +) -> Option { + match result { + Ok(result) => Some(result), + Err(error) => { + tracing::warn!( + projection, + reason = acquisition_failure(Some(error)).as_str(), + "external projection live acquisition failed" + ); + None + } + } +} + fn acquisition_failure(error: Option) -> RefreshFailure { match error { Some(ExternalSourceError::Grant) => RefreshFailure::Denied, @@ -1899,6 +2121,61 @@ mod tests { ); } + #[test] + fn a_failed_live_acquisition_is_logged_by_projection_and_reason() { + let captured = registry_platform_testing::CapturedDiagnosticLogs::default(); + let kept = tracing::dispatcher::with_default(&captured.subscriber(), || { + live_acquisition("person-live", Err(ExternalSourceError::Grant)) + }); + assert!(kept.is_none(), "the view answers unavailable"); + let logs = captured.text(); + assert_eq!(logs.lines().count(), 1, "{logs}"); + for expected in [ + r#""level":"WARN""#, + r#""projection":"person-live""#, + r#""reason":"denied""#, + ] { + assert!(logs.contains(expected), "{logs}"); + } + } + + #[tokio::test] + async fn a_failed_refresh_is_logged_by_projection_and_reason() { + let elapsed = tokio::time::timeout(Duration::ZERO, std::future::pending::<()>()) + .await + .unwrap_err(); + let captured = registry_platform_testing::CapturedDiagnosticLogs::default(); + tracing::dispatcher::with_default(&captured.subscriber(), || { + log_refresh("person-indexed", &Ok(Ok(json!({"published": 1})))); + log_refresh("person-indexed", &Ok(Err(ExternalViewError::InvalidCursor))); + log_refresh( + "person-labels", + &Ok(Err(ExternalViewError::ProjectionUnavailable( + "person-labels".to_owned(), + ))), + ); + log_refresh("person-indexed", &Err(elapsed)); + }); + let logs = captured.text(); + assert_eq!( + logs.lines().count(), + 3, + "a completed page adds no line: {logs}" + ); + for (line, (projection, reason)) in logs.lines().zip([ + ("person-indexed", "invalid-cursor"), + ("person-labels", "projection-unavailable"), + ("person-indexed", "deadline"), + ]) { + assert!(line.contains(r#""level":"WARN""#), "{line}"); + assert!( + line.contains(&format!(r#""projection":"{projection}""#)), + "{line}" + ); + assert!(line.contains(&format!(r#""reason":"{reason}""#)), "{line}"); + } + } + #[test] fn a_refresh_lease_outlives_the_page_deadline() { // The longest configurable request deadline is a minute. @@ -2650,4 +2927,223 @@ mod postgres_tests { } served.fixture.cleanup().await; } + + /// Source keys, labels and snapshot identities the fixtures below store. + /// None of them may reach an operational log line. + const FIXTURE_VALUES: [&str; 7] = [ + "\"first", + "\"second", + "\"third", + "\"hidden", + "\"subject", + "cached", + "canary", + ]; + + fn assert_value_free(logs: &str) { + for value in FIXTURE_VALUES { + assert!(!logs.contains(value), "{value} reached the log: {logs}"); + } + } + + fn log_line<'a>(logs: &'a str, projection: &str, reason: &str) -> &'a str { + logs.lines() + .find(|line| { + line.contains(&format!(r#""projection":"{projection}""#)) + && line.contains(&format!(r#""reason":"{reason}""#)) + }) + .unwrap_or_else(|| panic!("no {projection} {reason} line: {logs}")) + } + + #[tokio::test] + async fn a_recorded_refresh_page_logs_its_outcomes_by_reason_without_values() { + use tracing::instrument::WithSubscriber; + let served = Served::new(Duration::from_secs(2)).await; + let refresh = request("link"); + let plan = served.service.plan(&refresh, None).ok().unwrap(); + let claims = ClaimContext::for_compiled( + &served.service.registry, + &plan.root.id, + Some("reader-principal".to_owned()), + "reader", + Some("consultation".to_owned()), + vec![crate::postgres::context::RowBoundaryContext::Equals { + field: "tenant".to_owned(), + value: "allowed".to_owned(), + }], + ) + .unwrap(); + let page = RefreshRows { + rows: Vec::new(), + next: None, + eligibility: RootKeys::new(&plan).ok().unwrap().publication_eligibility, + supporting_until: None, + supporting: Vec::new(), + }; + let outcomes = [ + None, + Some(RefreshFailure::Denied), + Some(RefreshFailure::Unavailable), + ]; + let mut results = Vec::new(); + for ((id, key, _), failure) in ROOTS[..3].iter().zip(outcomes) { + let (ticket, snapshot) = served + .fixture + .label_refresh(&served.scope, key, "canary") + .await; + results.push(( + RefreshWork { + key: (*key).to_owned(), + roots: BTreeSet::from([Uuid::parse_str(id).unwrap()]), + ticket, + }, + failure.map_or(RefreshResult::Accepted(snapshot), RefreshResult::Failed), + )); + } + let captured = registry_platform_testing::CapturedDiagnosticLogs::default(); + let recorded = served + .service + .record_refresh( + &refresh, + &plan, + &claims, + &served.scope, + None, + &page, + results, + ) + .with_subscriber(captured.subscriber()) + .await; + assert_eq!(recorded.ok(), Some(1)); + let logs = captured.text(); + let recorded = logs + .lines() + .find(|line| line.contains(r#""published":1"#)) + .unwrap_or_else(|| panic!("{logs}")); + assert!(recorded.contains(r#""projection":"projection""#), "{logs}"); + for reason in ["denied", "unavailable"] { + let line = log_line(&logs, "projection", reason); + assert!(line.contains(r#""level":"WARN""#), "{line}"); + assert!(line.contains(r#""keys":1"#), "{line}"); + } + assert_eq!(logs.lines().count(), 3, "{logs}"); + assert_value_free(&logs); + served.fixture.cleanup().await; + } + + #[tokio::test] + async fn a_retention_tick_logs_what_it_purged_by_projection_and_reason() { + use tracing::instrument::WithSubscriber; + let served = Served::new(Duration::from_millis(300)).await; + served.publish_all().await; + let other = bound_scope(&served.fixture, "other").await; + served.fixture.cache(&other).await; + served + .fixture + .migration + .batch_execute( + "UPDATE registry_internal.registry_external_projection_snapshots + SET observed_at=0,expires_at=1 WHERE projection_id='projection'; + SELECT registry_internal.withdraw_external_projection( + projection_id,epoch,binding_digest,grant_digest) + FROM registry_internal.registry_external_projection_scopes + WHERE projection_id='other';", + ) + .await + .unwrap(); + + let captured = registry_platform_testing::CapturedDiagnosticLogs::default(); + assert!( + served + .service + .retention_tick() + .with_subscriber(captured.subscriber()) + .await + ); + let logs = captured.text(); + let expired = log_line(&logs, "projection", "expired"); + assert!(expired.contains(r#""snapshots":4"#), "{expired}"); + let withdrawn = log_line(&logs, "other", "withdrawn"); + assert!(withdrawn.contains(r#""snapshots":1"#), "{withdrawn}"); + assert!(withdrawn.contains(r#""receipts":1"#), "{withdrawn}"); + assert_value_free(&logs); + + // A migration holding the registry lock fails the tick by category. + let lock_key = RegistryLockKey::derive(served.service.registry.registry_id()) + .unwrap() + .get(); + served + .fixture + .migration + .execute("SELECT pg_advisory_lock($1)", &[&lock_key]) + .await + .unwrap(); + let captured = registry_platform_testing::CapturedDiagnosticLogs::default(); + assert!( + !served + .service + .retention_tick() + .with_subscriber(captured.subscriber()) + .await + ); + let logs = captured.text(); + assert_eq!(logs.lines().count(), 1, "{logs}"); + assert!(logs.contains(r#""level":"ERROR""#), "{logs}"); + assert!(logs.contains(r#""reason":"lock-timeout""#), "{logs}"); + served + .fixture + .migration + .execute("SELECT pg_advisory_unlock($1)", &[&lock_key]) + .await + .unwrap(); + served.fixture.cleanup().await; + } + + #[tokio::test] + async fn a_check_reports_each_grant_horizon_and_newest_publication() { + let served = Served::new(Duration::from_secs(2)).await; + served.publish_all().await; + let other = bound_scope(&served.fixture, "other").await; + served.fixture.cache(&other).await; + // A withdrawn scope's rows are not what the projection serves. + served + .fixture + .migration + .batch_execute( + "SELECT registry_internal.withdraw_external_projection( + projection_id,epoch,binding_digest,grant_digest) + FROM registry_internal.registry_external_projection_scopes + WHERE projection_id='other'", + ) + .await + .unwrap(); + let status = served.service.projection_status().await.ok().unwrap(); + assert_eq!( + status + .iter() + .map(|projection| projection.projection.as_str()) + .collect::>(), + ["other", "projection"] + ); + for projection in &status { + assert_eq!( + projection.grant_valid_until, + served + .service + .sources + .binding(&projection.projection) + .unwrap() + .grant_valid_until() + .timestamp() + ); + assert!(projection.indexed); + } + assert_eq!(status[0].newest_publication, None); + let newest = status[1].newest_publication.unwrap(); + assert!( + (Utc::now().timestamp() - newest).abs() <= 5, + "{newest} is the publication time" + ); + served.fixture.cleanup().await; + } } diff --git a/crates/registry-breg/src/startup.rs b/crates/registry-breg/src/startup.rs index 5519398e01..2a8eb39af2 100644 --- a/crates/registry-breg/src/startup.rs +++ b/crates/registry-breg/src/startup.rs @@ -853,6 +853,7 @@ pub struct PreparedServer { wasm_runtime: Option, postgres_advisories: Vec, unavailable_external_projections: BTreeMap, + external_projection_status: Vec, role_mode: RoleMode, #[cfg(all(feature = "postgres-test", feature = "tooling"))] fixture_pool: Option, @@ -942,6 +943,7 @@ impl PreparedServer { metrics: None, postgres_advisories: Vec::new(), unavailable_external_projections: BTreeMap::new(), + external_projection_status: Vec::new(), role_mode: RoleMode::Split, #[cfg(feature = "wasm")] wasm_runtime: None, @@ -984,6 +986,7 @@ impl PreparedServer { metrics: None, postgres_advisories: Vec::new(), unavailable_external_projections: BTreeMap::new(), + external_projection_status: Vec::new(), role_mode: RoleMode::Split, #[cfg(feature = "wasm")] wasm_runtime: None, @@ -1056,18 +1059,34 @@ pub async fn check_loaded(config: RuntimeConfig) -> Result { role_mode: prepared.role_mode(), unbound_review_authorities, unavailable_external_projections: prepared.unavailable_external_projections().clone(), + external_projections: prepared.external_projection_status.clone(), }) } /// What a startup check that passed decided without refusing: the -/// PostgreSQL baseline advisories, the role mode it verified, and the external -/// projections a server would start without, with the reason. +/// PostgreSQL baseline advisories, the role mode it verified, the external +/// projections a server would start without, with the reason, and the status +/// of each projection it would serve. #[derive(Debug, Clone)] pub struct CheckedStartup { pub postgres_advisories: Vec, pub role_mode: RoleMode, pub unbound_review_authorities: Vec, pub unavailable_external_projections: BTreeMap, + pub external_projections: Vec, +} + +/// One external projection a server would serve, as a startup check sees it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExternalProjectionStatus { + pub projection: String, + /// Whether the projection serves from cached snapshots rather than live. + pub indexed: bool, + /// When the source grant this deployment binds expires, in Unix seconds. + pub grant_valid_until: i64, + /// When the newest snapshot the projection holds was observed, in Unix + /// seconds. `None` when it holds none or reads live. + pub newest_publication: Option, } /// Prepare the clean database capability consumed by the production pre-sign @@ -1630,6 +1649,7 @@ async fn finish_prepared_server( None }; let mut unavailable_external_projections = BTreeMap::new(); + let mut external_projection_status = Vec::new(); let external_views = if registry.external_views().views.is_empty() && config.external_sources().is_empty() { None @@ -1667,6 +1687,12 @@ async fn finish_prepared_server( .await .map_err(|_| StartupError::ExternalSources)?; unavailable_external_projections = external.unavailable_projections().clone(); + if opening == AuditOpening::CheckOnly { + external_projection_status = external + .projection_status() + .await + .map_err(|_| StartupError::ExternalSources)?; + } Some(Arc::new(external)) }; let records = PostgresRecordReadService::new( @@ -1973,6 +1999,7 @@ async fn finish_prepared_server( metrics, postgres_advisories, unavailable_external_projections, + external_projection_status, role_mode: RoleMode::from_roles( config.database().roles().migration(), config.database().roles().runtime(), From 51c0863078aee3d2f1320e90a70073891355fa1b Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 14:17:18 +0700 Subject: [PATCH 21/64] feat(bregctl): report each external projection's grant horizon in doctor bregctl doctor now adds an external-projection.status advisory for each projection a server would serve. It names when the source grant expires and, for an indexed projection, when the newest snapshot it holds was observed, so an operator sees a lapsing grant or a stalled refresh sweep before readers do. Signed-off-by: Jeremi Joslin --- crates/registry-bregctl/src/lib.rs | 127 ++++++++++++++++++++++++++++- 1 file changed, 126 insertions(+), 1 deletion(-) diff --git a/crates/registry-bregctl/src/lib.rs b/crates/registry-bregctl/src/lib.rs index 95f1dbdbdf..0d7b4c9f40 100644 --- a/crates/registry-bregctl/src/lib.rs +++ b/crates/registry-bregctl/src/lib.rs @@ -3053,6 +3053,7 @@ where success.insecure_http_warning, &success.checked.unbound_review_authorities, &success.checked.unavailable_external_projections, + &success.checked.external_projections, format, stdout, stderr, @@ -12971,6 +12972,7 @@ fn write_doctor_success( insecure_http_warning: Option<&'static str>, unbound_review_authorities: &[String], unavailable_external_projections: &BTreeMap, + external_projections: &[registry_breg::startup::ExternalProjectionStatus], format: OutputFormat, stdout: &mut dyn Write, stderr: &mut dyn Write, @@ -13017,6 +13019,18 @@ fn write_doctor_success( observed: ObservedNumbers(&[]), }); } + let status_reports = external_projections + .iter() + .map(external_projection_status_report) + .collect::>(); + for (message, observed) in &status_reports { + doctor_advisories.push(DoctorAdvisory { + code: "external-projection.status".to_owned(), + severity: "information", + message, + observed: ObservedNumbers(observed), + }); + } let report = DoctorSuccessReport { ok: true, command: "doctor", @@ -13071,6 +13085,39 @@ fn write_doctor_success( } } +/// One served external projection's grant horizon and newest held snapshot, +/// so an operator sees a lapsing grant or a stalled refresh sweep. +fn external_projection_status_report( + status: ®istry_breg::startup::ExternalProjectionStatus, +) -> (String, Vec<(&'static str, i64)>) { + let instant = |seconds: i64| { + chrono::DateTime::from_timestamp(seconds, 0).map_or_else( + || format!("Unix second {seconds}"), + |value| value.to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + ) + }; + let holding = match (status.indexed, status.newest_publication) { + (false, _) => "It reads live and holds no snapshots.".to_owned(), + (true, None) => "It holds no snapshot; run a refresh sweep to fill it.".to_owned(), + (true, Some(observed)) => format!( + "The newest snapshot it holds was observed at {}.", + instant(observed) + ), + }; + let mut observed = vec![("grantValidUntilUnixSeconds", status.grant_valid_until)]; + if let Some(newest) = status.newest_publication { + observed.push(("newestPublicationUnixSeconds", newest)); + } + ( + format!( + "External projection `{}`: its source grant is valid until {}. {holding}", + status.projection, + instant(status.grant_valid_until) + ), + observed, + ) +} + fn write_field_encryption_keygen_success( outcome: &field_encryption::KeygenOutcome, format: OutputFormat, @@ -17061,7 +17108,7 @@ mod tests { let mut stderr = Vec::new(); assert_eq!( - write_doctor_success(&[], RoleMode::Split, None, &[], &BTreeMap::new(), format, &mut stdout, &mut stderr), + write_doctor_success(&[], RoleMode::Split, None, &[], &BTreeMap::new(), &[], format, &mut stdout, &mut stderr), ExitCode::SUCCESS ); assert_eq!( @@ -17083,6 +17130,7 @@ mod tests { None, &[], &BTreeMap::new(), + &[], OutputFormat::Human, &mut stdout, &mut stderr @@ -17108,6 +17156,7 @@ mod tests { None, &[], &BTreeMap::new(), + &[], OutputFormat::Json, &mut stdout, &mut stderr @@ -17142,6 +17191,7 @@ mod tests { None, &[], &BTreeMap::new(), + &[], OutputFormat::Human, &mut stdout, &mut stderr @@ -17182,6 +17232,7 @@ mod tests { None, &[], &BTreeMap::new(), + &[], OutputFormat::Json, &mut stdout, &mut stderr @@ -17240,6 +17291,7 @@ mod tests { None, &["local-review".into()], &BTreeMap::new(), + &[], format, &mut stdout, &mut stderr @@ -17280,6 +17332,7 @@ mod tests { None, &[], &unavailable, + &[], format, &mut stdout, &mut stderr @@ -17311,6 +17364,77 @@ mod tests { } } + #[test] + fn doctor_reports_each_external_projection_grant_horizon_and_newest_publication() { + use registry_breg::startup::ExternalProjectionStatus; + const GRANT: i64 = 1_798_761_600; + const PUBLISHED: i64 = 1_793_491_200; + let status = |projection: &str, indexed, newest_publication| ExternalProjectionStatus { + projection: projection.to_owned(), + indexed, + grant_valid_until: GRANT, + newest_publication, + }; + let projections = [ + status("person-indexed", true, Some(PUBLISHED)), + status("person-unrefreshed", true, None), + status("person-live", false, None), + ]; + for format in [OutputFormat::Human, OutputFormat::Json] { + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + assert_eq!( + write_doctor_success( + &[], + RoleMode::Split, + None, + &[], + &BTreeMap::new(), + &projections, + format, + &mut stdout, + &mut stderr, + ), + ExitCode::SUCCESS + ); + assert!(stderr.is_empty()); + let text = plain(&stdout) + .split_whitespace() + .collect::>() + .join(" "); + for expected in [ + "External projection `person-indexed`: its source grant is valid until 2027-01-01T00:00:00Z. The newest snapshot it holds was observed at 2026-11-01T00:00:00Z.", + "External projection `person-unrefreshed`: its source grant is valid until 2027-01-01T00:00:00Z. It holds no snapshot; run a refresh sweep to fill it.", + "External projection `person-live`: its source grant is valid until 2027-01-01T00:00:00Z. It reads live and holds no snapshots.", + ] { + assert!(text.contains(expected), "{text}"); + } + if format == OutputFormat::Json { + let report: Value = serde_json::from_slice(&stdout).unwrap(); + assert_eq!(report["ok"], true); + let advisories = report["advisories"].as_array().unwrap(); + assert_eq!(advisories.len(), 3); + for advisory in advisories { + assert_eq!(advisory["code"], "external-projection.status"); + assert_eq!(advisory["severity"], "information"); + } + assert_eq!( + advisories[0]["observed"], + json!({ + "grantValidUntilUnixSeconds": GRANT, + "newestPublicationUnixSeconds": PUBLISHED, + }) + ); + for advisory in &advisories[1..] { + assert_eq!( + advisory["observed"], + json!({"grantValidUntilUnixSeconds": GRANT}) + ); + } + } + } + } + #[test] fn doctor_reports_the_plain_http_issuer_opt_in_as_a_value_free_advisory() { const WARNING: &str = @@ -17325,6 +17449,7 @@ mod tests { Some(WARNING), &[], &BTreeMap::new(), + &[], format, &mut stdout, &mut stderr, From 7f3de7ec6c179f5cd95ca4c603e7acb6f5796a38 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 14:17:18 +0700 Subject: [PATCH 22/64] docs(breg): match external projection monitoring advice to what is emitted The storage guide told operators to monitor purge failures and backlog against an erasure margin that no setting defines. It now states the purge worker's actual bounds and lists the log lines, metric and doctor advisory BReg emits for refresh and purge, with their reason categories. Signed-off-by: Jeremi Joslin --- products/breg/external-projection-storage.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/products/breg/external-projection-storage.md b/products/breg/external-projection-storage.md index de6968c681..9e9cbdf151 100644 --- a/products/breg/external-projection-storage.md +++ b/products/breg/external-projection-storage.md @@ -29,7 +29,21 @@ Replaying an accepted result keeps its original expiry. A refresh that fails bec Reads check expiry independently of the refresh job. Indexed predicates and counts refuse incomplete coverage before selecting matching rows. Each page reads coverage, rows and any count from one statement snapshot, so a count always matches complete coverage at that moment. A refresh records each key in its own short transaction, so a read waits for at most one key's publication and can show new values for some keys of a page in flight and previous values for the rest. A `nextCursor` stays valid while refreshes publish or fail individual keys; the next page reads the values current when it runs. A row whose ordering value changed between pages can appear twice or not at all. Rotating, withdrawing or resetting the projection invalidates outstanding cursors. A complete empty related collection is covered; a partial collection, source failure, or protected-existence response is not evidence of emptiness or absence. -The bounded purge worker runs every few seconds and deletes snapshots once they expire. It also removes data from withdrawn or replaced epochs. Size database capacity for the backlog. Where a storage obligation is strict, choose a cache lifetime shorter than that obligation by more than the worker interval and the expected recovery delay. Freshness enforcement keeps expired values out of responses even when refresh or purge is delayed, but a delayed purge can still breach a storage obligation. Monitor purge failures, expiry backlog, and database downtime separately. No physical erasure guarantee holds during a database outage or when backlog exceeds that margin. Database WAL and physical snapshots need their own retention policy. +The purge worker runs every five seconds. Each pass deletes expired snapshots, refresh receipts whose lease has ended, and everything under withdrawn or replaced epochs, in at most eight batches of a thousand rows of each kind. Freshness enforcement keeps expired values out of responses even when refresh or purge is delayed, but an expired value stays stored until a pass deletes it. That happens while the database is down, while the server is stopped, and while a backlog larger than one pass drains. Size database capacity for the backlog. Where a storage obligation is strict, choose a cache lifetime shorter than that obligation by more than the worker interval and the time you need to recover from those conditions, and watch the signals in the next section. Database WAL and physical snapshots need their own retention policy. + +## Monitor refresh and purge + +Each signal names the projection and a fixed reason category. None carries a source key, a selector, a value or a secret. + +- A purge pass that deletes rows logs one `info` line per projection and reason, with `snapshots` and `receipts` counts. The reason is `expired`, `replaced` for a rotated epoch, or `withdrawn`. +- A purge pass that fails logs `external projection retention tick failed` at `error`, with a `reason` of `registry-not-ready`, `lock-timeout`, `query-cancelled`, `database` or `deadline`. It names no projection, because one pass covers them all. +- A purge pass that reaches its batch bound logs a `warn` line. The rest of the backlog waits for the next pass. +- On the metrics listener, `breg_worker_last_success_age_seconds{worker="external-projection-retention"}` reports the seconds since the last completed pass. It is absent until the first pass completes. Alert when it grows well past the five-second interval. +- A refresh page that acquired keys logs one `info` line with its `published` count, and one `warn` line per failure reason with a `keys` count. The reason is `denied`, `retention-refused`, `unavailable`, `incomplete`, `invalid-binding`, or `abandoned` when a newer result, a changed root or a changed collection superseded the result before publication. +- A refresh request that fails as a whole logs a `warn` line with a `reason` of `concealed`, `invalid-cursor`, `incomplete`, `unavailable`, `projection-unavailable` or `deadline`. +- A live read whose source acquisition fails logs a `warn` line with the same reasons as a refresh key, other than `abandoned`. The view answers unavailable. +- `bregctl doctor` reports each projection a server would serve as an `external-projection.status` advisory. It gives the time the source grant expires and, for an indexed projection, when the newest snapshot it holds was observed. Renew a grant before that time. An old or missing newest snapshot means the refresh sweep is not running. +- `bregctl external-projection inspect` reports the expired snapshots that purge has not yet deleted. ## Schedule refresh and inspect coverage From 99161de5b1f1f2c49f383b9925a50a3aef757855 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 14:36:27 +0700 Subject: [PATCH 23/64] fix(dev): check the opened state file instead of its path a second time `dev start` polls state.json while its supervisor saves it by renaming a successor over the path. A poll that opened the earlier file and then looked the path up again saw the successor's inode and refused the start with "local state changed while opening it". The read now checks the opened descriptor itself: owner-only, regular, and at most one link, where zero links means a writer replaced the path after the open and the bytes are the complete earlier version. The pre-open path check and NOFOLLOW stay. Signed-off-by: Jeremi Joslin --- crates/registry-bregctl/src/dev/private.rs | 28 +++++++++++++++---- crates/registry-bregctl/src/dev/tests.rs | 25 +++++++++++++++++ .../registry-caseworkctl/src/dev/private.rs | 28 +++++++++++++++---- crates/registry-caseworkctl/src/dev/tests.rs | 26 +++++++++++++++++ 4 files changed, 97 insertions(+), 10 deletions(-) diff --git a/crates/registry-bregctl/src/dev/private.rs b/crates/registry-bregctl/src/dev/private.rs index 67a20fb19b..59c6f220b6 100644 --- a/crates/registry-bregctl/src/dev/private.rs +++ b/crates/registry-bregctl/src/dev/private.rs @@ -5,6 +5,7 @@ use anyhow::{bail, Context, Result}; use std::{ fs::{self, File, OpenOptions}, io::{Read, Write}, + ops::RangeInclusive, os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt, PermissionsExt}, path::Path, }; @@ -25,10 +26,14 @@ pub(super) fn check(path: &Path, directory: bool) -> Result<()> { } pub(super) fn check_metadata(metadata: &fs::Metadata, directory: bool) -> Result<()> { + check_links(metadata, directory, 1..=1) +} + +fn check_links(metadata: &fs::Metadata, directory: bool, links: RangeInclusive) -> Result<()> { if metadata.uid() != rustix::process::geteuid().as_raw() || metadata.permissions().mode() & 0o077 != 0 || (directory && !metadata.is_dir()) - || (!directory && (!metadata.is_file() || metadata.nlink() != 1)) + || (!directory && (!metadata.is_file() || !links.contains(&metadata.nlink()))) { bail!("local state must use ordinary owner-only directories and single-link files"); } @@ -41,11 +46,15 @@ pub(super) fn read(path: &Path, maximum: u64) -> Result> { .read(true) .custom_flags(rustix::fs::OFlags::NOFOLLOW.bits() as i32) .open(path)?; - let before = fs::symlink_metadata(path)?; - let opened = file.metadata()?; - if before.ino() != opened.ino() || before.dev() != opened.dev() { - bail!("local state changed while opening it"); + #[cfg(test)] + if let Some(hook) = AFTER_OPEN.take() { + hook(path); } + // Check the file this read opened rather than looking its path up again. + // Another writer's atomic replace may rename a successor over the path + // after the open: the opened file keeps its complete bytes and has no + // link left, so no other path reaches it either. + check_links(&file.metadata()?, false, 0..=1)?; let mut bytes = Vec::new(); (&mut file).take(maximum + 1).read_to_end(&mut bytes)?; if bytes.len() as u64 > maximum { @@ -54,6 +63,15 @@ pub(super) fn read(path: &Path, maximum: u64) -> Result> { Ok(bytes) } +// Test-only seam that stands in for another writer acting on the path once +// `read` has opened it, as a supervisor saving state does while `dev start` +// polls that state. It runs once. +#[cfg(test)] +thread_local! { + pub(super) static AFTER_OPEN: std::cell::Cell> = + const { std::cell::Cell::new(None) }; +} + pub(super) fn create(path: &Path, bytes: &[u8]) -> Result<()> { check( path.parent() diff --git a/crates/registry-bregctl/src/dev/tests.rs b/crates/registry-bregctl/src/dev/tests.rs index b0fcb53917..1f6c0a62bc 100644 --- a/crates/registry-bregctl/src/dev/tests.rs +++ b/crates/registry-bregctl/src/dev/tests.rs @@ -2362,6 +2362,31 @@ fn private_files_refuse_symlinks_hardlinks_and_public_modes() { assert!(private::read(&root.join("public"), 20).is_err()); } +#[test] +fn private_reads_keep_an_atomic_replace_after_the_open_and_check_what_they_opened() { + let (temp, _, _, _) = fixture(); + let root = fs::canonicalize(temp.path()).unwrap(); + let state = root.join("state.json"); + private::create(&state, b"earlier").unwrap(); + // The supervisor saves state by renaming a successor over the path while + // `dev start` polls it. A read that has already opened the earlier file + // returns its complete bytes; the next read returns the successor. + private::AFTER_OPEN.set(Some(|path| private::replace(path, b"successor").unwrap())); + assert_eq!(private::read(&state, 20).unwrap(), b"earlier"); + assert_eq!(private::read(&state, 20).unwrap(), b"successor"); + // What was opened still has to be an owner-only, single-link file: a + // second link or a public mode that appears after the open is refused. + private::AFTER_OPEN.set(Some(|path| { + fs::hard_link(path, path.with_extension("alias")).unwrap() + })); + assert!(private::read(&state, 20).is_err()); + fs::remove_file(state.with_extension("alias")).unwrap(); + private::AFTER_OPEN.set(Some(|path| { + fs::set_permissions(path, fs::Permissions::from_mode(0o644)).unwrap() + })); + assert!(private::read(&state, 20).is_err()); +} + #[test] fn retained_database_inspection_accepts_only_verified_image_aliases() { let (_temp, mut state, _, _) = fixture(); diff --git a/crates/registry-caseworkctl/src/dev/private.rs b/crates/registry-caseworkctl/src/dev/private.rs index 67a20fb19b..59c6f220b6 100644 --- a/crates/registry-caseworkctl/src/dev/private.rs +++ b/crates/registry-caseworkctl/src/dev/private.rs @@ -5,6 +5,7 @@ use anyhow::{bail, Context, Result}; use std::{ fs::{self, File, OpenOptions}, io::{Read, Write}, + ops::RangeInclusive, os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt, PermissionsExt}, path::Path, }; @@ -25,10 +26,14 @@ pub(super) fn check(path: &Path, directory: bool) -> Result<()> { } pub(super) fn check_metadata(metadata: &fs::Metadata, directory: bool) -> Result<()> { + check_links(metadata, directory, 1..=1) +} + +fn check_links(metadata: &fs::Metadata, directory: bool, links: RangeInclusive) -> Result<()> { if metadata.uid() != rustix::process::geteuid().as_raw() || metadata.permissions().mode() & 0o077 != 0 || (directory && !metadata.is_dir()) - || (!directory && (!metadata.is_file() || metadata.nlink() != 1)) + || (!directory && (!metadata.is_file() || !links.contains(&metadata.nlink()))) { bail!("local state must use ordinary owner-only directories and single-link files"); } @@ -41,11 +46,15 @@ pub(super) fn read(path: &Path, maximum: u64) -> Result> { .read(true) .custom_flags(rustix::fs::OFlags::NOFOLLOW.bits() as i32) .open(path)?; - let before = fs::symlink_metadata(path)?; - let opened = file.metadata()?; - if before.ino() != opened.ino() || before.dev() != opened.dev() { - bail!("local state changed while opening it"); + #[cfg(test)] + if let Some(hook) = AFTER_OPEN.take() { + hook(path); } + // Check the file this read opened rather than looking its path up again. + // Another writer's atomic replace may rename a successor over the path + // after the open: the opened file keeps its complete bytes and has no + // link left, so no other path reaches it either. + check_links(&file.metadata()?, false, 0..=1)?; let mut bytes = Vec::new(); (&mut file).take(maximum + 1).read_to_end(&mut bytes)?; if bytes.len() as u64 > maximum { @@ -54,6 +63,15 @@ pub(super) fn read(path: &Path, maximum: u64) -> Result> { Ok(bytes) } +// Test-only seam that stands in for another writer acting on the path once +// `read` has opened it, as a supervisor saving state does while `dev start` +// polls that state. It runs once. +#[cfg(test)] +thread_local! { + pub(super) static AFTER_OPEN: std::cell::Cell> = + const { std::cell::Cell::new(None) }; +} + pub(super) fn create(path: &Path, bytes: &[u8]) -> Result<()> { check( path.parent() diff --git a/crates/registry-caseworkctl/src/dev/tests.rs b/crates/registry-caseworkctl/src/dev/tests.rs index 45e034c80b..f774d33b2a 100644 --- a/crates/registry-caseworkctl/src/dev/tests.rs +++ b/crates/registry-caseworkctl/src/dev/tests.rs @@ -2727,6 +2727,32 @@ fn invalid_service_journal_is_refused_before_the_child_starts() { assert!(!marker.exists()); } +#[test] +fn private_reads_keep_an_atomic_replace_after_the_open_and_check_what_they_opened() { + let root = crate::canonical_tempdir(); + let dev = root.path().join("dev"); + private::directory(&dev).unwrap(); + let state = dev.join("state.json"); + private::create(&state, b"earlier").unwrap(); + // The supervisor saves state by renaming a successor over the path while + // `dev start` polls it. A read that has already opened the earlier file + // returns its complete bytes; the next read returns the successor. + private::AFTER_OPEN.set(Some(|path| private::replace(path, b"successor").unwrap())); + assert_eq!(private::read(&state, 20).unwrap(), b"earlier"); + assert_eq!(private::read(&state, 20).unwrap(), b"successor"); + // What was opened still has to be an owner-only, single-link file: a + // second link or a public mode that appears after the open is refused. + private::AFTER_OPEN.set(Some(|path| { + fs::hard_link(path, path.with_extension("alias")).unwrap() + })); + assert!(private::read(&state, 20).is_err()); + fs::remove_file(state.with_extension("alias")).unwrap(); + private::AFTER_OPEN.set(Some(|path| { + fs::set_permissions(path, fs::Permissions::from_mode(0o644)).unwrap() + })); + assert!(private::read(&state, 20).is_err()); +} + #[test] fn service_guard_process_helper() { let Some(encoded) = std::env::var_os("CASEWORKCTL_TEST_SERVICE_GUARD_ARGV") else { From e480dbbba8490e53285646a46f1565023c6192a9 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 15:01:01 +0700 Subject: [PATCH 24/64] refactor(breg): move RetentionFailureReason to the postgres module Both retention workers and the external projection purge classify PostgreSQL failures with it, so it belongs with the PostgreSQL kernel rather than in the hook dispatcher. Signed-off-by: Jeremi Joslin --- crates/registry-breg/src/hooks.rs | 45 ++----------------- .../src/postgres/external_projection.rs | 6 ++- .../src/postgres/external_runtime.rs | 4 +- crates/registry-breg/src/postgres/mod.rs | 41 +++++++++++++++++ 4 files changed, 50 insertions(+), 46 deletions(-) diff --git a/crates/registry-breg/src/hooks.rs b/crates/registry-breg/src/hooks.rs index 91ff6235e9..9699a5c749 100644 --- a/crates/registry-breg/src/hooks.rs +++ b/crates/registry-breg/src/hooks.rs @@ -42,7 +42,9 @@ use crate::idempotency::IdempotencyPolicy; use crate::model::CompiledRegistry; use crate::mutation::{HookChangeApplication, HookChangeOutcome, MutationTransaction}; use crate::package::PackageError; -use crate::postgres::{ExpectedRegistryIdentity, RegistryLockKey, RuntimePool}; +use crate::postgres::{ + ExpectedRegistryIdentity, RegistryLockKey, RetentionFailureReason, RuntimePool, +}; use crate::runtime_config::load_runtime_config; use crate::startup::{HookStateTransitionCode, OperationalEvent}; @@ -367,47 +369,6 @@ struct ProposalCompletionBarrier { const MAX_RETENTION_BATCHES_PER_TICK: usize = 100; -#[derive(Clone, Copy)] -pub(crate) enum RetentionFailureReason { - RegistryNotReady, - LockTimeout, - QueryCancelled, - Database, -} - -impl RetentionFailureReason { - pub(crate) fn as_str(self) -> &'static str { - match self { - Self::RegistryNotReady => "registry-not-ready", - Self::LockTimeout => "lock-timeout", - Self::QueryCancelled => "query-cancelled", - Self::Database => "database", - } - } - - pub(crate) fn sqlstate(code: Option<&str>) -> Self { - match code { - Some("55P03") => Self::LockTimeout, - Some("57014") => Self::QueryCancelled, - _ => Self::Database, - } - } - - pub(crate) fn admission(error: crate::postgres::RegistryTransactionError) -> Self { - use crate::postgres::{PostgresKernelError, RegistryLockFailure, RegistryTransactionError}; - match error { - RegistryTransactionError::Kernel(PostgresKernelError::RegistryUnavailable) => { - Self::RegistryNotReady - } - RegistryTransactionError::Lock(RegistryLockFailure::Timeout) => Self::LockTimeout, - RegistryTransactionError::Lock(RegistryLockFailure::QueryCancelled) => { - Self::QueryCancelled - } - _ => Self::Database, - } - } -} - impl BregDeliverySeams { async fn verify_delivery_transaction( &self, diff --git a/crates/registry-breg/src/postgres/external_projection.rs b/crates/registry-breg/src/postgres/external_projection.rs index 9b176c1e9b..762584c7c3 100644 --- a/crates/registry-breg/src/postgres/external_projection.rs +++ b/crates/registry-breg/src/postgres/external_projection.rs @@ -11,8 +11,10 @@ use serde_json::Value; use tokio_postgres::GenericClient; use uuid::Uuid; -use super::{ExpectedRegistryIdentity, GuardedTransaction, RuntimeRevoke, SqlIdentifier}; -use crate::hooks::RetentionFailureReason; +use super::{ + ExpectedRegistryIdentity, GuardedTransaction, RetentionFailureReason, RuntimeRevoke, + SqlIdentifier, +}; pub(crate) const BIND_ONCE_FUNCTION: &str = "registry_internal.bind_external_projection_once(text,uuid,text,text,bigint,text,text,uuid)"; diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index b96f46b4aa..0531be5ba4 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -46,9 +46,9 @@ use super::external_view_query::{ use super::read::{strict_claim_context, ReadRelations}; use super::{ begin_record_transaction, begin_registry_transaction, begin_registry_transaction_with_reason, - ClaimContext, ExpectedRegistryIdentity, GuardedTransaction, RegistryLockKey, RuntimePool, + ClaimContext, ExpectedRegistryIdentity, GuardedTransaction, RegistryLockKey, + RetentionFailureReason, RuntimePool, }; -use crate::hooks::RetentionFailureReason; type Result = std::result::Result; /// Rows one retention tick committed, as (snapshots, receipts) by projection diff --git a/crates/registry-breg/src/postgres/mod.rs b/crates/registry-breg/src/postgres/mod.rs index fc2e195920..0c42be74c6 100644 --- a/crates/registry-breg/src/postgres/mod.rs +++ b/crates/registry-breg/src/postgres/mod.rs @@ -491,6 +491,47 @@ impl From for PostgresKernelError { /// Result returned by PostgreSQL kernel operations. pub type Result = std::result::Result; +#[derive(Clone, Copy)] +pub(crate) enum RetentionFailureReason { + RegistryNotReady, + LockTimeout, + QueryCancelled, + Database, +} + +impl RetentionFailureReason { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::RegistryNotReady => "registry-not-ready", + Self::LockTimeout => "lock-timeout", + Self::QueryCancelled => "query-cancelled", + Self::Database => "database", + } + } + + pub(crate) fn sqlstate(code: Option<&str>) -> Self { + match code { + Some("55P03") => Self::LockTimeout, + Some("57014") => Self::QueryCancelled, + _ => Self::Database, + } + } + + pub(crate) fn admission(error: crate::postgres::RegistryTransactionError) -> Self { + use crate::postgres::{PostgresKernelError, RegistryLockFailure, RegistryTransactionError}; + match error { + RegistryTransactionError::Kernel(PostgresKernelError::RegistryUnavailable) => { + Self::RegistryNotReady + } + RegistryTransactionError::Lock(RegistryLockFailure::Timeout) => Self::LockTimeout, + RegistryTransactionError::Lock(RegistryLockFailure::QueryCancelled) => { + Self::QueryCancelled + } + _ => Self::Database, + } + } +} + #[cfg(test)] mod tests { use super::sqlstate_ends_the_session; From 5b4dd362fedbb39c0ed5d9a2cd50a546bd38d0e3 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 15:01:22 +0700 Subject: [PATCH 25/64] test(breg): give harness databases and roles a random suffix Four library modules include the harness file, each with its own counter, so two copies could build the same role name from one process id and clock reading. A random suffix needs no state shared between the copies. Signed-off-by: Jeremi Joslin --- .../tests/support/postgres_harness.rs | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/crates/registry-breg/tests/support/postgres_harness.rs b/crates/registry-breg/tests/support/postgres_harness.rs index 7fbf881a95..b6711b3e0d 100644 --- a/crates/registry-breg/tests/support/postgres_harness.rs +++ b/crates/registry-breg/tests/support/postgres_harness.rs @@ -1,11 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -use std::{ - env, - str::FromStr, - sync::atomic::{AtomicU64, Ordering}, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; +use std::{env, str::FromStr, time::Duration}; use registry_breg::audit::{test_support::AuditCapture, RegistryAudit}; use registry_breg::postgres::{ @@ -329,14 +324,13 @@ async fn connect(config: Config) -> (Client, JoinHandle<()>) { (client, task) } +/// A random suffix for the database and role names. Several modules of one +/// test binary include this file, each copy with its own statics, so a counter +/// here would repeat across copies, and the clock does not separate two +/// copies that read it within the same tick. A random value needs no state +/// shared between the copies or between test processes on one server. fn unique_suffix() -> String { - static COUNTER: AtomicU64 = AtomicU64::new(0); - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system clock is after the Unix epoch") - .as_nanos(); - let counter = COUNTER.fetch_add(1, Ordering::Relaxed); - format!("{}_{nanos}_{counter}", std::process::id()) + uuid::Uuid::new_v4().simple().to_string() } /// Compare only the changed audit identity projection; the owning journeys From c0272f3cdbe7371bbef4f83f7f7c01daf6d67722 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 15:01:28 +0700 Subject: [PATCH 26/64] fix(breg): start the refresh continuation's base table scan at its boundary The refresh page query joins the source view to the base table on the record id and bounded only the view. PostgreSQL does not carry that inequality across the join, so the merge join scanned the base table from its first record on every page and paging cost grew with the square of the scope. In the dense test the base table scan read 9555 rows plus 497 filtered for a page of 101 after the midpoint; with the bound repeated on the base table it reads about one page. Signed-off-by: Jeremi Joslin --- .../src/postgres/external_runtime.rs | 146 ++++++++++++++++-- 1 file changed, 136 insertions(+), 10 deletions(-) diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index 0531be5ba4..29956ae094 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -1304,6 +1304,7 @@ struct RootKeys { select: String, publication_eligibility: String, id: String, + base_id: String, } impl RootKeys { @@ -1327,6 +1328,7 @@ impl RootKeys { relations.from_sql, relations.id_expression, key.sql, ), id: relations.id_expression, + base_id: format!("{}.record_id", relations.base_alias), }) } @@ -1344,9 +1346,13 @@ impl RootKeys { } fn page_sql(&self) -> String { + // The base table joins the source view on the record id, and + // PostgreSQL does not carry an inequality across that equality. + // Repeating the bound on the base table lets a merge join start its + // primary key scan at the continuation instead of at the first record. format!( - "{} WHERE ($1::uuid IS NULL OR {} > $1) ORDER BY {} LIMIT $2", - self.select, self.id, self.id + "{} WHERE ($1::uuid IS NULL OR ({} > $1 AND {} > $1)) ORDER BY {} LIMIT $2", + self.select, self.id, self.base_id, self.id ) } @@ -2911,23 +2917,143 @@ mod postgres_tests { .unwrap(); let page = keys.page(&transaction, after, 101).await.ok().unwrap(); assert_eq!(page.len(), 101); - let explained = transaction + let explained: Value = transaction .transaction() - .query( - &format!("EXPLAIN (COSTS OFF) {}", keys.page_sql()), + .query_one( + &format!( + "EXPLAIN (ANALYZE, FORMAT JSON, COSTS OFF, TIMING OFF, SUMMARY OFF) {}", + keys.page_sql() + ), &[&after, &101i64], ) .await .unwrap() - .into_iter() - .map(|row| row.get::<_, String>(0)) - .collect::>() - .join("\n"); - assert!(!explained.contains("Sort"), "after {after:?}: {explained}"); + .get(0); + let mut nodes = Vec::new(); + plan_nodes(&explained[0]["Plan"], &mut nodes); + assert!( + !nodes.iter().any(|node| node["Node Type"] + .as_str() + .is_some_and(|kind| kind.contains("Sort"))), + "after {after:?}: {explained}" + ); + // Both copies of the table start at the continuation: a scan of + // either from the first record would read about half of it here. + for node in &nodes { + let rows = node["Actual Rows"].as_f64().unwrap_or(0.0) + * node["Actual Loops"].as_f64().unwrap_or(1.0); + let removed = node["Rows Removed by Filter"].as_f64().unwrap_or(0.0) + * node["Actual Loops"].as_f64().unwrap_or(1.0); + assert!( + rows + removed <= 3.0 * 101.0, + "after {after:?} reads about one page at every node: {explained}" + ); + } } served.fixture.cleanup().await; } + fn plan_nodes<'a>(plan: &'a Value, nodes: &mut Vec<&'a Value>) { + nodes.push(plan); + for child in plan["Plans"].as_array().into_iter().flatten() { + plan_nodes(child, nodes); + } + } + + #[tokio::test] + async fn a_refresh_continuation_returns_each_root_once_in_order_past_an_insert_behind_it() { + let served = Served::new(Duration::from_secs(2)).await; + let entity = &served.service.registry.entities()["local-record"]; + let column = |id: &str| { + entity + .stored_fields + .iter() + .find(|field| field.logical.id == id) + .unwrap() + .physical_name + .clone() + }; + let insert = |rows: &[(&str, &str, &str)]| { + format!( + "INSERT INTO registry_data.\"{}\" (record_id,active_package_revision,\"{}\",\"{}\") VALUES {}", + entity.physical_table, + column("subject-key"), + column("tenant"), + rows.iter() + .map(|(id, key, tenant)| format!("('{id}','fixture','{key}','{tenant}')")) + .collect::>() + .join(",") + ) + }; + served + .fixture + .database + .admin + .batch_execute(&insert(&[ + ("00000000-0000-4000-8000-000000000005", "fifth", "allowed"), + ("00000000-0000-4000-8000-000000000006", "sixth", "other"), + ("00000000-0000-4000-8000-000000000007", "seventh", "allowed"), + ])) + .await + .unwrap(); + let refresh = request("link"); + let plan = served.service.plan(&refresh, None).ok().unwrap(); + let claims = ClaimContext::for_compiled( + &served.service.registry, + &plan.root.id, + Some("reader-principal".to_owned()), + "reader", + Some("consultation".to_owned()), + vec![crate::postgres::context::RowBoundaryContext::Equals { + field: "tenant".to_owned(), + value: "allowed".to_owned(), + }], + ) + .unwrap(); + let keys = RootKeys::new(&plan).ok().unwrap(); + let mut after = None; + let mut read = Vec::new(); + loop { + let mut session = served.service.session().await.ok().unwrap(); + let transaction = served + .service + .transaction( + &mut session, + &claims, + Instant::now() + Duration::from_secs(5), + ) + .await + .ok() + .unwrap(); + let page = keys.page(&transaction, after, 2).await.ok().unwrap(); + drop(transaction); + if after.is_none() { + // Another session commits one root behind the boundary and + // one ahead of it before the next page. + served + .fixture + .database + .admin + .batch_execute(&insert(&[ + ("00000000-0000-4000-8000-000000000000", "behind", "allowed"), + ("00000000-0000-4000-8000-000000000008", "ahead", "allowed"), + ])) + .await + .unwrap(); + } + let Some((last, _)) = page.last() else { + break; + }; + after = Some(*last); + read.extend(page.into_iter().map(|(_, key)| key.unwrap())); + } + assert_eq!( + read, + ["first", "second", "third", "fifth", "seventh", "ahead"] + ); + served.fixture.cleanup().await; + } + /// Source keys, labels and snapshot identities the fixtures below store. /// None of them may reach an operational log line. const FIXTURE_VALUES: [&str; 7] = [ From 213bbd5cd3c2cf1ef163b33af7011dba82de8587 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 15:11:18 +0700 Subject: [PATCH 27/64] test(breg): cover a live external view read over HTTP The live, no-store path had no end-to-end HTTP test. Read a live link through the router with a granted reader and a recorded source exchange: the answer comes from the source, nothing lands in the external projection tables or any other table, and reads the grant or row boundary does not cover are refused as problems before reaching the source. Signed-off-by: Jeremi Joslin --- crates/registry-breg/src/external_source.rs | 33 +++ .../src/postgres/external_runtime.rs | 245 +++++++++++++++++- 2 files changed, 268 insertions(+), 10 deletions(-) diff --git a/crates/registry-breg/src/external_source.rs b/crates/registry-breg/src/external_source.rs index d0e5807e0a..5def00dfc2 100644 --- a/crates/registry-breg/src/external_source.rs +++ b/crates/registry-breg/src/external_source.rs @@ -756,6 +756,39 @@ pub(crate) mod tests { service } + /// Add a live binding over the example pack's exact route, which retains + /// nothing. Each source call must match the next `transcript` exchange. + #[cfg(feature = "postgres-test")] + pub(crate) fn add_live_binding( + service: &mut ExternalSourceService, + projection: &str, + field: &str, + transcript: Arc, + ) { + let (pack, artifacts) = example(); + let program = + compile_projection_pack(&pack, "person-exact", &artifacts, EvidenceLimits::default()) + .unwrap() + .bind_replay(transcript) + .unwrap(); + service.bindings.insert( + projection.to_owned(), + BoundExternalSource { + program, + fields: BTreeSet::from([field.to_owned()]), + mode: ExternalProjectionMode::Live, + grant_valid_until: Utc::now() + chrono::Duration::days(1), + binding_fingerprint: "live-binding".to_owned(), + grant_fingerprint: "live-grant".to_owned(), + cache_ttl_seconds: None, + field_types: BTreeMap::from([( + field.to_owned(), + (ExternalFieldType::String, false), + )]), + }, + ); + } + struct Hooks; impl EvidenceHostHooks for Hooks { fn before_acquisition<'a>( diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index 29956ae094..035b29822e 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -2213,6 +2213,22 @@ mod postgres_tests { registry: &CompiledRegistry, fixture: &Activated, lock_timeout: Duration, + ) -> PostgresExternalViewService { + service_from(registry, fixture, lock_timeout, indexed_sources()) + } + + fn indexed_sources() -> crate::external_source::ExternalSourceService { + crate::external_source::tests::indexed_service(&[ + ("projection", "binding", "grant", 60), + ("other", "other-binding", "other-grant", 120), + ]) + } + + fn service_from( + registry: &CompiledRegistry, + fixture: &Activated, + lock_timeout: Duration, + sources: crate::external_source::ExternalSourceService, ) -> PostgresExternalViewService { PostgresExternalViewService::new( fixture.pool.clone(), @@ -2234,10 +2250,7 @@ mod postgres_tests { ) .unwrap(), ), - crate::external_source::tests::indexed_service(&[ - ("projection", "binding", "grant", 60), - ("other", "other-binding", "other-grant", 120), - ]), + sources, ) } @@ -2370,11 +2383,19 @@ mod postgres_tests { impl Served { async fn new(lock_timeout: Duration) -> Self { + Self::with(&two_projection_project(), indexed_sources(), lock_timeout).await + } + + async fn with( + project: &Value, + sources: crate::external_source::ExternalSourceService, + lock_timeout: Duration, + ) -> Self { let registry = Arc::new(super::super::external_view_query::tests::compile_project( - &two_projection_project(), + project, )); let fixture = Activated::new(®istry).await; - let mut service = service_with(®istry, &fixture, lock_timeout); + let mut service = service_from(®istry, &fixture, lock_timeout, sources); service.bind_configured_scopes(true).await.ok().unwrap(); let service = Arc::new(service); let scope = bound_scope(&fixture, "projection").await; @@ -2452,20 +2473,43 @@ mod postgres_tests { query: &str, claims: crate::api::VerifiedRequestClaims, ) -> (axum::http::StatusCode, Value) { + let (status, _, body) = self + .send( + &format!("/v1/external-views/view/link/records?accessProfile=reader{query}"), + claims, + ) + .await; + (status, body) + } + + /// GET `uri`, returning the status, the content type and the body. + async fn send( + &self, + uri: &str, + claims: crate::api::VerifiedRequestClaims, + ) -> (axum::http::StatusCode, String, Value) { use tower::ServiceExt; let mut request = axum::http::Request::builder() - .uri(format!( - "/v1/external-views/view/link/records?accessProfile=reader{query}" - )) + .uri(uri) .body(axum::body::Body::empty()) .unwrap(); request.extensions_mut().insert(claims); let response = self.router.clone().oneshot(request).await.unwrap(); let status = response.status(); + let content_type = response + .headers() + .get(axum::http::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_owned(); let bytes = axum::body::to_bytes(response.into_body(), 1024 * 1024) .await .unwrap(); - (status, serde_json::from_slice(&bytes).unwrap()) + ( + status, + content_type, + serde_json::from_slice(&bytes).unwrap(), + ) } } @@ -2855,6 +2899,187 @@ mod postgres_tests { served.fixture.cleanup().await; } + /// The fixture project with a live projection of a second external record + /// type, linked from the same view and readable by the reader profile. + fn live_and_indexed_project() -> Value { + let mut project = two_projection_project(); + project["externalRecordTypes"] + .as_array_mut() + .unwrap() + .push(json!({ + "id":"person","authority":"urn:test:authority","identifierScheme":"urn:test:keys", + "fields":[{"id":"family-name","type":"string","classification":"internal"}] + })); + project["externalProjections"] + .as_array_mut() + .unwrap() + .push(json!({ + "id":"holder","recordType":"person","operation":"person-exact","operationVersion":"1", + "purpose":"consultation","scope":"eligible","selectorScheme":"urn:test:keys", + "mode":"live","cardinality":"one","fields":["family-name"] + })); + project["externalViews"][0]["links"] + .as_array_mut() + .unwrap() + .push(json!({ + "id":"holder-link","projection":"holder","fromField":"subject-key", + "identifierScheme":"urn:test:keys","fields":["family-name"] + })); + project["accessProfiles"][0]["permissions"]["externalViews"][0]["links"] + .as_array_mut() + .unwrap() + .push(json!({"link":"holder-link","readableFields":["family-name"]})); + project + } + + fn assert_problem(content_type: &str, problem: &Value, code: &str) { + assert_eq!(content_type, "application/problem+json", "{problem}"); + assert_eq!(problem["code"], code, "{problem}"); + for member in ["type", "title", "status", "detail"] { + assert!(problem.get(member).is_some(), "{member}: {problem}"); + } + assert!(problem.get("records").is_none(), "{problem}"); + } + + /// Every table whose rows contain `needle` in their text form. + async fn tables_holding(client: &tokio_postgres::Client, needle: &str) -> Vec { + let tables = client + .query( + "SELECT format('%I.%I', table_schema, table_name) FROM information_schema.tables + WHERE table_type = 'BASE TABLE' + AND table_schema NOT IN ('pg_catalog', 'information_schema')", + &[], + ) + .await + .unwrap(); + let mut holding = Vec::new(); + for table in tables { + let name: String = table.get(0); + let found: bool = client + .query_one( + &format!( + "SELECT EXISTS (SELECT 1 FROM {name} AS t WHERE strpos(t::text, $1) > 0)" + ), + &[&needle], + ) + .await + .unwrap() + .get(0); + if found { + holding.push(name); + } + } + holding + } + + #[tokio::test] + async fn a_live_read_over_http_answers_from_the_source_and_retains_nothing() { + // Only the source holds this value, so finding it in any table + // afterwards would mean the read retained it. + const SOURCE_VALUE: &str = "synthetic-live-family-name"; + let transcript = Arc::new( + registry_adapter_engine::replay::ReplayTranscript::from_exchanges( + serde_json::from_value( + json!([{"type":"endpoint","endpoint":"synthetic-registry"}]), + ) + .unwrap(), + serde_json::from_value(json!([{ + "request":{"endpoint":"synthetic-registry","method":"POST","path":"/people/exact","body":{ + "type":"exact","key":"first","fields":["family-name"] + }}, + "response":{"status":200,"body":{ + "subject":"first","state":"found","properties":{"family-name":SOURCE_VALUE} + }} + }])) + .unwrap(), + ), + ); + let mut sources = indexed_sources(); + crate::external_source::tests::add_live_binding( + &mut sources, + "holder", + "family-name", + Arc::clone(&transcript), + ); + let served = + Served::with(&live_and_indexed_project(), sources, Duration::from_secs(2)).await; + let caller = reader("reader-principal", &["allowed"]); + let detail = |root: &str, query: &str| { + format!( + "/v1/external-views/view/holder-link/records/{root}?accessProfile=reader{query}" + ) + }; + + let (status, _, body) = served.send(&detail(ROOTS[0].0, ""), caller.clone()).await; + assert_eq!(status, axum::http::StatusCode::OK, "{body}"); + assert_eq!(body["status"], "available", "{body}"); + assert_eq!( + body["records"][0]["fields"]["family-name"], SOURCE_VALUE, + "{body}" + ); + + // A root outside the caller's row boundary, and a count the live + // link's grant does not allow, are refused before any source call. + let (status, content_type, problem) = + served.send(&detail(ROOTS[3].0, ""), caller.clone()).await; + assert_eq!(status, axum::http::StatusCode::NOT_FOUND, "{problem}"); + assert_problem(&content_type, &problem, "resource.not_found"); + let (status, content_type, problem) = served + .send(&detail(ROOTS[0].0, "&$count=true"), caller) + .await; + assert_eq!(status, axum::http::StatusCode::BAD_REQUEST, "{problem}"); + assert_problem(&content_type, &problem, "query.invalid"); + // The one recorded exchange was used, and nothing else reached it. + transcript.finish().unwrap(); + + let admin = &served.fixture.database.admin; + for table in [ + "registry_external_projection_refreshes", + "registry_external_projection_snapshots", + "registry_external_projection_values", + ] { + let rows: i64 = admin + .query_one( + &format!("SELECT count(*) FROM registry_internal.{table}"), + &[], + ) + .await + .unwrap() + .get(0); + assert_eq!(rows, 0, "{table}"); + } + let scoped: Vec = admin + .query( + "SELECT projection_id FROM registry_internal.registry_external_projection_scopes + ORDER BY projection_id", + &[], + ) + .await + .unwrap() + .into_iter() + .map(|row| row.get(0)) + .collect(); + assert_eq!( + scoped, + ["other", "projection"], + "only indexed projections bind" + ); + // The scan finds a value the fixture did store, so an empty answer + // below means the source value is absent, not that the scan is blind. + let stored = tables_holding(admin, ROOTS[3].1).await; + assert!( + stored + .iter() + .any(|table| table.starts_with("registry_data.")), + "{stored:?}" + ); + assert_eq!( + tables_holding(admin, SOURCE_VALUE).await, + Vec::::new() + ); + served.fixture.cleanup().await; + } + #[tokio::test] async fn a_refresh_pages_a_dense_scope_in_key_order_without_sorting_it() { let served = Served::new(Duration::from_secs(2)).await; From fb70708561f0e5b986dca1d05662fcb7652aff91 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 15:13:57 +0700 Subject: [PATCH 28/64] docs(breg): add external projection retention and doctor codes to monitoring Operators had no row for the retention worker an indexed external projection starts, and no entry for the two doctor codes it reports. Describe the worker's series, its pass signals and bound, the external-projection.status advisory, and the unavailable warning, in line with the storage guide's monitoring advice. Signed-off-by: Jeremi Joslin --- .../src/content/docs/operate/breg-monitor.mdx | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/docs/site/src/content/docs/operate/breg-monitor.mdx b/docs/site/src/content/docs/operate/breg-monitor.mdx index 98fdd7237f..af7e61179b 100644 --- a/docs/site/src/content/docs/operate/breg-monitor.mdx +++ b/docs/site/src/content/docs/operate/breg-monitor.mdx @@ -196,12 +196,14 @@ rise in `client-error`, and put the registry behind an upstream rate limit: it h | `review` | Submits Proposals for review and reads results. | | `attachment-verification` | Sends uploaded files to an external verifier, when enabled. | | `subject-access-log-retention` | Drops expired subject access log entries, once a minute. | +| `external-projection-retention` | Deletes expired, replaced, and withdrawn external projection snapshots and refresh receipts every five seconds, when the active package declares an indexed external projection. | `breg_worker_last_success_age_seconds` counts from the last iteration that did not fail, work or no work. The first three loop at least once a second, so an age of minutes means a worker fails every -time or is stuck. `breg_queue_oldest_pending_age_seconds` is how long the oldest due item has -waited; when it grows while the worker age stays small, the worker cannot keep up or keeps failing -one item. +time or is stuck. The `external-projection-retention` series is absent until its first pass +completes; alert when it grows well past five seconds. `breg_queue_oldest_pending_age_seconds` is +how long the oldest due item has waited; when it grows while the worker age stays small, the worker +cannot keep up or keeps failing one item. `breg_proposal_application_blocked_jobs` counts application jobs held or closed for the current submitted version while its Proposal has `package-changed`. A zero queue age does not mean these @@ -224,6 +226,34 @@ or the metrics listener panics or returns, the process logs a code naming it, su `review.worker.panicked`, drains requests, and exits with status 1. The same code on every restart names the worker to investigate. +An external projection retention pass deletes at most eight batches of a thousand rows of each kind. +None of its signals carries a source key, a value, or a secret: + +- A pass that deletes rows logs one `info` line per projection and reason, with `snapshots` and + `receipts` counts. The reason is `expired`, `replaced` for a rotated epoch, or `withdrawn`. +- A pass that fails logs `external projection retention tick failed` at `error`, with a `reason` of + `registry-not-ready`, `lock-timeout`, `query-cancelled`, `database`, or `deadline`. It names no + projection, because one pass covers them all. +- A pass that reaches its batch bound logs a `warn` line; the rest of the backlog waits for the next + pass. + +`bregctl doctor` adds an `external-projection.status` information advisory for each projection a +server would serve. It gives the time the source grant expires, as `grantValidUntilUnixSeconds`, and +for an indexed projection when the newest snapshot it holds was observed, as +`newestPublicationUnixSeconds`. A live projection holds no snapshots, and an indexed projection that +was never refreshed has no newest snapshot. Renew the grant before it expires. An old or missing +newest snapshot means the refresh sweep is not running. A projection without a usable binding is +reported as an `external-projection.unavailable` warning instead; see +[Troubleshooting](#troubleshooting). + +{/* Evidence: crates/registry-breg/src/postgres/external_runtime.rs, run_retention(), retention_tick(), + purge_tick(), projection_status(); + crates/registry-breg/src/postgres/external_projection.rs, enabled(); + crates/registry-breg/src/postgres/mod.rs, RetentionFailureReason; + crates/registry-breg/src/startup.rs, external_projection_retention and background_task_stop_code(); + crates/registry-bregctl/src/lib.rs, write_doctor_success() and external_projection_status_report(); + products/breg/external-projection-storage.md, Monitor refresh and purge. */} + {/* Evidence: crates/registry-breg/src/metrics.rs, ProgressWorker, PendingQueue, read_queues(); crates/registry-breg/src/startup.rs, OperationalEvent, RecordTimerIterationCause, background_task_stop_code(), SupervisedTask; @@ -301,6 +331,8 @@ Find the code the command, log, or HTTP problem printed. A `*` stands for any pa | `migration.review.*` | The reviewed migration directory was refused. | See [write a reviewed migration descriptor](../breg-changes/#write-a-reviewed-migration-descriptor). | | `migration.rehearsal.*` | PostgreSQL refused the rehearsal, so `apply` would too. | Correct the SQL or the project. | | `source.unavailable` | A read could not complete: the database is unreachable or has no free connection, an audit record could not be written, or history no longer covers the snapshot. | Check the database and the audit destination; for a snapshot read, take a fresh reference. | +| `external-projection.unavailable` | From `doctor`, a warning: an external projection the package declares or the runtime file binds has no usable binding, and the message names why. Its views answer `source.unavailable`; other views serve. | Add or correct its entry under `externalSources` in the runtime file, then restart. | +| `external-projection.status` | From `doctor`, information: when a projection's source grant expires and when its newest snapshot was observed. | Renew the grant before it expires; if the newest snapshot is old or missing, run the refresh sweep. | | `idempotency.expired` | The write committed; its receipt expired. | Read the record instead. | | `history.erase.acknowledgement.required` | `--acknowledge-irreversible` is missing. | Confirm, then add the flag. | | `history.rebaseline.live-rows.unverified` | A live row disagrees with its journal head. | Compare suspect records' revisions. | From 80b737123d9ba08cc671e0d3377e39abcf83b6f9 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 15:52:43 +0700 Subject: [PATCH 29/64] docs(breg): point the load-test guide at the external view scale fixture Signed-off-by: Jeremi Joslin --- products/breg/loadtest/README.md | 4 ++++ products/breg/loadtest/external-views/SCALE.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/products/breg/loadtest/README.md b/products/breg/loadtest/README.md index 6b4112c54a..70c77197ce 100644 --- a/products/breg/loadtest/README.md +++ b/products/breg/loadtest/README.md @@ -13,6 +13,10 @@ client to each access profile. The fixture continues to own that authorization proof. The load client carries the ordinary `business-operator` claims used by the remaining journey and workload. +The external projection scale fixture is separate. It runs paired candidate +binaries against its own PostgreSQL databases rather than this `bregctl dev` +environment; see [external-views/SCALE.md](external-views/SCALE.md). + ## Prerequisites - `cargo`, `docker`, `python3` diff --git a/products/breg/loadtest/external-views/SCALE.md b/products/breg/loadtest/external-views/SCALE.md index a9c78f1a60..055352f53b 100644 --- a/products/breg/loadtest/external-views/SCALE.md +++ b/products/breg/loadtest/external-views/SCALE.md @@ -1,7 +1,7 @@ # Native external projection scale fixture Run this acceptance fixture against the same paired candidate binaries as -the [small licensing example](README.md). The native run uses two fresh +the [small licensing example](../../examples/external-views/README.md). The native run uses two fresh loopback PostgreSQL 18 TLS databases, distinct migration and runtime roles, and a dedicated plan log. The predeclared machine is the shared M5 Max workstation with local storage. Load and timing on the shared From 5d4da5651d1888058d461c7c05d0ea53db6cd6d7 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 16:19:48 +0700 Subject: [PATCH 30/64] fix(breg): keep one bad link key from making an indexed view unavailable A root whose link field is empty or unset names no outside subject, but it counted as uncovered, so every list, filter, order and count over its view answered incomplete, and a live read of it answered unavailable. It now counts as covered with its external fields absent and is never sent to the source. A link key the store or the source contract refuses made the refresh page fail as a whole, so the sweep could not move past it. Each key is now checked before acquisition: an invalid key counts as unavailable for its root, is logged as invalid-key with a count and no value, and the rest of the page refreshes. The view stays incomplete until the key is corrected. Signed-off-by: Jeremi Joslin --- crates/registry-breg/src/external_source.rs | 34 ++ .../src/postgres/external_projection.rs | 3 +- .../src/postgres/external_runtime.rs | 424 +++++++++++++++++- .../src/postgres/external_view_query.rs | 4 +- products/breg/external-projection-storage.md | 5 +- 5 files changed, 455 insertions(+), 15 deletions(-) diff --git a/crates/registry-breg/src/external_source.rs b/crates/registry-breg/src/external_source.rs index 5def00dfc2..b36677fc6a 100644 --- a/crates/registry-breg/src/external_source.rs +++ b/crates/registry-breg/src/external_source.rs @@ -789,6 +789,40 @@ pub(crate) mod tests { ); } + /// Add an indexed binding over the example pack's batch route with a + /// sixty-second cache lifetime. Each source call must match the next + /// `transcript` exchange. + #[cfg(feature = "postgres-test")] + pub(crate) fn add_indexed_binding( + service: &mut ExternalSourceService, + projection: &str, + field: &str, + transcript: Arc, + ) { + let (pack, artifacts) = example(); + let program = + compile_projection_pack(&pack, "person-batch", &artifacts, EvidenceLimits::default()) + .unwrap() + .bind_replay(transcript) + .unwrap(); + service.bindings.insert( + projection.to_owned(), + BoundExternalSource { + program, + fields: BTreeSet::from([field.to_owned()]), + mode: ExternalProjectionMode::Indexed, + grant_valid_until: Utc::now() + chrono::Duration::days(1), + binding_fingerprint: "indexed-binding".to_owned(), + grant_fingerprint: "indexed-grant".to_owned(), + cache_ttl_seconds: Some(60), + field_types: BTreeMap::from([( + field.to_owned(), + (ExternalFieldType::String, false), + )]), + }, + ); + } + struct Hooks; impl EvidenceHostHooks for Hooks { fn before_acquisition<'a>( diff --git a/crates/registry-breg/src/postgres/external_projection.rs b/crates/registry-breg/src/postgres/external_projection.rs index 762584c7c3..fd307b37a0 100644 --- a/crates/registry-breg/src/postgres/external_projection.rs +++ b/crates/registry-breg/src/postgres/external_projection.rs @@ -568,7 +568,8 @@ impl RefreshFailure { } } -fn valid_key(value: &str) -> bool { +/// Whether the store accepts `value` as a subject or source key. +pub(crate) fn valid_key(value: &str) -> bool { !value.is_empty() && value.len() <= MAX_KEY_BYTES && !value.chars().any(char::is_control) } diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index 035b29822e..01dc49e56e 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -6,7 +6,9 @@ use std::sync::Arc; use std::time::Duration; use chrono::Utc; -use registry_adapter_core::{ProjectionRecord, ProjectionRequest, ProjectionResult}; +use registry_adapter_core::{ + projection_token_valid, ProjectionRecord, ProjectionRequest, ProjectionResult, +}; use registry_adapter_engine::evidence::{ EvidenceAcquisitionIntent, EvidenceHookError, EvidenceHookFuture, EvidenceHostHooks, EvidenceSendIntent, @@ -607,10 +609,12 @@ impl PostgresExternalViewService { let transaction = self .transaction(&mut session, claims, request.deadline) .await?; + // A root without a link key has no outside subject to fetch. let key = RootKeys::new(plan)? .for_root(&transaction, root) .await? - .ok_or(ExternalViewError::Concealed)?; + .ok_or(ExternalViewError::Concealed)? + .filter(|key| !key.is_empty()); transaction .commit() .await @@ -647,11 +651,19 @@ impl PostgresExternalViewService { let current = RootKeys::new(plan)? .for_root(&transaction, root) .await? - .ok_or(ExternalViewError::Concealed)?; - let (status, records) = if current == key { - self.live_records(request, plan, root, result)? - } else { + .ok_or(ExternalViewError::Concealed)? + .filter(|key| !key.is_empty()); + let (status, records) = if current != key { ("unavailable", Vec::new()) + } else if key.is_none() { + match plan.projection.cardinality { + ExternalCardinality::One => { + ("available", vec![response_row(root, "absent", json!({}))]) + } + ExternalCardinality::Many => ("available", Vec::new()), + } + } else { + self.live_records(request, plan, root, result)? }; self.log_roots(&transaction, request, plan, &[root.to_string()], operation) .await?; @@ -1621,15 +1633,30 @@ impl PostgresExternalViewService { query.page_size, ) .await?; + // A root without a link key is covered with no outside value, so it + // needs no acquisition. A key the store or the source contract would + // refuse is never sent: its root stays unavailable, and the rest of + // the page still refreshes. let mut by_key = BTreeMap::>::new(); - let mut unavailable = 0usize; + let mut invalid = 0usize; for (root, key) in &page.rows { - if let Some(key) = key.as_ref().filter(|value| !value.is_empty()) { - by_key.entry(key.clone()).or_default().insert(*root); - } else { - unavailable += 1; + match key.as_deref() { + None | Some("") => {} + Some(key) if store::valid_key(key) && projection_token_valid(key) => { + by_key.entry(key.to_owned()).or_default().insert(*root); + } + Some(_) => invalid += 1, } } + if invalid > 0 { + tracing::warn!( + projection = plan.projection.id.as_str(), + reason = "invalid-key", + keys = invalid, + "external projection refresh failed for some keys" + ); + } + let mut unavailable = invalid; let keys = by_key.keys().cloned().collect::>(); let due = store::refresh_due_keys( &transaction, @@ -3279,6 +3306,381 @@ mod postgres_tests { served.fixture.cleanup().await; } + /// Insert local roots as (id, link key, tenant). Each key is a statement + /// parameter, so a missing key or one with control characters is stored + /// exactly as given. + async fn insert_roots(served: &Served, rows: &[(&str, Option<&str>, &str)]) { + let entity = &served.service.registry.entities()["local-record"]; + let column = |id: &str| { + entity + .stored_fields + .iter() + .find(|field| field.logical.id == id) + .unwrap() + .physical_name + .clone() + }; + let sql = format!( + "INSERT INTO registry_data.\"{}\" (record_id,active_package_revision,\"{}\",\"{}\") VALUES ($1,'fixture',$2,$3)", + entity.physical_table, + column("subject-key"), + column("tenant"), + ); + for (id, key, tenant) in rows { + served + .fixture + .database + .admin + .execute(&sql, &[&Uuid::parse_str(id).unwrap(), key, tenant]) + .await + .unwrap(); + } + } + + /// (root, availability, fields) for each record of a view page. + fn availabilities(body: &Value) -> BTreeMap { + body["records"] + .as_array() + .unwrap() + .iter() + .map(|record| { + ( + record["rootId"].as_str().unwrap().to_owned(), + ( + record["availability"].as_str().unwrap().to_owned(), + record["fields"].clone(), + ), + ) + }) + .collect() + } + + #[tokio::test] + async fn a_root_without_a_link_key_is_covered_and_reads_absent() { + const UNLINKED: &str = "00000000-0000-4000-8000-000000000011"; + const EMPTY: &str = "00000000-0000-4000-8000-000000000012"; + const OUTSIDE: &str = "00000000-0000-4000-8000-000000000013"; + let served = Served::new(Duration::from_secs(2)).await; + served.publish_all().await; + insert_roots( + &served, + &[ + (UNLINKED, None, "allowed"), + (EMPTY, Some(""), "allowed"), + (OUTSIDE, None, "other"), + ], + ) + .await; + let caller = reader("reader-principal", &["allowed"]); + let mut expected = BTreeMap::new(); + for (id, key, _) in &ROOTS[..3] { + expected.insert( + (*id).to_owned(), + ("available".to_owned(), json!({ "label": key })), + ); + } + for id in [UNLINKED, EMPTY] { + expected.insert(id.to_owned(), ("absent".to_owned(), json!({}))); + } + + // A root with no link key has no outside value, so it is covered: + // lists, counts, ordering and filters keep serving. + let (status, page) = served.get("&$count=true", caller.clone()).await; + assert_eq!(status, axum::http::StatusCode::OK, "{page}"); + assert_eq!(page["status"], "available", "{page}"); + assert_eq!(page["count"], 5, "{page}"); + assert_eq!(availabilities(&page), expected); + let (status, page) = served + .get("&$orderby=label%20asc&$count=true", caller.clone()) + .await; + assert_eq!(status, axum::http::StatusCode::OK, "{page}"); + assert_eq!(availabilities(&page), expected); + let (status, page) = served + .get("&$filter=label%20eq%20'second'&$count=true", caller.clone()) + .await; + assert_eq!(status, axum::http::StatusCode::OK, "{page}"); + assert_eq!(page["count"], 1, "{page}"); + assert_eq!(labels(&page), ["second"]); + + for root in [UNLINKED, EMPTY] { + let (status, _, body) = served + .send( + &format!("/v1/external-views/view/link/records/{root}?accessProfile=reader"), + caller.clone(), + ) + .await; + assert_eq!(status, axum::http::StatusCode::OK, "{body}"); + assert_eq!(body["status"], "available", "{body}"); + assert_eq!( + availabilities(&body), + BTreeMap::from([(root.to_owned(), ("absent".to_owned(), json!({})))]) + ); + } + + // An unlinked root outside the caller's row boundary stays concealed + // and does not change the caller's count. + let (status, content_type, problem) = served + .send( + &format!("/v1/external-views/view/link/records/{OUTSIDE}?accessProfile=reader"), + caller, + ) + .await; + assert_eq!(status, axum::http::StatusCode::NOT_FOUND, "{problem}"); + assert_problem(&content_type, &problem, "resource.not_found"); + let (status, page) = served + .get("&$count=true", reader("reader-principal", &["other"])) + .await; + assert_eq!(status, axum::http::StatusCode::OK, "{page}"); + assert_eq!(page["count"], 2, "{page}"); + assert_eq!( + availabilities(&page), + BTreeMap::from([ + ( + ROOTS[3].0.to_owned(), + ("available".to_owned(), json!({"label": "hidden"})) + ), + (OUTSIDE.to_owned(), ("absent".to_owned(), json!({}))), + ]) + ); + served.fixture.cleanup().await; + } + + #[tokio::test] + async fn a_live_read_of_a_root_without_a_link_key_answers_absent_without_a_source_call() { + const UNLINKED: &str = "00000000-0000-4000-8000-000000000011"; + const EMPTY: &str = "00000000-0000-4000-8000-000000000012"; + let transcript = Arc::new( + registry_adapter_engine::replay::ReplayTranscript::from_exchanges( + serde_json::from_value( + json!([{"type":"endpoint","endpoint":"synthetic-registry"}]), + ) + .unwrap(), + serde_json::from_value(json!([])).unwrap(), + ), + ); + let mut sources = indexed_sources(); + crate::external_source::tests::add_live_binding( + &mut sources, + "holder", + "family-name", + Arc::clone(&transcript), + ); + let served = + Served::with(&live_and_indexed_project(), sources, Duration::from_secs(2)).await; + insert_roots( + &served, + &[(UNLINKED, None, "allowed"), (EMPTY, Some(""), "allowed")], + ) + .await; + let caller = reader("reader-principal", &["allowed"]); + let captured = registry_platform_testing::CapturedDiagnosticLogs::default(); + for root in [UNLINKED, EMPTY] { + use tracing::instrument::WithSubscriber; + let (status, _, body) = served + .send( + &format!( + "/v1/external-views/view/holder-link/records/{root}?accessProfile=reader" + ), + caller.clone(), + ) + .with_subscriber(captured.subscriber()) + .await; + assert_eq!(status, axum::http::StatusCode::OK, "{body}"); + assert_eq!(body["status"], "available", "{body}"); + assert_eq!( + availabilities(&body), + BTreeMap::from([(root.to_owned(), ("absent".to_owned(), json!({})))]) + ); + } + // Nothing reached the source, and no acquisition failure was logged. + transcript.finish().unwrap(); + let logs = captured.text(); + assert!(!logs.contains("live acquisition failed"), "{logs}"); + served.fixture.cleanup().await; + } + + /// The fixture registry with an indexed person projection over the example + /// pack's batch route, linked from the same local key. The local key + /// admits 512 characters, which can exceed the store's byte bound. + fn indexed_person_project() -> Value { + let mut project = two_projection_project(); + project["recordTypes"][0]["fields"][0]["maximumLength"] = json!(512); + project["externalRecordTypes"] + .as_array_mut() + .unwrap() + .push(json!({ + "id":"person","authority":"urn:test:authority","identifierScheme":"urn:test:keys", + "fields":[{"id":"family-name","type":"string","classification":"internal"}] + })); + project["externalProjections"] + .as_array_mut() + .unwrap() + .push(json!({ + "id":"people","recordType":"person","operation":"person-batch","operationVersion":"1", + "purpose":"consultation","scope":"eligible","selectorScheme":"urn:test:keys", + "mode":"indexed","cardinality":"one","cacheTtlSeconds":60,"fields":["family-name"] + })); + project["externalViews"][0]["links"] + .as_array_mut() + .unwrap() + .push(json!({ + "id":"people-link","projection":"people","fromField":"subject-key", + "identifierScheme":"urn:test:keys","fields":["family-name"] + })); + project["accessProfiles"][0]["permissions"]["externalViews"][0]["links"] + .as_array_mut() + .unwrap() + .push(json!({"link":"people-link","readableFields":["family-name"],"allowCount":true})); + project + } + + #[tokio::test] + async fn invalid_link_keys_are_unavailable_without_failing_the_rest_of_the_refresh_page() { + use tracing::instrument::WithSubscriber; + let oversized = "\u{1D518}".repeat(300); // 1200 bytes in 300 characters + let wide = "\u{E9}".repeat(300); // 600 bytes: over the source token bound + let rows = [ + ( + "00000000-0000-4000-8000-000000000011", + Some(oversized.as_str()), + ), + ("00000000-0000-4000-8000-000000000012", Some("bell\u{7}key")), + ("00000000-0000-4000-8000-000000000013", Some(wide.as_str())), + ("00000000-0000-4000-8000-000000000014", Some(" padded-key")), + ("00000000-0000-4000-8000-000000000015", None), + ("00000000-0000-4000-8000-000000000016", Some("")), + ("00000000-0000-4000-8000-000000000017", Some("fifth")), + ]; + let batch = |keys: &[&str]| { + json!({ + "request":{"endpoint":"synthetic-registry","method":"POST","path":"/people/batch","body":{ + "type":"batch","keys":keys,"fields":["family-name"] + }}, + "response":{"status":200,"body":{"records":keys.iter().map(|key| json!({ + "subject":key,"state":"found","properties":{"family-name":format!("family-{key}")} + })).collect::>()}} + }) + }; + let transcript = Arc::new( + registry_adapter_engine::replay::ReplayTranscript::from_exchanges( + serde_json::from_value( + json!([{"type":"endpoint","endpoint":"synthetic-registry"}]), + ) + .unwrap(), + serde_json::from_value(json!([ + batch(&["first", "second", "third"]), + batch(&["fifth"]) + ])) + .unwrap(), + ), + ); + let mut sources = indexed_sources(); + crate::external_source::tests::add_indexed_binding( + &mut sources, + "people", + "family-name", + Arc::clone(&transcript), + ); + let served = Served::with(&indexed_person_project(), sources, Duration::from_secs(2)).await; + insert_roots( + &served, + &rows + .iter() + .map(|(id, key)| (*id, *key, "allowed")) + .collect::>(), + ) + .await; + let mut refresh = request("people-link"); + refresh.query = ExternalQueryMode::First(ExternalQuery { + page_size: 9, + ..ExternalQuery::default() + }); + let plan = served.service.plan(&refresh, None).ok().unwrap(); + let claims = ClaimContext::for_compiled( + &served.service.registry, + &plan.root.id, + Some("reader-principal".to_owned()), + "reader", + Some("consultation".to_owned()), + vec![crate::postgres::context::RowBoundaryContext::Equals { + field: "tenant".to_owned(), + value: "allowed".to_owned(), + }], + ) + .unwrap(); + + // The first page holds the three valid fixture keys, four invalid + // keys and two roots without a key. + let captured = registry_platform_testing::CapturedDiagnosticLogs::default(); + let first = served + .service + .refresh_page(&refresh, &plan, &claims) + .with_subscriber(captured.subscriber()) + .await + .ok() + .unwrap() + .body; + assert_eq!(first["processed"], 9, "{first}"); + assert_eq!(first["published"], 3, "{first}"); + assert_eq!(first["unavailable"], 4, "{first}"); + let cursor = first["nextCursor"].as_str().unwrap().to_owned(); + let logs = captured.text(); + let line = log_line(&logs, "people", "invalid-key"); + assert!(line.contains(r#""level":"WARN""#), "{line}"); + assert!(line.contains(r#""keys":4"#), "{line}"); + assert_value_free(&logs); + for value in ["\u{1D518}", "\u{E9}", "bell", "padded", "family-"] { + assert!(!logs.contains(value), "{value} reached the log: {logs}"); + } + + // The sweep moves past the page. + refresh.query = ExternalQueryMode::Continue(cursor); + let second = served + .service + .refresh_page(&refresh, &plan, &claims) + .await + .ok() + .unwrap() + .body; + assert_eq!(second["processed"], 1, "{second}"); + assert_eq!(second["published"], 1, "{second}"); + assert_eq!(second["unavailable"], 0, "{second}"); + assert!(second.get("nextCursor").is_none(), "{second}"); + // Only the valid keys reached the source, in the two recorded calls. + transcript.finish().unwrap(); + let stored: Vec = served + .fixture + .database + .admin + .query( + "SELECT subject_key FROM registry_internal.registry_external_projection_snapshots + WHERE projection_id='people' ORDER BY subject_key COLLATE \"C\"", + &[], + ) + .await + .unwrap() + .into_iter() + .map(|row| row.get(0)) + .collect(); + assert_eq!(stored, ["fifth", "first", "second", "third"]); + + // A root whose key cannot be acquired stays uncovered, so its view + // keeps refusing lists and counts until the key is corrected. + let (status, _, problem) = served + .send( + "/v1/external-views/view/people-link/records?accessProfile=reader&$count=true", + reader("reader-principal", &["allowed"]), + ) + .await; + assert_eq!( + status, + axum::http::StatusCode::SERVICE_UNAVAILABLE, + "{problem}" + ); + assert_eq!(problem["code"], "external_view.incomplete", "{problem}"); + served.fixture.cleanup().await; + } + /// Source keys, labels and snapshot identities the fixtures below store. /// None of them may reach an operational log line. const FIXTURE_VALUES: [&str; 7] = [ diff --git a/crates/registry-breg/src/postgres/external_view_query.rs b/crates/registry-breg/src/postgres/external_view_query.rs index da1a3c81cd..7c0879ad4f 100644 --- a/crates/registry-breg/src/postgres/external_view_query.rs +++ b/crates/registry-breg/src/postgres/external_view_query.rs @@ -461,7 +461,7 @@ fn compile( ){label_state}, authorized_roots AS MATERIALIZED ({root_sql}), candidates AS MATERIALIZED ( SELECT r.root_id,r.subject_key,s.projection_id,s.epoch,o.kind,o.expires_at, - COALESCE(o.expires_at>{now}::bigint AND {expected_kind},false) AS covered + (COALESCE(r.subject_key,'')='' OR COALESCE(o.expires_at>{now}::bigint AND {expected_kind},false)) AS covered FROM authorized_roots AS r LEFT JOIN projection_state AS s ON true LEFT JOIN registry_internal.registry_external_projection_snapshots AS o ON o.projection_id=s.projection_id AND o.epoch=s.epoch AND o.subject_key=r.subject_key @@ -470,7 +470,7 @@ fn compile( count(*)>0 AS root_found,min(expires_at) FILTER (WHERE covered) AS expires_at FROM candidates ), matched AS MATERIALIZED ( SELECT c.root_id,COALESCE(v.source_key,'') AS source_key, - CASE WHEN c.covered AND c.kind='absent' THEN 'absent' WHEN c.covered THEN 'available' ELSE 'unavailable' END AS availability, + CASE WHEN c.covered AND (COALESCE(c.subject_key,'')='' OR c.kind='absent') THEN 'absent' WHEN c.covered THEN 'available' ELSE 'unavailable' END AS availability, CASE WHEN c.covered THEN v.fields ELSE NULL::jsonb END AS data FROM candidates AS c {membership} ), filtered AS MATERIALIZED ( diff --git a/products/breg/external-projection-storage.md b/products/breg/external-projection-storage.md index 9e9cbdf151..f6461141d4 100644 --- a/products/breg/external-projection-storage.md +++ b/products/breg/external-projection-storage.md @@ -29,6 +29,8 @@ Replaying an accepted result keeps its original expiry. A refresh that fails bec Reads check expiry independently of the refresh job. Indexed predicates and counts refuse incomplete coverage before selecting matching rows. Each page reads coverage, rows and any count from one statement snapshot, so a count always matches complete coverage at that moment. A refresh records each key in its own short transaction, so a read waits for at most one key's publication and can show new values for some keys of a page in flight and previous values for the rest. A `nextCursor` stays valid while refreshes publish or fail individual keys; the next page reads the values current when it runs. A row whose ordering value changed between pages can appear twice or not at all. Rotating, withdrawing or resetting the projection invalidates outstanding cursors. A complete empty related collection is covered; a partial collection, source failure, or protected-existence response is not evidence of emptiness or absence. +A root whose link field is empty or unset names no outside subject. It counts as covered, reads with its external fields absent, or as an empty related collection, and its refresh sends no source request. A root whose link key the store or the source contract would refuse is never sent to the source. The store refuses a key over 1024 bytes or with a control character; the source contract also refuses a key over 512 bytes or with leading or trailing white space. Such a root stays unavailable, so lists, predicates and counts over its view keep answering incomplete until the key is corrected. The rest of its refresh page still refreshes, and the sweep moves on. + The purge worker runs every five seconds. Each pass deletes expired snapshots, refresh receipts whose lease has ended, and everything under withdrawn or replaced epochs, in at most eight batches of a thousand rows of each kind. Freshness enforcement keeps expired values out of responses even when refresh or purge is delayed, but an expired value stays stored until a pass deletes it. That happens while the database is down, while the server is stopped, and while a backlog larger than one pass drains. Size database capacity for the backlog. Where a storage obligation is strict, choose a cache lifetime shorter than that obligation by more than the worker interval and the time you need to recover from those conditions, and watch the signals in the next section. Database WAL and physical snapshots need their own retention policy. ## Monitor refresh and purge @@ -40,6 +42,7 @@ Each signal names the projection and a fixed reason category. None carries a sou - A purge pass that reaches its batch bound logs a `warn` line. The rest of the backlog waits for the next pass. - On the metrics listener, `breg_worker_last_success_age_seconds{worker="external-projection-retention"}` reports the seconds since the last completed pass. It is absent until the first pass completes. Alert when it grows well past the five-second interval. - A refresh page that acquired keys logs one `info` line with its `published` count, and one `warn` line per failure reason with a `keys` count. The reason is `denied`, `retention-refused`, `unavailable`, `incomplete`, `invalid-binding`, or `abandoned` when a newer result, a changed root or a changed collection superseded the result before publication. +- A refresh page that holds link keys the store or the source contract would refuse logs a `warn` line with the reason `invalid-key` and a `keys` count. Those keys are never sent to the source. - A refresh request that fails as a whole logs a `warn` line with a `reason` of `concealed`, `invalid-cursor`, `incomplete`, `unavailable`, `projection-unavailable` or `deadline`. - A live read whose source acquisition fails logs a `warn` line with the same reasons as a refresh key, other than `abandoned`. The view answers unavailable. - `bregctl doctor` reports each projection a server would serve as an `external-projection.status` advisory. It gives the time the source grant expires and, for an indexed projection, when the newest snapshot it holds was observed. Renew a grant before that time. An old or missing newest snapshot means the refresh sweep is not running. @@ -49,7 +52,7 @@ Each signal names the projection and a fixed reason category. None carries a sou Run the named view’s `POST /v1/external-views/{view}/{link}/refresh` with the approved machine profile. Continue with the returned `nextCursor` until the sweep ends. For independent related labels, run `POST /v1/external-views/{view}/{link}/refresh-labels` after the related collection refresh. That machine profile needs explicit refresh grants on both projections under the same purpose. The label sweep selects keys only from authorised roots with fresh complete collections. Before each source call it confirms that the collections which supplied that page are still current and unexpired. A refresh of other collections leaves the sweep running; a refresh of one of its own collections stops acquisition for that page, and the next sweep picks up the new members. A rotated, withdrawn or reset collection projection ends the sweep, so start a fresh one. -The refresh report counts processed roots or related rows, published values, reused keys and unavailable results. `bregctl external-projection inspect` reports receipt counts by outcome and fresh, expired and invalidated snapshot counts. Use an authorised indexed query to test readiness for its local root scope. +The refresh report counts processed roots or related rows, published values, reused keys and unavailable results. A root without a link key counts only as processed. A root with an invalid link key counts as unavailable. `bregctl external-projection inspect` reports receipt counts by outcome and fresh, expired and invalidated snapshot counts. Use an authorised indexed query to test readiness for its local root scope. ## Stop acquisition, withdraw, and renew From 5bfc433a0c38925fe2f9108649b53a7f86a5c051 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 16:23:32 +0700 Subject: [PATCH 31/64] docs(breg): document projection backups, grant-bounded expiry and link authority The backup page gave the BReg dump without the four projection table-data exclusions the storage guide requires, so a backup taken from it would carry cached source values. The storage guide claimed a stored value also honours the source result's own expiry, but only the deployment grant cutoff and no-store bound it. Neither guide said that write access to a link field chooses which outside subject is fetched. Signed-off-by: Jeremi Joslin --- .../operate/advanced/back-up-and-restore.mdx | 21 +++++++++++++++++++ .../breg/examples/external-views/README.md | 9 ++++++++ products/breg/external-projection-storage.md | 4 +++- 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/docs/site/src/content/docs/operate/advanced/back-up-and-restore.mdx b/docs/site/src/content/docs/operate/advanced/back-up-and-restore.mdx index 49c09aaf37..ed28516175 100644 --- a/docs/site/src/content/docs/operate/advanced/back-up-and-restore.mdx +++ b/docs/site/src/content/docs/operate/advanced/back-up-and-restore.mdx @@ -34,6 +34,24 @@ up these alongside every database backup, on the same schedule: pg_dump --format=custom --file=/srv/backup/registry-before-build-2.dump registry ``` + When the package declares an indexed external projection, exclude the data of its four + projection tables. The backup then holds no source values, bindings, or refresh receipts. Keep + the tables themselves: use `--exclude-table-data`, never `--exclude-table`. + + ```sh + pg_dump --format=custom \ + --exclude-table-data=registry_internal.registry_external_projection_scopes \ + --exclude-table-data=registry_internal.registry_external_projection_refreshes \ + --exclude-table-data=registry_internal.registry_external_projection_snapshots \ + --exclude-table-data=registry_internal.registry_external_projection_values \ + --file=/srv/backup/registry-before-build-2.dump registry + ``` + + A restore then starts with an empty projection cache, as a fresh install does. Authorized refresh + rebuilds it under the current binding and source grant; nothing is restored from the backup. See + [Logical backups](https://github.com/registrystack/registry-stack/blob/main/products/breg/external-projection-storage.md#logical-backups) + in the external projection storage guide. + - The package directory, the runtime file, and the secret files it references. A restore needs the package the backup was taken under. - The field-encryption key when the project encrypts a field: the Transit key, or the local @@ -44,6 +62,9 @@ up these alongside every database backup, on the same schedule: - The attachment bucket, when `attachmentStorage` binds an S3-compatible backend. The database holds references; the bucket holds the bytes, so back it up at the same point. +{/* Evidence: products/breg/external-projection-storage.md, Logical backups and Restore; + crates/registry-breg/src/postgres/external_projection.rs, INSTALL_SQL. */} + ### Registry Casework state - The PostgreSQL database, with a logical backup before every activation and your chosen recovery points. Record a snapshot identifier with `caseworkctl apply --backup-reference`. diff --git a/products/breg/examples/external-views/README.md b/products/breg/examples/external-views/README.md index c9c66bbef4..7279082f3b 100644 --- a/products/breg/examples/external-views/README.md +++ b/products/breg/examples/external-views/README.md @@ -21,6 +21,15 @@ disclosure or query permission. The refresh profile uses a dedicated service identity, has only the approved refresh scopes, and can list local roots only in its assigned jurisdiction. +Write access to a link's `fromField` chooses which outside subject BReg +fetches. Whoever can set `holder-key` on a licence decides which Person the +source is asked for, under the deployment's source grant, and which Person +officers then see beside that licence. The same holds for `business-key` +and the holdings. Here only the `fixture-editor` profile can write those +fields. In a deployment, grant write access to a link field only to +identities trusted to assert that link, and review changes to it as you +would review a disclosure. + Check the project with the candidate `bregctl`: ```sh diff --git a/products/breg/external-projection-storage.md b/products/breg/external-projection-storage.md index f6461141d4..073aaf270f 100644 --- a/products/breg/external-projection-storage.md +++ b/products/breg/external-projection-storage.md @@ -19,9 +19,11 @@ The named link reserves `identifierBinding` for a future versioned resolver cont That resolver would take the authorised local reference and return a source-scoped handle before Adapter acquisition. The current checker refuses unknown binding configuration. +Write access to a link's `fromField` chooses which outside subject BReg fetches. Whoever can set that field on a root decides which subject the source is asked for under the deployment's source grant, and which subject's values readers of that root then see. Grant write access to a link field only to identities trusted to assert that link, and review changes to it as you would review a disclosure. + ## Freshness and purge -`cacheTtlSeconds` is the one cache lifetime of a source result. A deployment binding may set its own `cacheTtlSeconds` to shorten, but never extend, the authored value. The lifetime is measured from the original source observation time, not from refresh completion. The effective expiry is the earlier of that lifetime and the source result's own expiry, and the source must permit storage until then. There is no retention period after expiry: an expired value is purged. +`cacheTtlSeconds` is the one cache lifetime of a source result. A deployment binding may set its own `cacheTtlSeconds` to shorten, but never extend, the authored value. The lifetime is measured from the original source observation time, not from refresh completion. The effective expiry is the earlier of that lifetime and the deployment binding's source grant cutoff, `grantValidUntil`. From the source response, BReg honours only `no-store`; it reads no other expiry or storage permission there, so set `grantValidUntil` no later than the source permits storage. There is no retention period after expiry: an expired value is purged. A captured snapshot stores the observation time and a keyed revision reference when the source supplies a revision. The reference is scoped to the bound source. From 8fcb5e26b4a2d84aba35c192d900f48144cc5fa2 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 16:28:44 +0700 Subject: [PATCH 32/64] docs(breg): cite withdrawal proofs and the fresh-install restore model for SEC-207 Signed-off-by: Jeremi Joslin --- products/breg/contracts/security-invariant-matrix.yaml | 2 +- products/breg/contracts/security-test-traceability.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/products/breg/contracts/security-invariant-matrix.yaml b/products/breg/contracts/security-invariant-matrix.yaml index b8cdb9a653..14cbab732f 100644 --- a/products/breg/contracts/security-invariant-matrix.yaml +++ b/products/breg/contracts/security-invariant-matrix.yaml @@ -218,4 +218,4 @@ invariants: - {"id": "BREG-SEC-204", "state": "enforced", "targetWave": "W5", "threat": "An external operation uses a different source scope, identifier scheme or value contract, or retains data without source approval.", "enforcementPoint": "Startup binds the checked project to the pinned Adapter operation, the reviewed source grant and the authored cache lifetime, which deployment configuration may only shorten.", "refusal": "Refuse mismatched operation metadata or indexed acquisition without approved retention.", "negativeId": "BREG-NEG-204", "negativeTest": {"path": "crates/registry-breg/src/external_source.rs", "name": "project_operation_cannot_change_source_scope_scheme_type_or_retention"}} - {"id": "BREG-SEC-205", "state": "enforced", "targetWave": "W5", "threat": "An indexed external query leaks an ineligible root or reports a complete result over incomplete coverage.", "enforcementPoint": "Named view grants and PostgreSQL row policy determine eligibility. Coverage and page selection share a statement snapshot.", "refusal": "Refuse incomplete coverage before applying external predicates. Filter, sort, count and page over the same authorised population.", "negativeId": "BREG-NEG-205", "negativeTest": {"path": "crates/registry-breg/src/postgres/external_view_query.rs", "name": "root_rls_coverage_filter_count_and_bidirectional_null_paging_share_one_population"}} - {"id": "BREG-SEC-206", "state": "enforced", "targetWave": "W5", "threat": "An older acquisition restores a value after a newer denial, retention refusal or other terminal outcome.", "enforcementPoint": "The projection publication function checks the current scope and ordered refresh receipts before replacing retained data.", "refusal": "Refuse stale publication while an earlier attempt is still live, including after the newer receipt expires or removes the snapshot.", "negativeId": "BREG-NEG-206", "negativeTest": {"path": "crates/registry-breg/src/postgres/external_projection.rs", "name": "terminal_receipts_fence_older_results_after_snapshot_removal"}} - - {"id": "BREG-SEC-207", "state": "enforced", "targetWave": "W5", "threat": "An expired or withdrawn projection is returned after normal serving or backup restoration.", "enforcementPoint": "Indexed reads enforce the current scope and expiry. The split-role store supports bounded purge and excludes its data from the documented backup.", "refusal": "Withhold expired results, remove expired and withdrawn rows through the guarded purge, and rebuild restored coverage under the current binding.", "negativeId": "BREG-NEG-207", "negativeTest": {"path": "crates/registry-breg/src/postgres/external_projection.rs", "name": "projection_expiry_purge_and_excluded_backup_restore_are_closed"}} + - {"id": "BREG-SEC-207", "state": "enforced", "targetWave": "W5", "threat": "An expired or withdrawn projection is returned after normal serving or backup restoration.", "enforcementPoint": "Indexed reads enforce the current scope and expiry. The split-role store supports bounded purge, operator withdrawal erases a scope's rows, and the documented logical backup excludes the projection table data, so a restore starts with an empty cache as a fresh install does.", "refusal": "Withhold expired results, remove expired and withdrawn rows through the guarded purge, and after a restore rebuild projections through authorized refresh under the current binding; no projection data is restored.", "negativeId": "BREG-NEG-207", "negativeTest": {"path": "crates/registry-breg/src/postgres/external_projection.rs", "name": "projection_expiry_purge_and_excluded_backup_restore_are_closed"}} diff --git a/products/breg/contracts/security-test-traceability.yaml b/products/breg/contracts/security-test-traceability.yaml index aab38be61f..2918d7b8b7 100644 --- a/products/breg/contracts/security-test-traceability.yaml +++ b/products/breg/contracts/security-test-traceability.yaml @@ -221,4 +221,5 @@ traceability: - {"id": "BREG-SEC-204", "state": "enforced", "negativeId": "BREG-NEG-204", "negativeTest": {"path": "crates/registry-breg/src/external_source.rs", "name": "project_operation_cannot_change_source_scope_scheme_type_or_retention"}} - {"id": "BREG-SEC-205", "state": "enforced", "negativeId": "BREG-NEG-205", "negativeTest": {"path": "crates/registry-breg/src/postgres/external_view_query.rs", "name": "root_rls_coverage_filter_count_and_bidirectional_null_paging_share_one_population"}} - {"id": "BREG-SEC-206", "state": "enforced", "negativeId": "BREG-NEG-206", "negativeTest": {"path": "crates/registry-breg/src/postgres/external_projection.rs", "name": "terminal_receipts_fence_older_results_after_snapshot_removal"}} + # Withdrawal proofs: crates/registry-breg/src/postgres/external_projection.rs::package_catalog_and_bind_once_keep_operator_authority_separate and ::projection_publication_fences_scope_sequence_replay_and_atomic_membership; crates/registry-breg/src/postgres/external_runtime.rs::a_retention_tick_logs_what_it_purged_by_projection_and_reason and ::a_check_reports_each_grant_horizon_and_newest_publication. Backup exclusion proof: crates/registry-breg/src/postgres/external_projection.rs::backup_policy_excludes_every_projection_table. Restore follows the fresh-install model: the documented logical backup carries no projection data, and authorized refresh rebuilds coverage under the current binding. - {"id": "BREG-SEC-207", "state": "enforced", "negativeId": "BREG-NEG-207", "negativeTest": {"path": "crates/registry-breg/src/postgres/external_projection.rs", "name": "projection_expiry_purge_and_excluded_backup_restore_are_closed"}} From 0e5b807764cb1858e745e04e71d35cfd6ba0d863 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 16:28:44 +0700 Subject: [PATCH 33/64] docs(breg): list every external-projection bregctl code in BReg troubleshooting Signed-off-by: Jeremi Joslin --- docs/site/src/content/docs/operate/breg-monitor.mdx | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/site/src/content/docs/operate/breg-monitor.mdx b/docs/site/src/content/docs/operate/breg-monitor.mdx index af7e61179b..27165b9694 100644 --- a/docs/site/src/content/docs/operate/breg-monitor.mdx +++ b/docs/site/src/content/docs/operate/breg-monitor.mdx @@ -331,8 +331,13 @@ Find the code the command, log, or HTTP problem printed. A `*` stands for any pa | `migration.review.*` | The reviewed migration directory was refused. | See [write a reviewed migration descriptor](../breg-changes/#write-a-reviewed-migration-descriptor). | | `migration.rehearsal.*` | PostgreSQL refused the rehearsal, so `apply` would too. | Correct the SQL or the project. | | `source.unavailable` | A read could not complete: the database is unreachable or has no free connection, an audit record could not be written, or history no longer covers the snapshot. | Check the database and the audit destination; for a snapshot read, take a fresh reference. | -| `external-projection.unavailable` | From `doctor`, a warning: an external projection the package declares or the runtime file binds has no usable binding, and the message names why. Its views answer `source.unavailable`; other views serve. | Add or correct its entry under `externalSources` in the runtime file, then restart. | +| `external-projection.unavailable` | From `doctor`, a warning: an external projection the package declares or the runtime file binds has no usable binding, and the message names why. Its views answer `source.unavailable`; other views serve. From `bregctl external-projection`, the command could not read the runtime file, reach the database as the migration role, or confirm that the active package is ready. | For `doctor`, add or correct its entry under `externalSources` in the runtime file, then restart. For `bregctl external-projection`, pass the runtime file by absolute path and check the migration role and `bregctl status`, then rerun. | | `external-projection.status` | From `doctor`, information: when a projection's source grant expires and when its newest snapshot was observed. | Renew the grant before it expires; if the newest snapshot is old or missing, run the refresh sweep. | +| `external-projection.in-progress` | From `bregctl external-projection`, another session holds the migration lock. Nothing changed. | Wait, then rerun. | +| `external-projection.package.refused` | From `bregctl external-projection`, `package.expectedDigest` pins another package; both digests are named. | Deploy the pinned package or update the pin. | +| `external-projection.unknown` | The projection named to `bregctl external-projection` is not an indexed external projection of the active package. | Name one the package declares. | +| `external-projection.binding.unavailable` | `bregctl external-projection rotate` found no usable binding for the projection under `externalSources`; the message names the failed check. | Correct the binding, or remove it from the runtime file to stop acquisition. | +| `external-projection.not-bound` | `bregctl external-projection withdraw` found no bound scope for the projection. Nothing changed. | Nothing to withdraw. | | `idempotency.expired` | The write committed; its receipt expired. | Read the record instead. | | `history.erase.acknowledgement.required` | `--acknowledge-irreversible` is missing. | Confirm, then add the flag. | | `history.rebaseline.live-rows.unverified` | A live row disagrees with its journal head. | Compare suspect records' revisions. | @@ -368,6 +373,9 @@ Find the code the command, log, or HTTP problem printed. A `*` stands for any pa crates/registry-breg/src/postgres/context.rs, begin_registry_transaction(); crates/registry-breg/tests/postgres_startup.rs, live_old_server_drains_apply_and_exact_successor_restart_becomes_ready; + crates/registry-bregctl/src/lib.rs, external_projection_failure(); + crates/registry-breg/src/external_projection_maintenance.rs, ExternalProjectionMaintenanceError, + configured_scope(); crates/registry-breg/src/problem.rs. */} ## Next From 9cc7576624f5936ff151eb57ef83f02554cc2f63 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 16:41:50 +0700 Subject: [PATCH 34/64] fix(breg): bound label acquisition pages to the continuation A label refresh page sorted every eligible member of the scope and scanned from the first record on every continuation, so each page cost grew with the whole authorized population. Start the root scan at the continuation's root on both the source view and the base table, compare link keys in the snapshot key's collation so the snapshot primary key serves each root, and fold the selector checks into one nullable selector so PostgreSQL estimates the real member count. Publication eligibility keeps the unbounded root set and the same checks. Signed-off-by: Jeremi Joslin --- .../postgres/external_label_acquisition.rs | 307 +++++++++++++++++- 1 file changed, 296 insertions(+), 11 deletions(-) diff --git a/crates/registry-breg/src/postgres/external_label_acquisition.rs b/crates/registry-breg/src/postgres/external_label_acquisition.rs index e3b4a9828f..fd1c18ae70 100644 --- a/crates/registry-breg/src/postgres/external_label_acquisition.rs +++ b/crates/registry-breg/src/postgres/external_label_acquisition.rs @@ -165,25 +165,43 @@ impl LabelAcquisition { } let root_sql=format!("SELECT {} AS root_id,({})::text AS subject_key FROM {} WHERE current_setting('registry.access_profile',true)={} AND current_setting('registry.purpose',true)={}",relations.id_expression,key.sql,relations.from_sql,literal(profile_id),literal(purpose)); let state_sql=format!("SELECT projection_id,epoch FROM registry_internal.registry_external_projection_scopes WHERE projection_id={} AND epoch={}::uuid AND binding_digest={} AND grant_digest={} AND generation={} AND active",literal(&parent_scope.projection_id),literal(&parent_scope.epoch.to_string()),literal(&parent_scope.binding_digest),literal(&parent_scope.grant_digest),parent_generation.generation); + // A page resumes inside the continuation's root, so its roots start + // at that root. The base table joins the source view on the record + // id, and PostgreSQL does not carry an inequality across that + // equality; repeating the bound on the base table lets the primary + // key scan start there instead of at the first record. Publication + // checks named roots and keeps the whole root set. + let page_root_sql = format!( + "{root_sql} AND ($1::uuid IS NULL OR ({} >= $1::uuid AND {}.record_id >= $1::uuid))", + relations.id_expression, relations.base_alias, + ); Ok(Self::statements( + &page_root_sql, &root_sql, &state_sql, label.from_field.as_str(), )) } - fn statements(root_sql: &str, state_sql: &str, selector_field: &str) -> Self { + fn statements( + page_root_sql: &str, + root_sql: &str, + state_sql: &str, + selector_field: &str, + ) -> Self { // These replacements contain only compiled identifiers, profile names, // and scope fingerprints. Member positions and selectors are bound. - let eligible = ELIGIBLE_SQL - .replace("__ROOT_SQL__", root_sql) - .replace("__SELECTOR_FIELD__", &literal(selector_field)); + let eligible = |roots: &str| { + ELIGIBLE_SQL + .replace("__ROOT_SQL__", roots) + .replace("__SELECTOR_FIELD__", &literal(selector_field)) + }; let page_sql = PAGE_SQL .replace("__STATE_SQL__", state_sql) - .replace("__ELIGIBLE_SQL__", &eligible); + .replace("__ELIGIBLE_SQL__", &eligible(page_root_sql)); let publication_sql = PUBLICATION_SQL .replace("__STATE_SQL__", state_sql) - .replace("__ELIGIBLE_SQL__", &eligible); + .replace("__ELIGIBLE_SQL__", &eligible(root_sql)); Self { page_sql, publication_sql, @@ -260,20 +278,35 @@ fn literal(value: &str) -> String { format!("'{}'", value.replace('\'', "''")) } +// A page reads roots in record order and looks up each root's snapshot and +// members, so its cost follows the page, not the scope. Two details keep that +// plan available to PostgreSQL: +// - Root fields carry the generated text collation. Comparing the link key in +// the snapshot key's own collation lets the snapshot primary key serve the +// lookup; both collations are deterministic, so equality is unchanged. +// - PostgreSQL keeps no statistics for the selector checks. As separate +// conditions their default selectivities multiply to about one member in +// the whole scope, which makes sorting every eligible member look cheap. +// One nullable selector tested with IS NOT NULL keeps the estimate near the +// real member count. const ELIGIBLE_SQL: &str = r#" -SELECT roots.root_id,members.source_key AS member_key,members.fields->>__SELECTOR_FIELD__ AS selector, +SELECT roots.root_id,members.source_key AS member_key,checked.selector, snapshot.subject_key AS parent_key,snapshot.snapshot_id AS parent_snapshot,snapshot.expires_at AS parent_expires_at FROM (__ROOT_SQL__) AS roots JOIN parent_state AS scope ON true JOIN registry_internal.registry_external_projection_snapshots AS snapshot ON snapshot.projection_id=scope.projection_id AND snapshot.epoch=scope.epoch - AND snapshot.subject_key=roots.subject_key AND snapshot.kind='complete-collection' + AND snapshot.subject_key=roots.subject_key COLLATE "default" AND snapshot.kind='complete-collection' AND snapshot.expires_at>floor(extract(epoch FROM clock_timestamp()))::bigint JOIN registry_internal.registry_external_projection_values AS members ON members.projection_id=snapshot.projection_id AND members.epoch=snapshot.epoch AND members.subject_key=snapshot.subject_key -WHERE jsonb_typeof(members.fields->__SELECTOR_FIELD__)='string' - AND octet_length(members.fields->>__SELECTOR_FIELD__) BETWEEN 1 AND 1024 - AND (members.fields->>__SELECTOR_FIELD__) !~ '[[:cntrl:]]' +CROSS JOIN LATERAL ( + SELECT CASE WHEN jsonb_typeof(members.fields->__SELECTOR_FIELD__)='string' + AND octet_length(members.fields->>__SELECTOR_FIELD__) BETWEEN 1 AND 1024 + AND (members.fields->>__SELECTOR_FIELD__) !~ '[[:cntrl:]]' + THEN members.fields->>__SELECTOR_FIELD__ END AS selector +) AS checked +WHERE checked.selector IS NOT NULL "#; const PAGE_SQL: &str = r#" @@ -616,4 +649,256 @@ mod postgres_tests { drop(migration); database.cleanup().await; } + + const CLOCK: &str = "floor(extract(epoch FROM clock_timestamp()))::bigint"; + + /// A label plan over an installed store whose clock reads 1000, with a + /// runtime connection holding the refresh profile for tenant `allowed`. + struct LabelFixture { + database: harness::TestDatabase, + migration: tokio_postgres::Client, + runtime: tokio_postgres::Client, + tasks: [tokio::task::JoinHandle<()>; 2], + parent: ProjectionScope, + page_sql: String, + table: String, + key: String, + tenant: String, + } + + impl LabelFixture { + async fn new() -> Self { + let database = harness::TestDatabase::create(2).await; + let (migration, migration_task) = database.connect_migration().await; + let registry = super::tests::registry(&["parent", "label"]); + super::super::install_compiled_schema(&migration, ®istry, &database.runtime_role) + .await + .unwrap(); + migration + .batch_execute(&INSTALL_SQL.replace(CLOCK, "1000")) + .await + .unwrap(); + let parent = + ProjectionScope::new("parent", Uuid::new_v4(), "binding", "grant", 60).unwrap(); + configure_scope(&migration, &parent).await.unwrap(); + let entity = ®istry.entities()["local-record"]; + let column = |id: &str| { + entity + .stored_fields + .iter() + .find(|field| field.logical.id == id) + .unwrap() + .physical_name + .clone() + }; + let (runtime, connection) = database + .runtime_config + .postgres() + .connect(NoTls) + .await + .unwrap(); + let runtime_task = tokio::spawn(async move { + connection.await.unwrap(); + }); + runtime.batch_execute("SET registry.access_profile='refresh'; SET registry.principal='worker'; SET registry.purpose='consultation'; SET registry.row_boundaries='[{\"field\":\"tenant\",\"operator\":\"equals\",\"values\":[\"allowed\"]}]';").await.unwrap(); + let plan = LabelAcquisition::new( + ®istry, + "refresh", + "consultation", + "view", + "members", + &parent, + ProjectionGeneration { + epoch: parent.epoch, + generation: 0, + }, + ) + .unwrap(); + Self { + page_sql: plan.page_sql.replace(CLOCK, "1000"), + table: entity.physical_table.clone(), + key: column("subject-key"), + tenant: column("tenant"), + database, + migration, + runtime, + tasks: [migration_task, runtime_task], + parent, + } + } + + /// Add roots `first..=last`, each with a fresh complete collection + /// of `members`. Root `g` has record id `g` in its last digits and + /// subject key `parent-g`; every twentieth root is another tenant's. + async fn add_roots(&self, first: i32, last: i32, members: &[&str]) { + let members = members + .iter() + .map(|member| format!("('{member}')")) + .collect::>() + .join(","); + self.database + .admin + .batch_execute(&format!( + "INSERT INTO registry_data.\"{table}\" (record_id,active_package_revision,\"{key}\",\"{tenant}\") + SELECT ('00000000-0000-4000-8000-'||lpad(to_hex(g),12,'0'))::uuid,'fixture','parent-'||g, + CASE WHEN g % 20 = 0 THEN 'other' ELSE 'allowed' END + FROM generate_series({first},{last}) AS g; + INSERT INTO registry_internal.registry_external_projection_snapshots + (projection_id,epoch,subject_key,snapshot_id,observed_at,expires_at,refresh_sequence,kind) + SELECT 'parent','{epoch}'::uuid,'parent-'||g,'members-'||g,990,1050,g,'complete-collection' + FROM generate_series({first},{last}) AS g; + INSERT INTO registry_internal.registry_external_projection_values + (projection_id,epoch,subject_key,source_key,fields) + SELECT 'parent','{epoch}'::uuid,'parent-'||g,m,jsonb_build_object('holder','person-'||g||'-'||m) + FROM generate_series({first},{last}) AS g CROSS JOIN (VALUES {members}) AS members(m); + ANALYZE registry_data.\"{table}\"; + ANALYZE registry_internal.registry_external_projection_snapshots; + ANALYZE registry_internal.registry_external_projection_values;", + table = self.table, + key = self.key, + tenant = self.tenant, + epoch = self.parent.epoch, + )) + .await + .unwrap(); + } + + async fn page(&self, after: Option<&LabelAfter>, limit: u16) -> LabelSelectorPage { + let root = after.map(|position| position.root_id); + let member = after.map(|position| position.member_key.clone()); + let result = self + .runtime + .query_one(&self.page_sql, &[&root, &member, &(i64::from(limit) + 1)]) + .await + .unwrap(); + assert!(result.get::<_, bool>(0)); + bounded_page(serde_json::from_value(result.get(1)).unwrap(), limit) + } + + async fn cleanup(self) { + for task in self.tasks { + task.abort(); + } + drop(self.runtime); + drop(self.migration); + self.database.cleanup().await; + } + } + + fn root(g: u32) -> Uuid { + Uuid::parse_str(&format!("00000000-0000-4000-8000-{g:012x}")).unwrap() + } + + fn positions(page: &LabelSelectorPage) -> Vec<(Uuid, String)> { + page.rows + .iter() + .map(|row| (row.root_id, row.member_key.clone())) + .collect() + } + + fn plan_nodes<'a>(plan: &'a Value, nodes: &mut Vec<&'a Value>) { + nodes.push(plan); + for child in plan["Plans"].as_array().into_iter().flatten() { + plan_nodes(child, nodes); + } + } + + #[tokio::test] + async fn a_label_page_reads_about_one_page_of_a_dense_scope_at_any_continuation() { + let fixture = LabelFixture::new().await; + // Most of a large scope is eligible, so a page that started its root + // scan at the first record would read up to the whole scope. + fixture.add_roots(1, 20_000, &["a", "b"]).await; + let middle = LabelAfter { + root_id: root(10_001), + member_key: "a".to_owned(), + }; + for after in [None, Some(&middle)] { + let page = fixture.page(after, 101).await; + assert_eq!(page.rows.len(), 101); + if after.is_some() { + assert_eq!(page.rows[0].root_id, root(10_001)); + assert_eq!(page.rows[0].member_key, "b"); + } + let root_id = after.map(|position| position.root_id); + let member = after.map(|position| position.member_key.clone()); + let explained: Value = fixture + .runtime + .query_one( + &format!( + "EXPLAIN (ANALYZE, FORMAT JSON, COSTS OFF, TIMING OFF, SUMMARY OFF) {}", + fixture.page_sql + ), + &[&root_id, &member, &102_i64], + ) + .await + .unwrap() + .get(0); + let mut nodes = Vec::new(); + plan_nodes(&explained[0]["Plan"], &mut nodes); + // Rows arrive in root order; only the members of one root are + // put in key order. + assert!( + !nodes.iter().any(|node| node["Node Type"] == "Sort"), + "after {after:?}: {explained}" + ); + for node in &nodes { + let loops = node["Actual Loops"].as_f64().unwrap_or(1.0); + let rows = node["Actual Rows"].as_f64().unwrap_or(0.0) * loops; + let removed = node["Rows Removed by Filter"].as_f64().unwrap_or(0.0) * loops; + assert!( + rows + removed <= 3.0 * 101.0, + "after {after:?} reads about one page at every node: {explained}" + ); + } + } + fixture.cleanup().await; + } + + #[tokio::test] + async fn a_label_continuation_returns_each_key_once_in_order_past_an_insert_behind_it() { + let fixture = LabelFixture::new().await; + fixture.add_roots(3, 3, &["a", "b", "c"]).await; + fixture.add_roots(5, 5, &["a", "b"]).await; + fixture.add_roots(7, 7, &["a"]).await; + // Root 20 belongs to another tenant and is never selected. + fixture.add_roots(20, 20, &["a"]).await; + let mut seen = Vec::new(); + let first = fixture.page(None, 2).await; + seen.extend(positions(&first)); + let mut after = first.continuation; + // Records behind the continuation, and members added behind it in + // the root it stopped in, are not read again or late. + fixture.add_roots(1, 1, &["a"]).await; + fixture + .database + .admin + .execute( + "INSERT INTO registry_internal.registry_external_projection_values + (projection_id,epoch,subject_key,source_key,fields) + VALUES ('parent',$1,'parent-3','0','{\"holder\":\"person-3-0\"}')", + &[&fixture.parent.epoch], + ) + .await + .unwrap(); + // A root ahead of the continuation is read in its place. + fixture.add_roots(4, 4, &["a"]).await; + while let Some(position) = after { + let page = fixture.page(Some(&position), 2).await; + seen.extend(positions(&page)); + after = page.continuation; + } + let expected = [ + (3, "a"), + (3, "b"), + (3, "c"), + (4, "a"), + (5, "a"), + (5, "b"), + (7, "a"), + ] + .map(|(g, member)| (root(g), member.to_owned())); + assert_eq!(seen, expected); + fixture.cleanup().await; + } } From 9069e38b218405c03aac5944928eeba78decbc22 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 16:41:55 +0700 Subject: [PATCH 35/64] docs(breg): state that indexed page cost follows the authorized population Signed-off-by: Jeremi Joslin --- products/breg/external-projection-storage.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/products/breg/external-projection-storage.md b/products/breg/external-projection-storage.md index 073aaf270f..f0e7312e60 100644 --- a/products/breg/external-projection-storage.md +++ b/products/breg/external-projection-storage.md @@ -31,6 +31,8 @@ Replaying an accepted result keeps its original expiry. A refresh that fails bec Reads check expiry independently of the refresh job. Indexed predicates and counts refuse incomplete coverage before selecting matching rows. Each page reads coverage, rows and any count from one statement snapshot, so a count always matches complete coverage at that moment. A refresh records each key in its own short transaction, so a read waits for at most one key's publication and can show new values for some keys of a page in flight and previous values for the rest. A `nextCursor` stays valid while refreshes publish or fail individual keys; the next page reads the values current when it runs. A row whose ordering value changed between pages can appear twice or not at all. Rotating, withdrawing or resetting the projection invalidates outstanding cursors. A complete empty related collection is covered; a partial collection, source failure, or protected-existence response is not evidence of emptiness or absence. +A read of one record looks only at that record. Every other indexed page, including each continuation page, computes coverage, root presence, expiry and any count over every root the caller is authorized to read before it selects its rows, so its cost grows with the caller's whole authorized population, not with the page size. Grant indexed views only to access profiles whose row boundaries are narrowed to the records their callers need, and rate-limit the callers of indexed queries and of refresh at the gateway. + A root whose link field is empty or unset names no outside subject. It counts as covered, reads with its external fields absent, or as an empty related collection, and its refresh sends no source request. A root whose link key the store or the source contract would refuse is never sent to the source. The store refuses a key over 1024 bytes or with a control character; the source contract also refuses a key over 512 bytes or with leading or trailing white space. Such a root stays unavailable, so lists, predicates and counts over its view keep answering incomplete until the key is corrected. The rest of its refresh page still refreshes, and the sweep moves on. The purge worker runs every five seconds. Each pass deletes expired snapshots, refresh receipts whose lease has ended, and everything under withdrawn or replaced epochs, in at most eight batches of a thousand rows of each kind. Freshness enforcement keeps expired values out of responses even when refresh or purge is delayed, but an expired value stays stored until a pass deletes it. That happens while the database is down, while the server is stopped, and while a backlog larger than one pass drains. Size database capacity for the backlog. Where a storage obligation is strict, choose a cache lifetime shorter than that obligation by more than the worker interval and the time you need to recover from those conditions, and watch the signals in the next section. Database WAL and physical snapshots need their own retention policy. From 4040e217472775fd6aab211e030c343d576f5778 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 17:24:42 +0700 Subject: [PATCH 36/64] fix(breg): count invalid label keys instead of dropping them in SQL The label page dropped a selector with a control character or over 1024 bytes before Rust saw it, so the refresh never counted or logged it, and the PostgreSQL control-character class did not match the source token rule. The page now returns every string selector, cut to 513 characters, and the refresh page's store and source token checks decide. An invalid label key counts as unavailable and is logged as invalid-key; the rest of the label batch is acquired and the sweep moves on. Signed-off-by: Jeremi Joslin --- .../postgres/external_label_acquisition.rs | 10 +- .../src/postgres/external_projection.rs | 38 +++ .../src/postgres/external_runtime.rs | 234 ++++++++++++++++++ products/breg/external-projection-storage.md | 6 +- 4 files changed, 282 insertions(+), 6 deletions(-) diff --git a/crates/registry-breg/src/postgres/external_label_acquisition.rs b/crates/registry-breg/src/postgres/external_label_acquisition.rs index fd1c18ae70..94e765fd74 100644 --- a/crates/registry-breg/src/postgres/external_label_acquisition.rs +++ b/crates/registry-breg/src/postgres/external_label_acquisition.rs @@ -289,6 +289,12 @@ fn literal(value: &str) -> String { // the whole scope, which makes sorting every eligible member look cheap. // One nullable selector tested with IS NOT NULL keeps the estimate near the // real member count. +// Any string selector is returned. The refresh page validates each one with +// the store and source token rules, so a selector either rule refuses is +// counted and logged rather than dropped here. Cutting a selector to 513 +// characters bounds the page and keeps an overlong selector invalid: 513 +// characters are at least 513 bytes, over the source token bound, and a valid +// selector is never cut. const ELIGIBLE_SQL: &str = r#" SELECT roots.root_id,members.source_key AS member_key,checked.selector, snapshot.subject_key AS parent_key,snapshot.snapshot_id AS parent_snapshot,snapshot.expires_at AS parent_expires_at @@ -302,9 +308,7 @@ JOIN registry_internal.registry_external_projection_values AS members ON members.projection_id=snapshot.projection_id AND members.epoch=snapshot.epoch AND members.subject_key=snapshot.subject_key CROSS JOIN LATERAL ( SELECT CASE WHEN jsonb_typeof(members.fields->__SELECTOR_FIELD__)='string' - AND octet_length(members.fields->>__SELECTOR_FIELD__) BETWEEN 1 AND 1024 - AND (members.fields->>__SELECTOR_FIELD__) !~ '[[:cntrl:]]' - THEN members.fields->>__SELECTOR_FIELD__ END AS selector + THEN left(members.fields->>__SELECTOR_FIELD__,513) END AS selector ) AS checked WHERE checked.selector IS NOT NULL "#; diff --git a/crates/registry-breg/src/postgres/external_projection.rs b/crates/registry-breg/src/postgres/external_projection.rs index fd307b37a0..36ca6a13fa 100644 --- a/crates/registry-breg/src/postgres/external_projection.rs +++ b/crates/registry-breg/src/postgres/external_projection.rs @@ -2640,6 +2640,44 @@ pub(crate) mod postgres_tests { ); } + /// Publish one parent's complete member collection under migration + /// authority at the database clock, as (member key, fields) rows. + pub(crate) async fn publish_collection( + &self, + scope: &ProjectionScope, + key: &str, + members: &[(&str, Value)], + ) { + let now: i64 = self + .migration + .query_one( + "SELECT floor(extract(epoch FROM clock_timestamp()))::bigint", + &[], + ) + .await + .unwrap() + .get(0); + let ticket = allocate_inner(&self.migration, scope, key).await.unwrap(); + let snapshot = AcceptedSnapshot::complete_collection( + key, + &format!("{key}-members"), + now, + now + scope.cache_ttl_seconds, + RetentionPermission::PermittedUntil(now + scope.cache_ttl_seconds), + members + .iter() + .map(|(member, fields)| ProjectionRow::new(member, fields.clone()).unwrap()) + .collect(), + ) + .unwrap(); + assert_eq!( + publish_without_eligibility(&self.migration, &ticket, &snapshot) + .await + .unwrap(), + Publication::Published + ); + } + /// Allocate a refresh for one key and the accepted label snapshot a /// source would return for it, stamped at the database clock. pub(crate) async fn label_refresh( diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index 01dc49e56e..49b7980701 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -3681,6 +3681,240 @@ mod postgres_tests { served.fixture.cleanup().await; } + /// The person projection fixture plus a by-parent member projection whose + /// link labels each member through the person projection, and a service + /// profile that may refresh both. + fn labelled_members_project() -> Value { + let mut project = indexed_person_project(); + project["externalRecordTypes"] + .as_array_mut() + .unwrap() + .push(json!({ + "id":"member","authority":"urn:test:authority","identifierScheme":"urn:test:members", + "fields":[{"id":"holder","type":"string","classification":"internal","nullable":true}] + })); + project["externalProjections"] + .as_array_mut() + .unwrap() + .push(json!({ + "id":"members","recordType":"member","operation":"members","operationVersion":"1", + "purpose":"consultation","scope":"eligible","selectorScheme":"urn:test:keys", + "mode":"indexed","cardinality":"many","cacheTtlSeconds":60,"fields":["holder"] + })); + project["externalViews"][0]["links"] + .as_array_mut() + .unwrap() + .push(json!({ + "id":"members-link","projection":"members","fromField":"subject-key", + "identifierScheme":"urn:test:keys","fields":["holder"], + "label":{"projection":"people","fromField":"holder","identifierScheme":"urn:test:keys","fields":["family-name"]} + })); + project["accessProfiles"] + .as_array_mut() + .unwrap() + .push(json!({ + "id":"refresh","principalClaim":"registry_principal","actorKind":"service", + "requesterClients":["worker"],"requiredScopes":["registry.refresh"],"requiredPurposes":["consultation"], + "permissions":{ + "records":[{"recordType":"local-record","operations":["list"],"rowBoundaries":[{"field":"tenant","claim":"tenant","operator":"equals"}]}], + "refreshExternalProjections":["members","people"] + } + })); + project + } + + #[tokio::test] + async fn invalid_label_selectors_are_unavailable_without_failing_the_rest_of_the_label_batch() { + use tracing::instrument::WithSubscriber; + let long = "q".repeat(513); // one byte over the source token bound + let oversized = "\u{1D518}".repeat(300); // 1200 bytes: over the store key bound + let endpoints = || { + serde_json::from_value(json!([{"type":"endpoint","endpoint":"synthetic-registry"}])) + .unwrap() + }; + let batch = |keys: &[&str]| { + json!({ + "request":{"endpoint":"synthetic-registry","method":"POST","path":"/people/batch","body":{ + "type":"batch","keys":keys,"fields":["family-name"] + }}, + "response":{"status":200,"body":{"records":keys.iter().map(|key| json!({ + "subject":key,"state":"found","properties":{"family-name":format!("family-{key}")} + })).collect::>()}} + }) + }; + let transcript = Arc::new( + registry_adapter_engine::replay::ReplayTranscript::from_exchanges( + endpoints(), + serde_json::from_value(json!([batch(&["alpha", "beta"]), batch(&["delta"])])) + .unwrap(), + ), + ); + let mut sources = indexed_sources(); + crate::external_source::tests::add_indexed_binding( + &mut sources, + "people", + "family-name", + Arc::clone(&transcript), + ); + // A label refresh never calls the parent's source. + crate::external_source::tests::add_indexed_binding( + &mut sources, + "members", + "holder", + Arc::new( + registry_adapter_engine::replay::ReplayTranscript::from_exchanges( + endpoints(), + serde_json::from_value(json!([])).unwrap(), + ), + ), + ); + let served = + Served::with(&labelled_members_project(), sources, Duration::from_secs(2)).await; + let members = bound_scope(&served.fixture, "members").await; + served + .fixture + .publish_collection( + &members, + "first", + &[ + ("m1", json!({"holder":"alpha"})), + ("m2", json!({"holder":long})), + ("m3", json!({"holder":"beta"})), + ("m4", json!({"holder":"gamma "})), + ("m5", json!({"holder":"bell\u{7}key"})), + ("m6", json!({"holder":oversized})), + ], + ) + .await; + served + .fixture + .publish_collection(&members, "second", &[("m1", json!({"holder":"delta"}))]) + .await; + + let worker = crate::api::VerifiedRequestClaims::authenticated( + "registry_principal", + "worker-principal", + BTreeSet::from(["registry.refresh".to_owned()]), + Some("consultation".to_owned()), + BTreeMap::new(), + ) + .unwrap() + .with_contextual_authority( + Some(registry_platform_oidc::ActorKind::Service), + Some("worker".to_owned()), + None, + None, + BTreeMap::new(), + ) + .unwrap(); + let mut refresh = ExternalViewRequest { + context: crate::api::AuthorizedRequestContext::new( + Some("worker-principal".to_owned()), + Some("consultation".to_owned()), + "refresh".to_owned(), + Vec::new(), + ) + .with_grant_audit(&worker), + correlation: RequestCorrelation::breg_created(), + view: "view".to_owned(), + link: "members-link".to_owned(), + root: None, + query: ExternalQueryMode::First(ExternalQuery { + page_size: 6, + ..ExternalQuery::default() + }), + deadline: Instant::now() + Duration::from_secs(5), + }; + let plan = served + .service + .plan(&refresh, Some(RefreshMode::Labels)) + .ok() + .unwrap(); + let claims = ClaimContext::for_compiled( + &served.service.registry, + &plan.root.id, + Some("worker-principal".to_owned()), + "refresh", + Some("consultation".to_owned()), + vec![crate::postgres::context::RowBoundaryContext::Equals { + field: "tenant".to_owned(), + value: "allowed".to_owned(), + }], + ) + .unwrap(); + + // The first page holds the first parent's six members: two valid + // selectors and four the source contract or the store would refuse. + let captured = registry_platform_testing::CapturedDiagnosticLogs::default(); + let first = served + .service + .refresh_page(&refresh, &plan, &claims) + .with_subscriber(captured.subscriber()) + .await + .ok() + .unwrap() + .body; + assert_eq!(first["processed"], 6, "{first}"); + assert_eq!(first["published"], 2, "{first}"); + assert_eq!(first["unavailable"], 4, "{first}"); + let cursor = first["nextCursor"].as_str().unwrap().to_owned(); + let logs = captured.text(); + let line = log_line(&logs, "people", "invalid-key"); + assert!(line.contains(r#""level":"WARN""#), "{line}"); + assert!(line.contains(r#""keys":4"#), "{line}"); + assert_value_free(&logs); + for value in [ + "alpha", + "beta", + "gamma", + "bell", + "qqq", + "\u{1D518}", + "family-", + ] { + assert!(!logs.contains(value), "{value} reached the log: {logs}"); + } + + // The acquisition moves past the page. + refresh.query = ExternalQueryMode::Continue(cursor); + let second = served + .service + .refresh_page(&refresh, &plan, &claims) + .await + .ok() + .unwrap() + .body; + assert_eq!(second["processed"], 1, "{second}"); + assert_eq!(second["published"], 1, "{second}"); + assert_eq!(second["unavailable"], 0, "{second}"); + assert!(second.get("nextCursor").is_none(), "{second}"); + // Only the valid selectors reached the source, in the two recorded + // batches, and each of them now has its label. + transcript.finish().unwrap(); + let stored: Vec<(String, Value)> = served + .fixture + .database + .admin + .query( + "SELECT subject_key,fields FROM registry_internal.registry_external_projection_values + WHERE projection_id='people' ORDER BY subject_key COLLATE \"C\"", + &[], + ) + .await + .unwrap() + .into_iter() + .map(|row| (row.get(0), row.get(1))) + .collect(); + assert_eq!( + stored, + ["alpha", "beta", "delta"].map(|key| ( + key.to_owned(), + json!({"family-name": format!("family-{key}")}) + )) + ); + served.fixture.cleanup().await; + } + /// Source keys, labels and snapshot identities the fixtures below store. /// None of them may reach an operational log line. const FIXTURE_VALUES: [&str; 7] = [ diff --git a/products/breg/external-projection-storage.md b/products/breg/external-projection-storage.md index f0e7312e60..4efd7e33cb 100644 --- a/products/breg/external-projection-storage.md +++ b/products/breg/external-projection-storage.md @@ -33,7 +33,7 @@ Reads check expiry independently of the refresh job. Indexed predicates and coun A read of one record looks only at that record. Every other indexed page, including each continuation page, computes coverage, root presence, expiry and any count over every root the caller is authorized to read before it selects its rows, so its cost grows with the caller's whole authorized population, not with the page size. Grant indexed views only to access profiles whose row boundaries are narrowed to the records their callers need, and rate-limit the callers of indexed queries and of refresh at the gateway. -A root whose link field is empty or unset names no outside subject. It counts as covered, reads with its external fields absent, or as an empty related collection, and its refresh sends no source request. A root whose link key the store or the source contract would refuse is never sent to the source. The store refuses a key over 1024 bytes or with a control character; the source contract also refuses a key over 512 bytes or with leading or trailing white space. Such a root stays unavailable, so lists, predicates and counts over its view keep answering incomplete until the key is corrected. The rest of its refresh page still refreshes, and the sweep moves on. +A root whose link field is empty or unset names no outside subject. It counts as covered, reads with its external fields absent, or as an empty related collection, and its refresh sends no source request. A root whose link key the store or the source contract would refuse is never sent to the source. The store refuses a key over 1024 bytes or with a control character; the source contract also refuses a key over 512 bytes or with leading or trailing white space. Such a root stays unavailable, so lists, predicates and counts over its view keep answering incomplete until the key is corrected. The rest of its refresh page still refreshes, and the sweep moves on. A label sweep applies the same rules to each related row's label key, so one invalid label key leaves only that row without its label. The purge worker runs every five seconds. Each pass deletes expired snapshots, refresh receipts whose lease has ended, and everything under withdrawn or replaced epochs, in at most eight batches of a thousand rows of each kind. Freshness enforcement keeps expired values out of responses even when refresh or purge is delayed, but an expired value stays stored until a pass deletes it. That happens while the database is down, while the server is stopped, and while a backlog larger than one pass drains. Size database capacity for the backlog. Where a storage obligation is strict, choose a cache lifetime shorter than that obligation by more than the worker interval and the time you need to recover from those conditions, and watch the signals in the next section. Database WAL and physical snapshots need their own retention policy. @@ -46,7 +46,7 @@ Each signal names the projection and a fixed reason category. None carries a sou - A purge pass that reaches its batch bound logs a `warn` line. The rest of the backlog waits for the next pass. - On the metrics listener, `breg_worker_last_success_age_seconds{worker="external-projection-retention"}` reports the seconds since the last completed pass. It is absent until the first pass completes. Alert when it grows well past the five-second interval. - A refresh page that acquired keys logs one `info` line with its `published` count, and one `warn` line per failure reason with a `keys` count. The reason is `denied`, `retention-refused`, `unavailable`, `incomplete`, `invalid-binding`, or `abandoned` when a newer result, a changed root or a changed collection superseded the result before publication. -- A refresh page that holds link keys the store or the source contract would refuse logs a `warn` line with the reason `invalid-key` and a `keys` count. Those keys are never sent to the source. +- A refresh page that holds link keys or label keys the store or the source contract would refuse logs a `warn` line with the reason `invalid-key` and a `keys` count. Those keys are never sent to the source. - A refresh request that fails as a whole logs a `warn` line with a `reason` of `concealed`, `invalid-cursor`, `incomplete`, `unavailable`, `projection-unavailable` or `deadline`. - A live read whose source acquisition fails logs a `warn` line with the same reasons as a refresh key, other than `abandoned`. The view answers unavailable. - `bregctl doctor` reports each projection a server would serve as an `external-projection.status` advisory. It gives the time the source grant expires and, for an indexed projection, when the newest snapshot it holds was observed. Renew a grant before that time. An old or missing newest snapshot means the refresh sweep is not running. @@ -56,7 +56,7 @@ Each signal names the projection and a fixed reason category. None carries a sou Run the named view’s `POST /v1/external-views/{view}/{link}/refresh` with the approved machine profile. Continue with the returned `nextCursor` until the sweep ends. For independent related labels, run `POST /v1/external-views/{view}/{link}/refresh-labels` after the related collection refresh. That machine profile needs explicit refresh grants on both projections under the same purpose. The label sweep selects keys only from authorised roots with fresh complete collections. Before each source call it confirms that the collections which supplied that page are still current and unexpired. A refresh of other collections leaves the sweep running; a refresh of one of its own collections stops acquisition for that page, and the next sweep picks up the new members. A rotated, withdrawn or reset collection projection ends the sweep, so start a fresh one. -The refresh report counts processed roots or related rows, published values, reused keys and unavailable results. A root without a link key counts only as processed. A root with an invalid link key counts as unavailable. `bregctl external-projection inspect` reports receipt counts by outcome and fresh, expired and invalidated snapshot counts. Use an authorised indexed query to test readiness for its local root scope. +The refresh report counts processed roots or related rows, published values, reused keys and unavailable results. A root without a link key counts only as processed. A root with an invalid link key counts as unavailable. A related row with an invalid label key also counts as unavailable. `bregctl external-projection inspect` reports receipt counts by outcome and fresh, expired and invalidated snapshot counts. Use an authorised indexed query to test readiness for its local root scope. ## Stop acquisition, withdraw, and renew From d9e30612d5b1466e6717d7b1fec2006458756f58 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 17:31:40 +0700 Subject: [PATCH 37/64] fix(breg): look up indexed view snapshots by primary key per root Root fields carry the generated text collation, and an implicit non-default collation wins over the default one in a comparison. The indexed view joined each root's link key to the snapshot and value primary keys under the root's collation, so PostgreSQL could not use the key as an index condition and scanned every snapshot and value of the projection instead. Read the link key in the default collation, as the label acquisition already does. Both collations are deterministic, so equality is unchanged. A PostgreSQL test over 20,000 roots checks the EXPLAIN plan of a detail read and a narrow list read: each snapshot and value is a primary key lookup on the link key, no sort reads a projection table, and rows read outside the root table stay bounded. Signed-off-by: Jeremi Joslin --- .../src/postgres/external_view_query.rs | 187 +++++++++++++++++- 1 file changed, 186 insertions(+), 1 deletion(-) diff --git a/crates/registry-breg/src/postgres/external_view_query.rs b/crates/registry-breg/src/postgres/external_view_query.rs index 7c0879ad4f..eca5cc4232 100644 --- a/crates/registry-breg/src/postgres/external_view_query.rs +++ b/crates/registry-breg/src/postgres/external_view_query.rs @@ -318,7 +318,11 @@ fn compile( let root_param = parameter(&mut parameters, request.root_id.map(|id| id.to_string())); let now = parameter(&mut parameters, Some(request.now.to_string())); let scope = scope_predicate("s", request.scope, request.generation, &mut parameters); - let root_sql = format!("SELECT {} AS root_id,({})::text AS subject_key FROM {} WHERE current_setting('registry.access_profile',true)={profile_param} AND ({root_param}::uuid IS NULL OR {}={root_param}::uuid)",relations.id_expression,key.sql,relations.from_sql,relations.id_expression); + // Root fields carry the generated text collation. Reading the link key in + // the snapshot and value keys' own collation lets their primary keys serve + // the lookup for each root; both collations are deterministic, so + // equality is unchanged. + let root_sql = format!("SELECT {} AS root_id,({})::text COLLATE \"default\" AS subject_key FROM {} WHERE current_setting('registry.access_profile',true)={profile_param} AND ({root_param}::uuid IS NULL OR {}={root_param}::uuid)",relations.id_expression,key.sql,relations.from_sql,relations.id_expression); let many = projection.cardinality == ExternalCardinality::Many; let expected_kind = if many { "o.kind='complete-collection'" @@ -1216,4 +1220,185 @@ mod postgres_tests { drop(migration); database.cleanup().await; } + + fn plan_nodes<'a>(plan: &'a Value, nodes: &mut Vec<&'a Value>) { + nodes.push(plan); + for child in plan["Plans"].as_array().into_iter().flatten() { + plan_nodes(child, nodes); + } + } + + fn root(g: u32) -> Uuid { + Uuid::parse_str(&format!("00000000-0000-4000-8000-{g:012x}")).unwrap() + } + + #[tokio::test] + async fn an_indexed_read_looks_up_each_root_snapshot_by_its_primary_key() { + let database = harness::TestDatabase::create(2).await; + let (migration, migration_task) = database.connect_migration().await; + let registry = super::tests::registry(); + super::super::install_compiled_schema(&migration, ®istry, &database.runtime_role) + .await + .unwrap(); + install(&migration, &database.runtime_role).await.unwrap(); + migration + .batch_execute(&INSTALL_SQL.replace( + "floor(extract(epoch FROM clock_timestamp()))::bigint", + "1000", + )) + .await + .unwrap(); + let scope = + ProjectionScope::new("projection", Uuid::new_v4(), "binding", "grant", 60).unwrap(); + configure_scope(&migration, &scope).await.unwrap(); + let entity = ®istry.entities()["local-record"]; + let column = |id: &str| { + entity + .stored_fields + .iter() + .find(|field| field.logical.id == id) + .unwrap() + .physical_name + .clone() + }; + // A dense scope: every root holds a fresh value. Every thousandth + // root belongs to a narrow tenant. + database + .admin + .batch_execute(&format!( + "INSERT INTO registry_data.\"{table}\" (record_id,active_package_revision,\"{key}\",\"{tenant}\") + SELECT ('00000000-0000-4000-8000-'||lpad(to_hex(g),12,'0'))::uuid,'fixture','key-'||g, + CASE WHEN g % 1000 = 0 THEN 'narrow' ELSE 'allowed' END + FROM generate_series(1,20000) AS g; + INSERT INTO registry_internal.registry_external_projection_snapshots + (projection_id,epoch,subject_key,snapshot_id,observed_at,expires_at,refresh_sequence,kind) + SELECT 'projection','{epoch}'::uuid,'key-'||g,'snapshot-'||g,990,1050,g,'one' + FROM generate_series(1,20000) AS g; + INSERT INTO registry_internal.registry_external_projection_values + (projection_id,epoch,subject_key,source_key,fields) + SELECT 'projection','{epoch}'::uuid,'key-'||g,'key-'||g,jsonb_build_object('label','label-'||g) + FROM generate_series(1,20000) AS g; + ANALYZE registry_data.\"{table}\"; + ANALYZE registry_internal.registry_external_projection_snapshots; + ANALYZE registry_internal.registry_external_projection_values;", + table = entity.physical_table, + key = column("subject-key"), + tenant = column("tenant"), + epoch = scope.epoch, + )) + .await + .unwrap(); + let (runtime, connection) = database + .runtime_config + .postgres() + .connect(NoTls) + .await + .unwrap(); + let runtime_task = tokio::spawn(async move { + connection.await.expect("query fixture connection"); + }); + runtime.batch_execute("SET registry.access_profile='reader'; SET registry.principal='staff'; SET registry.purpose='consultation';").await.unwrap(); + + let mut detail = super::tests::request(&scope, 0); + detail.root_id = Some(root(10_001)); + let mut narrow = super::tests::request(&scope, 0); + narrow.include_count = true; + // A list page reads its rows and one lookahead row. + for (tenant, request, expected) in [("allowed", &detail, 1_usize), ("narrow", &narrow, 11)] + { + runtime + .batch_execute(&format!("SET registry.row_boundaries='[{{\"field\":\"tenant\",\"operator\":\"equals\",\"values\":[\"{tenant}\"]}}]';")) + .await + .unwrap(); + let statement = compile(®istry, request).unwrap(); + let result = run_statement(&runtime, &statement).await; + assert!(result.get::<_, bool>(1), "{tenant}: coverage is complete"); + let rows: Vec = serde_json::from_value(result.get(5)).unwrap(); + assert_eq!(rows.len(), expected, "{tenant}"); + assert!(rows + .iter() + .all(|row| row.availability == ExternalAvailability::Available)); + + let parameters = statement + .parameters + .iter() + .map(|value| { + ( + value as &(dyn tokio_postgres::types::ToSql + Sync), + Type::TEXT, + ) + }) + .collect::>(); + let explained: Value = runtime + .query_typed_one( + &format!( + "EXPLAIN (ANALYZE, FORMAT JSON, COSTS OFF, TIMING OFF, SUMMARY OFF) {}", + statement.sql + ), + ¶meters, + ) + .await + .unwrap() + .get(0); + let mut nodes = Vec::new(); + plan_nodes(&explained[0]["Plan"], &mut nodes); + let tables = [ + "registry_external_projection_snapshots", + "registry_external_projection_values", + ]; + for table in tables { + let scans = nodes + .iter() + .filter(|node| node["Relation Name"] == table) + .collect::>(); + assert!(!scans.is_empty(), "{tenant}: {explained}"); + for node in scans { + // Each root's snapshot and value are one primary key + // lookup, never a scan of the whole projection. + assert_eq!( + node["Index Name"], + format!("{table}_pkey"), + "{tenant}: {explained}" + ); + assert!( + node["Index Cond"] + .as_str() + .is_some_and(|condition| condition.contains("subject_key")), + "{tenant}: {explained}" + ); + } + } + // Only the page's own rows are put in order; no sort reads a + // projection table. + for sort in nodes.iter().filter(|node| node["Node Type"] == "Sort") { + let mut below = Vec::new(); + plan_nodes(sort, &mut below); + assert!( + !below.iter().any(|node| node["Relation Name"] + .as_str() + .is_some_and(|name| tables.contains(&name))), + "{tenant}: {explained}" + ); + } + for node in &nodes { + if node["Relation Name"] == entity.physical_table.as_str() { + // The root table read follows the authorized population, + // which this test does not bound. + continue; + } + let loops = node["Actual Loops"].as_f64().unwrap_or(1.0); + let rows = node["Actual Rows"].as_f64().unwrap_or(0.0) * loops; + let removed = node["Rows Removed by Filter"].as_f64().unwrap_or(0.0) * loops; + assert!( + rows + removed <= 60.0, + "{tenant}: each node reads about the authorized roots: {explained}" + ); + } + } + runtime_task.abort(); + migration_task.abort(); + drop(runtime); + drop(migration); + database.cleanup().await; + } } From 2fd1f2cfb0ecba9a1233209d3e5a4358d6c71902 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 17:34:31 +0700 Subject: [PATCH 38/64] test(breg): cover an unlinked root in an indexed related collection read A root whose link field is unset or empty names no outside subject, so its related collection reads as covered and empty. The query kernel had no test for that case on a Many projection. The PostgreSQL test reads one root at a time and contrasts the unlinked roots with a covered collection, a complete empty collection, an uncovered key and an unlinked root outside the caller's rows. Signed-off-by: Jeremi Joslin --- .../src/postgres/external_view_query.rs | 129 +++++++++++++++++- 1 file changed, 122 insertions(+), 7 deletions(-) diff --git a/crates/registry-breg/src/postgres/external_view_query.rs b/crates/registry-breg/src/postgres/external_view_query.rs index eca5cc4232..ac8316412d 100644 --- a/crates/registry-breg/src/postgres/external_view_query.rs +++ b/crates/registry-breg/src/postgres/external_view_query.rs @@ -866,7 +866,17 @@ mod postgres_tests { fields: Option, id: &str, ) { - let sequence: i64 = client + let sequence = allocate(client, scope, key).await; + let kind = if fields.is_some() { "one" } else { "absent" }; + let rows = fields + .map(|fields| serde_json::json!([{"source_key":key,"fields":fields}])) + .unwrap_or_else(|| serde_json::json!([])); + let result:String=client.query_one("SELECT registry_internal.publish_external_projection($1,$2,$3,$4,$5,$6,$7,999,1059,$8,$9)",&[&scope.projection_id,&scope.epoch,&scope.binding_digest,&scope.grant_digest,&sequence,&key,&id,&kind,&rows]).await.unwrap().get(0); + assert_eq!(result, "published"); + } + + async fn allocate(client: &Client, scope: &ProjectionScope, key: &str) -> i64 { + client .query_one( "SELECT registry_internal.allocate_external_projection_refresh($1,$2,$3,$4,$5,120)", &[ @@ -879,12 +889,25 @@ mod postgres_tests { ) .await .unwrap() - .get(0); - let kind = if fields.is_some() { "one" } else { "absent" }; - let rows = fields - .map(|fields| serde_json::json!([{"source_key":key,"fields":fields}])) - .unwrap_or_else(|| serde_json::json!([])); - let result:String=client.query_one("SELECT registry_internal.publish_external_projection($1,$2,$3,$4,$5,$6,$7,999,1059,$8,$9)",&[&scope.projection_id,&scope.epoch,&scope.binding_digest,&scope.grant_digest,&sequence,&key,&id,&kind,&rows]).await.unwrap().get(0); + .get(0) + } + + async fn publish_collection( + client: &Client, + scope: &ProjectionScope, + key: &str, + members: &[(&str, &str)], + ) { + let sequence = allocate(client, scope, key).await; + let rows = Value::Array( + members + .iter() + .map(|(member, label)| { + serde_json::json!({"source_key": member, "fields": {"label": label}}) + }) + .collect(), + ); + let result:String=client.query_one("SELECT registry_internal.publish_external_projection($1,$2,$3,$4,$5,$6,$7,999,1059,'complete-collection',$8)",&[&scope.projection_id,&scope.epoch,&scope.binding_digest,&scope.grant_digest,&sequence,&key,&key,&rows]).await.unwrap().get(0); assert_eq!(result, "published"); } @@ -1221,6 +1244,98 @@ mod postgres_tests { database.cleanup().await; } + #[tokio::test] + async fn an_unlinked_root_reads_as_a_complete_empty_related_collection() { + let database = harness::TestDatabase::create(2).await; + let (migration, migration_task) = database.connect_migration().await; + let mut project = super::tests::project(); + project["externalProjections"][0]["cardinality"] = Value::from("many"); + let registry = super::tests::compile_project(&project); + super::super::install_compiled_schema(&migration, ®istry, &database.runtime_role) + .await + .unwrap(); + install(&migration, &database.runtime_role).await.unwrap(); + migration + .batch_execute(&INSTALL_SQL.replace( + "floor(extract(epoch FROM clock_timestamp()))::bigint", + "1000", + )) + .await + .unwrap(); + let scope = + ProjectionScope::new("projection", Uuid::new_v4(), "binding", "grant", 60).unwrap(); + configure_scope(&migration, &scope).await.unwrap(); + let entity = ®istry.entities()["local-record"]; + let column = |id: &str| { + entity + .stored_fields + .iter() + .find(|field| field.logical.id == id) + .unwrap() + .physical_name + .clone() + }; + // Roots 2 and 3 name no outside subject: one leaves the link field + // unset, the other empty. Root 6 is unlinked too, but outside the + // caller's rows. + database.admin.batch_execute(&format!("INSERT INTO registry_data.\"{}\" (record_id,active_package_revision,\"{}\",\"{}\") VALUES ('{}','fixture','family','allowed'),('{}','fixture',NULL,'allowed'),('{}','fixture','','allowed'),('{}','fixture','pending','allowed'),('{}','fixture','empty','allowed'),('{}','fixture',NULL,'other')",entity.physical_table,column("subject-key"),column("tenant"),root(1),root(2),root(3),root(4),root(5),root(6))).await.unwrap(); + publish_collection(&migration, &scope, "family", &[("m2", "B"), ("m1", "A")]).await; + publish_collection(&migration, &scope, "empty", &[]).await; + let (runtime, connection) = database + .runtime_config + .postgres() + .connect(NoTls) + .await + .unwrap(); + let runtime_task = tokio::spawn(async move { + connection.await.expect("query fixture connection"); + }); + runtime.batch_execute("SET registry.access_profile='reader'; SET registry.principal='staff'; SET registry.purpose='consultation'; SET registry.row_boundaries='[{\"field\":\"tenant\",\"operator\":\"equals\",\"values\":[\"allowed\"]}]';").await.unwrap(); + let mut request = super::tests::request(&scope, 0); + // A related collection is read for one root at a time. + assert!(matches!( + compile(®istry, &request), + Err(ExternalQueryError::NotPermitted) + )); + // Each case: root, complete, found, earliest expiry, member labels. + let cases: [(u32, bool, bool, Option, &[&str]); 6] = [ + (1, true, true, Some(1059), &["A", "B"]), + (2, true, true, None, &[]), + (3, true, true, None, &[]), + (4, false, true, None, &[]), + (5, true, true, Some(1059), &[]), + (6, true, false, None, &[]), + ]; + for (id, complete, found, expires_at, labels) in cases { + let id = root(id); + request.root_id = Some(id); + let result = run(&runtime, ®istry, &request).await; + assert!(result.get::<_, bool>(0), "{id}: the generation holds"); + assert_eq!(result.get::<_, bool>(1), complete, "{id}: complete"); + assert_eq!(result.get::<_, bool>(2), found, "{id}: found"); + assert_eq!(result.get::<_, Option>(3), expires_at, "{id}: expiry"); + let rows: Vec = serde_json::from_value(result.get(5)).unwrap(); + assert!( + rows.iter() + .all(|row| row.root_id == id + && row.availability == ExternalAvailability::Available), + "{id}" + ); + assert_eq!( + rows.iter() + .map(|row| row.fields["label"].as_str().unwrap()) + .collect::>(), + labels, + "{id}: members" + ); + } + runtime_task.abort(); + migration_task.abort(); + drop(runtime); + drop(migration); + database.cleanup().await; + } + fn plan_nodes<'a>(plan: &'a Value, nodes: &mut Vec<&'a Value>) { nodes.push(plan); for child in plan["Plans"].as_array().into_iter().flatten() { From e96e04eb85472f6231605cbe1706372b0635337e Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 17:57:54 +0700 Subject: [PATCH 39/64] fix(bregctl): give external-projection prerequisite failures the shared codes `bregctl external-projection` reported an unreadable runtime file, an unreachable migration database and an active package that was not ready all as `external-projection.unavailable`, the code doctor uses for a projection without a usable binding. One code carried two meanings, and an operator could not tell a configuration mistake from an outage. The command now reports each prerequisite failure with the code other operator commands use for it: the shared runtime file reader codes, `runtime-config.path-invalid`, `package.refused` (a failed package verification as well as a missed pin), `database-configuration.refused`, `database.unavailable`, `migration-authority.refused`, `audit.unavailable` and `runtime.unavailable`. A database that is not ready for the active package, or whose activation moved after the command opened it, answers the distinct `external-projection.not-ready` and points at `bregctl doctor`. `external-projection.unavailable` stays the doctor's warning only, and the troubleshooting table lists each code once. Signed-off-by: Jeremi Joslin --- .../src/external_projection_maintenance.rs | 154 ++++++++++---- crates/registry-bregctl/src/lib.rs | 198 ++++++++++++++++-- crates/registry-bregctl/tests/cli.rs | 68 ++++++ .../tests/external_projection.rs | 101 ++++++--- .../src/content/docs/operate/breg-monitor.mdx | 11 +- 5 files changed, 439 insertions(+), 93 deletions(-) diff --git a/crates/registry-breg/src/external_projection_maintenance.rs b/crates/registry-breg/src/external_projection_maintenance.rs index e4f6f3638f..20d3b101e0 100644 --- a/crates/registry-breg/src/external_projection_maintenance.rs +++ b/crates/registry-breg/src/external_projection_maintenance.rs @@ -26,9 +26,11 @@ use crate::model::CompiledRegistry; use crate::package::PackageError; use crate::postgres::{ lock_wait_ended, verify_catalog_identity_for_catalog, verify_migration_role, ConnectionConfig, - ExpectedManagedCatalog, ExpectedRegistryIdentity, RegistryLockKey, SqlIdentifier, + ExpectedManagedCatalog, ExpectedRegistryIdentity, PostgresKernelError, RegistryLockKey, + SqlIdentifier, }; -use crate::runtime_config::{load_runtime_config, RuntimeConfig}; +use crate::runtime_config::{load_runtime_config, RuntimeConfig, RuntimeConfigError}; +use crate::startup::StartupError; /// Schema of the request and response entries of one external projection /// withdrawal or rotation. @@ -38,11 +40,27 @@ const MAINTENANCE: &str = "SELECT registry_internal.maintain_external_projection_scope($1,$2,$3,$4,$5,$6,$7,$8)"; /// Value-free failures. Projection IDs come from the reviewed package and a -/// binding reason names the failed check, never a configured value. +/// binding reason names the failed check, never a configured value. Each +/// prerequisite failure has its own variant so the command reports it with +/// the code other operator commands use for the same failure. #[derive(Debug, Error)] pub enum ExternalProjectionMaintenanceError { - #[error("the external projection maintenance request is invalid")] - InvalidRequest, + #[error("the runtime configuration path must be absolute")] + RuntimeConfigPath, + #[error("the runtime configuration was refused")] + RuntimeConfig(RuntimeConfigError), + #[error("the active package was refused")] + PackageRefused(PackageError), + #[error("the database connection configuration was refused")] + DatabaseConfiguration, + #[error("the registry database is unavailable")] + DatabaseUnavailable, + #[error("the migration connection does not hold the configured migration authority")] + MigrationAuthority, + #[error("the registry database is not ready for the active package")] + NotReady, + #[error("the external projection audit record is unavailable")] + AuditUnavailable, #[error("external projection `{0}` is not an indexed projection of the active package")] UnknownProjection(String), #[error("external projection `{projection}` has no usable binding: {reason}")] @@ -53,8 +71,6 @@ pub enum ExternalProjectionMaintenanceError { MigrationLockHeld, #[error("the active package does not match its configured pin")] PackagePinMismatch(registry_platform_config::blocks::PackageDigestMismatch), - #[error("external projection maintenance is unavailable")] - Unavailable, } type Result = std::result::Result; @@ -99,7 +115,7 @@ pub fn configured_scope( config: &RuntimeConfig, projection: &str, ) -> Result { - let resolver = config.secret_resolver().map_err(|_| Error::Unavailable)?; + let resolver = config.secret_resolver().map_err(Error::RuntimeConfig)?; let sources = ExternalSourceService::load(registry, config.external_sources(), Arc::new(resolver)) .map_err(|error| Error::BindingUnavailable { @@ -138,7 +154,7 @@ async fn scope_mark( &[&projection], ) .await - .map_err(|_| Error::Unavailable)? + .map_err(|_| Error::DatabaseUnavailable)? .map(|row| ScopeMark { epoch: row.get(0), generation: row.get(1), @@ -201,19 +217,19 @@ impl ExternalProjectionOperatorService { /// The service, its configuration and the active package's registry. async fn open(path: &Path) -> Result<(Self, RuntimeConfig, CompiledRegistry)> { if !path.is_absolute() { - return Err(Error::InvalidRequest); + return Err(Error::RuntimeConfigPath); } - let config = load_runtime_config(path).map_err(|_| Error::Unavailable)?; + let config = load_runtime_config(path).map_err(Error::RuntimeConfig)?; let package = config.load_active_package().map_err(|error| match error { PackageError::ExpectedDigestMismatch(mismatch) => Error::PackagePinMismatch(mismatch), - _ => Error::Unavailable, + error => Error::PackageRefused(error), })?; let pool = config .runtime_database_connection_config() - .map_err(|_| Error::Unavailable)? + .map_err(|_| Error::DatabaseConfiguration)? .build_pool() - .map_err(|_| Error::Unavailable)?; - let mut client = pool.get().await.map_err(|_| Error::Unavailable)?; + .map_err(|_| Error::DatabaseUnavailable)?; + let mut client = pool.get().await.map_err(|_| Error::DatabaseUnavailable)?; let startup = crate::startup::prepare_loaded_startup( package, config.identity().database_id(), @@ -222,7 +238,15 @@ impl ExternalProjectionOperatorService { config.database().roles().runtime(), ) .await - .map_err(|_| Error::Unavailable)?; + .map_err(|error| match error { + StartupError::RuntimeConfig(error) => Error::RuntimeConfig(error), + StartupError::PackageRefused(error) => Error::PackageRefused(error), + StartupError::PackageEnvelopeRefused(_) => { + Error::PackageRefused(PackageError::Envelope) + } + StartupError::DatabaseConnection(_) => Error::DatabaseUnavailable, + _ => Error::NotReady, + })?; let registry = startup.package().registry().clone(); let service = Self { indexed: indexed_projections(®istry), @@ -231,14 +255,14 @@ impl ExternalProjectionOperatorService { lock_key: startup.lock_key(), migration_connection: config .migration_database_connection_config() - .map_err(|_| Error::Unavailable)?, + .map_err(|_| Error::DatabaseConfiguration)?, migration_role: config.database().roles().migration().clone(), runtime_role: config.database().roles().runtime().clone(), lock_timeout: config.operational_timeouts().migration_lock, statement_timeout: config.operational_timeouts().migration_statement, audit: RegistryAudit::open_companion(&config) .await - .map_err(|_| Error::Unavailable)?, + .map_err(|_| Error::AuditUnavailable)?, }; Ok((service, config, registry)) } @@ -283,8 +307,8 @@ impl ExternalProjectionOperatorService { let pool = self .migration_connection .build_pool() - .map_err(|_| Error::Unavailable)?; - let mut client = pool.get().await.map_err(|_| Error::Unavailable)?; + .map_err(|_| Error::DatabaseUnavailable)?; + let mut client = pool.get().await.map_err(|_| Error::DatabaseUnavailable)?; let client: &mut tokio_postgres::Client = &mut client; let transaction = self.open_transaction(client).await?; transaction @@ -301,7 +325,7 @@ impl ExternalProjectionOperatorService { transaction .rollback() .await - .map_err(|_| Error::Unavailable)?; + .map_err(|_| Error::DatabaseUnavailable)?; Ok(scope) } @@ -355,7 +379,7 @@ impl ExternalProjectionOperatorService { record("terminal", "unfinished"), ) .await - .map_err(|_| Error::Unavailable)?; + .map_err(|_| Error::AuditUnavailable)?; let changed = match self .change_in_transaction(action, projection, configured) .await @@ -381,7 +405,7 @@ impl ExternalProjectionOperatorService { "the unacknowledged external projection maintenance's response audit entry was not recorded" ); } - return Err(Error::Unavailable); + return Err(Error::DatabaseUnavailable); } }, Err(error) => Err(error), @@ -390,12 +414,12 @@ impl ExternalProjectionOperatorService { Ok(scope) => { let mut response = record("terminal", action.outcome()); response["epoch"] = scope["epoch"].clone(); - // The change committed; a refused entry reports the command + // The change committed; a refused entry reports the audit // unavailable, and the writer then refuses every later entry. attempt .respond(response) .await - .map_err(|_| Error::Unavailable)?; + .map_err(|_| Error::AuditUnavailable)?; Ok(scope) } Err(error) => { @@ -427,15 +451,16 @@ impl ExternalProjectionOperatorService { let pool = self .migration_connection .build_pool() - .map_err(|_| Error::Unavailable)?; - let mut client = pool.get().await.map_err(|_| Error::Unavailable)?; + .map_err(|_| Error::DatabaseUnavailable)?; + let mut client = pool.get().await.map_err(|_| Error::DatabaseUnavailable)?; let client: &mut tokio_postgres::Client = &mut client; let transaction = self.open_transaction(client).await?; lock_registry(&transaction, self.lock_key) .await .map_err(|error| match error { HistoryMaintenanceError::MigrationLockHeld => Error::MigrationLockHeld, - _ => Error::Unavailable, + HistoryMaintenanceError::MigrationAuthority => Error::MigrationAuthority, + _ => Error::DatabaseUnavailable, })?; self.verify_catalog(&transaction).await?; let before = scope_mark(&transaction, projection).await?; @@ -468,8 +493,14 @@ impl ExternalProjectionOperatorService { ) -> Result> { verify_migration_role(client, &self.migration_role) .await - .map_err(|_| Error::Unavailable)?; - let transaction = client.transaction().await.map_err(|_| Error::Unavailable)?; + .map_err(|error| match error { + PostgresKernelError::RoleInvariant(_) => Error::MigrationAuthority, + _ => Error::DatabaseUnavailable, + })?; + let transaction = client + .transaction() + .await + .map_err(|_| Error::DatabaseUnavailable)?; transaction .query_one( "SELECT set_config('lock_timeout', $1, true), set_config('statement_timeout', $2, true)", @@ -479,7 +510,7 @@ impl ExternalProjectionOperatorService { ], ) .await - .map_err(|_| Error::Unavailable)?; + .map_err(|_| Error::DatabaseUnavailable)?; Ok(transaction) } @@ -493,7 +524,7 @@ impl ExternalProjectionOperatorService { ) .await .map(|_| ()) - .map_err(|_| Error::Unavailable) + .map_err(catalog_failure) } /// One call to the migration-only maintenance function, bound to the @@ -508,7 +539,7 @@ impl ExternalProjectionOperatorService { configured: Option<&ConfiguredScope>, ) -> Result { let activation = - Uuid::parse_str(&self.expected.activation_id).map_err(|_| Error::Unavailable)?; + Uuid::parse_str(&self.expected.activation_id).map_err(|_| Error::NotReady)?; Ok(transaction .query_one( MAINTENANCE, @@ -533,7 +564,22 @@ fn lock_failure(error: tokio_postgres::Error) -> Error { if lock_wait_ended(&error) { Error::MigrationLockHeld } else { - Error::Unavailable + Error::DatabaseUnavailable + } +} + +/// A catalog that no longer records the package and activation this command +/// opened against, or whose managed objects drifted, is not ready for it; only +/// a connection or statement failure is a database outage. +fn catalog_failure(error: PostgresKernelError) -> Error { + match error { + PostgresKernelError::MigrationLockHeld => Error::MigrationLockHeld, + PostgresKernelError::Configuration(_) + | PostgresKernelError::RoleInvariant(_) + | PostgresKernelError::CatalogInvariant(_) + | PostgresKernelError::SchemaFingerprintMismatch { .. } + | PostgresKernelError::RegistryUnavailable => Error::NotReady, + _ => Error::DatabaseUnavailable, } } @@ -742,11 +788,11 @@ mod postgres_tests { ); assert!(matches!( runtime.withdraw("projection").await, - Err(Error::Unavailable) + Err(Error::MigrationAuthority) )); assert!(matches!( runtime.inspect("projection").await, - Err(Error::Unavailable) + Err(Error::MigrationAuthority) )); assert!( fixture @@ -762,4 +808,40 @@ mod postgres_tests { ); fixture.cleanup().await; } + + /// A database whose activation moved since the command opened is not + /// ready for it; that is neither an outage nor a refused binding. + #[tokio::test] + async fn a_moved_activation_is_reported_as_not_ready() { + let registry = compile_project(&two_projection_project()); + let fixture = Activated::new(®istry).await; + fixture + .bind(®istry, "projection", "binding", "grant", 60) + .await + .unwrap(); + let mut moved = fixture.expected.clone(); + moved.activation_id = Uuid::new_v4().to_string(); + let stale = ExternalProjectionOperatorService::new_for_test( + ®istry, + moved, + fixture.database.migration_config.clone(), + fixture.database.migration_role.clone(), + fixture.database.runtime_role.clone(), + audit(&fixture), + ); + assert!(matches!( + stale.inspect("projection").await, + Err(Error::NotReady) + )); + assert!(matches!( + stale.withdraw("projection").await, + Err(Error::NotReady) + )); + assert!( + fixture + .serves(®istry, "projection", "binding", "grant") + .await + ); + fixture.cleanup().await; + } } diff --git a/crates/registry-bregctl/src/lib.rs b/crates/registry-bregctl/src/lib.rs index 0d7b4c9f40..a3c0166809 100644 --- a/crates/registry-bregctl/src/lib.rs +++ b/crates/registry-bregctl/src/lib.rs @@ -3283,25 +3283,39 @@ where }; let path = target.runtime_config.as_path(); let projection = target.projection.as_str(); - let outcome = tokio::runtime::Builder::new_current_thread() + let Ok(runtime) = tokio::runtime::Builder::new_current_thread() .enable_all() .build() - .map_err(|_| maintenance::ExternalProjectionMaintenanceError::Unavailable) - .and_then(|runtime| { - runtime.block_on(async { - match &args.command { - ExternalProjectionCommand::Inspect(_) => { - maintenance::inspect(path, projection).await - } - ExternalProjectionCommand::Withdraw(_) => { - maintenance::withdraw(path, projection).await - } - ExternalProjectionCommand::Rotate(_) => { - maintenance::rotate(path, projection).await - } - } - }) - }); + else { + return write_failure( + &source_failure( + command, + diagnostic( + "external-projection.runtime.unavailable", + "runtime", + "the external projection maintenance runtime is unavailable", + ), + DiagnosticArtifact::DatabaseMigration, + SuggestedAction::VerifyMigrationAuthority, + ), + format, + stdout, + stderr, + ); + }; + let outcome = runtime.block_on(async { + match &args.command { + ExternalProjectionCommand::Inspect(_) => { + maintenance::inspect(path, projection).await + } + ExternalProjectionCommand::Withdraw(_) => { + maintenance::withdraw(path, projection).await + } + ExternalProjectionCommand::Rotate(_) => { + maintenance::rotate(path, projection).await + } + } + }); return match outcome { Ok(scope) => { let result = if format == OutputFormat::Json { @@ -3852,7 +3866,62 @@ fn external_projection_failure( SuggestedAction::VerifyExternalProjectionOperation, ) }; + let migration = |code: &str, message: &str| { + source_failure( + command, + diagnostic(code, "database", message), + DiagnosticArtifact::DatabaseMigration, + SuggestedAction::VerifyMigrationAuthority, + ) + }; match error { + Error::RuntimeConfigPath => source_failure( + command, + diagnostic( + "external-projection.runtime-config.path-invalid", + "runtimeConfig", + "the runtime configuration path must be absolute", + ), + DiagnosticArtifact::RuntimeConfiguration, + SuggestedAction::CorrectRuntimeConfiguration, + ), + Error::RuntimeConfig(error) => runtime_config_failure(command, error), + Error::PackageRefused(error) => source_failure( + command, + diagnostic( + "external-projection.package.refused", + "package", + package_refusal_message(&error, "the active runtime package was refused"), + ), + DiagnosticArtifact::VerifiedPackage, + package_refusal_action(&error), + ), + Error::DatabaseConfiguration => migration( + "external-projection.database-configuration.refused", + "the database connection configuration was refused: database.runtimeUrlRef or database.migrationUrlRef did not resolve to a PostgreSQL URL that names its configured role under database.roles. Nothing was changed", + ), + Error::DatabaseUnavailable => migration( + "external-projection.database.unavailable", + "the database could not be reached or did not complete a statement or the commit. A withdrawal or rotation whose commit was not acknowledged may have taken effect: run `bregctl external-projection inspect` before rerunning it", + ), + Error::MigrationAuthority => migration( + "external-projection.migration-authority.refused", + "external projection maintenance requires the configured migration authority: database.migrationUrlRef must connect as database.roles.migration, without administrative authority, owning the registry schemas", + ), + Error::NotReady => migration( + "external-projection.not-ready", + "the database is not ready for the active package, or its activation changed after the command opened it. Nothing was changed. Run `bregctl doctor` to name the cause, then rerun the command", + ), + Error::AuditUnavailable => source_failure( + command, + diagnostic( + "external-projection.audit.unavailable", + "audit", + "the audit destination could not be opened or refused an entry; check the audit path and its directory permissions. When it refused the outcome entry the change committed: run `bregctl external-projection inspect` before retrying", + ), + DiagnosticArtifact::RuntimeConfiguration, + SuggestedAction::CorrectRuntimeConfiguration, + ), Error::MigrationLockHeld => source_failure( command, diagnostic( @@ -3889,10 +3958,6 @@ fn external_projection_failure( "external-projection.not-bound", &format!("{error}; nothing was changed"), ), - Error::InvalidRequest | Error::Unavailable => operation( - "external-projection.unavailable", - "Verify the absolute runtime configuration, migration authority and that the active package is ready.", - ), } } @@ -19798,3 +19863,94 @@ fn plan_timer_preview_is_value_free_in_text_and_json() { assert!(!serialized.contains(forbidden), "{serialized}"); } } + +/// Each `bregctl external-projection` failure carries its own code, the one +/// other commands use for the same failure where one exists; none of them is +/// the doctor's unusable-binding warning. +#[cfg(test)] +#[test] +fn external_projection_failures_each_carry_one_code() { + use registry_breg::external_projection_maintenance::ExternalProjectionMaintenanceError as Error; + let cases = [ + ( + Error::RuntimeConfigPath, + "external-projection.runtime-config.path-invalid", + "runtimeConfig", + ), + ( + Error::PackageRefused(PackageError::Integrity), + "external-projection.package.refused", + "package", + ), + ( + Error::DatabaseConfiguration, + "external-projection.database-configuration.refused", + "database", + ), + ( + Error::DatabaseUnavailable, + "external-projection.database.unavailable", + "database", + ), + ( + Error::MigrationAuthority, + "external-projection.migration-authority.refused", + "database", + ), + (Error::NotReady, "external-projection.not-ready", "database"), + ( + Error::AuditUnavailable, + "external-projection.audit.unavailable", + "audit", + ), + ( + Error::MigrationLockHeld, + "external-projection.in-progress", + "database", + ), + ( + Error::UnknownProjection("projection".to_owned()), + "external-projection.unknown", + "externalProjection", + ), + ( + Error::BindingUnavailable { + projection: "projection".to_owned(), + reason: "no external source is bound to this projection".to_owned(), + }, + "external-projection.binding.unavailable", + "externalSources", + ), + ( + Error::NotBound("projection".to_owned()), + "external-projection.not-bound", + "externalProjection", + ), + ]; + let mut codes = BTreeSet::new(); + for (error, code, path) in cases { + let report = external_projection_failure("external-projection inspect", error); + assert!(!report.ok); + assert_eq!(report.command, "external-projection inspect"); + let [diagnostic] = report.diagnostics.as_slice() else { + panic!("one diagnostic for {code}"); + }; + assert_eq!(diagnostic.code, code); + assert_eq!(diagnostic.path, path, "{code}"); + assert!(codes.insert(code), "{code} names one failure"); + } + assert!(!codes.contains("external-projection.unavailable")); + + let package = external_projection_failure( + "external-projection rotate", + Error::PackageRefused(PackageError::Permissions), + ); + assert_eq!( + package.diagnostics[0].suggested_action, + SuggestedAction::VerifyPackagePermissions + ); + assert_eq!( + package.diagnostics[0].artifact, + DiagnosticArtifact::VerifiedPackage + ); +} diff --git a/crates/registry-bregctl/tests/cli.rs b/crates/registry-bregctl/tests/cli.rs index 6bd8df78c7..f9e9fe41b8 100644 --- a/crates/registry-bregctl/tests/cli.rs +++ b/crates/registry-bregctl/tests/cli.rs @@ -9345,6 +9345,74 @@ fn external_projection_operations_name_both_digests_when_the_active_package_miss } } +/// The database prerequisites of `bregctl external-projection` carry the +/// codes `status` and `apply` use for the same failures, never the doctor's +/// unusable-binding warning. +#[test] +fn external_projection_operations_report_database_prerequisites_with_the_shared_codes() { + let fixture = RuntimePackageFixture::production("127.0.0.1:1".parse().unwrap()); + for command in ["inspect", "withdraw", "rotate"] { + let arguments = [ + "--format", + "json", + "external-projection", + command, + "--runtime-config", + path(&fixture.runtime_config), + "--projection", + "holdings-indexed", + ]; + // The runtime file names connection secrets nothing provides. + let refused = bregctl(&arguments); + // The secrets name an address nothing listens on. + let unreachable = Command::new(env!("CARGO_BIN_EXE_bregctl")) + .env( + VERIFY_RUNTIME_DATABASE_SECRET_CANARY, + "postgresql://registry_runtime@127.0.0.1:1/breg", + ) + .env( + VERIFY_MIGRATION_DATABASE_SECRET_CANARY, + "postgresql://registry_migration@127.0.0.1:1/breg", + ) + .args(arguments) + .output() + .expect("bregctl starts"); + for (output, exit, code) in [ + ( + &refused, + 1, + "external-projection.database-configuration.refused", + ), + (&unreachable, 3, "external-projection.database.unavailable"), + ] { + assert_eq!(output.status.code(), Some(exit), "{output:?}"); + assert!(output.stderr.is_empty(), "{output:?}"); + let report = json_stdout(output); + assert_eq!(report["ok"], false); + assert_eq!(report["command"], format!("external-projection {command}")); + assert_eq!(report["diagnostics"].as_array().unwrap().len(), 1); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["code"], code); + assert_eq!(diagnostic["path"], "database"); + assert_tool_diagnostic( + diagnostic, + "database-migration", + "verify-migration-authority", + ); + let rendered = String::from_utf8_lossy(&output.stdout); + for forbidden in [ + path(&fixture.runtime_config), + path(&fixture.package), + VERIFY_RUNTIME_DATABASE_SECRET_CANARY, + VERIFY_MIGRATION_DATABASE_SECRET_CANARY, + "127.0.0.1", + ] { + assert!(!rendered.contains(forbidden), "{rendered}"); + } + } + } +} + #[test] fn canonical_package_tampering_is_refused_without_rendering_package_values() { let fixture = RuntimePackageFixture::production("127.0.0.1:1".parse().unwrap()); diff --git a/crates/registry-bregctl/tests/external_projection.rs b/crates/registry-bregctl/tests/external_projection.rs index d2ae8c4697..27ffeb5e84 100644 --- a/crates/registry-bregctl/tests/external_projection.rs +++ b/crates/registry-bregctl/tests/external_projection.rs @@ -2,11 +2,34 @@ use std::ffi::OsStr; use std::io::Write; +use std::process::ExitCode; use serde_json::Value; +fn run_json<'a>(arguments: impl IntoIterator) -> (ExitCode, Value) { + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let status = registry_bregctl::run_from( + [ + OsStr::new("bregctl"), + OsStr::new("--format"), + OsStr::new("json"), + ] + .into_iter() + .chain(arguments), + &mut stdout, + &mut stderr, + ); + assert!(stderr.is_empty()); + let output = String::from_utf8(stdout).unwrap(); + (status, serde_json::from_str(&output).unwrap()) +} + +/// A refused runtime file is reported with the shared reader's diagnostics, +/// exactly as `status` reports the same file, and never as the doctor's +/// unusable-binding warning. #[test] -fn invalid_runtime_configuration_reports_external_projection_recovery_without_values() { +fn a_refused_runtime_configuration_is_reported_as_status_reports_it() { const PATH_CANARY: &str = "external-projection-path-canary"; const CONFIG_CANARY: &str = "external-projection-config-canary"; for command in ["inspect", "withdraw", "rotate"] { @@ -15,41 +38,43 @@ fn invalid_runtime_configuration_reports_external_projection_recovery_without_va .tempfile() .unwrap(); writeln!(config, "invalid: [{CONFIG_CANARY}").unwrap(); - let mut stdout = Vec::new(); - let mut stderr = Vec::new(); - let status = registry_bregctl::run_from( - [ - OsStr::new("bregctl"), - OsStr::new("--format"), - OsStr::new("json"), - OsStr::new("external-projection"), - OsStr::new(command), - OsStr::new("--runtime-config"), - config.path().as_os_str(), - OsStr::new("--projection"), - OsStr::new("holdings-indexed"), - ], - &mut stdout, - &mut stderr, - ); - assert_ne!(status, std::process::ExitCode::SUCCESS); - assert!(stderr.is_empty()); - let output = String::from_utf8(stdout).unwrap(); + let (status, report) = run_json([ + OsStr::new("external-projection"), + OsStr::new(command), + OsStr::new("--runtime-config"), + config.path().as_os_str(), + OsStr::new("--projection"), + OsStr::new("holdings-indexed"), + ]); + let (status_exit, status_report) = run_json([ + OsStr::new("status"), + OsStr::new("--runtime-config"), + config.path().as_os_str(), + ]); + let rendered = report.to_string(); for value in [PATH_CANARY, CONFIG_CANARY] { - assert!(!output.contains(value)); + assert!(!rendered.contains(value), "{rendered}"); } - let report: Value = serde_json::from_str(&output).unwrap(); assert_eq!(report["ok"], false); assert_eq!(report["command"], format!("external-projection {command}")); - assert_eq!(report["diagnostics"].as_array().unwrap().len(), 1); - let diagnostic = &report["diagnostics"][0]; - assert_eq!(diagnostic["code"], "external-projection.unavailable"); - assert_eq!(diagnostic["path"], "externalProjection"); - assert_eq!(diagnostic["artifact"], "external-projection-operation"); - assert_eq!( - diagnostic["suggestedAction"], - "verify-external-projection-operation" - ); + assert_eq!(status, status_exit); + assert_ne!(status, ExitCode::SUCCESS); + assert!(!report["diagnostics"].as_array().unwrap().is_empty()); + assert_eq!(report["diagnostics"], status_report["diagnostics"]); + for diagnostic in report["diagnostics"].as_array().unwrap() { + assert!( + !diagnostic["code"] + .as_str() + .unwrap() + .starts_with("external-projection."), + "{rendered}" + ); + assert_eq!(diagnostic["artifact"], "runtime-configuration"); + assert_eq!( + diagnostic["suggestedAction"], + "correct-runtime-configuration" + ); + } } } @@ -78,8 +103,16 @@ fn a_relative_runtime_configuration_is_refused_before_it_is_read() { let report: Value = serde_json::from_str(&output).unwrap(); assert_eq!(report["ok"], false); assert_eq!(report["command"], "external-projection inspect"); + assert_eq!(report["diagnostics"].as_array().unwrap().len(), 1); + let diagnostic = &report["diagnostics"][0]; + assert_eq!( + diagnostic["code"], + "external-projection.runtime-config.path-invalid" + ); + assert_eq!(diagnostic["path"], "runtimeConfig"); + assert_eq!(diagnostic["artifact"], "runtime-configuration"); assert_eq!( - report["diagnostics"][0]["code"], - "external-projection.unavailable" + diagnostic["suggestedAction"], + "correct-runtime-configuration" ); } diff --git a/docs/site/src/content/docs/operate/breg-monitor.mdx b/docs/site/src/content/docs/operate/breg-monitor.mdx index 27165b9694..402cb696f9 100644 --- a/docs/site/src/content/docs/operate/breg-monitor.mdx +++ b/docs/site/src/content/docs/operate/breg-monitor.mdx @@ -331,10 +331,17 @@ Find the code the command, log, or HTTP problem printed. A `*` stands for any pa | `migration.review.*` | The reviewed migration directory was refused. | See [write a reviewed migration descriptor](../breg-changes/#write-a-reviewed-migration-descriptor). | | `migration.rehearsal.*` | PostgreSQL refused the rehearsal, so `apply` would too. | Correct the SQL or the project. | | `source.unavailable` | A read could not complete: the database is unreachable or has no free connection, an audit record could not be written, or history no longer covers the snapshot. | Check the database and the audit destination; for a snapshot read, take a fresh reference. | -| `external-projection.unavailable` | From `doctor`, a warning: an external projection the package declares or the runtime file binds has no usable binding, and the message names why. Its views answer `source.unavailable`; other views serve. From `bregctl external-projection`, the command could not read the runtime file, reach the database as the migration role, or confirm that the active package is ready. | For `doctor`, add or correct its entry under `externalSources` in the runtime file, then restart. For `bregctl external-projection`, pass the runtime file by absolute path and check the migration role and `bregctl status`, then rerun. | +| `external-projection.unavailable` | From `doctor`, a warning: an external projection the package declares or the runtime file binds has no usable binding, and the message names why. Its views answer `source.unavailable`; other views serve. | Add or correct its entry under `externalSources` in the runtime file, then restart. | | `external-projection.status` | From `doctor`, information: when a projection's source grant expires and when its newest snapshot was observed. | Renew the grant before it expires; if the newest snapshot is old or missing, run the refresh sweep. | | `external-projection.in-progress` | From `bregctl external-projection`, another session holds the migration lock. Nothing changed. | Wait, then rerun. | -| `external-projection.package.refused` | From `bregctl external-projection`, `package.expectedDigest` pins another package; both digests are named. | Deploy the pinned package or update the pin. | +| `external-projection.runtime-config.path-invalid` | `bregctl external-projection` was given a relative runtime file path. A runtime file it cannot read or accept reports the same reader codes `bregctl status` does. | Pass the runtime file by absolute path. | +| `external-projection.package.refused` | From `bregctl external-projection`, the active package was refused: `package.expectedDigest` pins another package (both digests are named), or the package at `package.root` failed verification (the message names why). | Deploy the pinned package or update the pin; for a failed verification, restore the reviewed package directory. | +| `external-projection.database-configuration.refused` | `bregctl external-projection` could not build its database connections: `database.runtimeUrlRef` or `database.migrationUrlRef` did not resolve to a PostgreSQL URL that names its configured role under `database.roles`. Nothing changed. | Correct the reference or its secret, then rerun. | +| `external-projection.database.unavailable` | `bregctl external-projection` could not reach the database, or a statement or the commit did not complete. A withdrawal or rotation whose commit was not acknowledged may have taken effect. | Check the database, then run `bregctl external-projection inspect` before rerunning a change. | +| `external-projection.migration-authority.refused` | The migration connection does not hold the migration authority: it connects as another role than `database.roles.migration`, holds database-level administrative authority, or does not own the registry schemas. Nothing changed. | Point `database.migrationUrlRef` at the migration role, then rerun. | +| `external-projection.not-ready` | The database is not ready for the active package, or its activation changed after the command opened it. Nothing changed. | Run `bregctl doctor` to name the cause, fix it, then rerun. | +| `external-projection.audit.unavailable` | The audit destination could not be opened or refused an entry. When it refused the outcome entry, the change committed. | Repair the audit path and directory permissions, then run `bregctl external-projection inspect` before retrying. | +| `external-projection.runtime.unavailable` | `bregctl external-projection` could not start its async runtime. Nothing changed. | Rerun the command. | | `external-projection.unknown` | The projection named to `bregctl external-projection` is not an indexed external projection of the active package. | Name one the package declares. | | `external-projection.binding.unavailable` | `bregctl external-projection rotate` found no usable binding for the projection under `externalSources`; the message names the failed check. | Correct the binding, or remove it from the runtime file to stop acquisition. | | `external-projection.not-bound` | `bregctl external-projection withdraw` found no bound scope for the projection. Nothing changed. | Nothing to withdraw. | From 6aea4ba1582071901d69d37c626c498dd022c55e Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 18:24:22 +0700 Subject: [PATCH 40/64] test(breg): name the unlinked-root case type so clippy accepts it The unlinked-root collection test spelled its five-field case tuple inline, which clippy's type-complexity lint refuses under the postgres-test feature. A local type alias keeps the same cases. Signed-off-by: Jeremi Joslin --- crates/registry-breg/src/postgres/external_view_query.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/registry-breg/src/postgres/external_view_query.rs b/crates/registry-breg/src/postgres/external_view_query.rs index ac8316412d..1acab8ae3c 100644 --- a/crates/registry-breg/src/postgres/external_view_query.rs +++ b/crates/registry-breg/src/postgres/external_view_query.rs @@ -1298,7 +1298,8 @@ mod postgres_tests { Err(ExternalQueryError::NotPermitted) )); // Each case: root, complete, found, earliest expiry, member labels. - let cases: [(u32, bool, bool, Option, &[&str]); 6] = [ + type Case = (u32, bool, bool, Option, &'static [&'static str]); + let cases: [Case; 6] = [ (1, true, true, Some(1059), &["A", "B"]), (2, true, true, None, &[]), (3, true, true, None, &[]), From 639d491639ef162ab565aa0a107c9fd4a1ed6f9e Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 18:55:44 +0700 Subject: [PATCH 41/64] test: register the external view incomplete code in the client write boundary Signed-off-by: Jeremi Joslin --- crates/registry-breg-client/tests/write_http_boundary.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/registry-breg-client/tests/write_http_boundary.rs b/crates/registry-breg-client/tests/write_http_boundary.rs index e271875170..7af3a17ea9 100644 --- a/crates/registry-breg-client/tests/write_http_boundary.rs +++ b/crates/registry-breg-client/tests/write_http_boundary.rs @@ -2134,6 +2134,9 @@ fn problem_detail(code: BRegProblemCode) -> &'static str { Code::ActionHandlerFailed => "The action handler could not produce an accepted result.", Code::ActionRefused => REFUSAL_LABEL, Code::AuthenticationRefused => "The bearer credential is missing or refused.", + Code::ExternalViewIncomplete => { + "The external view's indexed coverage is incomplete for this request." + } Code::IdempotencyConflict => "The idempotency key is bound to another request.", Code::IdempotencyExpired => { "The held response of the idempotency key expired; the key stays spent." From c561fcb15c9d34d1f51a677fb6ff74b49f8c600b Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 19:02:05 +0700 Subject: [PATCH 42/64] fix: align the BReg runtime schema and external config with the conventions lint Signed-off-by: Jeremi Joslin --- crates/registry-breg/src/external_source.rs | 31 +- crates/registry-breg/src/external_views.rs | 10 +- crates/registry-breg/src/runtime_config.rs | 94 ++++ .../explain/ModelExplanation.schema.json | 6 +- .../external-views/synthetic_source.py | 2 +- .../authoring/registry-project.schema.json | 3 +- .../generated/runtime/runtime.schema.json | 420 ++++++++---------- 7 files changed, 325 insertions(+), 241 deletions(-) diff --git a/crates/registry-breg/src/external_source.rs b/crates/registry-breg/src/external_source.rs index b36677fc6a..b3e669c749 100644 --- a/crates/registry-breg/src/external_source.rs +++ b/crates/registry-breg/src/external_source.rs @@ -50,7 +50,7 @@ const MAXIMUM_CLOSURE_BYTES: usize = 16 * 1_048_576; const MAXIMUM_RESPONSE_BYTES: u64 = 16 * 1_048_576; const MAXIMUM_CA_BYTES: usize = registry_platform_httputil::destination::MAX_DESTINATION_CA_BUNDLE_BYTES; -const MAXIMUM_CACHE_TTL_SECONDS: u64 = 31_536_000; +const MAXIMUM_CACHE_TTL_SECONDS: u64 = crate::external_views::MAXIMUM_CACHE_TTL_SECONDS as u64; /// Operator binding for one already declared projection. The reviewed pack /// provides the acquisition ceiling; runtime configuration can narrow it. @@ -85,7 +85,7 @@ impl Identified for ExternalSourceBinding { #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ExternalSourceAcquisition { pub maximum_data_operations: BoundedU64<1, 1000>, - pub timeout_milliseconds: BoundedU64<1, 60000>, + pub attempt_timeout_milliseconds: BoundedU64<1, 60000>, pub maximum_response_bytes: BoundedU64<1, MAXIMUM_RESPONSE_BYTES>, } @@ -270,7 +270,7 @@ fn bind_one<'a>( let (pack, artifacts) = load_pack(Path::new(&binding.pack_file), &binding.pack_digest)?; let limits = EvidenceLimits::try_new( binding.acquisition.maximum_data_operations.get(), - binding.acquisition.timeout_milliseconds.get(), + binding.acquisition.attempt_timeout_milliseconds.get(), binding.acquisition.maximum_response_bytes.get() as usize, ) .map_err(|_| ExternalSourceError::Configuration)?; @@ -486,12 +486,10 @@ fn effective_ttl( match (projection.mode, projection.cache_ttl_seconds, deployment) { (ExternalProjectionMode::Live, None, None) => Ok(None), (ExternalProjectionMode::Indexed, Some(authored), deployment) => { + let authored = authored.get(); let shortened = deployment.map_or(authored, |ttl| { u32::try_from(ttl).map_or(authored, |ttl| authored.min(ttl)) }); - if shortened == 0 || u64::from(shortened) > MAXIMUM_CACHE_TTL_SECONDS { - return Err(ExternalSourceError::Configuration); - } Ok(Some(shortened)) } _ => Err(ExternalSourceError::Configuration), @@ -1083,7 +1081,7 @@ pub(crate) mod tests { )); let mut indexed = live; indexed.mode = ExternalProjectionMode::Indexed; - indexed.cache_ttl_seconds = Some(300); + indexed.cache_ttl_seconds = Some(registry_platform_yaml::BoundedU32::new(300).unwrap()); assert!(matches!(effective_ttl(&indexed, None), Ok(Some(300)))); assert!(matches!(effective_ttl(&indexed, Some(60)), Ok(Some(60)))); assert!(matches!(effective_ttl(&indexed, Some(900)), Ok(Some(300)))); @@ -1116,7 +1114,7 @@ pub(crate) mod tests { "tlsTrustProfile":"source-ca","clientIdentity":"forbidden", "authentication":{"type":"static-api-key"}}], "allowedFields":fields,"grantValidUntil":"2030-01-01T00:00:00Z", - "acquisition":{"maximumDataOperations":8,"timeoutMilliseconds":8000,"maximumResponseBytes":262144} + "acquisition":{"maximumDataOperations":8,"attemptTimeoutMilliseconds":8000,"maximumResponseBytes":262144} }) } @@ -1181,7 +1179,7 @@ pub(crate) mod tests { renewed["grantValidUntil"] = json!("2031-06-01T00:00:00Z"); renewed["connection"]["endpoints"][0]["baseUrl"] = json!("https://renewed.example.test"); renewed["endpointPolicies"][0]["baseUrl"] = json!("https://renewed.example.test"); - renewed["acquisition"] = json!({"maximumDataOperations":2,"timeoutMilliseconds":3000,"maximumResponseBytes":65536}); + renewed["acquisition"] = json!({"maximumDataOperations":2,"attemptTimeoutMilliseconds":3000,"maximumResponseBytes":65536}); renewed["cacheTtlSeconds"] = json!(120); assert_eq!( fingerprints(&load_values(®istry, &[renewed]), "person-indexed"), @@ -1267,7 +1265,7 @@ pub(crate) mod tests { "clientIdentity":"forbidden","authentication":{"type":"static-api-key"}}], "allowedFields":["family-name"],"grantValidUntil":"2030-01-01T00:00:00Z", "cacheTtlSeconds":60, - "acquisition":{"maximumDataOperations":8,"timeoutMilliseconds":8000,"maximumResponseBytes":262144} + "acquisition":{"maximumDataOperations":8,"attemptTimeoutMilliseconds":8000,"maximumResponseBytes":262144} }); let parsed: ExternalSourceBinding = serde_json::from_value(binding.clone()).unwrap(); assert_eq!(parsed.cache_ttl_seconds.map(|ttl| ttl.get()), Some(60)); @@ -1283,4 +1281,17 @@ pub(crate) mod tests { assert!(serde_json::from_value::(retired).is_err()); } } + + #[test] + fn the_acquisition_timeout_is_spelled_as_an_attempt_timeout() { + let acquisition = json!({ + "maximumDataOperations":8,"attemptTimeoutMilliseconds":8000,"maximumResponseBytes":262144 + }); + let parsed: ExternalSourceAcquisition = serde_json::from_value(acquisition).unwrap(); + assert_eq!(parsed.attempt_timeout_milliseconds.get(), 8000); + let refused = json!({ + "maximumDataOperations":8,"timeoutMilliseconds":8000,"maximumResponseBytes":262144 + }); + assert!(serde_json::from_value::(refused).is_err()); + } } diff --git a/crates/registry-breg/src/external_views.rs b/crates/registry-breg/src/external_views.rs index 03de33d2b1..299052e629 100644 --- a/crates/registry-breg/src/external_views.rs +++ b/crates/registry-breg/src/external_views.rs @@ -3,7 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; -use registry_platform_yaml::{ExternalId, LocalId}; +use registry_platform_yaml::{BoundedU32, ExternalId, LocalId}; use serde::{Deserialize, Serialize}; use crate::contract::{ @@ -65,6 +65,10 @@ pub enum ExternalCardinality { Many, } +/// The longest cache lifetime, in seconds, of an indexed projection: one year. +/// The deployment binding and the stored scope hold the same bound. +pub const MAXIMUM_CACHE_TTL_SECONDS: u32 = 31_536_000; + #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields, rename_all = "camelCase")] @@ -80,7 +84,7 @@ pub struct ExternalProjectionSource { pub cardinality: ExternalCardinality, pub fields: UniqueSet, #[serde(default, skip_serializing_if = "Option::is_none")] - pub cache_ttl_seconds: Option, + pub cache_ttl_seconds: Option>, } #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] @@ -262,7 +266,7 @@ pub(crate) fn compile( format!("{path}/cacheTtlSeconds"), "Remove the cache lifetime for a live projection.", ), - (ExternalProjectionMode::Indexed, None | Some(0)) => error( + (ExternalProjectionMode::Indexed, None) => error( &mut diagnostics, "indexed-cache", format!("{path}/cacheTtlSeconds"), diff --git a/crates/registry-breg/src/runtime_config.rs b/crates/registry-breg/src/runtime_config.rs index a7cb220b96..2e79826a38 100644 --- a/crates/registry-breg/src/runtime_config.rs +++ b/crates/registry-breg/src/runtime_config.rs @@ -3660,6 +3660,14 @@ pub fn runtime_config_schema() -> std::result::Result ]), ); } + for definition in ["RuntimeCredential", "EvidenceAuthenticationPolicy"] { + internally_tagged( + schema + .pointer_mut(&format!("/$defs/{definition}")) + .expect("the runtime schema defines each tagged union"), + "type", + ); + } for pointer in [ "/properties/destinations", "/$defs/RawDestinationsConfig/properties/hooks", @@ -3674,6 +3682,67 @@ pub fn runtime_config_schema() -> std::result::Result Ok(schema) } +/// Rewrite one union whose serde representation the `registry-platform-yaml` +/// union macros implement by hand: `schemars` renders the external form +/// (`{"static-api-key": {...}}`), the reader accepts the internally tagged +/// form (`{"type": "static-api-key", ...}`). Every variant keeps its +/// description and fields and gains the required constant tag member. +#[cfg(feature = "schema")] +fn internally_tagged(definition: &mut Value, tag: &str) { + let object = definition + .as_object_mut() + .expect("a definition is an object"); + let Some(Value::Array(variants)) = object.remove("oneOf") else { + panic!("schemars renders the tagged union as oneOf"); + }; + let tagged = variants + .iter() + .map(|variant| tagged_variant(variant, tag)) + .collect(); + object.insert("oneOf".to_owned(), Value::Array(tagged)); +} + +/// The internally tagged form of one externally rendered variant. +#[cfg(feature = "schema")] +fn tagged_variant(variant: &Value, tag: &str) -> Value { + let properties = variant + .get("properties") + .and_then(Value::as_object) + .expect("schemars renders each variant under its kind name"); + let mut kinds = properties.iter(); + let (kind, fields) = kinds.next().expect("schemars renders one kind per variant"); + assert!( + kinds.next().is_none(), + "schemars renders one kind per variant" + ); + let mut members = serde_json::Map::new(); + members.insert( + tag.to_owned(), + serde_json::json!({"const": kind, "type": "string"}), + ); + if let Some(fields) = fields.get("properties").and_then(Value::as_object) { + members.extend( + fields + .iter() + .map(|(name, field)| (name.clone(), field.clone())), + ); + } + let mut required = vec![tag.to_owned()]; + if let Some(names) = fields.get("required").and_then(Value::as_array) { + required.extend(names.iter().filter_map(Value::as_str).map(str::to_owned)); + } + required.sort(); + let mut tagged = serde_json::Map::new(); + if let Some(description) = variant.get("description") { + tagged.insert("description".to_owned(), description.clone()); + } + tagged.insert("type".to_owned(), Value::String("object".to_owned())); + tagged.insert("additionalProperties".to_owned(), Value::Bool(false)); + tagged.insert("properties".to_owned(), Value::Object(members)); + tagged.insert("required".to_owned(), serde_json::json!(required)); + Value::Object(tagged) +} + #[cfg(feature = "schema")] fn install_schema_constraints(schema: &mut Value) { // `AuditDestination::from_settings`: the `stdout` destination refuses @@ -4208,6 +4277,31 @@ mod schema_tests { } } + #[test] + fn schema_renders_the_tagged_unions_the_reader_reads() { + let schema = super::runtime_config_schema().unwrap(); + for (definition, accepted, refused) in [ + ( + "RuntimeCredential", + json!({"type": "static-api-key", "valueRef": "secret:file/source-key"}), + json!({"static-api-key": {"valueRef": "secret:file/source-key"}}), + ), + ( + "EvidenceAuthenticationPolicy", + json!({"type": "static-api-key"}), + json!({"static-api-key": {}}), + ), + ] { + let document = json!({ + "$defs": schema["$defs"], + "$ref": format!("#/$defs/{definition}") + }); + let validator = jsonschema::JSONSchema::compile(&document).unwrap(); + assert!(validator.is_valid(&accepted), "{definition}: {accepted}"); + assert!(!validator.is_valid(&refused), "{definition}: {refused}"); + } + } + #[test] fn schema_refuses_a_client_listed_with_no_assertion_issuer() { let schema = super::runtime_config_schema().unwrap(); diff --git a/products/breg/contracts/explain/ModelExplanation.schema.json b/products/breg/contracts/explain/ModelExplanation.schema.json index b9bcc4866e..67a156a54f 100644 --- a/products/breg/contracts/explain/ModelExplanation.schema.json +++ b/products/breg/contracts/explain/ModelExplanation.schema.json @@ -586,8 +586,10 @@ "ExternalDeclarations": { "type": "object", "propertyNames": { "$ref": "#/$defs/LocalId" }, - "additionalProperties": { "type": "object" }, - "x-registry-passthrough": "Normalized external view declarations and grants emitted by the compiler." + "additionalProperties": { + "type": "object", + "x-registry-passthrough": "Normalized external view declarations and grants emitted by the compiler." + } }, "Digest": { "description": "A SHA-256 digest: `sha256:` followed by 64 lowercase hex digits.", diff --git a/products/breg/examples/external-views/synthetic_source.py b/products/breg/examples/external-views/synthetic_source.py index abb70d7a1b..a77a375435 100644 --- a/products/breg/examples/external-views/synthetic_source.py +++ b/products/breg/examples/external-views/synthetic_source.py @@ -363,7 +363,7 @@ def runtime_fragment(origin, key_file_name, grant_cutoff): "connection": connection, "endpointPolicies": policy, "allowedFields": sorted(fields), "grantValidUntil": grant_cutoff, - "acquisition": {"maximumDataOperations": 8, "timeoutMilliseconds": 8000, + "acquisition": {"maximumDataOperations": 8, "attemptTimeoutMilliseconds": 8000, "maximumResponseBytes": 262144}, } bindings.append(entry) diff --git a/products/breg/generated/authoring/registry-project.schema.json b/products/breg/generated/authoring/registry-project.schema.json index e27a99885c..b7934975d0 100644 --- a/products/breg/generated/authoring/registry-project.schema.json +++ b/products/breg/generated/authoring/registry-project.schema.json @@ -2810,7 +2810,8 @@ "properties": { "cacheTtlSeconds": { "format": "uint32", - "minimum": 0, + "maximum": 31536000, + "minimum": 1, "type": "integer" }, "cardinality": { diff --git a/products/breg/generated/runtime/runtime.schema.json b/products/breg/generated/runtime/runtime.schema.json index 521c272270..9cf3e78355 100644 --- a/products/breg/generated/runtime/runtime.schema.json +++ b/products/breg/generated/runtime/runtime.schema.json @@ -207,119 +207,111 @@ { "additionalProperties": false, "properties": { - "none": { - "additionalProperties": false, - "type": "object" + "type": { + "const": "none", + "type": "string" } }, "required": [ - "none" + "type" ], "type": "object" }, { "additionalProperties": false, "properties": { - "basic": { - "additionalProperties": false, - "type": "object" + "type": { + "const": "basic", + "type": "string" } }, "required": [ - "basic" + "type" ], "type": "object" }, { "additionalProperties": false, "properties": { - "static-authorization": { - "additionalProperties": false, - "type": "object" + "type": { + "const": "static-authorization", + "type": "string" } }, "required": [ - "static-authorization" + "type" ], "type": "object" }, { "additionalProperties": false, "properties": { - "static-api-key": { - "additionalProperties": false, - "type": "object" + "type": { + "const": "static-api-key", + "type": "string" } }, "required": [ - "static-api-key" + "type" ], "type": "object" }, { "additionalProperties": false, "properties": { - "oauth2-client-credentials": { - "additionalProperties": false, - "properties": { - "audience": { - "type": "string" - }, - "credentialPlacement": { - "$ref": "#/$defs/CredentialPlacement" - }, - "resource": { - "type": "string" - }, - "scope": { - "type": "string" - }, - "tokenEndpoint": { - "$ref": "#/$defs/Url" - } - }, - "required": [ - "tokenEndpoint", - "credentialPlacement" - ], - "type": "object" + "audience": { + "type": "string" + }, + "credentialPlacement": { + "$ref": "#/$defs/CredentialPlacement" + }, + "resource": { + "type": "string" + }, + "scope": { + "type": "string" + }, + "tokenEndpoint": { + "$ref": "#/$defs/Url" + }, + "type": { + "const": "oauth2-client-credentials", + "type": "string" } }, "required": [ - "oauth2-client-credentials" + "credentialPlacement", + "tokenEndpoint", + "type" ], "type": "object" }, { "additionalProperties": false, "properties": { - "oauth2-private-key-jwt": { - "additionalProperties": false, - "properties": { - "audience": { - "type": "string" - }, - "clientAssertionAudience": { - "type": "string" - }, - "resource": { - "type": "string" - }, - "scope": { - "type": "string" - }, - "tokenEndpoint": { - "$ref": "#/$defs/Url" - } - }, - "required": [ - "tokenEndpoint" - ], - "type": "object" + "audience": { + "type": "string" + }, + "clientAssertionAudience": { + "type": "string" + }, + "resource": { + "type": "string" + }, + "scope": { + "type": "string" + }, + "tokenEndpoint": { + "$ref": "#/$defs/Url" + }, + "type": { + "const": "oauth2-private-key-jwt", + "type": "string" } }, "required": [ - "oauth2-private-key-jwt" + "tokenEndpoint", + "type" ], "type": "object" } @@ -461,28 +453,28 @@ "ExternalSourceAcquisition": { "additionalProperties": false, "properties": { - "maximumDataOperations": { + "attemptTimeoutMilliseconds": { "format": "uint64", - "maximum": 1000, + "maximum": 60000, "minimum": 1, "type": "integer" }, - "maximumResponseBytes": { + "maximumDataOperations": { "format": "uint64", - "maximum": 16777216, + "maximum": 1000, "minimum": 1, "type": "integer" }, - "timeoutMilliseconds": { + "maximumResponseBytes": { "format": "uint64", - "maximum": 60000, + "maximum": 16777216, "minimum": 1, "type": "integer" } }, "required": [ "maximumDataOperations", - "timeoutMilliseconds", + "attemptTimeoutMilliseconds", "maximumResponseBytes" ], "type": "object" @@ -1786,13 +1778,13 @@ "additionalProperties": false, "description": "No credential; matches a `none` endpoint.", "properties": { - "none": { - "additionalProperties": false, - "type": "object" + "type": { + "const": "none", + "type": "string" } }, "required": [ - "none" + "type" ], "type": "object" }, @@ -1800,27 +1792,23 @@ "additionalProperties": false, "description": "The Basic credential pair.", "properties": { - "basic": { - "additionalProperties": false, - "properties": { - "passwordRef": { - "$ref": "#/$defs/SecretReference", - "description": "The password secret reference." - }, - "usernameRef": { - "$ref": "#/$defs/SecretReference", - "description": "The username secret reference." - } - }, - "required": [ - "usernameRef", - "passwordRef" - ], - "type": "object" + "passwordRef": { + "$ref": "#/$defs/SecretReference", + "description": "The password secret reference." + }, + "type": { + "const": "basic", + "type": "string" + }, + "usernameRef": { + "$ref": "#/$defs/SecretReference", + "description": "The username secret reference." } }, "required": [ - "basic" + "passwordRef", + "type", + "usernameRef" ], "type": "object" }, @@ -1828,22 +1816,18 @@ "additionalProperties": false, "description": "The static token, presented under the endpoint's declared `scheme`.", "properties": { - "static-authorization": { - "additionalProperties": false, - "properties": { - "tokenRef": { - "$ref": "#/$defs/SecretReference", - "description": "The token secret reference." - } - }, - "required": [ - "tokenRef" - ], - "type": "object" + "tokenRef": { + "$ref": "#/$defs/SecretReference", + "description": "The token secret reference." + }, + "type": { + "const": "static-authorization", + "type": "string" } }, "required": [ - "static-authorization" + "tokenRef", + "type" ], "type": "object" }, @@ -1851,22 +1835,18 @@ "additionalProperties": false, "description": "The static API key, sent in the endpoint's declared header.", "properties": { - "static-api-key": { - "additionalProperties": false, - "properties": { - "valueRef": { - "$ref": "#/$defs/SecretReference", - "description": "The key secret reference." - } - }, - "required": [ - "valueRef" - ], - "type": "object" + "type": { + "const": "static-api-key", + "type": "string" + }, + "valueRef": { + "$ref": "#/$defs/SecretReference", + "description": "The key secret reference." } }, "required": [ - "static-api-key" + "type", + "valueRef" ], "type": "object" }, @@ -1874,64 +1854,60 @@ "additionalProperties": false, "description": "The client-credentials binding, Evidence's fields: the token\nendpoint, the client id, and the shared secret, all as references\nor fixed strings.", "properties": { - "oauth2-client-credentials": { - "additionalProperties": false, - "properties": { - "assumedLifetimeSeconds": { - "description": "Lifetime assumed when the provider omits `expires_in`.", - "format": "uint64", - "maximum": 86400, - "minimum": 1, - "type": "integer" - }, - "audience": { - "description": "Fixed `audience` form parameter.", - "type": "string" - }, - "clientIdRef": { - "$ref": "#/$defs/SecretReference", - "description": "The client id secret reference." - }, - "clientSecretRef": { - "$ref": "#/$defs/SecretReference", - "description": "The shared client secret reference." - }, - "credentialPlacement": { - "$ref": "#/$defs/CredentialPlacement", - "description": "Where the shared client secret travels." - }, - "maximumCacheSeconds": { - "description": "The token cache ceiling, Evidence's bound.", - "format": "uint64", - "maximum": 86400, - "minimum": 0, - "type": "integer" - }, - "resource": { - "description": "RFC 8707 resource indicator sent as a token-request form\nparameter.", - "type": "string" - }, - "scope": { - "description": "The token-request scope parameter.", - "type": "string" - }, - "tokenEndpoint": { - "$ref": "#/$defs/Url", - "description": "The token endpoint, a fixed destination never derived from a\nresponse; an absolute `http`/`https` URL the origin check\nconfines like any other destination." - } - }, - "required": [ - "tokenEndpoint", - "clientIdRef", - "clientSecretRef", - "credentialPlacement", - "maximumCacheSeconds" - ], - "type": "object" + "assumedLifetimeSeconds": { + "description": "Lifetime assumed when the provider omits `expires_in`.", + "format": "uint64", + "maximum": 86400, + "minimum": 1, + "type": "integer" + }, + "audience": { + "description": "Fixed `audience` form parameter.", + "type": "string" + }, + "clientIdRef": { + "$ref": "#/$defs/SecretReference", + "description": "The client id secret reference." + }, + "clientSecretRef": { + "$ref": "#/$defs/SecretReference", + "description": "The shared client secret reference." + }, + "credentialPlacement": { + "$ref": "#/$defs/CredentialPlacement", + "description": "Where the shared client secret travels." + }, + "maximumCacheSeconds": { + "description": "The token cache ceiling, Evidence's bound.", + "format": "uint64", + "maximum": 86400, + "minimum": 0, + "type": "integer" + }, + "resource": { + "description": "RFC 8707 resource indicator sent as a token-request form\nparameter.", + "type": "string" + }, + "scope": { + "description": "The token-request scope parameter.", + "type": "string" + }, + "tokenEndpoint": { + "$ref": "#/$defs/Url", + "description": "The token endpoint, a fixed destination never derived from a\nresponse; an absolute `http`/`https` URL the origin check\nconfines like any other destination." + }, + "type": { + "const": "oauth2-client-credentials", + "type": "string" } }, "required": [ - "oauth2-client-credentials" + "clientIdRef", + "clientSecretRef", + "credentialPlacement", + "maximumCacheSeconds", + "tokenEndpoint", + "type" ], "type": "object" }, @@ -1939,58 +1915,54 @@ "additionalProperties": false, "description": "The private-key-jwt binding, Evidence's fields. The signed\nassertion always travels in the form body; there is no placement\nparameter.", "properties": { - "oauth2-private-key-jwt": { - "additionalProperties": false, - "properties": { - "assumedLifetimeSeconds": { - "format": "uint64", - "maximum": 86400, - "minimum": 1, - "type": "integer" - }, - "audience": { - "type": "string" - }, - "clientAssertionAudience": { - "description": "Audience claim of the signed assertion, defaulting to the\ntoken endpoint.", - "type": "string" - }, - "clientAssertionKeyRef": { - "$ref": "#/$defs/SecretReference", - "description": "The private JWK the client assertion is signed with." - }, - "clientIdRef": { - "$ref": "#/$defs/SecretReference", - "description": "The client id secret reference." - }, - "maximumCacheSeconds": { - "format": "uint64", - "maximum": 86400, - "minimum": 0, - "type": "integer" - }, - "resource": { - "type": "string" - }, - "scope": { - "type": "string" - }, - "tokenEndpoint": { - "$ref": "#/$defs/Url", - "description": "The token endpoint, a fixed destination never derived from a\nresponse; an absolute `http`/`https` URL the origin check\nconfines like any other destination." - } - }, - "required": [ - "tokenEndpoint", - "clientIdRef", - "clientAssertionKeyRef", - "maximumCacheSeconds" - ], - "type": "object" + "assumedLifetimeSeconds": { + "format": "uint64", + "maximum": 86400, + "minimum": 1, + "type": "integer" + }, + "audience": { + "type": "string" + }, + "clientAssertionAudience": { + "description": "Audience claim of the signed assertion, defaulting to the\ntoken endpoint.", + "type": "string" + }, + "clientAssertionKeyRef": { + "$ref": "#/$defs/SecretReference", + "description": "The private JWK the client assertion is signed with." + }, + "clientIdRef": { + "$ref": "#/$defs/SecretReference", + "description": "The client id secret reference." + }, + "maximumCacheSeconds": { + "format": "uint64", + "maximum": 86400, + "minimum": 0, + "type": "integer" + }, + "resource": { + "type": "string" + }, + "scope": { + "type": "string" + }, + "tokenEndpoint": { + "$ref": "#/$defs/Url", + "description": "The token endpoint, a fixed destination never derived from a\nresponse; an absolute `http`/`https` URL the origin check\nconfines like any other destination." + }, + "type": { + "const": "oauth2-private-key-jwt", + "type": "string" } }, "required": [ - "oauth2-private-key-jwt" + "clientAssertionKeyRef", + "clientIdRef", + "maximumCacheSeconds", + "tokenEndpoint", + "type" ], "type": "object" } From 6c3bffdd6914e0cea320e3c898f1aa1afb873d3c Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 19:02:22 +0700 Subject: [PATCH 43/64] docs: add the external view obligations to the BReg security model Signed-off-by: Jeremi Joslin --- products/breg/SECURITY-MODEL.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/products/breg/SECURITY-MODEL.md b/products/breg/SECURITY-MODEL.md index edbe948f83..63e29e8dfd 100644 --- a/products/breg/SECURITY-MODEL.md +++ b/products/breg/SECURITY-MODEL.md @@ -136,6 +136,27 @@ For the invariants to hold, a deployment must do the following. or `lease-expired`, for the current submitted Proposal. Run it after the cause is fixed; it refuses a withdrawn, unsubmitted, or detail-erased Proposal and one blocked as `package-changed` or `application-refused`. +17. **Grant write access to a link's `fromField` only to identities trusted + to assert that link.** Whoever can set that field on a root chooses the + outside subject BReg asks the source for, under the deployment's source + grant, and which subject's values readers of that root then see. Review a + change to it as you would review a disclosure. See + [External projection storage](external-projection-storage.md). +18. **Keep the external projection cache out of logical backups.** Dump with + the four `--exclude-table-data` options for the + `registry_external_projection_*` tables, so a restore starts with no + source values, old bindings, or refresh receipts. Physical backups and + storage snapshots cannot exclude them and need a separately approved + erasure and restore procedure + ([logical backups](external-projection-storage.md#logical-backups)). +19. **Withdraw a source grant in deployment configuration.** Remove the + binding and restart; that is the only cutoff for acquisition. + `bregctl external-projection withdraw` purges retained values and refresh + receipts but does not stop acquisition, and a binding that stays + configured acquires again at the next start. Record the withdrawal in + deployment configuration or at the source authority so it also holds after + a restore + ([stop acquisition](external-projection-storage.md#stop-acquisition-withdraw-and-renew)). ## 3. Accepted residuals From 19b875b17d2e95989728a94cdfbe292575750498 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 19:08:23 +0700 Subject: [PATCH 44/64] fix: serve external decimal fields as strings on live and indexed paths Signed-off-by: Jeremi Joslin --- crates/registry-breg/src/external_source.rs | 170 +++++++++++++++++- .../src/external_view_artifacts.rs | 10 +- 2 files changed, 176 insertions(+), 4 deletions(-) diff --git a/crates/registry-breg/src/external_source.rs b/crates/registry-breg/src/external_source.rs index b3e669c749..45942af794 100644 --- a/crates/registry-breg/src/external_source.rs +++ b/crates/registry-breg/src/external_source.rs @@ -238,11 +238,12 @@ impl ExternalSourceService { let grant_deadline = tokio::time::Instant::now() .checked_add(remaining) .ok_or(ExternalSourceError::Grant)?; - let result = bound + let mut result = bound .program .acquire_until(hooks, mode, request, fields, Some(grant_deadline)) .await?; check_result(&result, &bound.field_types)?; + decimals_to_wire(&mut result, &bound.field_types)?; if clock() >= bound.grant_valid_until { return Err(ExternalSourceError::Grant); } @@ -520,6 +521,61 @@ fn check_result(result: &ProjectionResult, types: &FieldTypes) -> Result<(), Ext } } +/// The source contract states a decimal as a JSON number; BReg serves and +/// stores it as the JSON string its own decimal fields use on the wire. A +/// number with no plain decimal spelling (an exponent form) is refused. +fn decimals_to_wire( + result: &mut ProjectionResult, + types: &FieldTypes, +) -> Result<(), ExternalSourceError> { + let convert = |fields: &mut Map| { + for (name, value) in fields.iter_mut() { + let is_decimal = types + .get(name) + .is_some_and(|&(value_type, _)| value_type == ExternalFieldType::Decimal); + let Value::Number(number) = value else { + continue; + }; + if !is_decimal { + continue; + } + let text = number.to_string(); + if !plain_decimal(&text) { + return Err(ExternalSourceError::Result); + } + *value = Value::String(text); + } + Ok(()) + }; + match result { + ProjectionResult::Exact { record } => match record { + ProjectionRecord::Present { fields, .. } => convert(fields), + _ => Ok(()), + }, + ProjectionResult::Batch { records } => { + records.iter_mut().try_for_each(|record| match record { + ProjectionRecord::Present { fields, .. } => convert(fields), + _ => Ok(()), + }) + } + ProjectionResult::ByParent { rows, .. } => { + rows.iter_mut().try_for_each(|row| convert(&mut row.fields)) + } + } +} + +/// Whether `text` is an optional minus sign, digits, and an optional fraction +/// of digits: the pattern the served decimal schema states. +fn plain_decimal(text: &str) -> bool { + let digits = text.strip_prefix('-').unwrap_or(text); + let mut parts = digits.split('.'); + let digits_only = + |part: &str| !part.is_empty() && part.bytes().all(|byte| byte.is_ascii_digit()); + parts.next().is_some_and(digits_only) + && parts.next().is_none_or(digits_only) + && parts.next().is_none() +} + fn typed_value(value_type: ExternalFieldType, nullable: bool, value: &Value) -> bool { if value.is_null() { return nullable; @@ -708,6 +764,7 @@ fn read_relative(_: &File, _: &Path, _: usize) -> Result, ExternalSource #[cfg(test)] pub(crate) mod tests { use super::*; + use registry_adapter_core::ProjectionRow; use registry_adapter_engine::evidence::{EvidenceAcquisitionIntent, EvidenceHookFuture}; use registry_platform_yaml::ExternalId; @@ -1087,6 +1144,117 @@ pub(crate) mod tests { assert!(matches!(effective_ttl(&indexed, Some(900)), Ok(Some(300)))); } + fn decimal_fields(value: Value) -> Map { + let Value::Object(fields) = json!({"balance": value, "label": "x"}) else { + unreachable!() + }; + fields + } + + fn decimal_types() -> FieldTypes { + BTreeMap::from([ + ("balance".to_owned(), (ExternalFieldType::Decimal, true)), + ("label".to_owned(), (ExternalFieldType::String, false)), + ]) + } + + fn present(key: &str, value: Value) -> ProjectionRecord { + ProjectionRecord::Present { + key: key.to_owned(), + fields: decimal_fields(value), + source_revision: None, + } + } + + fn balance_of(record: &ProjectionRecord) -> Value { + match record { + ProjectionRecord::Present { fields, .. } => fields["balance"].clone(), + _ => panic!("not present"), + } + } + + /// Every result shape, the exact (live) and batch and by-parent (indexed) + /// ones, serves a decimal as a string the generated schema accepts. + #[test] + fn decimals_are_served_as_strings_the_generated_schema_accepts() { + let validator = jsonschema::JSONSchema::options() + .with_draft(jsonschema::Draft::Draft202012) + .compile(&crate::external_view_artifacts::decimal_schema()) + .unwrap(); + let types = decimal_types(); + let spelled: [(Value, &str); 4] = [ + (json!(12.5), "12.5"), + (json!(-3), "-3"), + (json!(0.1), "0.1"), + (json!(100), "100"), + ]; + for (number, text) in spelled { + let mut exact = ProjectionResult::Exact { + record: present("a", number.clone()), + }; + decimals_to_wire(&mut exact, &types).unwrap(); + let ProjectionResult::Exact { record } = &exact else { + unreachable!() + }; + assert_eq!(balance_of(record), json!(text)); + assert!(validator.is_valid(&balance_of(record))); + + let mut batch = ProjectionResult::Batch { + records: vec![present("a", number.clone()), present("b", Value::Null)], + }; + decimals_to_wire(&mut batch, &types).unwrap(); + let ProjectionResult::Batch { records } = &batch else { + unreachable!() + }; + assert_eq!(balance_of(&records[0]), json!(text)); + assert_eq!(balance_of(&records[1]), Value::Null); + + let Value::Object(fields) = json!({"balance": number}) else { + unreachable!() + }; + let mut by_parent = ProjectionResult::ByParent { + parent_key: "p".to_owned(), + snapshot: "s".to_owned(), + rows: vec![ProjectionRow { + key: "r".to_owned(), + parent_key: "p".to_owned(), + fields, + source_revision: None, + }], + }; + decimals_to_wire(&mut by_parent, &types).unwrap(); + let ProjectionResult::ByParent { rows, .. } = &by_parent else { + unreachable!() + }; + assert_eq!(rows[0].fields["balance"], json!(text)); + } + assert!(!validator.is_valid(&json!(12.5))); + } + + #[test] + fn a_decimal_with_no_plain_spelling_is_refused() { + let types = decimal_types(); + for number in [json!(1e30), json!(1.5e-9)] { + let mut result = ProjectionResult::Exact { + record: present("a", number), + }; + assert!(matches!( + decimals_to_wire(&mut result, &types), + Err(ExternalSourceError::Result) + )); + } + } + + #[test] + fn plain_decimal_accepts_only_the_served_pattern() { + for good in ["0", "-1", "12.50", "007"] { + assert!(plain_decimal(good), "{good}"); + } + for bad in ["", "-", "1.", ".5", "1e5", "1.2.3", "+1", "--1", "1 "] { + assert!(!plain_decimal(bad), "{bad}"); + } + } + fn example_registry() -> CompiledRegistry { let source = std::fs::read( Path::new(env!("CARGO_MANIFEST_DIR")) diff --git a/crates/registry-breg/src/external_view_artifacts.rs b/crates/registry-breg/src/external_view_artifacts.rs index 85766e32a0..168bafe18e 100644 --- a/crates/registry-breg/src/external_view_artifacts.rs +++ b/crates/registry-breg/src/external_view_artifacts.rs @@ -216,6 +216,12 @@ fn read_operation( }) } +/// The wire schema of a decimal field: the string form BReg's own decimal +/// fields use. +pub(crate) fn decimal_schema() -> Value { + json!({"type":"string","pattern":"^-?[0-9]+(\\.[0-9]+)?$"}) +} + fn fields(external: &ExternalRecordTypeSource, readable: &BTreeSet<&str>) -> Value { let properties = external .fields @@ -226,9 +232,7 @@ fn fields(external: &ExternalRecordTypeSource, readable: &BTreeSet<&str>) -> Val ExternalFieldType::String => json!({"type":"string"}), ExternalFieldType::Integer => json!({"type":"integer"}), ExternalFieldType::Boolean => json!({"type":"boolean"}), - ExternalFieldType::Decimal => { - json!({"type":"string","pattern":"^-?[0-9]+(\\.[0-9]+)?$"}) - } + ExternalFieldType::Decimal => decimal_schema(), ExternalFieldType::Date => json!({"type":"string","format":"date"}), ExternalFieldType::Uuid => json!({"type":"string","format":"uuid"}), }; From 8b98a620f99137f2aae3bd8f133e73e96a79ca4f Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 19:11:16 +0700 Subject: [PATCH 45/64] fix: resume external sort continuations from long sort values Signed-off-by: Jeremi Joslin --- .../src/postgres/external_runtime.rs | 34 +++++++++++++ .../src/postgres/external_view_query.rs | 51 ++++++++++++++++++- products/breg/external-projection-storage.md | 2 +- 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index 49b7980701..ddce7403c4 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -2120,6 +2120,40 @@ fn snapshot_identity( mod tests { use super::*; + /// A sort value past the 4096-byte filter bound travels inside the + /// sealed continuation and opens unchanged. + #[test] + fn a_continuation_carries_a_long_sort_value() { + let codec = CursorCodec::new( + zeroize::Zeroizing::new(vec![42; 32]), + Duration::from_secs(120), + ) + .unwrap(); + let sort_value = "s".repeat(5000); + let token = codec + .seal_external(&Continuation { + binding: "binding".to_owned(), + expires_at: Utc::now().timestamp() + 60, + query: ExternalQuery::default(), + position: Position::Read { + root_id: Uuid::nil(), + source_key: "row".to_owned(), + sort_value: Some(sort_value.clone()), + }, + }) + .unwrap(); + assert!(token.len() <= crate::query::MAX_CURSOR_TOKEN_BYTES); + let opened: Continuation = codec.open_external(&token).unwrap(); + let Position::Read { + sort_value: Some(opened), + .. + } = opened.position + else { + panic!("position changed"); + }; + assert_eq!(opened, sort_value); + } + #[test] fn a_new_capture_preserves_a_scoped_source_revision_reference() { let codec = CursorCodec::new( diff --git a/crates/registry-breg/src/postgres/external_view_query.rs b/crates/registry-breg/src/postgres/external_view_query.rs index 1acab8ae3c..c0219200e8 100644 --- a/crates/registry-breg/src/postgres/external_view_query.rs +++ b/crates/registry-breg/src/postgres/external_view_query.rs @@ -552,7 +552,22 @@ fn sql_type(kind: ExternalFieldType) -> &'static str { } } +/// Largest string a request filter may carry. +const MAXIMUM_FILTER_STRING_BYTES: usize = 4096; +/// Largest sort value a continuation resumes from. Only a cursor BReg sealed +/// carries one, and the cursor's own payload bound (8 KiB, shared with the +/// request's key, query and binding) is the effective limit above this. +pub(crate) const MAXIMUM_CURSOR_SORT_VALUE_BYTES: usize = 6144; + fn scalar(value: &Value, kind: ExternalFieldType) -> Result, ExternalQueryError> { + bounded_scalar(value, kind, MAXIMUM_FILTER_STRING_BYTES) +} + +fn bounded_scalar( + value: &Value, + kind: ExternalFieldType, + maximum_string_bytes: usize, +) -> Result, ExternalQueryError> { if value.is_null() { return Ok(None); } @@ -560,7 +575,7 @@ fn scalar(value: &Value, kind: ExternalFieldType) -> Result, Exte let result = match kind { ExternalFieldType::String => value .as_str() - .filter(|s| s.len() <= 4096) + .filter(|s| s.len() <= maximum_string_bytes) .map(str::to_owned) .ok_or_else(invalid)?, ExternalFieldType::Integer => value.as_i64().map(|v| v.to_string()).ok_or_else(invalid)?, @@ -630,7 +645,8 @@ fn continuation_sql( .map_err(|_| ExternalQueryError::InvalidQuery)?, _ => Value::String(value.clone()), }; - let value = scalar(&parsed, kind)?.ok_or(ExternalQueryError::InvalidQuery)?; + let value = bounded_scalar(&parsed, kind, MAXIMUM_CURSOR_SORT_VALUE_BYTES)? + .ok_or(ExternalQueryError::InvalidQuery)?; let value = parameter(parameters, Some(value)); let cast = sql_type(kind); Ok(format!("({column} {} {value}::{cast} OR {column} IS NULL OR ({column}={value}::{cast} AND {tie}))",direction.comparison())) @@ -767,6 +783,37 @@ pub(crate) mod tests { ); } + #[test] + fn a_sort_value_past_the_filter_bound_resumes_within_the_cursor_bound() { + let order = |value: String| { + let after = PageAfter { + root_id: Uuid::nil(), + source_key: "row".into(), + sort_value: Some(value), + }; + let mut params = vec![]; + continuation_sql( + Some(( + &after, + Some(( + "sort_key", + ExternalFieldType::String, + SortDirection::Ascending, + )), + )), + &mut params, + ) + }; + assert!(order("a".repeat(5000)).is_ok()); + assert!(order("a".repeat(MAXIMUM_CURSOR_SORT_VALUE_BYTES)).is_ok()); + assert!(matches!( + order("a".repeat(MAXIMUM_CURSOR_SORT_VALUE_BYTES + 1)), + Err(ExternalQueryError::InvalidQuery) + )); + // A request filter keeps its own, smaller bound. + assert!(scalar(&json!("a".repeat(5000)), ExternalFieldType::String).is_err()); + } + #[test] fn a_continuation_preserves_nulls_last_in_both_directions() { let after = PageAfter { diff --git a/products/breg/external-projection-storage.md b/products/breg/external-projection-storage.md index 4efd7e33cb..33667a4ceb 100644 --- a/products/breg/external-projection-storage.md +++ b/products/breg/external-projection-storage.md @@ -29,7 +29,7 @@ A captured snapshot stores the observation time and a keyed revision reference w Replaying an accepted result keeps its original expiry. A refresh that fails because the source is unavailable, returns an incomplete result, or cannot be acquired under the current binding keeps the previous snapshot of either kind until its own expiry, without extending its lifetime. Explicit grant denial or no-store removes the previous snapshot, making coverage unproven, fenced against newer publications. A source response marked `no-store` cannot enter the indexed store. If it supersedes an earlier retained result for the same subject, the guarded retention-refusal path removes that result; an older refresh cannot erase a newer publication. -Reads check expiry independently of the refresh job. Indexed predicates and counts refuse incomplete coverage before selecting matching rows. Each page reads coverage, rows and any count from one statement snapshot, so a count always matches complete coverage at that moment. A refresh records each key in its own short transaction, so a read waits for at most one key's publication and can show new values for some keys of a page in flight and previous values for the rest. A `nextCursor` stays valid while refreshes publish or fail individual keys; the next page reads the values current when it runs. A row whose ordering value changed between pages can appear twice or not at all. Rotating, withdrawing or resetting the projection invalidates outstanding cursors. A complete empty related collection is covered; a partial collection, source failure, or protected-existence response is not evidence of emptiness or absence. +Reads check expiry independently of the refresh job. Indexed predicates and counts refuse incomplete coverage before selecting matching rows. Each page reads coverage, rows and any count from one statement snapshot, so a count always matches complete coverage at that moment. A refresh records each key in its own short transaction, so a read waits for at most one key's publication and can show new values for some keys of a page in flight and previous values for the rest. A `nextCursor` stays valid while refreshes publish or fail individual keys; the next page reads the values current when it runs. A row whose ordering value changed between pages can appear twice or not at all. A cursor is encrypted and authenticated with the deployment's cursor key and bound to its caller, query and projection generation, so its contents (including the sort value it resumes from) cannot be altered or read by the client. A sort value resumes from up to 6144 bytes; a stored value, or a cursor whose key, query and sort value together exceed the 8 KiB cursor payload, ends the listing with `source.unavailable`. Rotating, withdrawing or resetting the projection invalidates outstanding cursors. A complete empty related collection is covered; a partial collection, source failure, or protected-existence response is not evidence of emptiness or absence. A read of one record looks only at that record. Every other indexed page, including each continuation page, computes coverage, root presence, expiry and any count over every root the caller is authorized to read before it selects its rows, so its cost grows with the caller's whole authorized population, not with the page size. Grant indexed views only to access profiles whose row boundaries are narrowed to the records their callers need, and rate-limit the callers of indexed queries and of refresh at the gateway. From 2a865dfd45d75cee781c9e0155f1fc8859714dca Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 19:25:37 +0700 Subject: [PATCH 46/64] fix: serve malformed external link keys as no outside value and refuse them on write Signed-off-by: Jeremi Joslin --- crates/registry-breg/src/compiler.rs | 2 + crates/registry-breg/src/external_views.rs | 42 +++ crates/registry-breg/src/history_schema.rs | 1 + crates/registry-breg/src/model.rs | 5 + crates/registry-breg/src/mutation.rs | 34 ++- .../src/postgres/external_runtime.rs | 78 +++++- .../src/postgres/external_view_query.rs | 76 +++++- crates/registry-breg/tests/external_views.rs | 15 ++ .../registry-breg/tests/postgres_mutation.rs | 243 ++++++++++++++++++ products/breg/external-projection-storage.md | 6 +- 10 files changed, 485 insertions(+), 17 deletions(-) diff --git a/crates/registry-breg/src/compiler.rs b/crates/registry-breg/src/compiler.rs index 8040c4402b..b667f2286c 100644 --- a/crates/registry-breg/src/compiler.rs +++ b/crates/registry-breg/src/compiler.rs @@ -378,6 +378,7 @@ pub fn compile_project_with_assets( "external view compilation", ) })?; + crate::external_views::mark_link_fields(&external_views, &mut entities); crate::membership::compile(&mut entities); let recipients = crate::consent::compile(project, &sources, &mut entities); crate::event_record::compile(&sources, &mut entities); @@ -7915,6 +7916,7 @@ fn compile_entities( display: source.display.display(), grouping: source.display.grouping(), event_record: None, + external_link_fields: BTreeSet::new(), definition_revision: record_definition_revision(source, &constraints), id: source.id.clone(), source_module, diff --git a/crates/registry-breg/src/external_views.rs b/crates/registry-breg/src/external_views.rs index 299052e629..b03ccea2c7 100644 --- a/crates/registry-breg/src/external_views.rs +++ b/crates/registry-breg/src/external_views.rs @@ -166,6 +166,48 @@ impl CompiledExternalViews { } } +/// Record, on each root entity, the stored fields its views link from, so a +/// write path can check a link key without the view definitions. +pub(crate) fn mark_link_fields( + views: &CompiledExternalViews, + entities: &mut BTreeMap, +) { + for view in views.views.values() { + let Some(entity) = entities.get_mut(view.record_type.as_str()) else { + continue; + }; + for link in &view.links { + entity + .external_link_fields + .insert(link.from_field.as_str().to_owned()); + } + } +} + +/// Whether `key` is a link key BReg accepts: the form the source contract +/// accepts for a selector. The write path refuses any other key on a field an +/// external view links from, and every read treats one that is already stored +/// as no outside value. An empty key is not a key; it names no outside +/// subject and is accepted. +pub(crate) fn link_key_acceptable(key: &str) -> bool { + key.is_empty() || registry_adapter_core::projection_token_valid(key) +} + +/// The SQL form of [`link_key_acceptable`] over the column or expression +/// `key`: the key itself when it is NULL, empty or acceptable, otherwise +/// NULL. The character classes are the explicit code point sets of Rust's +/// `char::is_control` and `str::trim`, so both forms agree on every string. +pub(crate) fn acceptable_link_key_sql(key: &str) -> String { + const WHITESPACE: &str = + r"\u0009-\u000d\u0020\u0085\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000"; + const CONTROL: &str = r"\u0001-\u001f\u007f-\u009f"; + format!( + "CASE WHEN {key} IS NULL OR {key}='' OR (octet_length({key})<=512 \ + AND {key} !~ '^[{WHITESPACE}]' AND {key} !~ '[{WHITESPACE}]$' \ + AND {key} !~ '[{CONTROL}]') THEN {key} END" + ) +} + pub(crate) fn compile( project: &RegistryProject, entities: &BTreeMap, diff --git a/crates/registry-breg/src/history_schema.rs b/crates/registry-breg/src/history_schema.rs index def4716c8c..dd6b388a2e 100644 --- a/crates/registry-breg/src/history_schema.rs +++ b/crates/registry-breg/src/history_schema.rs @@ -832,6 +832,7 @@ mod tests { display: Default::default(), grouping: None, event_record: None, + external_link_fields: BTreeSet::new(), definition_revision: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_owned(), relationships: Default::default(), diff --git a/crates/registry-breg/src/model.rs b/crates/registry-breg/src/model.rs index 1e620e193c..8c0bdc432a 100644 --- a/crates/registry-breg/src/model.rs +++ b/crates/registry-breg/src/model.rs @@ -1039,6 +1039,11 @@ pub struct CompiledEntity { /// identity. #[serde(default, skip_serializing_if = "Option::is_none")] pub event_record: Option, + /// The stored fields an external view links from. A value written to one + /// is a link key sent to a source, so it must satisfy the same rule as at + /// send time. + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub external_link_fields: BTreeSet, pub definition_revision: String, } diff --git a/crates/registry-breg/src/mutation.rs b/crates/registry-breg/src/mutation.rs index 4970a1b831..e1e5fc5d34 100644 --- a/crates/registry-breg/src/mutation.rs +++ b/crates/registry-breg/src/mutation.rs @@ -2990,9 +2990,10 @@ async fn insert_current_row( } continue; } - values.push(BoundValue::Text(sql_value( + values.push(BoundValue::Text(written_sql_value( + entity, + field, &data[&field.id], - &field.field_type, )?)); field_columns.push(quote_identifier(&field.physical_name)); field_parameters.push(typed_parameter(values.len(), &field.field_type)); @@ -3073,9 +3074,10 @@ async fn update_current_row( } continue; } - values.push(BoundValue::Text(sql_value( + values.push(BoundValue::Text(written_sql_value( + entity, + field, &data[&field.id], - &field.field_type, )?)); assignments.push(format!( "{} = {}", @@ -3456,7 +3458,8 @@ fn apply_patch_document( if value.is_null() && field.required { return Err(at(PatchMember::Value)); } - sql_value(value, &field.field_type).map_err(|_| at(PatchMember::Value))?; + written_sql_value(&request.plan.entity, field, value) + .map_err(|_| at(PatchMember::Value))?; materialized.insert(field_id.clone(), value.clone()); changed.insert(field_id, value.clone()); mutated = true; @@ -3754,7 +3757,7 @@ fn validate_request( if value.is_null() && field.required { return Err(location()); } - sql_value(value, &field.field_type).map_err(|_| location())?; + written_sql_value(&request.plan.entity, field, value).map_err(|_| location())?; } } Ok(()) @@ -3996,7 +3999,8 @@ fn validate_patch_static( if value.is_null() && field.required { return Err(at(PatchMember::Value)); } - sql_value(value, &field.field_type).map_err(|_| at(PatchMember::Value))?; + written_sql_value(&request.plan.entity, field, value) + .map_err(|_| at(PatchMember::Value))?; mutated = true; } PatchOperation::Remove { .. } => { @@ -4134,6 +4138,22 @@ fn record_reference( .map_err(|_| MutationError::Unavailable) } +/// The SQL text of a value written to `field`, refusing a link key an +/// external view could not send to its source. Every record write passes +/// through here, so the check holds on each write path. +fn written_sql_value( + entity: &CompiledEntity, + field: &CompiledField, + value: &Value, +) -> Result, MutationError> { + if entity.external_link_fields.contains(&field.id) + && matches!(value, Value::String(key) if !crate::external_views::link_key_acceptable(key)) + { + return Err(MutationError::InvalidRequest); + } + sql_value(value, &field.field_type) +} + pub(crate) fn sql_value( value: &Value, field_type: &FieldTypeSource, diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index ddce7403c4..f5bd2fca74 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -609,12 +609,12 @@ impl PostgresExternalViewService { let transaction = self .transaction(&mut session, claims, request.deadline) .await?; - // A root without a link key has no outside subject to fetch. + // A root without a usable link key has no outside subject to fetch. let key = RootKeys::new(plan)? .for_root(&transaction, root) .await? .ok_or(ExternalViewError::Concealed)? - .filter(|key| !key.is_empty()); + .filter(|key| !key.is_empty() && crate::external_views::link_key_acceptable(key)); transaction .commit() .await @@ -652,7 +652,7 @@ impl PostgresExternalViewService { .for_root(&transaction, root) .await? .ok_or(ExternalViewError::Concealed)? - .filter(|key| !key.is_empty()); + .filter(|key| !key.is_empty() && crate::external_views::link_key_acceptable(key)); let (status, records) = if current != key { ("unavailable", Vec::new()) } else if key.is_none() { @@ -3480,9 +3480,71 @@ mod postgres_tests { } #[tokio::test] - async fn a_live_read_of_a_root_without_a_link_key_answers_absent_without_a_source_call() { + async fn a_malformed_link_key_reads_as_no_outside_value_and_the_view_keeps_serving() { + const PADDED: &str = "00000000-0000-4000-8000-000000000011"; + const BELL: &str = "00000000-0000-4000-8000-000000000012"; + const IDEOGRAPHIC_SPACE: &str = "00000000-0000-4000-8000-000000000014"; + let served = Served::new(Duration::from_secs(2)).await; + served.publish_all().await; + insert_roots( + &served, + &[ + (PADDED, Some(" padded-key"), "allowed"), + (BELL, Some("bell\u{7}key"), "allowed"), + (IDEOGRAPHIC_SPACE, Some("trailing\u{3000}"), "allowed"), + ], + ) + .await; + let caller = reader("reader-principal", &["allowed"]); + let mut expected = BTreeMap::new(); + for (id, key, _) in &ROOTS[..3] { + expected.insert( + (*id).to_owned(), + ("available".to_owned(), json!({ "label": key })), + ); + } + for id in [PADDED, BELL, IDEOGRAPHIC_SPACE] { + expected.insert(id.to_owned(), ("absent".to_owned(), json!({}))); + } + + let (status, page) = served.get("&$count=true", caller.clone()).await; + assert_eq!(status, axum::http::StatusCode::OK, "{page}"); + assert_eq!(page["status"], "available", "{page}"); + assert_eq!(page["count"], 6, "{page}"); + assert_eq!(availabilities(&page), expected); + let (status, page) = served + .get("&$orderby=label%20asc&$count=true", caller.clone()) + .await; + assert_eq!(status, axum::http::StatusCode::OK, "{page}"); + assert_eq!(availabilities(&page), expected); + // A filter on an outside field leaves the malformed records out. + let (status, page) = served + .get("&$filter=label%20eq%20'second'&$count=true", caller.clone()) + .await; + assert_eq!(status, axum::http::StatusCode::OK, "{page}"); + assert_eq!(page["count"], 1, "{page}"); + assert_eq!(labels(&page), ["second"]); + let (status, _, body) = served + .send( + &format!("/v1/external-views/view/link/records/{BELL}?accessProfile=reader"), + caller, + ) + .await; + assert_eq!(status, axum::http::StatusCode::OK, "{body}"); + assert_eq!(body["status"], "available", "{body}"); + assert_eq!( + availabilities(&body), + BTreeMap::from([(BELL.to_owned(), ("absent".to_owned(), json!({})))]) + ); + served.fixture.cleanup().await; + } + + #[tokio::test] + async fn a_live_read_of_a_root_without_a_usable_link_key_answers_absent_without_a_source_call() + { const UNLINKED: &str = "00000000-0000-4000-8000-000000000011"; const EMPTY: &str = "00000000-0000-4000-8000-000000000012"; + const MALFORMED: &str = "00000000-0000-4000-8000-000000000013"; let transcript = Arc::new( registry_adapter_engine::replay::ReplayTranscript::from_exchanges( serde_json::from_value( @@ -3503,12 +3565,16 @@ mod postgres_tests { Served::with(&live_and_indexed_project(), sources, Duration::from_secs(2)).await; insert_roots( &served, - &[(UNLINKED, None, "allowed"), (EMPTY, Some(""), "allowed")], + &[ + (UNLINKED, None, "allowed"), + (EMPTY, Some(""), "allowed"), + (MALFORMED, Some("bell\u{7}key"), "allowed"), + ], ) .await; let caller = reader("reader-principal", &["allowed"]); let captured = registry_platform_testing::CapturedDiagnosticLogs::default(); - for root in [UNLINKED, EMPTY] { + for root in [UNLINKED, EMPTY, MALFORMED] { use tracing::instrument::WithSubscriber; let (status, _, body) = served .send( diff --git a/crates/registry-breg/src/postgres/external_view_query.rs b/crates/registry-breg/src/postgres/external_view_query.rs index c0219200e8..37fe2a6fdf 100644 --- a/crates/registry-breg/src/postgres/external_view_query.rs +++ b/crates/registry-breg/src/postgres/external_view_query.rs @@ -322,7 +322,10 @@ fn compile( // the snapshot and value keys' own collation lets their primary keys serve // the lookup for each root; both collations are deterministic, so // equality is unchanged. - let root_sql = format!("SELECT {} AS root_id,({})::text COLLATE \"default\" AS subject_key FROM {} WHERE current_setting('registry.access_profile',true)={profile_param} AND ({root_param}::uuid IS NULL OR {}={root_param}::uuid)",relations.id_expression,key.sql,relations.from_sql,relations.id_expression); + // A link key the write path would refuse reads as no outside value, like + // an empty one. + let subject_key = crate::external_views::acceptable_link_key_sql("raw_key"); + let root_sql = format!("SELECT root_id,{subject_key} AS subject_key FROM (SELECT {} AS root_id,({})::text COLLATE \"default\" AS raw_key FROM {} WHERE current_setting('registry.access_profile',true)={profile_param} AND ({root_param}::uuid IS NULL OR {}={root_param}::uuid)) AS linked",relations.id_expression,key.sql,relations.from_sql,relations.id_expression); let many = projection.cardinality == ExternalCardinality::Many; let expected_kind = if many { "o.kind='complete-collection'" @@ -1200,6 +1203,77 @@ mod postgres_tests { database.cleanup().await; } + /// The SQL form of the link-key rule agrees with the Rust predicate on + /// every boundary: byte length, control characters, and the leading and + /// trailing white space `str::trim` removes. + #[tokio::test] + async fn the_sql_link_key_rule_agrees_with_the_rust_predicate() { + let database = harness::TestDatabase::create(2).await; + let sql = format!( + "SELECT {} FROM (SELECT $1::text AS raw_key) AS linked", + crate::external_views::acceptable_link_key_sql("raw_key") + ); + let max = "a".repeat(512); + let over = "a".repeat(513); + let wide_max = "\u{E9}".repeat(256); + let wide_over = "\u{E9}".repeat(257); + let mut keys = vec![ + "plain".to_owned(), + "inner space".to_owned(), + "dash-and_dot.9".to_owned(), + max, + over, + wide_max, + wide_over, + "\u{1D518}key".to_owned(), + ]; + for edge in [ + '\u{1}', '\u{7}', '\u{1f}', '\u{7f}', '\u{80}', '\u{9f}', '\u{a0}', '\u{a1}', '\u{85}', + '\u{1680}', '\u{2000}', '\u{200a}', '\u{200b}', '\u{2028}', '\u{2029}', '\u{202f}', + '\u{205f}', '\u{3000}', ' ', '\t', '\n', '\r', '\u{b}', '\u{c}', '\u{e000}', + '\u{feff}', + ] { + for key in [ + format!("{edge}key"), + format!("key{edge}"), + format!("ke{edge}y"), + ] { + keys.push(key); + } + } + for key in keys { + let accepted: Option = database + .admin + .query_one(&sql, &[&key]) + .await + .unwrap() + .get(0); + assert_eq!( + accepted.is_some(), + crate::external_views::link_key_acceptable(&key), + "{:?}", + key.chars().take(8).collect::() + ); + } + for key in ["", "ok"] { + let accepted: Option = database + .admin + .query_one(&sql, &[&key]) + .await + .unwrap() + .get(0); + assert_eq!(accepted.as_deref(), Some(key)); + } + let none: Option = database + .admin + .query_one(&sql, &[&Option::::None]) + .await + .unwrap() + .get(0); + assert_eq!(none, None); + database.cleanup().await; + } + #[tokio::test] async fn a_date_cursor_reads_the_same_under_any_date_style() { let database = harness::TestDatabase::create(2).await; diff --git a/crates/registry-breg/tests/external_views.rs b/crates/registry-breg/tests/external_views.rs index c4e7f8a4ed..fba7ebc586 100644 --- a/crates/registry-breg/tests/external_views.rs +++ b/crates/registry-breg/tests/external_views.rs @@ -168,3 +168,18 @@ fn external_openapi_uses_named_view_and_field_grants() { assert!(schema["properties"].get("count").is_none()); assert!(document["paths"]["/v1/external-views/local-with-subject/subject/refresh"].is_null()); } + +#[test] +fn the_field_a_view_links_from_is_marked_on_its_root_entity() { + let registry = compile(PROJECT).expect("an explicitly granted external view compiles"); + let entity = ®istry.entities()["local-record"]; + assert_eq!( + entity + .external_link_fields + .iter() + .map(String::as_str) + .collect::>(), + ["subject-key"], + "only the linked field carries the link-key rule" + ); +} diff --git a/crates/registry-breg/tests/postgres_mutation.rs b/crates/registry-breg/tests/postgres_mutation.rs index 6615fa1dc3..fcc342b93e 100644 --- a/crates/registry-breg/tests/postgres_mutation.rs +++ b/crates/registry-breg/tests/postgres_mutation.rs @@ -5971,3 +5971,246 @@ async fn real_postgres_receipt_sweep_clears_the_raw_caller_of_every_expired_spen drop(client); database.cleanup().await; } + +fn linked_registry() -> registry_breg::CompiledRegistry { + let project = parse_project_json( + br#"{"datasets":[{"id":"local-records"}], + "apiVersion":"id.registrystack.org/formats/breg/project/v1alpha1", + "kind":"BRegProject", + "project":{"id":"mutation-registry","version":"1","defaultLanguage":"en","canonicalBaseIri":"https://authoring.example.test"}, + "recordTypes":[{ + "id":"local-record","primaryDataset":"local-records","route":"local-records","mutationMode":"mutable","classification":"internal", + "fields":[ + {"id":"subjectkey","type":"string","maximumLength":32,"classification":"internal"}, + {"id":"localtitle","type":"string","maximumLength":120,"classification":"internal"} + ] + }], + "externalRecordTypes":[{ + "id":"external-subject","authority":"urn:example:source","identifierScheme":"urn:example:subject-key", + "fields":[{"id":"label","type":"string","classification":"internal"}] + }], + "externalProjections":[{ + "id":"approved-subjects","recordType":"external-subject","operation":"subjects-by-key","operationVersion":"1", + "purpose":"local-consultation","scope":"approved-subjects","selectorScheme":"urn:example:subject-key", + "mode":"indexed","cardinality":"one","cacheTtlSeconds":600,"fields":["label"] + }], + "externalViews":[{ + "id":"local-with-subject","recordType":"local-record", + "links":[{"id":"subject","projection":"approved-subjects","fromField":"subjectkey", + "identifierScheme":"urn:example:subject-key","fields":["label"]}] + }], + "accessProfiles":[{ + "id":"operator","default":true,"principalClaim":"registry_principal","requiredScopes":"unrestricted", + "requiredPurposes":["local-consultation"], + "permissions":{"records":[{ + "recordType":"local-record","operations":["create","get","list","patch"], + "readableFields":["subjectkey","localtitle"], + "writableFields":["subjectkey","localtitle"], + "rowBoundaries":"unrestricted" + }], + "externalViews":[{"view":"local-with-subject","links":[{"link":"subject","readableFields":["label"]}]}]} + }] + }"#, + ) + .expect("linked fixture parses"); + compile_project(&project, &[], CompileProfile::Authoring).expect("linked fixture compiles") +} + +fn linked_create<'a>( + plan: &'a MutationPlan, + key: &'a str, + claims: &'a ClaimContext, + subject_key: &str, + title: &str, +) -> MutationRequest<'a> { + MutationRequest { + plan, + idempotency_key: key, + claims, + record_id: None, + expected_etag: None, + body: MutationBody::Create(Map::from_iter([ + ( + "subjectkey".to_owned(), + Value::String(subject_key.to_owned()), + ), + ("localtitle".to_owned(), Value::String(title.to_owned())), + ])), + response_fields: BTreeSet::from(["subjectkey".to_owned(), "localtitle".to_owned()]), + representation: registry_breg::record_profile::RecordRepresentation::Json, + correlation: registry_breg::correlation::RequestCorrelation::breg_created(), + } +} + +fn linked_patch<'a>( + plan: &'a MutationPlan, + key: &'a str, + claims: &'a ClaimContext, + record_id: &'a str, + expected_etag: &'a str, + field: &str, + value: &str, +) -> MutationRequest<'a> { + MutationRequest { + plan, + idempotency_key: key, + claims, + record_id: Some(record_id), + expected_etag: Some(expected_etag), + body: MutationBody::Patch(vec![PatchOperation::Replace { + path: format!("/data/{field}"), + value: Value::String(value.to_owned()), + }]), + response_fields: BTreeSet::from(["subjectkey".to_owned(), "localtitle".to_owned()]), + representation: registry_breg::record_profile::RecordRepresentation::Json, + correlation: registry_breg::correlation::RequestCorrelation::breg_created(), + } +} + +/// A key that cannot be sent to the source is refused on write to the +/// field an external view links from, and accepted anywhere else. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_malformed_external_link_key_is_refused_on_write_to_the_linked_field_only() { + let database = TestDatabase::create(10).await; + let (migration, migration_task) = database.connect_migration().await; + let compiled = linked_registry(); + install_compiled_schema(&migration, &compiled, &database.runtime_role) + .await + .expect("migration installs the compiler-owned schema"); + let identity = initialize_compiled_registry_state_for_test( + &migration, + &database.runtime_role, + &compiled, + RegistryStateTestIdentity { + package_id: PACKAGE_ID, + database_id: DATABASE_ID, + label: "package-mutation-1", + }, + ) + .await + .expect("migration initializes the active package"); + migration_task.abort(); + let pool = database + .runtime_config + .build_pool() + .expect("bounded runtime pool builds"); + let coordinator = audited_coordinator(&database, &identity); + let create_plan = MutationPlan::from_compiled(&compiled, "records.local-record.create") + .expect("create plan comes from the compiled inventory"); + let patch_plan = MutationPlan::from_compiled(&compiled, "records.local-record.patch") + .expect("patch plan comes from the compiled inventory"); + let claims = ClaimContext::for_compiled( + &compiled, + "local-record", + Some(PRINCIPAL_CANARY.to_owned()), + "operator", + Some("local-consultation".to_owned()), + Vec::new(), + ) + .expect("claim context is compiler-bound"); + let table = &compiled.entities()["local-record"].physical_table; + let mut client = pool + .get_for_test() + .await + .expect("runtime connection is available"); + + coordinator + .execute( + &mut client, + linked_create( + &create_plan, + "well-formed-create", + &claims, + "key-0", + "title", + ), + ) + .await + .expect("a well-formed key is accepted on the linked field"); + + for (index, malformed) in [" padded", "trailing ", "bell\u{7}key", "trailing\u{3000}"] + .into_iter() + .enumerate() + { + let key = format!("malformed-create-{index}"); + let before = durable_counts(&database, table).await; + let refused = coordinator + .execute( + &mut client, + linked_create(&create_plan, &key, &claims, malformed, "title"), + ) + .await; + assert_eq!( + refused, + Err(MutationError::InvalidRequestAt( + RequestLocation::data_field("subjectkey") + )), + "{malformed:?} is refused on create" + ); + let after = durable_counts(&database, table).await; + assert_eq!(after.current, before.current, "nothing is written"); + assert_eq!(after.revisions, before.revisions, "no revision is written"); + } + + let long = "k".repeat(513); + // maximumLength is 32, so the length rule is exercised by the unit + // tests; here the character rule alone decides. + assert!(long.len() > 32); + + // The same malformed value in an unlinked field is accepted. + let accepted = coordinator + .execute( + &mut client, + linked_create(&create_plan, "unlinked-create", &claims, "key-1", " padded"), + ) + .await + .expect("a malformed-looking value is accepted in an unlinked field"); + let record_id = response_id(&accepted); + let etag = response_etag(&accepted); + + let before = durable_counts(&database, table).await; + let refused = coordinator + .execute( + &mut client, + linked_patch( + &patch_plan, + "malformed-patch", + &claims, + &record_id, + &etag, + "subjectkey", + "bell\u{7}key", + ), + ) + .await; + assert_eq!( + refused, + Err(MutationError::InvalidRequestAt( + RequestLocation::patch_operation(0, Some(PatchMember::Value)) + )), + "a malformed key is refused on patch" + ); + let after = durable_counts(&database, table).await; + assert_eq!(after.current, before.current); + assert_eq!( + after.revisions, before.revisions, + "the patch writes nothing" + ); + + let patched = coordinator + .execute( + &mut client, + linked_patch( + &patch_plan, + "unlinked-patch", + &claims, + &record_id, + &etag, + "localtitle", + "bell\u{7}title", + ), + ) + .await + .expect("the same value is accepted in an unlinked field"); + assert!(!patched.replayed()); +} diff --git a/products/breg/external-projection-storage.md b/products/breg/external-projection-storage.md index 33667a4ceb..d502fefdc6 100644 --- a/products/breg/external-projection-storage.md +++ b/products/breg/external-projection-storage.md @@ -33,7 +33,7 @@ Reads check expiry independently of the refresh job. Indexed predicates and coun A read of one record looks only at that record. Every other indexed page, including each continuation page, computes coverage, root presence, expiry and any count over every root the caller is authorized to read before it selects its rows, so its cost grows with the caller's whole authorized population, not with the page size. Grant indexed views only to access profiles whose row boundaries are narrowed to the records their callers need, and rate-limit the callers of indexed queries and of refresh at the gateway. -A root whose link field is empty or unset names no outside subject. It counts as covered, reads with its external fields absent, or as an empty related collection, and its refresh sends no source request. A root whose link key the store or the source contract would refuse is never sent to the source. The store refuses a key over 1024 bytes or with a control character; the source contract also refuses a key over 512 bytes or with leading or trailing white space. Such a root stays unavailable, so lists, predicates and counts over its view keep answering incomplete until the key is corrected. The rest of its refresh page still refreshes, and the sweep moves on. A label sweep applies the same rules to each related row's label key, so one invalid label key leaves only that row without its label. +A root whose link field is empty or unset names no outside subject. It counts as covered, reads with its external fields absent, or as an empty related collection, and its refresh sends no source request. A root whose link key the store or the source contract would refuse is never sent to the source. The store refuses a key over 1024 bytes or with a control character; the source contract also refuses a key over 512 bytes or with leading or trailing white space. Such a root reads as having no outside value, like a root with an empty link field, for that record alone: it is covered, its external fields are absent, and lists, predicates, sorts and counts over its view keep serving the other records. The refresh still logs and counts the key as `invalid-key` and as unavailable in its report, so an operator can find and correct it. A write of such a key to a field an external view links from is refused as `request.invalid` on every write path (create, patch, action, proposal, batch and import), and writes nothing; the same value in a field no view links from is accepted. A key stored before the view was configured is read under the rule above until it is corrected. A label sweep applies the same rules to each related row's label key, so one invalid label key leaves only that row without its label. The purge worker runs every five seconds. Each pass deletes expired snapshots, refresh receipts whose lease has ended, and everything under withdrawn or replaced epochs, in at most eight batches of a thousand rows of each kind. Freshness enforcement keeps expired values out of responses even when refresh or purge is delayed, but an expired value stays stored until a pass deletes it. That happens while the database is down, while the server is stopped, and while a backlog larger than one pass drains. Size database capacity for the backlog. Where a storage obligation is strict, choose a cache lifetime shorter than that obligation by more than the worker interval and the time you need to recover from those conditions, and watch the signals in the next section. Database WAL and physical snapshots need their own retention policy. @@ -46,7 +46,7 @@ Each signal names the projection and a fixed reason category. None carries a sou - A purge pass that reaches its batch bound logs a `warn` line. The rest of the backlog waits for the next pass. - On the metrics listener, `breg_worker_last_success_age_seconds{worker="external-projection-retention"}` reports the seconds since the last completed pass. It is absent until the first pass completes. Alert when it grows well past the five-second interval. - A refresh page that acquired keys logs one `info` line with its `published` count, and one `warn` line per failure reason with a `keys` count. The reason is `denied`, `retention-refused`, `unavailable`, `incomplete`, `invalid-binding`, or `abandoned` when a newer result, a changed root or a changed collection superseded the result before publication. -- A refresh page that holds link keys or label keys the store or the source contract would refuse logs a `warn` line with the reason `invalid-key` and a `keys` count. Those keys are never sent to the source. +- A refresh page that holds link keys or label keys the store or the source contract would refuse logs a `warn` line with the reason `invalid-key` and a `keys` count. Those keys are never sent to the source, and the records that hold them read as having no outside value until the keys are corrected. - A refresh request that fails as a whole logs a `warn` line with a `reason` of `concealed`, `invalid-cursor`, `incomplete`, `unavailable`, `projection-unavailable` or `deadline`. - A live read whose source acquisition fails logs a `warn` line with the same reasons as a refresh key, other than `abandoned`. The view answers unavailable. - `bregctl doctor` reports each projection a server would serve as an `external-projection.status` advisory. It gives the time the source grant expires and, for an indexed projection, when the newest snapshot it holds was observed. Renew a grant before that time. An old or missing newest snapshot means the refresh sweep is not running. @@ -56,7 +56,7 @@ Each signal names the projection and a fixed reason category. None carries a sou Run the named view’s `POST /v1/external-views/{view}/{link}/refresh` with the approved machine profile. Continue with the returned `nextCursor` until the sweep ends. For independent related labels, run `POST /v1/external-views/{view}/{link}/refresh-labels` after the related collection refresh. That machine profile needs explicit refresh grants on both projections under the same purpose. The label sweep selects keys only from authorised roots with fresh complete collections. Before each source call it confirms that the collections which supplied that page are still current and unexpired. A refresh of other collections leaves the sweep running; a refresh of one of its own collections stops acquisition for that page, and the next sweep picks up the new members. A rotated, withdrawn or reset collection projection ends the sweep, so start a fresh one. -The refresh report counts processed roots or related rows, published values, reused keys and unavailable results. A root without a link key counts only as processed. A root with an invalid link key counts as unavailable. A related row with an invalid label key also counts as unavailable. `bregctl external-projection inspect` reports receipt counts by outcome and fresh, expired and invalidated snapshot counts. Use an authorised indexed query to test readiness for its local root scope. +The refresh report counts processed roots or related rows, published values, reused keys and unavailable results. A root without a link key counts only as processed. A root with an invalid link key counts as unavailable, although a read of it answers as having no outside value. A related row with an invalid label key also counts as unavailable. `bregctl external-projection inspect` reports receipt counts by outcome and fresh, expired and invalidated snapshot counts. Use an authorised indexed query to test readiness for its local root scope. ## Stop acquisition, withdraw, and renew From 10698e98ac070b15a61894e0d309cc19ed5edc3b Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 19:29:55 +0700 Subject: [PATCH 47/64] fix: keep the link key rule inside registry-breg without the runtime feature Signed-off-by: Jeremi Joslin --- crates/registry-breg/src/external_views.rs | 11 ++++++++++- .../registry-breg/src/postgres/external_view_query.rs | 5 +++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/crates/registry-breg/src/external_views.rs b/crates/registry-breg/src/external_views.rs index b03ccea2c7..1be3d4ad0d 100644 --- a/crates/registry-breg/src/external_views.rs +++ b/crates/registry-breg/src/external_views.rs @@ -184,19 +184,28 @@ pub(crate) fn mark_link_fields( } } +/// The longest key the source contract accepts for a selector. +#[cfg(feature = "runtime")] +const MAXIMUM_LINK_KEY_BYTES: usize = 512; + /// Whether `key` is a link key BReg accepts: the form the source contract /// accepts for a selector. The write path refuses any other key on a field an /// external view links from, and every read treats one that is already stored /// as no outside value. An empty key is not a key; it names no outside /// subject and is accepted. +#[cfg(feature = "runtime")] pub(crate) fn link_key_acceptable(key: &str) -> bool { - key.is_empty() || registry_adapter_core::projection_token_valid(key) + key.is_empty() + || (key.len() <= MAXIMUM_LINK_KEY_BYTES + && key.trim() == key + && !key.chars().any(char::is_control)) } /// The SQL form of [`link_key_acceptable`] over the column or expression /// `key`: the key itself when it is NULL, empty or acceptable, otherwise /// NULL. The character classes are the explicit code point sets of Rust's /// `char::is_control` and `str::trim`, so both forms agree on every string. +#[cfg(feature = "runtime")] pub(crate) fn acceptable_link_key_sql(key: &str) -> String { const WHITESPACE: &str = r"\u0009-\u000d\u0020\u0085\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000"; diff --git a/crates/registry-breg/src/postgres/external_view_query.rs b/crates/registry-breg/src/postgres/external_view_query.rs index 37fe2a6fdf..a2bc17ac13 100644 --- a/crates/registry-breg/src/postgres/external_view_query.rs +++ b/crates/registry-breg/src/postgres/external_view_query.rs @@ -1254,6 +1254,11 @@ mod postgres_tests { "{:?}", key.chars().take(8).collect::() ); + assert_eq!( + crate::external_views::link_key_acceptable(&key), + registry_adapter_core::projection_token_valid(&key), + "the write-time rule is the source contract's selector rule" + ); } for key in ["", "ok"] { let accepted: Option = database From fe249cc47080f2afa38adfe06364e0b01aaba213 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 20:05:26 +0700 Subject: [PATCH 48/64] test: expect a malformed link key to read as no outside value after refresh Signed-off-by: Jeremi Joslin --- .../src/postgres/external_runtime.rs | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index f5bd2fca74..78a8ec90c5 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -3635,7 +3635,7 @@ mod postgres_tests { } #[tokio::test] - async fn invalid_link_keys_are_unavailable_without_failing_the_rest_of_the_refresh_page() { + async fn invalid_link_keys_are_counted_without_failing_the_rest_of_the_refresh_page() { use tracing::instrument::WithSubscriber; let oversized = "\u{1D518}".repeat(300); // 1200 bytes in 300 characters let wide = "\u{E9}".repeat(300); // 600 bytes: over the source token bound @@ -3764,20 +3764,23 @@ mod postgres_tests { .collect(); assert_eq!(stored, ["fifth", "first", "second", "third"]); - // A root whose key cannot be acquired stays uncovered, so its view - // keeps refusing lists and counts until the key is corrected. - let (status, _, problem) = served + // A root whose key cannot be acquired reads as having no outside + // value, so its view keeps serving the other records. + let (status, _, body) = served .send( "/v1/external-views/view/people-link/records?accessProfile=reader&$count=true", reader("reader-principal", &["allowed"]), ) .await; - assert_eq!( - status, - axum::http::StatusCode::SERVICE_UNAVAILABLE, - "{problem}" - ); - assert_eq!(problem["code"], "external_view.incomplete", "{problem}"); + assert_eq!(status, axum::http::StatusCode::OK, "{body}"); + assert_eq!(body["count"], 10, "{body}"); + let available = body["records"] + .as_array() + .unwrap() + .iter() + .filter(|record| record["availability"] == "available") + .count(); + assert_eq!(available, 4, "{body}"); served.fixture.cleanup().await; } From 3c931ed276439ceb9db9b7de1d9d960132e20a72 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 20:14:20 +0700 Subject: [PATCH 49/64] chore(identifiers): regenerate the catalog for the BReg schema changes Signed-off-by: Jeremi Joslin --- products/identifiers/generated/catalog.v1.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/products/identifiers/generated/catalog.v1.json b/products/identifiers/generated/catalog.v1.json index 598c91119c..3d2db6bac2 100644 --- a/products/identifiers/generated/catalog.v1.json +++ b/products/identifiers/generated/catalog.v1.json @@ -4938,11 +4938,11 @@ "description": "Base Registry Engine bregctl explain output JSON Schemas, written by hand and held to bregctl's output by its contract test.", "source": { "path": "products/breg/contracts/explain/ModelExplanation.schema.json", - "sha256": "d559294995dfc1049aad49574d9ca9b185971f5c6201054dc143cee96a3395e7" + "sha256": "e988715404f48672c0c130a1f4fc706ff9e4b86399cc874652221347537fe62d" }, "artifact": { "path": "products/breg/contracts/explain/ModelExplanation.schema.json", - "sha256": "d559294995dfc1049aad49574d9ca9b185971f5c6201054dc143cee96a3395e7", + "sha256": "e988715404f48672c0c130a1f4fc706ff9e4b86399cc874652221347537fe62d", "mediaType": "application/schema+json" } }, @@ -4996,7 +4996,7 @@ }, "artifact": { "path": "products/breg/generated/authoring/registry-project.schema.json", - "sha256": "9d432a58cfcb304ae9116be9f96b2a00f35c87da1e543d3598b2bf9207a8bd36", + "sha256": "f4b28508be761c19821a7c578bd7a71d0656494d8fafcfad0b1a713c514bf40c", "mediaType": "application/schema+json" } }, @@ -5086,7 +5086,7 @@ }, "artifact": { "path": "products/breg/generated/runtime/runtime.schema.json", - "sha256": "29266d2d65e256cd54d27641dca372a18c4380771e09740f56a81fa59289d8bf", + "sha256": "e6015b288d731aa6621d318abb17e40e95ccd988304c28ba9a77f1730acb2402", "mediaType": "application/schema+json" } }, From 698dbdbe524c3a5a94b1e6bce64007a37348f2f9 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 18:57:08 +0700 Subject: [PATCH 50/64] fix: observe projection responses after credential recovery and let no-store win over failures A 401 answered by a successful OAuth credential refresh no longer poisons the acquisition, and a no-store response now yields NoStore even when the status or route failed. Signed-off-by: Jeremi Joslin --- .../registry-adapter-engine/src/projection.rs | 9 +- crates/registry-adapter-engine/src/route.rs | 20 +- .../tests/projection_acquisition.rs | 240 +++++++++++++++++- 3 files changed, 255 insertions(+), 14 deletions(-) diff --git a/crates/registry-adapter-engine/src/projection.rs b/crates/registry-adapter-engine/src/projection.rs index f3d9a9a5c9..8b8c4c15fa 100644 --- a/crates/registry-adapter-engine/src/projection.rs +++ b/crates/registry-adapter-engine/src/projection.rs @@ -392,15 +392,18 @@ impl ProjectionProgram { .ok_or(ProjectionError::Contract)?; let previous_successes = http.successful.load(Ordering::Relaxed); let result = self.program.execute_raw(acquisition, input).await; + // A response that forbids retention outranks every failure: BReg + // drops the cached snapshot for `NoStore` but keeps it for + // `Unavailable`. + if mode == ProjectionMode::Indexed && http.retention_restricted.load(Ordering::Relaxed) { + return Err(ProjectionError::NoStore); + } if http.failed.load(Ordering::Relaxed) || http.successful.load(Ordering::Relaxed) == previous_successes { return Err(ProjectionError::Unavailable); } let value = result.map_err(|_| ProjectionError::Unavailable)?; - if mode == ProjectionMode::Indexed && http.retention_restricted.load(Ordering::Relaxed) { - return Err(ProjectionError::NoStore); - } match serde_json::from_value(value) { Ok(SourceResult::Unavailable {}) => Err(ProjectionError::Unavailable), Ok(result) => Ok(result), diff --git a/crates/registry-adapter-engine/src/route.rs b/crates/registry-adapter-engine/src/route.rs index b5302ce06d..f92e0fb491 100644 --- a/crates/registry-adapter-engine/src/route.rs +++ b/crates/registry-adapter-engine/src/route.rs @@ -3535,14 +3535,6 @@ impl Run<'_, '_> { ) }; self.ensure_deadline()?; - if let Some(projection) = self - .evidence - .and_then(|acquisition| acquisition.projection_http.as_ref()) - { - let successful = matches!(&reply, PhysicalSendReply::Ordinary(Ok(reply)) - if (200..300).contains(&reply.status) && reply.status != 202); - projection.observe_response(successful); - } if let Some(acquisition) = self.evidence { let response_bytes = reply.response_bytes(); if response_bytes != 0 { @@ -3581,6 +3573,18 @@ impl Run<'_, '_> { { action = UpstreamStatusAction::CredentialRejected; } + // A response that triggers a credential refresh is not terminal: + // the retry's response is the one the projection observes. + if action != UpstreamStatusAction::RetryWithFreshCredential { + if let Some(projection) = self + .evidence + .and_then(|acquisition| acquisition.projection_http.as_ref()) + { + let successful = matches!(&reply, PhysicalSendReply::Ordinary(Ok(reply)) + if (200..300).contains(&reply.status) && reply.status != 202); + projection.observe_response(successful); + } + } if action == UpstreamStatusAction::RetryWithFreshCredential { let mut sent = json!({ "endpoint": to.uri.id, diff --git a/crates/registry-adapter-engine/tests/projection_acquisition.rs b/crates/registry-adapter-engine/tests/projection_acquisition.rs index 2c76a39d3a..786478e030 100644 --- a/crates/registry-adapter-engine/tests/projection_acquisition.rs +++ b/crates/registry-adapter-engine/tests/projection_acquisition.rs @@ -21,6 +21,14 @@ impl EvidenceHostHooks for Hooks { } fn operation(retention: &str) -> (AdapterPack, PackArtifactBytes) { + operation_with_authentication(retention, json!({"type":"none"}), 1) +} + +fn operation_with_authentication( + retention: &str, + authentication: Value, + step_count: usize, +) -> (AdapterPack, PackArtifactBytes) { let schema = serde_json::to_vec(&json!({ "type":"object", "additionalProperties":false, @@ -28,12 +36,13 @@ fn operation(retention: &str) -> (AdapterPack, PackArtifactBytes) { })) .unwrap(); let digest = registry_platform_config::sha256_uri(&schema); - let steps = json!([{"to":{"uri":"http:source","parameters":{ + let step = json!({"to":{"uri":"http:source","parameters":{ "httpMethod":"GET", "path":"/people" - }}}]); + }}}); + let steps = json!(vec![step; step_count]); let pack: AdapterPack = serde_json::from_value(json!({ "id":"synthetic","version":"1","testedVersions":[], - "endpoints":[{"id":"source","component":"http","authentication":{"type":"none"},"attemptTimeoutMilliseconds":1000,"maximumResponseBytes":4096}], + "endpoints":[{"id":"source","component":"http","authentication":authentication,"attemptTimeoutMilliseconds":1000,"maximumResponseBytes":4096}], "sourceRoutes":[{"id":"lookup","projection":{ "operation":"exact","operationVersion":"1","authority":"population-register","purpose":"licensing", "scope":"licensing-subjects","selectorScheme":"person-number","recordKeyScheme":"person-number", @@ -505,3 +514,228 @@ async fn an_audit_delay_cannot_start_source_io_after_the_grant_deadline() { assert!(matches!(result, Err(ProjectionError::Unavailable))); exchanges.finish().unwrap(); } + +/// Serves `/token` and `/people` on loopback. Each `/people` request takes +/// the next `(status, extra header lines)` entry; each `/token` request mints +/// a numbered credential. Returns the origin and the count of token requests. +#[cfg(unix)] +async fn oauth_source( + data: Vec<(u16, &'static str)>, +) -> ( + String, + Arc, + tokio::task::JoinHandle<()>, +) { + use std::sync::atomic::{AtomicUsize, Ordering}; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let tokens = Arc::new(AtomicUsize::new(0)); + let observed = Arc::clone(&tokens); + let task = tokio::spawn(async move { + let mut data = data.into_iter(); + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut buffer = Vec::new(); + let mut chunk = [0; 1024]; + let (header_end, content_length) = loop { + let count = socket.read(&mut chunk).await.unwrap(); + assert!(count > 0 && buffer.len() < 16384); + buffer.extend_from_slice(&chunk[..count]); + if let Some(end) = buffer.windows(4).position(|bytes| bytes == b"\r\n\r\n") { + let header = std::str::from_utf8(&buffer[..end]).unwrap(); + let length = header + .lines() + .filter_map(|line| line.split_once(':')) + .find(|(name, _)| name.eq_ignore_ascii_case("content-length")) + .map_or(0, |(_, value)| value.trim().parse::().unwrap()); + break (end + 4, length); + } + }; + while buffer.len() < header_end + content_length { + let count = socket.read(&mut chunk).await.unwrap(); + assert!(count > 0); + buffer.extend_from_slice(&chunk[..count]); + } + let path = std::str::from_utf8(&buffer[..header_end]) + .unwrap() + .split_whitespace() + .nth(1) + .unwrap() + .to_owned(); + let (status, extra, body) = if path == "/token" { + let number = observed.fetch_add(1, Ordering::SeqCst) + 1; + ( + 200, + "", + json!({"access_token":format!("token-{number}"),"token_type":"Bearer","expires_in":300}) + .to_string(), + ) + } else { + let (status, extra) = data.next().expect("an unexpected source request"); + let body = if status == 200 { + json!({"type":"exact","record":{"outcome":"present","key":"person-7","fields":{"familyName":"Lee"}}}) + .to_string() + } else { + "{}".to_owned() + }; + (status, extra, body) + }; + socket + .write_all( + format!( + "HTTP/1.1 {status} Synthetic\r\nContent-Type: application/json\r\n{extra}Content-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + .as_bytes(), + ) + .await + .unwrap(); + } + }); + (origin, tokens, task) +} + +#[cfg(unix)] +async fn oauth_exact( + mode: ProjectionMode, + step_count: usize, + data: Vec<(u16, &'static str)>, +) -> (Result, usize) { + use registry_adapter_core::{AdapterEndpointBindings, EvidenceConnectionPolicy}; + use registry_adapter_engine::evidence::EvidenceRuntimeBindings; + use registry_platform_config::{SecretProvider, SecretResolver}; + use std::os::unix::fs::PermissionsExt; + + let (origin, tokens, source) = oauth_source(data).await; + let root = tempfile::tempdir().unwrap(); + for (name, value) in [("client-id", "client"), ("client-secret", "secret")] { + let path = root.path().join(name); + std::fs::write(&path, value).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap(); + } + let runtime = EvidenceRuntimeBindings::new( + serde_json::from_value::(json!({ + "id":"synthetic-connection", + "endpoints":[{ + "endpoint":"source","baseUrl":origin, + "credential":{ + "type":"oauth2-client-credentials", + "tokenEndpoint":format!("{origin}/token"), + "clientIdRef":"secret:file/client-id", + "clientSecretRef":"secret:file/client-secret", + "credentialPlacement":"form-body", + "maximumCacheSeconds":300 + } + }] + })) + .unwrap(), + SecretResolver::new([SecretProvider::File], root.path()).unwrap(), + ); + let policy: EvidenceConnectionPolicy = serde_json::from_value(json!({ + "endpoint":"source","baseUrl":origin,"clientIdentity":"forbidden", + "authentication":{ + "type":"oauth2-client-credentials", + "tokenEndpoint":format!("{origin}/token"), + "credentialPlacement":"form-body" + } + })) + .unwrap(); + let (pack, artifacts) = operation_with_authentication( + "retain", + json!({"type":"oauth2-client-credentials"}), + step_count, + ); + let program = compile_projection_pack(&pack, "lookup", &artifacts, EvidenceLimits::default()) + .unwrap() + .bind_live(&runtime, "synthetic-connection", &[policy]) + .unwrap(); + let result = program + .acquire( + Arc::new(Hooks), + mode, + ProjectionRequest::Exact { + key: "person-7".to_owned(), + }, + &BTreeSet::from(["familyName".to_owned()]), + ) + .await; + source.abort(); + (result, tokens.load(std::sync::atomic::Ordering::SeqCst)) +} + +#[cfg(unix)] +#[tokio::test] +async fn a_credential_refresh_after_a_401_does_not_fail_the_acquisition() { + let (result, tokens) = oauth_exact(ProjectionMode::Live, 1, vec![(401, ""), (200, "")]).await; + assert!( + matches!( + result, + Ok(ProjectionResult::Exact { + record: ProjectionRecord::Present { .. } + }) + ), + "{result:?}" + ); + assert_eq!(tokens, 2, "the 401 triggers exactly one refresh"); +} + +#[cfg(unix)] +#[tokio::test] +async fn a_401_that_survives_the_credential_refresh_still_fails() { + let (result, tokens) = oauth_exact(ProjectionMode::Live, 1, vec![(401, ""), (401, "")]).await; + assert!(result.is_err(), "{result:?}"); + assert_eq!(tokens, 2); +} + +#[cfg(unix)] +#[tokio::test] +async fn a_non_401_source_failure_still_fails_without_a_refresh() { + let (result, tokens) = oauth_exact(ProjectionMode::Live, 1, vec![(503, "")]).await; + assert!( + matches!(result, Err(ProjectionError::Unavailable)), + "{result:?}" + ); + assert_eq!(tokens, 1); +} + +#[cfg(unix)] +#[tokio::test] +async fn no_store_wins_over_an_unsuccessful_source_status() { + let (result, _) = oauth_exact( + ProjectionMode::Indexed, + 1, + vec![(503, "Cache-Control: no-store\r\n")], + ) + .await; + assert!( + matches!(result, Err(ProjectionError::NoStore)), + "{result:?}" + ); +} + +#[cfg(unix)] +#[tokio::test] +async fn no_store_wins_over_a_route_failure_after_the_header_arrived() { + let (result, _) = oauth_exact( + ProjectionMode::Indexed, + 2, + vec![(200, "Cache-Control: no-store\r\n"), (503, "")], + ) + .await; + assert!( + matches!(result, Err(ProjectionError::NoStore)), + "{result:?}" + ); +} + +#[cfg(unix)] +#[tokio::test] +async fn an_unsuccessful_source_status_without_no_store_stays_unavailable() { + let (result, _) = oauth_exact(ProjectionMode::Indexed, 1, vec![(503, "")]).await; + assert!( + matches!(result, Err(ProjectionError::Unavailable)), + "{result:?}" + ); +} From 61ac119635b8f1329a27c57d57ae6edef8cfd50c Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 18:57:47 +0700 Subject: [PATCH 51/64] fix: bind recovery names before use and call the fixture path helper in the load-test runners Resolves the CodeQL uninitialized-local and unused-import alerts without changing what the runners do. Signed-off-by: Jeremi Joslin --- products/breg/loadtest/external-views/fixture_path.py | 11 +++++++++-- products/breg/loadtest/external-views/recovery.py | 3 ++- .../breg/loadtest/external-views/run_candidate.py | 8 +++++--- .../breg/loadtest/external-views/scale_exercise.py | 3 ++- products/breg/loadtest/external-views/test_runners.py | 3 ++- products/breg/loadtest/external-views/test_scale.py | 3 ++- 6 files changed, 22 insertions(+), 9 deletions(-) diff --git a/products/breg/loadtest/external-views/fixture_path.py b/products/breg/loadtest/external-views/fixture_path.py index 994892067b..d0f6f01f74 100644 --- a/products/breg/loadtest/external-views/fixture_path.py +++ b/products/breg/loadtest/external-views/fixture_path.py @@ -6,5 +6,12 @@ from pathlib import Path EXAMPLE = Path(__file__).resolve().parents[2] / "examples/external-views" -if str(EXAMPLE) not in sys.path: - sys.path.insert(0, str(EXAMPLE)) + + +def ensure_importable(): + """Put the example on `sys.path` once and return its directory.""" + if not EXAMPLE.is_dir(): + raise RuntimeError(f"the external views example is missing: {EXAMPLE}") + if str(EXAMPLE) not in sys.path: + sys.path.insert(0, str(EXAMPLE)) + return EXAMPLE diff --git a/products/breg/loadtest/external-views/recovery.py b/products/breg/loadtest/external-views/recovery.py index d96ec3d212..8dd66c2ddd 100644 --- a/products/breg/loadtest/external-views/recovery.py +++ b/products/breg/loadtest/external-views/recovery.py @@ -7,7 +7,8 @@ import re import subprocess -import fixture_path # noqa: F401 +import fixture_path +fixture_path.ensure_importable() from journey import JourneyError, expect, records, request PROJECTION = "holdings-indexed" diff --git a/products/breg/loadtest/external-views/run_candidate.py b/products/breg/loadtest/external-views/run_candidate.py index 0f00d8bbe2..d1afd02e31 100644 --- a/products/breg/loadtest/external-views/run_candidate.py +++ b/products/breg/loadtest/external-views/run_candidate.py @@ -22,7 +22,9 @@ from urllib.parse import unquote, urlsplit from uuid import UUID -import fixture_path # noqa: F401 +import fixture_path +fixture_path.ensure_importable() +from recovery import ScopeRecovery from journey import HTTP, Journey, JourneyError, expect, request from synthetic_source import Fixture, handler, runtime_fragment @@ -288,8 +290,6 @@ def run(options): "the scale exercise requires an existing PostgreSQL plan log") else: scale = None - if options.recovery: - from recovery import ScopeRecovery work = options.work_dir.resolve() work.mkdir(mode=0o700) os.umask(0o077) @@ -328,6 +328,7 @@ def run(options): source_key = (keys / "source-key").read_text(encoding="ascii") control_key = (keys / "control-key").read_text(encoding="ascii") fixture = scale.ScaleFixture() if scale else Fixture() + recovery = None with ThreadingHTTPServer(("127.0.0.1", 0), handler(fixture, source_key, control_key)) as source: thread = Thread(target=source.serve_forever, kwargs={"poll_interval": 0.05}) thread.start() @@ -374,6 +375,7 @@ def run(options): recovery = ScopeRecovery(journey, fixture, work, facts, ctl, runtime, write_json) recovery.withdraw_and_rotate() if options.recovery: + expect(recovery is not None, "scope recovery did not run before the restart") print("Restarting and rebuilding the rotated holdings scope", flush=True) with serving(breg, runtime, work, "breg-restarted.log", origin): recovery.verify_rebuild() diff --git a/products/breg/loadtest/external-views/scale_exercise.py b/products/breg/loadtest/external-views/scale_exercise.py index f1adeb318e..8386bb952e 100644 --- a/products/breg/loadtest/external-views/scale_exercise.py +++ b/products/breg/loadtest/external-views/scale_exercise.py @@ -13,7 +13,8 @@ import time from urllib.parse import unquote, urlsplit -import fixture_path # noqa: F401 +import fixture_path +fixture_path.ensure_importable() from journey import JourneyError, expect, fixture_credential, records, request from synthetic_source import Fixture, digest, require diff --git a/products/breg/loadtest/external-views/test_runners.py b/products/breg/loadtest/external-views/test_runners.py index 90399d3db3..c91db42aec 100644 --- a/products/breg/loadtest/external-views/test_runners.py +++ b/products/breg/loadtest/external-views/test_runners.py @@ -7,7 +7,8 @@ import tempfile import unittest -import fixture_path # noqa: F401 +import fixture_path +fixture_path.ensure_importable() from journey import JourneyError from recovery import operator_report from run_candidate import URL_FILES, database_urls, stage_project, verify_source_facts diff --git a/products/breg/loadtest/external-views/test_scale.py b/products/breg/loadtest/external-views/test_scale.py index 2f15535b1e..1368449de5 100644 --- a/products/breg/loadtest/external-views/test_scale.py +++ b/products/breg/loadtest/external-views/test_scale.py @@ -5,7 +5,8 @@ import unittest -import fixture_path # noqa: F401 +import fixture_path +fixture_path.ensure_importable() from scale_exercise import ( ABSENT_KEYS, BATCH_KEYS, DENIED_ROOTS, ELIGIBLE_ROOTS, HOLDER_KEYS, MATCH_KEYS, PERSON_REQUEST_BUDGET, REFRESH_PAGE_SIZE, ScaleFixture, From 6300b0e171e1b7bd136d36303199ad0ee40167ea Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 19:13:27 +0700 Subject: [PATCH 52/64] fix: honour no-store only on successful responses and fail the projection on credential failures Signed-off-by: Jeremi Joslin --- crates/registry-adapter-engine/src/egress.rs | 11 +- .../registry-adapter-engine/src/projection.rs | 6 + crates/registry-adapter-engine/src/route.rs | 24 +- .../tests/projection_acquisition.rs | 213 +++++++++++++----- 4 files changed, 198 insertions(+), 56 deletions(-) diff --git a/crates/registry-adapter-engine/src/egress.rs b/crates/registry-adapter-engine/src/egress.rs index 90f46130a4..272657aec6 100644 --- a/crates/registry-adapter-engine/src/egress.rs +++ b/crates/registry-adapter-engine/src/egress.rs @@ -1478,7 +1478,12 @@ impl EndpointEgress { .map_err(destination_send_outcome)?; let status = reply.status().as_u16(); - if let Some(projection) = projection { + // Only a successful data response can forbid retention: a no-store on + // an error, redirect or credential-refresh response describes that + // response alone and must not purge the previous snapshot. + if let Some(projection) = + projection.filter(|_| crate::projection::ProjectionHttpState::status_successful(status)) + { // Read the raw transport headers before the pack's allowlist or // status handling can discard the source's retention restriction. match reply @@ -1898,7 +1903,9 @@ impl EndpointEgress { .map_err(EgressError::Refused)?; let status = u16::try_from(response.status.get()) .expect("a recorded status is bounded to an HTTP status"); - if let Some(projection) = projection { + if let Some(projection) = + projection.filter(|_| crate::projection::ProjectionHttpState::status_successful(status)) + { projection.observe_cache_control( response .headers diff --git a/crates/registry-adapter-engine/src/projection.rs b/crates/registry-adapter-engine/src/projection.rs index 8b8c4c15fa..4cf854bb25 100644 --- a/crates/registry-adapter-engine/src/projection.rs +++ b/crates/registry-adapter-engine/src/projection.rs @@ -80,6 +80,12 @@ impl ProjectionHttpState { } } + /// The status test shared by success counting and no-store handling: a + /// 2xx that is not a 202 acceptance carries data. + pub(crate) fn status_successful(status: u16) -> bool { + (200..300).contains(&status) && status != 202 + } + pub(crate) fn observe_response(&self, successful: bool) { if successful { self.successful.fetch_add(1, Ordering::Relaxed); diff --git a/crates/registry-adapter-engine/src/route.rs b/crates/registry-adapter-engine/src/route.rs index f92e0fb491..19566fe8fb 100644 --- a/crates/registry-adapter-engine/src/route.rs +++ b/crates/registry-adapter-engine/src/route.rs @@ -2843,6 +2843,18 @@ impl Run<'_, '_> { } } + /// A credential failure in a projection acquisition is terminal for the + /// projection even when a route step catches the resulting failure, so a + /// partial result is never released. + fn observe_credential_failure(&self) { + if let Some(projection) = self + .evidence + .and_then(|acquisition| acquisition.projection_http.as_ref()) + { + projection.observe_response(false); + } + } + /// Refuse work that begins at or after the route-wide deadline. The /// outer timeout remains the final guard for an in-flight operation. pub(crate) fn ensure_deadline(&self) -> Result<(), RouteFailure> { @@ -3265,7 +3277,10 @@ impl Run<'_, '_> { .map_err(|_| RouteFailure::of(ErrorClass::BudgetExhausted))?; // The send is announced only once its credential is in hand; a // credential failure never reads as a send that began. - let mut credential = authorized.map_err(|error| auth_failure(&error, &auth_events))?; + let mut credential = authorized.map_err(|error| { + self.observe_credential_failure(); + auth_failure(&error, &auth_events) + })?; let headers = self.exchange.message.string_headers(); let system_traceparent = self.exchange.system_traceparent().cloned(); let source_idempotency_key = match resume_send { @@ -3581,7 +3596,7 @@ impl Run<'_, '_> { .and_then(|acquisition| acquisition.projection_http.as_ref()) { let successful = matches!(&reply, PhysicalSendReply::Ordinary(Ok(reply)) - if (200..300).contains(&reply.status) && reply.status != 202); + if crate::projection::ProjectionHttpState::status_successful(reply.status)); projection.observe_response(successful); } } @@ -3683,7 +3698,10 @@ impl Run<'_, '_> { }; credential = refreshed .map_err(|_| RouteFailure::of(ErrorClass::BudgetExhausted))? - .map_err(|error| auth_failure(&error, &auth_events))?; + .map_err(|error| { + self.observe_credential_failure(); + auth_failure(&error, &auth_events) + })?; continue; } if action == UpstreamStatusAction::CredentialRejected { diff --git a/crates/registry-adapter-engine/tests/projection_acquisition.rs b/crates/registry-adapter-engine/tests/projection_acquisition.rs index 786478e030..dd2e1abc3d 100644 --- a/crates/registry-adapter-engine/tests/projection_acquisition.rs +++ b/crates/registry-adapter-engine/tests/projection_acquisition.rs @@ -28,6 +28,17 @@ fn operation_with_authentication( retention: &str, authentication: Value, step_count: usize, +) -> (AdapterPack, PackArtifactBytes) { + let step = json!({"to":{"uri":"http:source","parameters":{ + "httpMethod":"GET", "path":"/people" + }}}); + operation_with_steps(retention, authentication, json!(vec![step; step_count])) +} + +fn operation_with_steps( + retention: &str, + authentication: Value, + steps: Value, ) -> (AdapterPack, PackArtifactBytes) { let schema = serde_json::to_vec(&json!({ "type":"object", @@ -36,10 +47,6 @@ fn operation_with_authentication( })) .unwrap(); let digest = registry_platform_config::sha256_uri(&schema); - let step = json!({"to":{"uri":"http:source","parameters":{ - "httpMethod":"GET", "path":"/people" - }}}); - let steps = json!(vec![step; step_count]); let pack: AdapterPack = serde_json::from_value(json!({ "id":"synthetic","version":"1","testedVersions":[], "endpoints":[{"id":"source","component":"http","authentication":authentication,"attemptTimeoutMilliseconds":1000,"maximumResponseBytes":4096}], @@ -515,26 +522,33 @@ async fn an_audit_delay_cannot_start_source_io_after_the_grant_deadline() { exchanges.finish().unwrap(); } +/// Request counts observed by the loopback source. +#[cfg(unix)] +#[derive(Default)] +struct Served { + tokens: std::sync::atomic::AtomicUsize, + data: std::sync::atomic::AtomicUsize, +} + /// Serves `/token` and `/people` on loopback. Each `/people` request takes -/// the next `(status, extra header lines)` entry; each `/token` request mints -/// a numbered credential. Returns the origin and the count of token requests. +/// the next `(status, extra header lines)` entry; each `/token` request takes +/// the next status from `token_statuses` (200 once it is exhausted) and mints +/// a numbered credential on 200. Returns the origin and the request counts. #[cfg(unix)] async fn oauth_source( data: Vec<(u16, &'static str)>, -) -> ( - String, - Arc, - tokio::task::JoinHandle<()>, -) { - use std::sync::atomic::{AtomicUsize, Ordering}; + token_statuses: Vec, +) -> (String, Arc, tokio::task::JoinHandle<()>) { + use std::sync::atomic::Ordering; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let origin = format!("http://{}", listener.local_addr().unwrap()); - let tokens = Arc::new(AtomicUsize::new(0)); - let observed = Arc::clone(&tokens); + let served = Arc::new(Served::default()); + let observed = Arc::clone(&served); let task = tokio::spawn(async move { let mut data = data.into_iter(); + let mut token_statuses = token_statuses.into_iter(); loop { let (mut socket, _) = listener.accept().await.unwrap(); let mut buffer = Vec::new(); @@ -565,14 +579,18 @@ async fn oauth_source( .unwrap() .to_owned(); let (status, extra, body) = if path == "/token" { - let number = observed.fetch_add(1, Ordering::SeqCst) + 1; - ( - 200, - "", - json!({"access_token":format!("token-{number}"),"token_type":"Bearer","expires_in":300}) - .to_string(), - ) + let number = observed.tokens.fetch_add(1, Ordering::SeqCst) + 1; + match token_statuses.next().unwrap_or(200) { + 200 => ( + 200, + "", + json!({"access_token":format!("token-{number}"),"token_type":"Bearer","expires_in":300}) + .to_string(), + ), + status => (status, "", "{}".to_owned()), + } } else { + observed.data.fetch_add(1, Ordering::SeqCst); let (status, extra) = data.next().expect("an unexpected source request"); let body = if status == 200 { json!({"type":"exact","record":{"outcome":"present","key":"person-7","fields":{"familyName":"Lee"}}}) @@ -594,7 +612,7 @@ async fn oauth_source( .unwrap(); } }); - (origin, tokens, task) + (origin, served, task) } #[cfg(unix)] @@ -602,13 +620,26 @@ async fn oauth_exact( mode: ProjectionMode, step_count: usize, data: Vec<(u16, &'static str)>, -) -> (Result, usize) { +) -> (Result, Arc) { + let step = json!({"to":{"uri":"http:source","parameters":{ + "httpMethod":"GET", "path":"/people" + }}}); + oauth_exact_steps(mode, json!(vec![step; step_count]), data, vec![]).await +} + +#[cfg(unix)] +async fn oauth_exact_steps( + mode: ProjectionMode, + steps: Value, + data: Vec<(u16, &'static str)>, + token_statuses: Vec, +) -> (Result, Arc) { use registry_adapter_core::{AdapterEndpointBindings, EvidenceConnectionPolicy}; use registry_adapter_engine::evidence::EvidenceRuntimeBindings; use registry_platform_config::{SecretProvider, SecretResolver}; use std::os::unix::fs::PermissionsExt; - let (origin, tokens, source) = oauth_source(data).await; + let (origin, served, source) = oauth_source(data, token_statuses).await; let root = tempfile::tempdir().unwrap(); for (name, value) in [("client-id", "client"), ("client-secret", "secret")] { let path = root.path().join(name); @@ -642,11 +673,8 @@ async fn oauth_exact( } })) .unwrap(); - let (pack, artifacts) = operation_with_authentication( - "retain", - json!({"type":"oauth2-client-credentials"}), - step_count, - ); + let (pack, artifacts) = + operation_with_steps("retain", json!({"type":"oauth2-client-credentials"}), steps); let program = compile_projection_pack(&pack, "lookup", &artifacts, EvidenceLimits::default()) .unwrap() .bind_live(&runtime, "synthetic-connection", &[policy]) @@ -662,63 +690,128 @@ async fn oauth_exact( ) .await; source.abort(); - (result, tokens.load(std::sync::atomic::Ordering::SeqCst)) + (result, served) +} + +#[cfg(unix)] +fn count(counter: &std::sync::atomic::AtomicUsize) -> usize { + counter.load(std::sync::atomic::Ordering::SeqCst) +} + +#[cfg(unix)] +fn is_present(result: &Result) -> bool { + matches!( + result, + Ok(ProjectionResult::Exact { + record: ProjectionRecord::Present { .. } + }) + ) } #[cfg(unix)] #[tokio::test] async fn a_credential_refresh_after_a_401_does_not_fail_the_acquisition() { - let (result, tokens) = oauth_exact(ProjectionMode::Live, 1, vec![(401, ""), (200, "")]).await; - assert!( - matches!( - result, - Ok(ProjectionResult::Exact { - record: ProjectionRecord::Present { .. } - }) - ), - "{result:?}" + let (result, served) = oauth_exact(ProjectionMode::Live, 1, vec![(401, ""), (200, "")]).await; + assert!(is_present(&result), "{result:?}"); + assert_eq!( + count(&served.tokens), + 2, + "the 401 triggers exactly one refresh" ); - assert_eq!(tokens, 2, "the 401 triggers exactly one refresh"); + assert_eq!(count(&served.data), 2); } #[cfg(unix)] #[tokio::test] async fn a_401_that_survives_the_credential_refresh_still_fails() { - let (result, tokens) = oauth_exact(ProjectionMode::Live, 1, vec![(401, ""), (401, "")]).await; - assert!(result.is_err(), "{result:?}"); - assert_eq!(tokens, 2); + let (result, served) = oauth_exact(ProjectionMode::Live, 1, vec![(401, ""), (401, "")]).await; + assert!( + matches!(result, Err(ProjectionError::Unavailable)), + "{result:?}" + ); + assert_eq!(count(&served.tokens), 2); + assert_eq!(count(&served.data), 2); } #[cfg(unix)] #[tokio::test] async fn a_non_401_source_failure_still_fails_without_a_refresh() { - let (result, tokens) = oauth_exact(ProjectionMode::Live, 1, vec![(503, "")]).await; + let (result, served) = oauth_exact(ProjectionMode::Live, 1, vec![(503, "")]).await; + assert!( + matches!(result, Err(ProjectionError::Unavailable)), + "{result:?}" + ); + assert_eq!(count(&served.tokens), 1); + assert_eq!(count(&served.data), 1); +} + +#[cfg(unix)] +fn enrich_after_a_primary_send() -> Value { + json!([ + {"to":{"uri":"http:source","parameters":{"httpMethod":"GET","path":"/people"}}}, + {"enrich":{ + "uri":"http:source", + "aggregateOnException":true, + "parameters":{"httpMethod":"GET","path":"/people"} + }} + ]) +} + +#[cfg(unix)] +#[tokio::test] +async fn a_caught_token_endpoint_failure_during_a_refresh_fails_the_acquisition() { + // The primary send succeeds; the enrich gets a 401 and the token refresh + // then fails. The route catches that failure, so only the projection's + // own record of it keeps the partial result from being released. + let (result, served) = oauth_exact_steps( + ProjectionMode::Indexed, + enrich_after_a_primary_send(), + vec![(200, ""), (401, "")], + vec![200, 503], + ) + .await; assert!( matches!(result, Err(ProjectionError::Unavailable)), "{result:?}" ); - assert_eq!(tokens, 1); + assert_eq!(count(&served.tokens), 2); + assert_eq!(count(&served.data), 2); } #[cfg(unix)] #[tokio::test] -async fn no_store_wins_over_an_unsuccessful_source_status() { - let (result, _) = oauth_exact( +async fn an_error_response_that_marks_itself_no_store_does_not_purge_the_snapshot() { + let (result, served) = oauth_exact( ProjectionMode::Indexed, 1, vec![(503, "Cache-Control: no-store\r\n")], ) .await; assert!( - matches!(result, Err(ProjectionError::NoStore)), + matches!(result, Err(ProjectionError::Unavailable)), "{result:?}" ); + assert_eq!(count(&served.data), 1); } #[cfg(unix)] #[tokio::test] -async fn no_store_wins_over_a_route_failure_after_the_header_arrived() { - let (result, _) = oauth_exact( +async fn a_no_store_401_that_is_refreshed_does_not_restrict_the_clean_retry() { + let (result, served) = oauth_exact( + ProjectionMode::Indexed, + 1, + vec![(401, "Cache-Control: no-store\r\n"), (200, "")], + ) + .await; + assert!(is_present(&result), "{result:?}"); + assert_eq!(count(&served.tokens), 2); + assert_eq!(count(&served.data), 2); +} + +#[cfg(unix)] +#[tokio::test] +async fn no_store_on_a_successful_response_wins_over_a_later_route_failure() { + let (result, served) = oauth_exact( ProjectionMode::Indexed, 2, vec![(200, "Cache-Control: no-store\r\n"), (503, "")], @@ -728,14 +821,32 @@ async fn no_store_wins_over_a_route_failure_after_the_header_arrived() { matches!(result, Err(ProjectionError::NoStore)), "{result:?}" ); + assert_eq!(count(&served.data), 2, "the failing second step ran"); +} + +#[cfg(unix)] +#[tokio::test] +async fn an_unreadable_cache_control_on_a_successful_response_fails_closed() { + let (result, served) = oauth_exact( + ProjectionMode::Indexed, + 1, + vec![(200, "Cache-Control: caf\u{e9}\r\n")], + ) + .await; + assert!( + matches!(result, Err(ProjectionError::NoStore)), + "{result:?}" + ); + assert_eq!(count(&served.data), 1); } #[cfg(unix)] #[tokio::test] async fn an_unsuccessful_source_status_without_no_store_stays_unavailable() { - let (result, _) = oauth_exact(ProjectionMode::Indexed, 1, vec![(503, "")]).await; + let (result, served) = oauth_exact(ProjectionMode::Indexed, 1, vec![(503, "")]).await; assert!( matches!(result, Err(ProjectionError::Unavailable)), "{result:?}" ); + assert_eq!(count(&served.data), 1); } From 4a33bd8468df0df0cf4d2f23443fb2fd21797633 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 19:13:27 +0700 Subject: [PATCH 53/64] chore: tidy the external views load-test helpers Signed-off-by: Jeremi Joslin --- products/breg/loadtest/external-views/fixture_path.py | 3 +-- products/breg/loadtest/external-views/run_candidate.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/products/breg/loadtest/external-views/fixture_path.py b/products/breg/loadtest/external-views/fixture_path.py index d0f6f01f74..241141ad46 100644 --- a/products/breg/loadtest/external-views/fixture_path.py +++ b/products/breg/loadtest/external-views/fixture_path.py @@ -9,9 +9,8 @@ def ensure_importable(): - """Put the example on `sys.path` once and return its directory.""" + """Put the example on `sys.path` once.""" if not EXAMPLE.is_dir(): raise RuntimeError(f"the external views example is missing: {EXAMPLE}") if str(EXAMPLE) not in sys.path: sys.path.insert(0, str(EXAMPLE)) - return EXAMPLE diff --git a/products/breg/loadtest/external-views/run_candidate.py b/products/breg/loadtest/external-views/run_candidate.py index d1afd02e31..11bbb3cf16 100644 --- a/products/breg/loadtest/external-views/run_candidate.py +++ b/products/breg/loadtest/external-views/run_candidate.py @@ -24,8 +24,8 @@ import fixture_path fixture_path.ensure_importable() -from recovery import ScopeRecovery from journey import HTTP, Journey, JourneyError, expect, request +from recovery import ScopeRecovery from synthetic_source import Fixture, handler, runtime_fragment PROJECT = Path(__file__).resolve().parents[2] / "examples/external-views" From 48d35a3d4eaadba7ceb69d99d4c598ebc5b4186f Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 20:15:29 +0700 Subject: [PATCH 54/64] docs: say only a successful source response's no-store removes the kept snapshot Signed-off-by: Jeremi Joslin --- products/breg/external-projection-storage.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/products/breg/external-projection-storage.md b/products/breg/external-projection-storage.md index d502fefdc6..03ea57beae 100644 --- a/products/breg/external-projection-storage.md +++ b/products/breg/external-projection-storage.md @@ -23,11 +23,11 @@ Write access to a link's `fromField` chooses which outside subject BReg fetches. ## Freshness and purge -`cacheTtlSeconds` is the one cache lifetime of a source result. A deployment binding may set its own `cacheTtlSeconds` to shorten, but never extend, the authored value. The lifetime is measured from the original source observation time, not from refresh completion. The effective expiry is the earlier of that lifetime and the deployment binding's source grant cutoff, `grantValidUntil`. From the source response, BReg honours only `no-store`; it reads no other expiry or storage permission there, so set `grantValidUntil` no later than the source permits storage. There is no retention period after expiry: an expired value is purged. +`cacheTtlSeconds` is the one cache lifetime of a source result. A deployment binding may set its own `cacheTtlSeconds` to shorten, but never extend, the authored value. The lifetime is measured from the original source observation time, not from refresh completion. The effective expiry is the earlier of that lifetime and the deployment binding's source grant cutoff, `grantValidUntil`. From a successful source response, BReg honours only `no-store`; it reads no other expiry or storage permission there, so set `grantValidUntil` no later than the source permits storage. There is no retention period after expiry: an expired value is purged. A captured snapshot stores the observation time and a keyed revision reference when the source supplies a revision. The reference is scoped to the bound source. -Replaying an accepted result keeps its original expiry. A refresh that fails because the source is unavailable, returns an incomplete result, or cannot be acquired under the current binding keeps the previous snapshot of either kind until its own expiry, without extending its lifetime. Explicit grant denial or no-store removes the previous snapshot, making coverage unproven, fenced against newer publications. A source response marked `no-store` cannot enter the indexed store. If it supersedes an earlier retained result for the same subject, the guarded retention-refusal path removes that result; an older refresh cannot erase a newer publication. +Replaying an accepted result keeps its original expiry. A refresh that fails because the source is unavailable, returns an incomplete result, or cannot be acquired under the current binding keeps the previous snapshot of either kind until its own expiry, without extending its lifetime. Explicit grant denial, or a successful source response marked `no-store`, removes the previous snapshot, making coverage unproven, fenced against newer publications. A `no-store` on an error, a redirect or the authentication challenge that starts a credential refresh changes nothing: that refresh fails as unavailable and keeps the previous snapshot. A successful source response marked `no-store` cannot enter the indexed store, even when a later step of the same acquisition fails. If it supersedes an earlier retained result for the same subject, the guarded retention-refusal path removes that result; an older refresh cannot erase a newer publication. Reads check expiry independently of the refresh job. Indexed predicates and counts refuse incomplete coverage before selecting matching rows. Each page reads coverage, rows and any count from one statement snapshot, so a count always matches complete coverage at that moment. A refresh records each key in its own short transaction, so a read waits for at most one key's publication and can show new values for some keys of a page in flight and previous values for the rest. A `nextCursor` stays valid while refreshes publish or fail individual keys; the next page reads the values current when it runs. A row whose ordering value changed between pages can appear twice or not at all. A cursor is encrypted and authenticated with the deployment's cursor key and bound to its caller, query and projection generation, so its contents (including the sort value it resumes from) cannot be altered or read by the client. A sort value resumes from up to 6144 bytes; a stored value, or a cursor whose key, query and sort value together exceed the 8 KiB cursor payload, ends the listing with `source.unavailable`. Rotating, withdrawing or resetting the projection invalidates outstanding cursors. A complete empty related collection is covered; a partial collection, source failure, or protected-existence response is not evidence of emptiness or absence. From ff1a7aca95d2343b46fa4c352c14fe0f6d1764df Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 20:39:15 +0700 Subject: [PATCH 55/64] fix: refuse a malformed link key when a proposal is prepared or an action builds its record Signed-off-by: Jeremi Joslin --- crates/registry-breg/src/mutation.rs | 13 +++- crates/registry-breg/src/mutation/action.rs | 38 +++++++++- crates/registry-breg/src/proposal_prepare.rs | 10 ++- .../src/proposal_prepare/tests.rs | 73 +++++++++++++++++++ 4 files changed, 127 insertions(+), 7 deletions(-) diff --git a/crates/registry-breg/src/mutation.rs b/crates/registry-breg/src/mutation.rs index e1e5fc5d34..3cdd2a9350 100644 --- a/crates/registry-breg/src/mutation.rs +++ b/crates/registry-breg/src/mutation.rs @@ -4146,14 +4146,21 @@ fn written_sql_value( field: &CompiledField, value: &Value, ) -> Result, MutationError> { - if entity.external_link_fields.contains(&field.id) - && matches!(value, Value::String(key) if !crate::external_views::link_key_acceptable(key)) - { + if link_key_refused(entity, &field.id, value) { return Err(MutationError::InvalidRequest); } sql_value(value, &field.field_type) } +/// Whether `value` written to `field_id` is a link key an external view could +/// not send to its source. Proposal preparation and action input apply it +/// beside the type check, so a malformed key is refused before anything is +/// stored rather than at apply time. +pub(crate) fn link_key_refused(entity: &CompiledEntity, field_id: &str, value: &Value) -> bool { + entity.external_link_fields.contains(field_id) + && matches!(value, Value::String(key) if !crate::external_views::link_key_acceptable(key)) +} + pub(crate) fn sql_value( value: &Value, field_type: &FieldTypeSource, diff --git a/crates/registry-breg/src/mutation/action.rs b/crates/registry-breg/src/mutation/action.rs index bb8b514666..a60e57029a 100644 --- a/crates/registry-breg/src/mutation/action.rs +++ b/crates/registry-breg/src/mutation/action.rs @@ -3379,7 +3379,9 @@ fn action_effect_document( Value::String(row.record_id.clone()) } }; - if !validate_field_value(FieldValue::Json(&value), &field_source.field_type) { + if !validate_field_value(FieldValue::Json(&value), &field_source.field_type) + || link_key_refused(entity, field, &value) + { return Err(MutationError::InvalidRequest); } data.insert(field.clone(), value); @@ -4315,6 +4317,40 @@ mod tests { )); } + #[test] + fn a_malformed_link_key_is_refused_when_an_action_builds_its_effect_document() { + let project = crate::contract::parse_project_yaml(include_bytes!( + "../../tests/fixtures/fixed-optional-action-input.yaml" + )) + .unwrap(); + let registry = crate::compiler::compile_project( + &project, + &[], + crate::compiler::CompileProfile::Authoring, + ) + .unwrap(); + let action = ®istry.actions().actions[0]; + let unlinked = ®istry.entities()["entry"]; + let mut linked = unlinked.clone(); + linked.external_link_fields.insert("label".to_owned()); + let effect = &action.effects[0]; + for malformed in [" padded", "trailing ", "bell\u{7}key"] { + let input = Map::from_iter([("label".to_owned(), json!(malformed))]); + assert!( + matches!( + action_effect_document(action, &linked, effect, &input, &BTreeMap::new(), None), + Err(MutationError::InvalidRequest) + ), + "{malformed:?} is refused on the linked field" + ); + action_effect_document(action, unlinked, effect, &input, &BTreeMap::new(), None) + .expect("the same value is accepted on an unlinked field"); + } + let input = Map::from_iter([("label".to_owned(), json!("key-1"))]); + action_effect_document(action, &linked, effect, &input, &BTreeMap::new(), None) + .expect("a well-formed key is accepted on the linked field"); + } + #[test] fn absent_optional_fixed_inputs_leave_their_fields_out_of_create_and_patch_documents() { let project = crate::contract::parse_project_yaml(include_bytes!( diff --git a/crates/registry-breg/src/proposal_prepare.rs b/crates/registry-breg/src/proposal_prepare.rs index db90821658..db427b2c5e 100644 --- a/crates/registry-breg/src/proposal_prepare.rs +++ b/crates/registry-breg/src/proposal_prepare.rs @@ -8,7 +8,7 @@ use crate::model::{ CompiledAction, CompiledActionExecution, CompiledPlanEffect, CompiledPlanFunctionKind, CompiledPlanMutation, CompiledPlanTargetBinding, CompiledPlanValue, CompiledRegistry, }; -use crate::mutation::MutationError; +use crate::mutation::{link_key_refused, MutationError}; use crate::proposal_workflow::{ ContractFingerprint, EffectId, EntityId, FieldId, FieldValue, FrozenApplicationPreconditions, FrozenGuardTargetSnapshot, FrozenInputTarget, FrozenPlannerKind, FrozenPlanningBinding, @@ -209,7 +209,9 @@ pub(crate) fn materialize_create_fields( CompiledPlanMutation::Clear { .. } if !field.required => Value::Null, CompiledPlanMutation::Clear { .. } => return Err(MutationError::InvalidRequest), }; - if !validate_field_value(DataFieldValue::Json(&value), &field.field_type) { + if !validate_field_value(DataFieldValue::Json(&value), &field.field_type) + || link_key_refused(entity, field_id, &value) + { return Err(MutationError::InvalidRequest); } fields.insert(field_id.clone(), value); @@ -406,7 +408,9 @@ pub(crate) fn prepare_with_reserved_create_fields( let change = match mutation { CompiledPlanMutation::Set { value, .. } => { let value = resolve_plan_value(input, resolved, value)?; - if !validate_field_value(DataFieldValue::Json(&value), &compiled.field_type) { + if !validate_field_value(DataFieldValue::Json(&value), &compiled.field_type) + || link_key_refused(entity, field, &value) + { return Err(MutationError::InvalidRequest); } snapshot.after.insert(field.clone(), value.clone()); diff --git a/crates/registry-breg/src/proposal_prepare/tests.rs b/crates/registry-breg/src/proposal_prepare/tests.rs index f2a9890507..76c67ea4cb 100644 --- a/crates/registry-breg/src/proposal_prepare/tests.rs +++ b/crates/registry-breg/src/proposal_prepare/tests.rs @@ -583,3 +583,76 @@ fn rhai_plan_refuses_authority_ceiling_escape_before_target_locks() { Err(MutationError::InvalidRequest) )); } + +/// Links `target.first` to an external subject, so a value written there must +/// be a key the view can send to its source. +fn link_first(source: &mut Value) { + source["externalRecordTypes"] = json!([{ + "id":"external-subject","authority":"urn:example:source","identifierScheme":"urn:example:subject-key", + "fields":[{"id":"label","type":"string","classification":"internal"}] + }]); + source["externalProjections"] = json!([{ + "id":"approved-subjects","recordType":"external-subject","operation":"subjects-by-key","operationVersion":"1", + "purpose":"local-consultation","scope":"approved-subjects","selectorScheme":"urn:example:subject-key", + "mode":"indexed","cardinality":"one","cacheTtlSeconds":600,"fields":["label"] + }]); + source["externalViews"] = json!([{ + "id":"target-with-subject","recordType":"target", + "links":[{"id":"subject","projection":"approved-subjects","fromField":"first", + "identifierScheme":"urn:example:subject-key","fields":["label"]}] + }]); +} + +const MALFORMED_KEYS: [&str; 3] = [" padded", "trailing ", "bell\u{7}key"]; + +#[test] +fn a_malformed_link_key_is_refused_when_a_patch_proposal_is_prepared() { + let linked = fixture_with(json!([patch_effect()]), link_first); + let unlinked = fixture(json!([patch_effect()])); + for malformed in MALFORMED_KEYS { + let input = map(json!({"one":TARGET,"value":malformed})); + assert!( + matches!( + existing(&linked, input.clone(), map(json!({"first":"old"}))), + Err(MutationError::InvalidRequest) + ), + "{malformed:?} is refused on the linked field" + ); + existing(&unlinked, input, map(json!({"first":"old"}))) + .expect("the same value is accepted on an unlinked field"); + } + existing( + &linked, + map(json!({"one":TARGET,"value":"key-1"})), + map(json!({"first":"old"})), + ) + .expect("a well-formed key is accepted on the linked field"); +} + +#[test] +fn a_malformed_link_key_is_refused_when_a_create_proposal_is_prepared() { + let linked = fixture_with(create_effects(), link_first); + let unlinked = fixture(create_effects()); + let reserved = BTreeMap::from([ + ("parent".into(), Uuid::new_v4()), + ("child".into(), Uuid::new_v4()), + ]); + for malformed in MALFORMED_KEYS { + let input = map(json!({"value":malformed})); + for (registry, accepted) in [(&linked, false), (&unlinked, true)] { + let resolved = resolve_targets( + registry, + action(registry), + &input, + candidate(registry, &input).unwrap(), + &reserved, + ) + .unwrap(); + assert_eq!( + materialize_create_fields(registry, &input, &resolved).is_ok(), + accepted, + "{malformed:?} on a create" + ); + } + } +} From 79946d9ea26c3e024f266fa70b1d9592bf71217a Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 20:41:10 +0700 Subject: [PATCH 56/64] test: pin the write-time link key check at the record write choke point Signed-off-by: Jeremi Joslin --- crates/registry-breg/src/mutation.rs | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/crates/registry-breg/src/mutation.rs b/crates/registry-breg/src/mutation.rs index 3cdd2a9350..663d2e6b84 100644 --- a/crates/registry-breg/src/mutation.rs +++ b/crates/registry-breg/src/mutation.rs @@ -4549,6 +4549,46 @@ impl From for MutationError { mod tests { use super::*; + /// Every record write converts its values here. A value a caller never + /// submitted (a proposal prepared before a view linked the field, applied + /// as stored) reaches the write without the earlier located checks, so this + /// is the check that holds for it. + #[test] + fn the_write_path_refuses_a_malformed_link_key_on_the_linked_field_only() { + let project = crate::contract::parse_project_yaml(include_bytes!( + "../tests/fixtures/fixed-optional-action-input.yaml" + )) + .unwrap(); + let registry = crate::compiler::compile_project( + &project, + &[], + crate::compiler::CompileProfile::Authoring, + ) + .unwrap(); + let unlinked = registry.entities()["entry"].clone(); + let mut linked = unlinked.clone(); + linked.external_link_fields.insert("label".to_owned()); + let field = &unlinked.fields["label"]; + for malformed in [" padded", "trailing ", "bell\u{7}key"] { + let value = Value::String(malformed.to_owned()); + assert_eq!( + written_sql_value(&linked, field, &value), + Err(MutationError::InvalidRequest), + "{malformed:?} is refused on the linked field" + ); + assert_eq!( + written_sql_value(&unlinked, field, &value), + Ok(Some(malformed.to_owned())), + "the same value is written to an unlinked field" + ); + } + assert_eq!( + written_sql_value(&linked, field, &Value::String("key-1".to_owned())), + Ok(Some("key-1".to_owned())) + ); + assert_eq!(written_sql_value(&linked, field, &Value::Null), Ok(None)); + } + #[test] fn response_etag_binds_package_authority_boundary_and_projection() { let profile = AuditProfile::production_from_secret_bytes(vec![0x4d; 32].into()) From 472d637b3e762046524f6b3d117b59627f411012 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 20:41:10 +0700 Subject: [PATCH 57/64] test: drop an unused value from the linked field write test Signed-off-by: Jeremi Joslin --- crates/registry-breg/tests/postgres_mutation.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/crates/registry-breg/tests/postgres_mutation.rs b/crates/registry-breg/tests/postgres_mutation.rs index fcc342b93e..1ae0199fbb 100644 --- a/crates/registry-breg/tests/postgres_mutation.rs +++ b/crates/registry-breg/tests/postgres_mutation.rs @@ -6152,11 +6152,6 @@ async fn a_malformed_external_link_key_is_refused_on_write_to_the_linked_field_o assert_eq!(after.revisions, before.revisions, "no revision is written"); } - let long = "k".repeat(513); - // maximumLength is 32, so the length rule is exercised by the unit - // tests; here the character rule alone decides. - assert!(long.len() > 32); - // The same malformed value in an unlinked field is accepted. let accepted = coordinator .execute( From 33a69d20c94ca695eb411ae480476cfae795d54c Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 20:43:15 +0700 Subject: [PATCH 58/64] fix: serve exponent-range external decimals as plain strings instead of failing the batch Signed-off-by: Jeremi Joslin --- crates/registry-breg/src/external_source.rs | 49 +++++++++++++++----- products/breg/external-projection-storage.md | 2 + 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/crates/registry-breg/src/external_source.rs b/crates/registry-breg/src/external_source.rs index 45942af794..0b3966b375 100644 --- a/crates/registry-breg/src/external_source.rs +++ b/crates/registry-breg/src/external_source.rs @@ -522,8 +522,9 @@ fn check_result(result: &ProjectionResult, types: &FieldTypes) -> Result<(), Ext } /// The source contract states a decimal as a JSON number; BReg serves and -/// stores it as the JSON string its own decimal fields use on the wire. A -/// number with no plain decimal spelling (an exponent form) is refused. +/// stores it as the JSON string its own decimal fields use on the wire. An +/// integer keeps its exact digits; any other number is read as a double and +/// spelled without an exponent. fn decimals_to_wire( result: &mut ProjectionResult, types: &FieldTypes, @@ -539,7 +540,12 @@ fn decimals_to_wire( if !is_decimal { continue; } - let text = number.to_string(); + let text = if number.is_f64() { + // `f64` Display never uses an exponent form. + number.as_f64().map_or_else(String::new, |f| f.to_string()) + } else { + number.to_string() + }; if !plain_decimal(&text) { return Err(ExternalSourceError::Result); } @@ -1231,17 +1237,36 @@ pub(crate) mod tests { assert!(!validator.is_valid(&json!(12.5))); } + /// A source number whose shortest spelling is an exponent form (below 1e-5 + /// or from 1e16) is still served as a plain decimal string, and one such + /// value does not fail the batch that carries it. #[test] - fn a_decimal_with_no_plain_spelling_is_refused() { + fn exponent_range_decimals_are_served_as_plain_strings() { + let validator = jsonschema::JSONSchema::options() + .with_draft(jsonschema::Draft::Draft202012) + .compile(&crate::external_view_artifacts::decimal_schema()) + .unwrap(); let types = decimal_types(); - for number in [json!(1e30), json!(1.5e-9)] { - let mut result = ProjectionResult::Exact { - record: present("a", number), - }; - assert!(matches!( - decimals_to_wire(&mut result, &types), - Err(ExternalSourceError::Result) - )); + let spelled: [(Value, &str); 5] = [ + (json!(0.000001), "0.000001"), + (json!(1e16), "10000000000000000"), + (json!(-1e-7), "-0.0000001"), + (json!(1.5e-9), "0.0000000015"), + (json!(12.5), "12.5"), + ]; + let mut batch = ProjectionResult::Batch { + records: spelled + .iter() + .map(|(number, _)| present("a", number.clone())) + .collect(), + }; + decimals_to_wire(&mut batch, &types).expect("one extreme value does not fail the batch"); + let ProjectionResult::Batch { records } = &batch else { + unreachable!() + }; + for (record, (_, text)) in records.iter().zip(spelled) { + assert_eq!(balance_of(record), json!(text)); + assert!(validator.is_valid(&balance_of(record)), "{text}"); } } diff --git a/products/breg/external-projection-storage.md b/products/breg/external-projection-storage.md index 03ea57beae..9153ccfeb0 100644 --- a/products/breg/external-projection-storage.md +++ b/products/breg/external-projection-storage.md @@ -21,6 +21,8 @@ before Adapter acquisition. The current checker refuses unknown binding configur Write access to a link's `fromField` chooses which outside subject BReg fetches. Whoever can set that field on a root decides which subject the source is asked for under the deployment's source grant, and which subject's values readers of that root then see. Grant write access to a link field only to identities trusted to assert that link, and review changes to it as you would review a disclosure. +A source states a decimal field as a JSON number, and BReg serves it as a string. An integer keeps its exact digits. Any other number is read as a double, so a value with more than about 15 significant digits is rounded before BReg serves it. A source that needs exact decimals beyond that cannot send them as JSON numbers; BReg does not accept decimal strings from a source. + ## Freshness and purge `cacheTtlSeconds` is the one cache lifetime of a source result. A deployment binding may set its own `cacheTtlSeconds` to shorten, but never extend, the authored value. The lifetime is measured from the original source observation time, not from refresh completion. The effective expiry is the earlier of that lifetime and the deployment binding's source grant cutoff, `grantValidUntil`. From a successful source response, BReg honours only `no-store`; it reads no other expiry or storage permission there, so set `grantValidUntil` no later than the source permits storage. There is no retention period after expiry: an expired value is purged. From 023aa4a48b98b54048410c0f5ca153b62f365285 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 20:45:59 +0700 Subject: [PATCH 59/64] fix: issue a continuation cursor only for a sort value the next page accepts Signed-off-by: Jeremi Joslin --- .../src/postgres/external_runtime.rs | 82 ++++++++++++++++++- .../src/postgres/external_view_query.rs | 10 +++ products/breg/external-projection-storage.md | 4 +- 3 files changed, 90 insertions(+), 6 deletions(-) diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index 78a8ec90c5..8693c2c3be 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -1108,17 +1108,35 @@ impl PostgresExternalViewService { .timestamp() .saturating_add(self.cursors.max_age().as_secs() as i64), ); - self.cursors - .seal_external(&Continuation { + seal_continuation( + &self.cursors, + &Continuation { binding, query, position, expires_at, - }) - .map_err(|_| ExternalViewError::Unavailable) + }, + ) } } +/// Seal a continuation, refusing a sort value the next page would refuse, so +/// every cursor the server issues is one it accepts. +fn seal_continuation(cursors: &CursorCodec, continuation: &Continuation) -> Result { + if let Position::Read { + sort_value: Some(value), + .. + } = &continuation.position + { + if !indexed::sort_value_resumable(value) { + return Err(ExternalViewError::Unavailable); + } + } + cursors + .seal_external(continuation) + .map_err(|_| ExternalViewError::Unavailable) +} + impl ExternalViewService for PostgresExternalViewService { fn read(&self, request: ExternalViewRequest) -> ServiceFuture<'_, Result> { Box::pin(async move { @@ -2154,6 +2172,62 @@ mod tests { assert_eq!(opened, sort_value); } + fn continuation_at(sort_value: String) -> Continuation { + Continuation { + binding: "binding".to_owned(), + expires_at: Utc::now().timestamp() + 60, + query: ExternalQuery::default(), + position: Position::Read { + root_id: Uuid::nil(), + source_key: "row".to_owned(), + sort_value: Some(sort_value), + }, + } + } + + /// The server never issues a cursor it would refuse: a sort value past the + /// resume bound fails when the cursor is issued, and every cursor that is + /// issued opens and carries a value the continuation accepts. + #[test] + fn a_cursor_is_issued_only_for_a_sort_value_it_will_accept() { + let codec = CursorCodec::new( + zeroize::Zeroizing::new(vec![42; 32]), + Duration::from_secs(120), + ) + .unwrap(); + let bound = indexed::MAXIMUM_CURSOR_SORT_VALUE_BYTES; + for value in [ + "a".repeat(bound), + "\"".repeat(1000), + "\u{7}".repeat(500), + "\u{e9}".repeat(bound / 2), + ] { + let token = seal_continuation(&codec, &continuation_at(value.clone())) + .unwrap_or_else(|_| panic!("a {}-byte value is issued", value.len())); + let opened: Continuation = codec.open_external(&token).unwrap(); + let Position::Read { + sort_value: Some(opened), + .. + } = opened.position + else { + panic!("position changed"); + }; + assert_eq!(opened, value); + assert!(indexed::sort_value_resumable(&opened)); + } + // Past the bound, and a value whose JSON escaping overflows the 8 KiB + // payload, fail when issued rather than on the next page. + for value in ["a".repeat(bound + 1), "\"".repeat(bound)] { + assert!( + matches!( + seal_continuation(&codec, &continuation_at(value)), + Err(ExternalViewError::Unavailable) + ), + "an oversized sort value is refused when the cursor is issued" + ); + } + } + #[test] fn a_new_capture_preserves_a_scoped_source_revision_reference() { let codec = CursorCodec::new( diff --git a/crates/registry-breg/src/postgres/external_view_query.rs b/crates/registry-breg/src/postgres/external_view_query.rs index a2bc17ac13..140a9d7393 100644 --- a/crates/registry-breg/src/postgres/external_view_query.rs +++ b/crates/registry-breg/src/postgres/external_view_query.rs @@ -562,6 +562,13 @@ const MAXIMUM_FILTER_STRING_BYTES: usize = 4096; /// request's key, query and binding) is the effective limit above this. pub(crate) const MAXIMUM_CURSOR_SORT_VALUE_BYTES: usize = 6144; +/// Whether a continuation may resume from `value`. The server applies it when +/// it issues a cursor and again when it opens one, so a cursor it issues is +/// always one it accepts. +pub(crate) fn sort_value_resumable(value: &str) -> bool { + value.len() <= MAXIMUM_CURSOR_SORT_VALUE_BYTES +} + fn scalar(value: &Value, kind: ExternalFieldType) -> Result, ExternalQueryError> { bounded_scalar(value, kind, MAXIMUM_FILTER_STRING_BYTES) } @@ -637,6 +644,9 @@ fn continuation_sql( match &after.sort_value { None => Ok(format!("{column} IS NULL AND {tie}")), Some(value) => { + if !sort_value_resumable(value) { + return Err(ExternalQueryError::InvalidQuery); + } let parsed = match kind { ExternalFieldType::Integer => value .parse::() diff --git a/products/breg/external-projection-storage.md b/products/breg/external-projection-storage.md index 9153ccfeb0..a903ddc826 100644 --- a/products/breg/external-projection-storage.md +++ b/products/breg/external-projection-storage.md @@ -31,11 +31,11 @@ A captured snapshot stores the observation time and a keyed revision reference w Replaying an accepted result keeps its original expiry. A refresh that fails because the source is unavailable, returns an incomplete result, or cannot be acquired under the current binding keeps the previous snapshot of either kind until its own expiry, without extending its lifetime. Explicit grant denial, or a successful source response marked `no-store`, removes the previous snapshot, making coverage unproven, fenced against newer publications. A `no-store` on an error, a redirect or the authentication challenge that starts a credential refresh changes nothing: that refresh fails as unavailable and keeps the previous snapshot. A successful source response marked `no-store` cannot enter the indexed store, even when a later step of the same acquisition fails. If it supersedes an earlier retained result for the same subject, the guarded retention-refusal path removes that result; an older refresh cannot erase a newer publication. -Reads check expiry independently of the refresh job. Indexed predicates and counts refuse incomplete coverage before selecting matching rows. Each page reads coverage, rows and any count from one statement snapshot, so a count always matches complete coverage at that moment. A refresh records each key in its own short transaction, so a read waits for at most one key's publication and can show new values for some keys of a page in flight and previous values for the rest. A `nextCursor` stays valid while refreshes publish or fail individual keys; the next page reads the values current when it runs. A row whose ordering value changed between pages can appear twice or not at all. A cursor is encrypted and authenticated with the deployment's cursor key and bound to its caller, query and projection generation, so its contents (including the sort value it resumes from) cannot be altered or read by the client. A sort value resumes from up to 6144 bytes; a stored value, or a cursor whose key, query and sort value together exceed the 8 KiB cursor payload, ends the listing with `source.unavailable`. Rotating, withdrawing or resetting the projection invalidates outstanding cursors. A complete empty related collection is covered; a partial collection, source failure, or protected-existence response is not evidence of emptiness or absence. +Reads check expiry independently of the refresh job. Indexed predicates and counts refuse incomplete coverage before selecting matching rows. Each page reads coverage, rows and any count from one statement snapshot, so a count always matches complete coverage at that moment. A refresh records each key in its own short transaction, so a read waits for at most one key's publication and can show new values for some keys of a page in flight and previous values for the rest. A `nextCursor` stays valid while refreshes publish or fail individual keys; the next page reads the values current when it runs. A row whose ordering value changed between pages can appear twice or not at all. A cursor is encrypted and authenticated with the deployment's cursor key and bound to its caller, query and projection generation, so its contents (including the sort value it resumes from) cannot be altered or read by the client. A sort value resumes from up to 6144 bytes. When the last row of a page carries a longer sort value, or its cursor (key, query and sort value, with quotes, backslashes and control characters counted as their JSON escapes) would exceed the 8 KiB cursor payload, that page's request is answered `source.unavailable` and no `nextCursor` is issued; the earlier pages were served, and the listing stops there. Every cursor the server issues is one the next page accepts. Rotating, withdrawing or resetting the projection invalidates outstanding cursors. A complete empty related collection is covered; a partial collection, source failure, or protected-existence response is not evidence of emptiness or absence. A read of one record looks only at that record. Every other indexed page, including each continuation page, computes coverage, root presence, expiry and any count over every root the caller is authorized to read before it selects its rows, so its cost grows with the caller's whole authorized population, not with the page size. Grant indexed views only to access profiles whose row boundaries are narrowed to the records their callers need, and rate-limit the callers of indexed queries and of refresh at the gateway. -A root whose link field is empty or unset names no outside subject. It counts as covered, reads with its external fields absent, or as an empty related collection, and its refresh sends no source request. A root whose link key the store or the source contract would refuse is never sent to the source. The store refuses a key over 1024 bytes or with a control character; the source contract also refuses a key over 512 bytes or with leading or trailing white space. Such a root reads as having no outside value, like a root with an empty link field, for that record alone: it is covered, its external fields are absent, and lists, predicates, sorts and counts over its view keep serving the other records. The refresh still logs and counts the key as `invalid-key` and as unavailable in its report, so an operator can find and correct it. A write of such a key to a field an external view links from is refused as `request.invalid` on every write path (create, patch, action, proposal, batch and import), and writes nothing; the same value in a field no view links from is accepted. A key stored before the view was configured is read under the rule above until it is corrected. A label sweep applies the same rules to each related row's label key, so one invalid label key leaves only that row without its label. +A root whose link field is empty or unset names no outside subject. It counts as covered, reads with its external fields absent, or as an empty related collection, and its refresh sends no source request. A root whose link key the store or the source contract would refuse is never sent to the source. The store refuses a key over 1024 bytes or with a control character; the source contract also refuses a key over 512 bytes or with leading or trailing white space. Such a root reads as having no outside value, like a root with an empty link field, for that record alone: it is covered, its external fields are absent, and lists, predicates, sorts and counts over its view keep serving the other records. The refresh still logs and counts the key as `invalid-key` and as unavailable in its report, so an operator can find and correct it. A write of such a key to a field an external view links from is refused as `request.invalid` on create, patch, action, proposal, batch and import writes, and writes nothing. Two paths differ: a mutation returned in an action hook's answer is refused as `hook.proposal.mutation_refused`, and the SQL steps of a history migration write current rows without this check, so reads guard those rows. The same value in a field no view links from is accepted. A key stored before the view was configured is read under the rule above until it is corrected. A label sweep applies the same rules to each related row's label key, so one invalid label key leaves only that row without its label. The purge worker runs every five seconds. Each pass deletes expired snapshots, refresh receipts whose lease has ended, and everything under withdrawn or replaced epochs, in at most eight batches of a thousand rows of each kind. Freshness enforcement keeps expired values out of responses even when refresh or purge is delayed, but an expired value stays stored until a pass deletes it. That happens while the database is down, while the server is stopped, and while a backlog larger than one pass drains. Size database capacity for the backlog. Where a storage obligation is strict, choose a cache lifetime shorter than that obligation by more than the worker interval and the time you need to recover from those conditions, and watch the signals in the next section. Database WAL and physical snapshots need their own retention policy. From 83c082f62a982fd1cb57d45a66014899c9efc40b Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 20:48:11 +0700 Subject: [PATCH 60/64] fix: log a live external read that skips a malformed link key Signed-off-by: Jeremi Joslin --- .../src/postgres/external_runtime.rs | 28 +++++++++++++++++-- products/breg/external-projection-storage.md | 1 + 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index 8693c2c3be..5db8a31e09 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -610,11 +610,23 @@ impl PostgresExternalViewService { .transaction(&mut session, claims, request.deadline) .await?; // A root without a usable link key has no outside subject to fetch. - let key = RootKeys::new(plan)? + let stored = RootKeys::new(plan)? .for_root(&transaction, root) .await? - .ok_or(ExternalViewError::Concealed)? - .filter(|key| !key.is_empty() && crate::external_views::link_key_acceptable(key)); + .ok_or(ExternalViewError::Concealed)?; + if stored + .as_deref() + .is_some_and(|key| !key.is_empty() && !crate::external_views::link_key_acceptable(key)) + { + tracing::warn!( + view = request.view.as_str(), + reason = "invalid-key", + keys = 1, + "external view live read skipped a key it would not send to the source" + ); + } + let key = + stored.filter(|key| !key.is_empty() && crate::external_views::link_key_acceptable(key)); transaction .commit() .await @@ -3670,6 +3682,16 @@ mod postgres_tests { transcript.finish().unwrap(); let logs = captured.text(); assert!(!logs.contains("live acquisition failed"), "{logs}"); + // Only the malformed key is logged, once, with the view and a count. + let skipped = logs + .lines() + .filter(|line| line.contains(r#""reason":"invalid-key""#)) + .collect::>(); + assert_eq!(skipped.len(), 1, "{logs}"); + assert!(skipped[0].contains(r#""level":"WARN""#), "{logs}"); + assert!(skipped[0].contains(r#""view":"view""#), "{logs}"); + assert!(skipped[0].contains(r#""keys":1"#), "{logs}"); + assert!(!logs.contains("bell"), "the key reached the log: {logs}"); served.fixture.cleanup().await; } diff --git a/products/breg/external-projection-storage.md b/products/breg/external-projection-storage.md index a903ddc826..a5b9b495b1 100644 --- a/products/breg/external-projection-storage.md +++ b/products/breg/external-projection-storage.md @@ -49,6 +49,7 @@ Each signal names the projection and a fixed reason category. None carries a sou - On the metrics listener, `breg_worker_last_success_age_seconds{worker="external-projection-retention"}` reports the seconds since the last completed pass. It is absent until the first pass completes. Alert when it grows well past the five-second interval. - A refresh page that acquired keys logs one `info` line with its `published` count, and one `warn` line per failure reason with a `keys` count. The reason is `denied`, `retention-refused`, `unavailable`, `incomplete`, `invalid-binding`, or `abandoned` when a newer result, a changed root or a changed collection superseded the result before publication. - A refresh page that holds link keys or label keys the store or the source contract would refuse logs a `warn` line with the reason `invalid-key` and a `keys` count. Those keys are never sent to the source, and the records that hold them read as having no outside value until the keys are corrected. +- A live read whose root holds a link key the store or the source contract would refuse logs one `warn` line with the `view`, the reason `invalid-key` and `keys` of 1. The key is never sent to the source and the record reads as having no outside value. - A refresh request that fails as a whole logs a `warn` line with a `reason` of `concealed`, `invalid-cursor`, `incomplete`, `unavailable`, `projection-unavailable` or `deadline`. - A live read whose source acquisition fails logs a `warn` line with the same reasons as a refresh key, other than `abandoned`. The view answers unavailable. - `bregctl doctor` reports each projection a server would serve as an `external-projection.status` advisory. It gives the time the source grant expires and, for an indexed projection, when the newest snapshot it holds was observed. Renew a grant before that time. An old or missing newest snapshot means the refresh sweep is not running. From a662acc881367432c8cb493979d1fcd83860222b Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 20:49:16 +0700 Subject: [PATCH 61/64] test: pin credential failure and no-store handling in projection acquisition Signed-off-by: Jeremi Joslin --- .../tests/projection_acquisition.rs | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/crates/registry-adapter-engine/tests/projection_acquisition.rs b/crates/registry-adapter-engine/tests/projection_acquisition.rs index dd2e1abc3d..bc27d6ddfb 100644 --- a/crates/registry-adapter-engine/tests/projection_acquisition.rs +++ b/crates/registry-adapter-engine/tests/projection_acquisition.rs @@ -850,3 +850,68 @@ async fn an_unsuccessful_source_status_without_no_store_stays_unavailable() { ); assert_eq!(count(&served.data), 1); } + +#[cfg(unix)] +#[tokio::test] +async fn a_caught_failure_of_the_initial_authorization_fails_the_acquisition() { + // The enrich is the first send and its credential cannot be obtained; the + // route catches that failure and the next send then succeeds with a fresh + // token. Only the projection's own record of the credential failure keeps + // the partial result from being released. + let steps = json!([ + {"enrich":{ + "uri":"http:source", + "aggregateOnException":true, + "parameters":{"httpMethod":"GET","path":"/people"} + }}, + {"to":{"uri":"http:source","parameters":{"httpMethod":"GET","path":"/people"}}} + ]); + let (result, served) = + oauth_exact_steps(ProjectionMode::Indexed, steps, vec![(200, "")], vec![503]).await; + assert!( + matches!(result, Err(ProjectionError::Unavailable)), + "{result:?}" + ); + assert_eq!(count(&served.tokens), 2); + assert_eq!(count(&served.data), 1, "the second send ran"); +} + +fn replayed_exact(status: u16, headers: Value) -> Value { + json!({ + "request":{"endpoint":"source","method":"GET","path":"/people"}, + "response":{"status":status,"headers":headers,"body":{}} + }) +} + +/// A recorded response that is not a data response cannot forbid retention, +/// so the previous snapshot is kept (the acquisition is unavailable, not +/// no-store), whatever its Cache-Control says. +#[tokio::test] +async fn a_replayed_response_that_is_not_data_cannot_forbid_retention() { + for status in [503, 202] { + let reply = replayed_exact(status, json!({"Cache-Control":"no-store"})); + let exchanges = transcript(vec![reply]); + let result = exact(ProjectionMode::Indexed, "retain", Arc::clone(&exchanges)).await; + assert!( + matches!(result, Err(ProjectionError::Unavailable)), + "{status}: {result:?}" + ); + exchanges.finish().unwrap(); + } +} + +#[cfg(unix)] +#[tokio::test] +async fn a_202_marked_no_store_keeps_the_previous_snapshot() { + let (result, served) = oauth_exact( + ProjectionMode::Indexed, + 1, + vec![(202, "Cache-Control: no-store\r\n")], + ) + .await; + assert!( + matches!(result, Err(ProjectionError::Unavailable)), + "{result:?}" + ); + assert_eq!(count(&served.data), 1); +} From dc631342e9c6690fd932410b09eb2bee6c765154 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 21:23:03 +0700 Subject: [PATCH 62/64] test(adapter): give the projection-invalid code its bad-corpus member The corpus gate requires every registered authoring code to have a member; adapter.pack.projection-invalid had none, so the Adapter contract script failed once the conventions lint passed. Signed-off-by: Jeremi Joslin --- products/adapter/fixtures/corpus/README.md | 3 +- .../adapter.pack.projection-invalid/pack.yaml | 65 +++++++++++++++++++ .../profile.yaml | 39 +++++++++++ .../project.yaml | 43 ++++++++++++ .../schemas/person.json | 8 +++ 5 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 products/adapter/fixtures/corpus/bad/adapter.pack.projection-invalid/pack.yaml create mode 100644 products/adapter/fixtures/corpus/bad/adapter.pack.projection-invalid/profile.yaml create mode 100644 products/adapter/fixtures/corpus/bad/adapter.pack.projection-invalid/project.yaml create mode 100644 products/adapter/fixtures/corpus/bad/adapter.pack.projection-invalid/schemas/person.json diff --git a/products/adapter/fixtures/corpus/README.md b/products/adapter/fixtures/corpus/README.md index 63c6cf306e..a2d1a96edd 100644 --- a/products/adapter/fixtures/corpus/README.md +++ b/products/adapter/fixtures/corpus/README.md @@ -73,7 +73,7 @@ digest pins and the GraphQL query-file checks). Route findings report once at the authored position: `/routeTemplates/N` when the defect sits in a template body, `/routes/{id}` otherwise, so every member refuses with exactly one -finding per code. The corpus holds 181 members. +finding per code. The corpus holds 191 members. `base-job/` is the clean governed-ingestion control (CONCEPT section 17): a pack whose source route polls a bounded `file` endpoint through @@ -183,6 +183,7 @@ and deployment custody aligned. Each refuses with its directory's exact code. | `adapter.pack.unknown-template-parameter` | Template body references `{{missing-param}}`, no declared parameter provides it | `adapter.pack.unknown-template-parameter` | checks | | `adapter.pack.map-parameter-misplaced` | The `map` parameter `{{labels}}` sits inside longer text (`look up {{labels}}`), where an object cannot be rendered | `adapter.pack.map-parameter-misplaced` | checks | | `adapter.pack.read-support-invalid` | A `read-one` route declares no native identity input | `adapter.pack.read-support-invalid` | checks | +| `adapter.pack.projection-invalid` | An `exact` projection selects by `urn:example:person-number` but keys its records by `urn:example:tax-number` | `adapter.pack.projection-invalid` | checks | | `adapter.pack.digest-pin-required` | Script file's bytes do not hash to its pin (the pin is valid hex, wrong value) | `adapter.pack.digest-pin-required` | tool | | `adapter.pack.script-uncompilable` | A pinned script does not parse (missing closing brace; its pin matches the broken bytes) | `adapter.pack.script-uncompilable` | tool | | `adapter.pack.entrypoint-parameter` | A pinned script's entrypoint takes `record`, not the `exchange` the slot passes (CONCEPT 8.11.3); the script parses and hashes to its pin | `adapter.pack.entrypoint-parameter` | tool | diff --git a/products/adapter/fixtures/corpus/bad/adapter.pack.projection-invalid/pack.yaml b/products/adapter/fixtures/corpus/bad/adapter.pack.projection-invalid/pack.yaml new file mode 100644 index 0000000000..8d8f692726 --- /dev/null +++ b/products/adapter/fixtures/corpus/bad/adapter.pack.projection-invalid/pack.yaml @@ -0,0 +1,65 @@ +# yaml-language-server: $schema=https://id.registrystack.org/schemas/adapter/pack/pack.v1alpha1.schema.json +# +# Bad-corpus member adapter.pack.projection-invalid: this artifact set must be refused with +# exactly adapter.pack.projection-invalid. An exact projection reads records by a selector +# scheme other than its record key scheme. +# +# Refuse with: +# target/debug/adapterctl check products/adapter/fixtures/corpus/bad/adapter.pack.projection-invalid/project.yaml +apiVersion: id.registrystack.org/formats/adapter/pack/v1alpha1 +kind: AdapterPack +id: case-api-pack +version: "1" +testedVersions: + - "1.0" +endpoints: + - id: cases + component: http + authentication: + type: basic + attemptTimeoutMilliseconds: 10000 + maximumResponseBytes: 1048576 +routeTemplates: + - id: find-cases + parameters: + - name: type + type: string + required: false + - name: case_id + type: string + required: false + steps: + - to: + uri: http:cases + parameters: + httpMethod: GET + path: /v2/cases/{{type}}/{{case_id}} + pathParameters: + type: + constant: welfare + case_id: + header: recordId + query: + _format: + constant: json +sourceRoutes: + - id: person-exact + projection: + operation: exact + operationVersion: "1" + authority: urn:example:population-register + purpose: licensing + scope: licensing-holders + selectorScheme: urn:example:person-number + recordKeyScheme: urn:example:tax-number + retention: live-only + absence: authoritative + localDisclosure: host-profiles + fieldsSchema: + path: schemas/person.json + digest: sha256:1c1ad6f4c266d0470b8a28c054d360744c6a9f36ba61c379d6e75e0c791035b7 + maximumBatchKeys: 1 + maximumPages: 1 + maximumRows: 1 + steps: [] +capabilities: {} diff --git a/products/adapter/fixtures/corpus/bad/adapter.pack.projection-invalid/profile.yaml b/products/adapter/fixtures/corpus/bad/adapter.pack.projection-invalid/profile.yaml new file mode 100644 index 0000000000..bb7a9d4728 --- /dev/null +++ b/products/adapter/fixtures/corpus/bad/adapter.pack.projection-invalid/profile.yaml @@ -0,0 +1,39 @@ +# yaml-language-server: $schema=https://id.registrystack.org/schemas/adapter/profile/profile.v1alpha1.schema.json +# +# Bad-corpus member adapter.pack.projection-invalid: this artifact set must be refused with +# exactly adapter.pack.projection-invalid. Clean baseline profile. +# +# Refuse with: +# target/debug/adapterctl check products/adapter/fixtures/corpus/bad/adapter.pack.projection-invalid/project.yaml +apiVersion: id.registrystack.org/formats/adapter/profile/v1alpha1 +kind: AdapterProfile +id: case-api +version: "1" +operations: + - id: cases-get + method: GET + path: /v2/cases/{type}/{case_id} + pathParameters: + - type + - case_id + recordTypeParameter: type + recordIdParameter: case_id + responseEnvelope: '{"case": {{record}}, "fetchedAt": "{{requestId}}"}' +problems: + - code: cases.upstream + status: 502 + title: the source could not be reached + classes: [upstream, breaker.open, throttle.rejected, loop.maximum-iterations-reached] + - code: cases.mapping + status: 422 + title: the request or its mapping was refused + classes: [validation, conversion.unmapped, aggregation.duplicate] + - code: cases.exhausted + status: 502 + title: the run exhausted a budget + classes: [script, budget.exhausted] + - code: cases.boundary + status: 500 + title: the facade refused the call + classes: [disclosure.violation, egress.refused, credential.rejected, outcome, facade] +problemEnvelope: '{"family": "{{problemCode}}"}' diff --git a/products/adapter/fixtures/corpus/bad/adapter.pack.projection-invalid/project.yaml b/products/adapter/fixtures/corpus/bad/adapter.pack.projection-invalid/project.yaml new file mode 100644 index 0000000000..f8be69a17f --- /dev/null +++ b/products/adapter/fixtures/corpus/bad/adapter.pack.projection-invalid/project.yaml @@ -0,0 +1,43 @@ +# yaml-language-server: $schema=https://id.registrystack.org/schemas/adapter/project/project.v1alpha1.schema.json +# +# Bad-corpus member adapter.pack.projection-invalid: this artifact set must be refused with +# exactly adapter.pack.projection-invalid. Clean baseline project; the defect sits in the pack. +# +# Refuse with: +# target/debug/adapterctl check products/adapter/fixtures/corpus/bad/adapter.pack.projection-invalid/project.yaml +apiVersion: id.registrystack.org/formats/adapter/project/v1alpha1 +kind: AdapterProject +project: + id: case-api-example + version: "1" +profile: + id: case-api + version: "1" + digest: sha256:a3b93ff65445b06216fa71312c18250a16eb3bc7d596adbe7e4cb5e348f5eeaf +pack: + id: case-api-pack + version: "1" + digest: sha256:3ffc43076e75f5995ee70aa378fed3bcdb499d4131366ca2696d3a531de107a1 +recordTypes: + - name: case + fields: + - name: case_type + type: string + required: true +templatedRoutes: + - routeTemplate: find-cases + routeId: case-find +bindings: + - id: case-get + operation: cases-get + recordType: case + route: case-find + maximumUpstreamCalls: 4 +accessProfiles: + - id: case-reader + scopes: + - cases.read + operations: + - cases-get + recordTypes: + - case diff --git a/products/adapter/fixtures/corpus/bad/adapter.pack.projection-invalid/schemas/person.json b/products/adapter/fixtures/corpus/bad/adapter.pack.projection-invalid/schemas/person.json new file mode 100644 index 0000000000..c0ac160b3f --- /dev/null +++ b/products/adapter/fixtures/corpus/bad/adapter.pack.projection-invalid/schemas/person.json @@ -0,0 +1,8 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "given-name": {"type": "string"}, + "family-name": {"type": "string"} + } +} From 2fa8a0fa93ca7b6f7afb9c255c44b4f9a70cab97 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 21:51:51 +0700 Subject: [PATCH 63/64] fix(breg): audit external view reads with the per-client request context The request audit now records a client reference for every request (BREG-SEC-203). External view reads and their test worker take the same context, so the journal tells OAuth clients apart on these reads too. Signed-off-by: Jeremi Joslin --- crates/registry-breg/src/postgres/external_runtime.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/registry-breg/src/postgres/external_runtime.rs b/crates/registry-breg/src/postgres/external_runtime.rs index 5db8a31e09..affed311e8 100644 --- a/crates/registry-breg/src/postgres/external_runtime.rs +++ b/crates/registry-breg/src/postgres/external_runtime.rs @@ -956,7 +956,7 @@ impl PostgresExternalViewService { self.audit.profile(), ReadTerminalAudit { terminal: TerminalAudit { - grant: request.context.grant_audit().cloned(), + context: request.context.audit_context().cloned(), outcome: if permitted { TerminalAuditOutcome::Returned } else { @@ -4013,7 +4013,7 @@ mod postgres_tests { "refresh".to_owned(), Vec::new(), ) - .with_grant_audit(&worker), + .with_audit_context(&worker), correlation: RequestCorrelation::breg_created(), view: "view".to_owned(), link: "members-link".to_owned(), From 322a624fad388411032ab9f5c7e1dd926d94fe3a Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Tue, 10 Nov 2026 21:55:28 +0700 Subject: [PATCH 64/64] chore(identifiers): regenerate the catalog after the rebase onto the audit context change Signed-off-by: Jeremi Joslin --- .../identifiers/generated/catalog.v1.json | 114 +++++++++--------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/products/identifiers/generated/catalog.v1.json b/products/identifiers/generated/catalog.v1.json index 3d2db6bac2..913fab30cb 100644 --- a/products/identifiers/generated/catalog.v1.json +++ b/products/identifiers/generated/catalog.v1.json @@ -1429,7 +1429,7 @@ "description": "Starts stay held after a restore until an operator releases the admission hold.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "admission.held", @@ -1448,7 +1448,7 @@ "description": "This instance runs in recovery-only mode and admits no runs. Send starts to the ordinary service.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "admission.recovery-only", @@ -1467,7 +1467,7 @@ "description": "The operation may have completed, but its audited result could not be released. Inspect the run before repeating the operation.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "audit.response-unavailable", @@ -1486,7 +1486,7 @@ "description": "The bearer credential is missing, invalid, or expired.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "authentication.refused", @@ -1505,7 +1505,7 @@ "description": "The run's original connection settings differ from this service's runtime configuration.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "binding.conflict", @@ -1524,7 +1524,7 @@ "description": "Connection settings that runs in progress use differ from this service's runtime configuration.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "binding.live-conflict", @@ -1543,7 +1543,7 @@ "description": "That step is not waiting.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "check.not-waiting", @@ -1562,7 +1562,7 @@ "description": "The workflow's workflow definition, one of its mappings or one of its functions refused the request.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "definition.refused", @@ -1581,7 +1581,7 @@ "description": "An apply changed the active packages after this service started. Restart it with the runtime file that apply used.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "deployment.activation-changed", @@ -1600,7 +1600,7 @@ "description": "The deployment's activation, runtime configuration or state key custody check failed.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "deployment.refused", @@ -1619,7 +1619,7 @@ "description": "An evaluation may have completed at its provider, so the run is never evaluated again.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "evaluation.uncertain", @@ -1638,7 +1638,7 @@ "description": "The signed event must satisfy the accepted canonical envelope and typed event contract.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "event.invalid", @@ -1657,7 +1657,7 @@ "description": "Restore the configured event source signing key and its enabled secret provider before retrying delivery.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "event.source-unavailable", @@ -1676,7 +1676,7 @@ "description": "The signed event exceeds the event value profile: 131072 canonical bytes, 32 levels of nesting, 4096 values, 256 items per array, 256 members per object, and 16384 bytes per string. Have the producer project fewer or smaller fields into the event.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "event.too-large", @@ -1695,7 +1695,7 @@ "description": "Send a fresh signed delivery from the exact configured source to its accepted trigger.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "event.unauthenticated", @@ -1714,7 +1714,7 @@ "description": "Execution is under restore hold until an operator releases it.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "execution.held", @@ -1733,7 +1733,7 @@ "description": "Supply an Idempotency-Key header of 1 to 256 bytes.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "idempotency.key-invalid", @@ -1752,7 +1752,7 @@ "description": "This idempotency key already started a run with different input.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "idempotency.key-reused", @@ -1771,7 +1771,7 @@ "description": "The start input does not satisfy the workflow's declared input schema and value bounds.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "input.invalid", @@ -1790,7 +1790,7 @@ "description": "An unkeyed mutation may have completed at its provider, so the run never sends it again.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "mutation.uncertain", @@ -1809,7 +1809,7 @@ "description": "The caller's policy does not allow this operation.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "operation.not-authorized", @@ -1828,7 +1828,7 @@ "description": "The original command's receipt can no longer be recovered. This is not evidence that the command was not sent.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "receipt.expired", @@ -1847,7 +1847,7 @@ "description": "The run's progress changed while it was being reconciled. Inspect it again.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "reconcile.raced", @@ -1866,7 +1866,7 @@ "description": "Reconcile applies only to a held mutation with a prepared command, its original binding and no active lease.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "reconcile.refused", @@ -1885,7 +1885,7 @@ "description": "The held operation has no original-result lookup.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "reconcile.unsupported", @@ -1904,7 +1904,7 @@ "description": "The request body exceeds the accepted size: 65536 bytes for a JSON request, or 1048576 bytes for a signed event delivery.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "request.body-too-large", @@ -1923,7 +1923,7 @@ "description": "The request could not be read as a Coordinator request.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "request.invalid", @@ -1942,7 +1942,7 @@ "description": "The list limit must be between 1 and 100.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "request.limit-out-of-range", @@ -1961,7 +1961,7 @@ "description": "The route exists but not for this method.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "request.method-not-allowed", @@ -1980,7 +1980,7 @@ "description": "The requested route does not exist.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "request.not-found", @@ -1999,7 +1999,7 @@ "description": "The reason or recovery reference must be a bounded reference without personal data or control characters.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "request.reason-invalid", @@ -2018,7 +2018,7 @@ "description": "The retention cutoff must be in the past and the limit between 1 and 100.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "request.retention-invalid", @@ -2037,7 +2037,7 @@ "description": "The request body could not be processed.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "request.unprocessable", @@ -2056,7 +2056,7 @@ "description": "The request body is not JSON.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "request.unsupported-media-type", @@ -2075,7 +2075,7 @@ "description": "This recovery action requires execution to be under restore hold.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "restore.hold-required", @@ -2094,7 +2094,7 @@ "description": "Held work no longer matches its protected definition or prepared command.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "restore.lease-incompatible", @@ -2113,7 +2113,7 @@ "description": "Execution history must be complete and the prior deployment fenced before recovery is reviewed.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "restore.review-required", @@ -2132,7 +2132,7 @@ "description": "Held work remains unresolved, so the restore hold cannot be released.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "restore.unresolved", @@ -2151,7 +2151,7 @@ "description": "Retry-same applies only to held work before its original deadline.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "retry.refused", @@ -2170,7 +2170,7 @@ "description": "This service's Coordinator version cannot interpret the run's pinned definition.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "run.definition-incompatible", @@ -2189,7 +2189,7 @@ "description": "No run with this identifier is visible to the caller.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "run.not-visible", @@ -2208,7 +2208,7 @@ "description": "Retention erased this run's payloads, so it can no longer be retried or reconciled.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "run.payload-erased", @@ -2227,7 +2227,7 @@ "description": "Coordinator could not complete the request.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "runtime.failure", @@ -2246,7 +2246,7 @@ "description": "A dependency of the Coordinator, such as its database, audit stream or token verifier, is unavailable. Try again after the service recovers.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "service.unavailable", @@ -2265,7 +2265,7 @@ "description": "This deployment does not serve the named workflow.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "workflow.not-served", @@ -2284,7 +2284,7 @@ "description": "The expected workflow revision differs from the active revision or the original admission.", "source": { "path": "crates/registry-coordinator/src/problem.rs", - "sha256": "8c1a94809b8a61a3b15e1d913d70b64f29bfefe15010e74be0cc1561ed854a24" + "sha256": "5dcf42811a37fccaa9cd975c4845c0b3c2d695a66db26fb58fd2a828b41d55e7" }, "problem": { "code": "workflow.revision-mismatch", @@ -4978,7 +4978,7 @@ }, "artifact": { "path": "products/breg/generated/authoring/registry-module.schema.json", - "sha256": "2948465f69e8dd8856e245826adb47addbb0b0b3ae5a4a3eece660dfa77ba060", + "sha256": "7beb51728884e600dccd3b47fd1c46e47936423883079f7cffb9dba147f0339a", "mediaType": "application/schema+json" } }, @@ -4996,7 +4996,7 @@ }, "artifact": { "path": "products/breg/generated/authoring/registry-project.schema.json", - "sha256": "f4b28508be761c19821a7c578bd7a71d0656494d8fafcfad0b1a713c514bf40c", + "sha256": "673823f6efdd44555b18b3ca4ab52aeed102b82657eb0b49956268f738cee020", "mediaType": "application/schema+json" } }, @@ -5136,7 +5136,7 @@ "description": "Registry Casework fixture JSON Schema.", "source": { "path": "crates/registry-casework-core/src/schema.rs", - "sha256": "1604cf79433172ea4553b2f56365ea9e999a515aa3f16d60959cc652972ae185" + "sha256": "aae3e2891d651ad99989ae4720c08991d954c0f3232e194ec68d18e30642a711" }, "artifact": { "path": "products/casework/generated/fixture/fixture.schema.json", @@ -5154,7 +5154,7 @@ "description": "Registry Casework holiday set JSON Schema.", "source": { "path": "crates/registry-casework-core/src/schema.rs", - "sha256": "1604cf79433172ea4553b2f56365ea9e999a515aa3f16d60959cc652972ae185" + "sha256": "aae3e2891d651ad99989ae4720c08991d954c0f3232e194ec68d18e30642a711" }, "artifact": { "path": "products/casework/generated/holiday-set/holiday-set.schema.json", @@ -5172,11 +5172,11 @@ "description": "Registry Casework project JSON Schema.", "source": { "path": "crates/registry-casework-core/src/schema.rs", - "sha256": "1604cf79433172ea4553b2f56365ea9e999a515aa3f16d60959cc652972ae185" + "sha256": "aae3e2891d651ad99989ae4720c08991d954c0f3232e194ec68d18e30642a711" }, "artifact": { "path": "products/casework/generated/project/project.schema.json", - "sha256": "87a1a61127de6eb20c130f17b2d804c8b209c7343e7e4275f480f37833b83fcf", + "sha256": "3973b27d8d378de8f11877013c36c0bacc0aec6db003f70f1375c52b21857505", "mediaType": "application/schema+json" } }, @@ -5194,7 +5194,7 @@ }, "artifact": { "path": "products/casework/contracts/cli/report.schema.json", - "sha256": "2cbf5e3d2fd943abfa3777b63f204feac0dc0f5c02241a121aa3a135f3d420c7", + "sha256": "442c29fe6db3bd93c2a35a5303e341d8aaa70c95d58dcd589fee9fff529fdff4", "mediaType": "application/schema+json" } }, @@ -5226,11 +5226,11 @@ "description": "Registry Coordinator portable project authoring JSON Schema.", "source": { "path": "crates/registry-coordinator/src/authoring.rs", - "sha256": "032aa502fa63b5e7fe1cfb8f400be84d0802637409ac82ddf9f40152c482b91f" + "sha256": "8aff778324c1f05ca2615121980be6e3212d3e80f1bc99104b7256dfe18445a5" }, "artifact": { "path": "products/coordinator/generated/project/project.schema.json", - "sha256": "b22c06678f15e4a5aa028df61819cde8d37ce2311e1249a3a8fcc18d00aa3eab", + "sha256": "c4bc1e7a160804ec03dc958eed753e6cc59fb3132b702bf6bd43638f0c6829a1", "mediaType": "application/schema+json" } }, @@ -5262,11 +5262,11 @@ "description": "Registry Coordinator deterministic scenario authoring JSON Schema.", "source": { "path": "crates/registry-coordinator/src/scenarios.rs", - "sha256": "e20158e5382f690d46879161f32b5adda80385767b2c99b6efb11f166d43a394" + "sha256": "5e5de3c3a4ca9d36a2286b0abd1d3f551acb28691ae4a16439ee3fafcb83d4ef" }, "artifact": { "path": "products/coordinator/generated/scenarios/scenarios.schema.json", - "sha256": "e42e044eb828ff7e12f0a199fc95065c04baeadf91773c2d28ba9a1b0252090e", + "sha256": "7a91cc37393fabc0bd05d3ea22708a382a7ef11f4b7c63cdeedba5b939cf12fa", "mediaType": "application/schema+json" } },