From 98ca20dc6f457bd3ecfc93736026fc4aa67b44bf Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 11:31:50 +0530 Subject: [PATCH 01/39] docs(reporting): specify official finality policies --- ...iven-official-reporting-finality-design.md | 219 ++++++++++++++++++ 1 file changed, 219 insertions(+) create mode 100644 docs/superpowers/specs/2026-11-18-definition-driven-official-reporting-finality-design.md diff --git a/docs/superpowers/specs/2026-11-18-definition-driven-official-reporting-finality-design.md b/docs/superpowers/specs/2026-11-18-definition-driven-official-reporting-finality-design.md new file mode 100644 index 000000000..3efe27c82 --- /dev/null +++ b/docs/superpowers/specs/2026-11-18-definition-driven-official-reporting-finality-design.md @@ -0,0 +1,219 @@ +# Definition-Driven Official Reporting Finality + +Status: approved for implementation +Tracking: AI-22659 + +## Context + +The reporting producer can currently emit an official revision without retaining the +specific finality policy selected from the pinned report definition. It derives a +`finality_basis` from source-manifest evidence and constructs a synthetic +`finality_policy_id`. That is not sufficient for AdCP official finality: the emitted ID +and basis must name an exact policy in the content-addressed report definition, and the +seller must prove the requirements of that policy before publishing the irreversible +official close. + +The TypeScript SDK already models an explicitly pinned `finalityPolicy` for +`source_final` and `contractual_cutoff`. The Python model will use the same policy +shapes and add the protocol's third `stabilized` basis. A later TypeScript change can +add the same third shape and proof flow without translating between different policy +concepts. + +## Goals + +- Require an explicit report-definition policy for new official reporting. +- Support `source_final`, `contractual_cutoff`, and `stabilized` without inventing an + SDK default. +- Emit the exact selected policy ID and basis on official revisions. +- Derive contractual and stabilized finality from verified snapshots; only + `source_final` requires an authoritative upstream signal. +- Retain enough durable observation evidence to prove stabilization across restarts. +- Preserve snapshot scheduling, unchanged-content suppression, reconciliation, + materialization, and status behavior outside the official-finality decision. + +## Non-goals + +- The SDK will not choose among multiple policies in a report definition. +- The SDK will not treat the existence of `finality_policies[]` as a request for an + official close. +- The SDK will not infer a policy from source-manifest evidence, an offering ID, + `official_close_lag`, or a source-provided `stable_since` value. +- This change will not modify the TypeScript SDK. Its matching `stabilized` support is + separate work. +- The SDK will not approximate calendar-relative years or months as fixed durations. + +## Policy Model + +Add a frozen, public Python policy union with protocol field names: + +- source-final policy: `finality_policy_id`, `basis="source_final"`, `source_signal` +- contractual-cutoff policy: `finality_policy_id`, + `basis="contractual_cutoff"`, `duration_after_period_end` +- stabilized policy: `finality_policy_id`, `basis="stabilized"`, `minimum_age`, + `unchanged_for` + +The selected policy is a trusted projection from the content-addressed report +definition. It is retained with `ReportingDefinitionBinding`, which already stores +trusted non-wire projections of that definition. This keeps the selected policy in the +existing configuration and obligation definition JSON rather than changing the shared +PostgreSQL configuration/obligation column layout. `to_wire()` remains unchanged; +`to_storage()` includes the selected policy. + +There is no default policy and no automatic selection. New production admission +requires a selected policy when `required_finality` is `official`. A snapshot +configuration does not select a policy unless it explicitly enables an official-close +path. An explicit snapshot-to-official path without a selected policy fails admission +instead of emitting a synthetic policy ID. + +Policy identifiers must satisfy the report-definition identifier constraint. Durations +must use the SDK's exact day/hour/minute/second profile. A policy containing calendar +years or months is valid AdCP data but unsupported by this fixed-clock producer and is +rejected during admission rather than approximated. + +## Admission and Source Routing + +Production offering validation uses the selected policy to prove that the configured +source can supply the needed evidence: + +- `source_final` requires an `AuthoritativeOfferingV1` and the configured official + source offering. +- `contractual_cutoff` requires a `ProvisionalSnapshotOfferingV1` and the configured + snapshot source offering. +- `stabilized` requires a `ProvisionalSnapshotOfferingV1`, durable provisional + observation support, and the configured snapshot source offering. + +The public delivery offering must still advertise support for the configuration's +required finality. The source route is an implementation detail: a seller may publish +an official revision derived from a snapshot when the selected report-definition +policy, rather than the source, establishes finality. + +The selected policy is copied unchanged from configuration to obligation through the +retained definition binding. Restarts and configuration successors therefore cannot +reinterpret an existing obligation under a different policy. + +## Runtime Flows + +### Source-final + +The producer waits for the authoritative source offering, requests an authoritative +publication, and validates all existing manifest and coverage invariants. It additionally +requires: + +- finality evidence observed no earlier than the period end; +- an authoritative evidence basis (`provider_declared` or `provider_job_terminal`); +- `evidence_ref` equal to the selected `source_signal`. + +The official revision uses the selected policy ID, `source_final`, and the source +evidence observation time as `finalized_at`. A mismatched signal is a manifest mismatch, +not a reason to substitute another policy. + +### Contractual cutoff + +The producer does not dispatch the final read before +`period.end + duration_after_period_end`. At or after that boundary it requests a full, +verified snapshot through the snapshot offering. A successful read is published as an +official revision under the selected contractual policy. + +`finalized_at` is the trusted producer publication time after the qualifying acquisition +completes. This avoids backdating finality to a cutoff at which the SDK had not yet +verified the published content. Source availability and retry rules remain in force; a +cutoff makes a close permissible, not magically available. + +### Stabilized + +The producer begins verified snapshot observations when the snapshot source is +available. Every successful observation is retained, including unchanged rereads. The +existing content-change suppression remains in force: an unchanged observation advances +the durable observation ordinal and history but does not create another snapshot +revision. + +A stability run begins when the current snapshot content is first observed. It continues +only while: + +- each later observation has the same content fingerprint and immutable row binding; +- no scheduled observation is missed; and +- the current snapshot revision and observation history remain readable. + +A content change or a gap past the frozen next-observation deadline starts a new run. +Source-provided `stable_since` is retained as source evidence but never establishes the +run. + +The content qualifies when the latest retained observation is at or after both +`period.end + minimum_age` and `run_start + unchanged_for`. The producer then publishes +the verified current content as an official revision with the selected policy ID, +`stabilized`, and the qualifying observation time as `finalized_at`. + +The final qualifying observation is committed before the official revision. A crash +between those commits can delay the close but cannot publish one without durable proof; +the next worker turn resumes from the retained history and performs an idempotent close. + +## Durable Observation Evidence + +Extend the provisional observation store protocol with a bounded, account-scoped list +operation for one obligation. Memory and PostgreSQL stores already retain observations +by observation ordinal, so no schema migration is required. The PostgreSQL implementation +will return them in ordinal order from the existing immutable observation table. + +The stabilization evaluator consumes this retained sequence and validates each record +against its referenced snapshot revision. It does not trust a derived checkpoint alone. +The current restatement checkpoint remains the scheduling cursor; the observations are +the proof. + +Observation listing is bounded by the policy interval and cadence. The evaluator asks +only for the suffix needed to cover `unchanged_for`, plus the immediately preceding +observation needed to establish the run boundary. Store implementations must not scan +unrelated accounts or obligations. + +## Revision Construction + +Official revision construction receives a validated selected policy and a proven +`finalized_at`; it no longer derives either from the offering ID. The revision fields are: + +- `finality="official"` +- `finality_basis=selected_policy.basis` +- `finality_policy_id=selected_policy.finality_policy_id` +- `finalized_at=the policy-specific proven instant` + +All existing content hashing, source publication references, control totals, row staging, +terminal-official behavior, and adjustment-only corrections remain unchanged. + +## Error Handling + +Admission fails before obligations are created when: + +- official finality has no selected policy; +- the selected policy is malformed or uses an unsupported duration; +- the source offering class cannot support the selected proof flow; +- an explicit snapshot-to-official path has no selected policy; or +- production offering, definition binding, and selected policy disagree. + +Runtime publication fails closed when source-final evidence names the wrong signal, +retained stabilization history is missing or unreadable, observation identities conflict, +or the producer clock would make `finalized_at` later than `created_at`. Temporary source +unavailability continues through the existing retry and escalation path. + +Legacy retained obligations without a selected policy remain readable and projectable. +They are not reinterpreted or rewritten. A settled legacy official revision stays +settled; unfinished legacy official work requires a new configuration generation with an +explicit policy. + +## Tests + +Add focused tests for: + +- policy construction, identifier validation, duration validation, and storage round trips; +- official admission requiring a selected policy and the correct source-offering class; +- source-final success, wrong signal, early evidence, and exact emitted policy fields; +- contractual cutoff refusing early dispatch and promoting the first qualifying verified + snapshot; +- stabilization across unchanged observations without duplicate snapshot revisions; +- stabilization reset after content change or missed observation deadline; +- restart recovery from retained memory and PostgreSQL observation history; +- unreadable or inconsistent retained evidence failing closed; +- exact `finalized_at <= created_at` behavior; +- unchanged snapshot-only reporting, settling, reconciliation, materialization, and status + projections; and +- the package-grain Reliable Reporting 1.2 snapshot offering remaining snapshot-only. + +Verification will run targeted unit and conformance suites first, followed by the full +Python test, type-check, lint, and package-storage suites used by CI. From 4f551fb45f29c1bfd3ba3832744c68cf98511e88 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 11:40:16 +0530 Subject: [PATCH 02/39] docs(reporting): plan official finality support --- ...tion-driven-official-reporting-finality.md | 721 ++++++++++++++++++ 1 file changed, 721 insertions(+) create mode 100644 docs/superpowers/plans/2026-11-18-definition-driven-official-reporting-finality.md diff --git a/docs/superpowers/plans/2026-11-18-definition-driven-official-reporting-finality.md b/docs/superpowers/plans/2026-11-18-definition-driven-official-reporting-finality.md new file mode 100644 index 000000000..c089057f1 --- /dev/null +++ b/docs/superpowers/plans/2026-11-18-definition-driven-official-reporting-finality.md @@ -0,0 +1,721 @@ +# Definition-Driven Official Reporting Finality Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the Python reporting producer publish official revisions only after proving the exact selected report-definition finality policy. + +**Architecture:** Add an immutable three-variant finality-policy model to the retained definition binding, validate its source requirements during production admission, and pass a policy-specific proof into revision construction. Source-final uses authoritative source evidence; contractual cutoff promotes a post-cutoff snapshot; stabilized promotes snapshot content only after a bounded durable observation history proves an uninterrupted unchanged interval. + +**Tech Stack:** Python 3.10+, frozen dataclasses, Pydantic source contracts, asyncio, in-memory and PostgreSQL reporting stores, pytest/pytest-asyncio, mypy, pyright, Ruff. + +**Spec:** `docs/superpowers/specs/2026-11-18-definition-driven-official-reporting-finality-design.md` + +## Global Constraints + +- Never infer or default a finality policy. +- Only `source_final` requires an authoritative upstream signal. +- `contractual_cutoff` and `stabilized` promote verified snapshot content. +- `stable_since`, `official_close_lag`, source offering IDs, and manifest evidence never select a policy. +- Reject calendar-relative years and months instead of approximating them. +- Keep the shared PostgreSQL configuration and obligation column layouts unchanged. +- Preserve unchanged-content suppression: successful unchanged observations advance history without creating revisions. +- Legacy retained records remain readable; unfinished official work without a policy fails closed. +- Use `apply_patch` for edits and explicitly stage named files only. + +--- + +### Task 1: Immutable finality-policy model and retained binding + +**Files:** +- Create: `src/adcp/reporting/ledger/finality.py` +- Modify: `src/adcp/reporting/ledger/models.py:120-190` +- Modify: `src/adcp/reporting/ledger/_record_rows.py:100-160` +- Modify: `src/adcp/reporting/ledger/__init__.py:90-330` +- Test: `tests/test_reporting_ledger.py` +- Test: `tests/type_checks/reporting_finality.py` + +**Interfaces:** +- Produces: `ReportingSourceFinalPolicy`, `ReportingContractualCutoffPolicy`, `ReportingStabilizedPolicy`, and `ReportingFinalityPolicy`. +- Produces: `reporting_finality_policy_from_storage(value: Mapping[str, Any]) -> ReportingFinalityPolicy`. +- Produces: `ReportingDefinitionBinding.finality_policy: ReportingFinalityPolicy | None`. +- Consumes later: admission and producer tasks read the selected policy only from the obligation's retained definition binding. + +- [ ] **Step 1: Write failing policy validation and storage tests** + +```python +def test_definition_binding_retains_selected_finality_policy_off_wire() -> None: + policy = ReportingStabilizedPolicy( + finality_policy_id="daily-stable-v1", + minimum_age="P2D", + unchanged_for="PT12H", + ) + binding = replace(DEFINITION, finality_policy=policy) + + assert binding.to_wire() == DEFINITION.to_wire() + assert binding.to_storage()["finality_policy"] == { + "finality_policy_id": "daily-stable-v1", + "basis": "stabilized", + "minimum_age": "P2D", + "unchanged_for": "PT12H", + } + assert _definition_from_payload(binding.to_storage()) == binding + + +@pytest.mark.parametrize("value", ["P1M", "P1Y", "not-a-duration"]) +def test_finality_policy_refuses_unsupported_duration(value: str) -> None: + with pytest.raises(ValueError, match="supported reporting duration"): + ReportingContractualCutoffPolicy("cutoff-v1", value) +``` + +Add a type-check fixture that constructs all three variants, assigns each to `ReportingFinalityPolicy`, and passes it to `ReportingDefinitionBinding`. + +- [ ] **Step 2: Run the new tests and confirm the missing API failure** + +```bash +uv run pytest tests/test_reporting_ledger.py -k 'finality_policy or selected_finality' -q +uv run mypy tests/type_checks/reporting_finality.py +``` + +Expected: pytest collection/import fails because the policy classes do not exist; mypy reports missing exports. + +- [ ] **Step 3: Implement the frozen policy union** + +```python +@dataclass(frozen=True) +class ReportingSourceFinalPolicy: + finality_policy_id: str + source_signal: str + basis: Literal["source_final"] = field(default="source_final", init=False) + + def __post_init__(self) -> None: + _validate_policy_id(self.finality_policy_id) + if not 1 <= len(self.source_signal) <= 512: + raise ValueError("source_signal must contain 1 to 512 characters") + + +@dataclass(frozen=True) +class ReportingContractualCutoffPolicy: + finality_policy_id: str + duration_after_period_end: str + basis: Literal["contractual_cutoff"] = field(default="contractual_cutoff", init=False) + + def __post_init__(self) -> None: + _validate_policy_id(self.finality_policy_id) + iso_duration_milliseconds_v1(self.duration_after_period_end) + + +@dataclass(frozen=True) +class ReportingStabilizedPolicy: + finality_policy_id: str + minimum_age: str + unchanged_for: str + basis: Literal["stabilized"] = field(default="stabilized", init=False) + + def __post_init__(self) -> None: + _validate_policy_id(self.finality_policy_id) + iso_duration_milliseconds_v1(self.minimum_age) + iso_duration_milliseconds_v1(self.unchanged_for) + + +ReportingFinalityPolicy = ( + ReportingSourceFinalPolicy | ReportingContractualCutoffPolicy | ReportingStabilizedPolicy +) +``` + +Implement `to_storage()` on each variant and a basis-discriminated closed decoder. Reject unknown keys and bases so retained policy bytes cannot be silently reinterpreted. + +- [ ] **Step 4: Add the policy to `ReportingDefinitionBinding` storage only** + +Add `finality_policy: ReportingFinalityPolicy | None = None`, validate its exact runtime type in `__post_init__`, append it only in `to_storage()`, and leave `to_wire()` byte-for-byte unchanged. `_definition_from_payload()` must pop and decode `finality_policy` before constructing the binding. + +- [ ] **Step 5: Export the policy API and run focused tests** + +```bash +uv run pytest tests/test_reporting_ledger.py -k 'finality_policy or selected_finality' -q +uv run mypy tests/type_checks/reporting_finality.py +uv run ruff check src/adcp/reporting/ledger/finality.py src/adcp/reporting/ledger/models.py src/adcp/reporting/ledger/_record_rows.py tests/test_reporting_ledger.py tests/type_checks/reporting_finality.py +``` + +Expected: all commands pass. + +- [ ] **Step 6: Commit the policy model** + +```bash +git add src/adcp/reporting/ledger/finality.py src/adcp/reporting/ledger/models.py src/adcp/reporting/ledger/_record_rows.py src/adcp/reporting/ledger/__init__.py tests/test_reporting_ledger.py tests/type_checks/reporting_finality.py +git commit -m "feat(reporting): retain selected finality policies" +``` + +--- + +### Task 2: Definition-driven production admission and source routing + +**Files:** +- Modify: `src/adcp/reporting/service.py:790-875` +- Modify: `src/adcp/reporting/production/offerings.py:40-240` +- Modify: `src/adcp/reporting/production/configuration.py:35-105` +- Test: `tests/test_reliable_reporting_service.py` +- Test: `tests/conformance/reporting/test_reporting_production_configuration.py` + +**Interfaces:** +- Consumes: `ReportingDefinitionBinding.finality_policy` and the policy variants from Task 1. +- Produces: one source rule mapping `source_final -> official offering` and `contractual_cutoff|stabilized -> snapshot offering`. +- Produces: fail-fast configuration errors before obligation creation. + +- [ ] **Step 1: Write failing admission tests for all three bases** + +```python +@pytest.mark.parametrize( + ("policy", "expected_offering"), + [ + (ReportingSourceFinalPolicy("source-v1", "delivery.is_final"), OFFICIAL_OFFERING_ID), + (ReportingContractualCutoffPolicy("cutoff-v1", "P2D"), SNAPSHOT_OFFERING_ID), + (ReportingStabilizedPolicy("stable-v1", "P1D", "PT6H"), SNAPSHOT_OFFERING_ID), + ], +) +async def test_official_configuration_routes_by_selected_policy(policy, expected_offering): + configuration = official_configuration(policy) + await service.configure(configuration) + binding = service._bindings[configuration.generation_key] + assert binding.producer._offerings.offering_for(finality_policy_source_class(policy)) == expected_offering + + +async def test_official_configuration_without_policy_fails_before_obligation(): + with pytest.raises(ReliableReportingConfigurationError, match="selected finality policy"): + await service.configure(official_configuration(None)) + assert await service.store.list_obligations(account_id=ACCOUNT) == [] +``` + +Add the explicit-close admission case: + +```python +async def test_snapshot_official_close_requires_selected_policy() -> None: + configuration = snapshot_configuration(definition=DEFINITION) + context = account_context_with_official_close(configuration) + service = ReliableReportingService.memory(account_context=lambda _: context) + service.sources.register("gam", closing_adapter()) + + with pytest.raises(ReliableReportingConfigurationError, match="selected finality policy"): + await service.configure(configuration) +``` + +- [ ] **Step 2: Run the admission tests and verify current routing fails** + +```bash +uv run pytest tests/test_reliable_reporting_service.py -k 'official_configuration or selected_policy' -q +uv run pytest tests/conformance/reporting/test_reporting_production_configuration.py -k finality -q +``` + +Expected: missing-policy admission succeeds or the service insists on an authoritative offering for derived policies. + +- [ ] **Step 3: Centralize the source-finality requirement** + +```python +def finality_policy_source_class(policy: ReportingFinalityPolicy) -> Literal["snapshot", "official"]: + return "official" if isinstance(policy, ReportingSourceFinalPolicy) else "snapshot" +``` + +Use this helper in service and production offering validation. Obtain the policy from the exact `ReportingDefinitionBinding` shared by the configuration and verification key. Reject `required_finality="official"` with no policy, reject the wrong source offering class, and never fall back to `required_finality` to choose a source. + +- [ ] **Step 4: Validate explicit snapshot-to-official configuration** + +Keep ordinary snapshot-only configurations unchanged. When a snapshot source declares an explicit official-close path, require `definition.finality_policy`; validate that policy's source class exactly as for an official-required configuration. Do not interpret `official_close_lag` as a policy or policy duration. + +- [ ] **Step 5: Run production admission regressions** + +```bash +uv run pytest tests/test_reliable_reporting_service.py -q +uv run pytest tests/conformance/reporting/test_reporting_production_configuration.py -q +uv run ruff check src/adcp/reporting/service.py src/adcp/reporting/production/offerings.py src/adcp/reporting/production/configuration.py tests/test_reliable_reporting_service.py tests/conformance/reporting/test_reporting_production_configuration.py +``` + +Expected: all commands pass; snapshot-only fixtures without an official-close path need no policy. + +- [ ] **Step 6: Commit admission and routing** + +```bash +git add src/adcp/reporting/ledger/finality.py src/adcp/reporting/service.py src/adcp/reporting/production/offerings.py src/adcp/reporting/production/configuration.py tests/test_reliable_reporting_service.py tests/conformance/reporting/test_reporting_production_configuration.py +git commit -m "feat(reporting): validate official finality admission" +``` + +--- + +### Task 3: Bounded durable observation-history reads + +**Files:** +- Modify: `src/adcp/reporting/ledger/provisional.py:190-220` +- Modify: `src/adcp/reporting/ledger/store.py:830-850,1320-1440` +- Modify: `packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py:1785-1820` +- Test: `tests/test_reporting_provisional_observations.py` + +**Interfaces:** +- Produces: `ProvisionalObservationStore.list_provisional_observations(*, account_id: str, reporting_obligation_id: str, limit: int) -> tuple[ProvisionalObservation, ...]`. +- Guarantees: account/obligation scoped, bounded by `limit`, ascending observation ordinal. +- Consumes later: stabilization requests `ceil(unchanged_for / cadence) + 2` records. + +- [ ] **Step 1: Write failing memory/PostgreSQL history tests** + +```python +async def test_observation_history_is_bounded_ordered_and_account_scoped(make_harness): + producer, store, _, clock = await make_harness( + _capabilities(restatement_window="P3D", restatement_cadence="PT1H") + ) + await producer.run_worker() + clock[0] += timedelta(hours=1) + await producer.run_worker() + clock[0] += timedelta(hours=1) + await producer.run_worker() + obligation = await _only_obligation(store) + + history = await store.list_provisional_observations( + account_id=ACCOUNT, + reporting_obligation_id=obligation.reporting_obligation_id, + limit=2, + ) + + assert [item.acquisition.ordinal for item in history] == [1, 2] + assert all(item.acquisition.account_id == ACCOUNT for item in history) +``` + +Also assert `limit <= 0` raises `ValueError` and a different account returns an empty tuple. + +- [ ] **Step 2: Run the test and confirm the protocol method is absent** + +```bash +uv run pytest tests/test_reporting_provisional_observations.py -k observation_history -q +``` + +Expected: FAIL with `AttributeError` for `list_provisional_observations`. + +- [ ] **Step 3: Implement the protocol and in-memory suffix read** + +```python +async def list_provisional_observations( + self, *, account_id: str, reporting_obligation_id: str, limit: int +) -> tuple[ProvisionalObservation, ...]: + if type(limit) is not int or limit <= 0: + raise ValueError("observation history limit must be a positive integer") + matches = sorted( + ( + (ordinal, value) + for (account, obligation, ordinal), value in self._provisional_observations.items() + if account == account_id and obligation == reporting_obligation_id + ), + key=lambda item: item[0], + ) + return tuple(value for _, value in matches[-limit:]) +``` + +- [ ] **Step 4: Implement the PostgreSQL suffix read** + +Query the immutable table with exact owner predicates and `ORDER BY ordinal DESC LIMIT %s`, decode rows, then reverse the tuple so both stores expose ascending order. Bind `limit` as a SQL parameter. + +- [ ] **Step 5: Run memory and PostgreSQL observation tests** + +```bash +uv run pytest tests/test_reporting_provisional_observations.py -k 'observation_history or unchanged_scheduled_read' -q +uv run ruff check src/adcp/reporting/ledger/provisional.py src/adcp/reporting/ledger/store.py packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py tests/test_reporting_provisional_observations.py +``` + +Expected: memory passes; PostgreSQL passes when `ADCP_PG_TEST_URL` is configured and otherwise follows the existing skip behavior. + +- [ ] **Step 6: Commit observation history support** + +```bash +git add src/adcp/reporting/ledger/provisional.py src/adcp/reporting/ledger/store.py packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py tests/test_reporting_provisional_observations.py +git commit -m "feat(reporting): expose retained observation history" +``` + +--- + +### Task 4: Source-final and contractual-cutoff proof flows + +**Files:** +- Modify: `src/adcp/reporting/ledger/finality.py` +- Modify: `src/adcp/reporting/ledger/producer.py:700-920,1095-1325,1720-1885,2050-2145` +- Test: `tests/test_reporting_official_finality.py` +- Test: `tests/test_reporting_source_availability.py` + +**Interfaces:** +- Produces: `FinalityProof(finality_policy_id: str, basis: ..., finalized_at: datetime)`. +- Produces: `contractual_cutoff_at(obligation, policy) -> datetime`. +- Produces: `source_finality_proof(obligation, manifest, policy) -> FinalityProof`. +- Changes: `commit_revision_from_manifest(..., finality_proof: FinalityProof | None = None)` requires proof for new official revisions. + +- [ ] **Step 1: Write failing source-final tests** + +```python +async def test_source_final_emits_exact_selected_policy() -> None: + policy = ReportingSourceFinalPolicy("delivery-final-v1", "delivery.is_final") + producer, store, configuration, obligation, authoritative_manifest = await official_harness(policy) + + revision = await producer.acquire_obligation(configuration, obligation) + + assert revision is not None + assert revision.finality_basis == "source_final" + assert revision.finality_policy_id == "delivery-final-v1" + assert revision.finalized_at == authoritative_manifest.finality_evidence.observed_at + + +async def test_source_final_rejects_a_different_source_signal() -> None: + with pytest.raises(LedgerConflictError, match="source signal"): + await producer.acquire_obligation(configuration, obligation) +``` + +- [ ] **Step 2: Write failing contractual-cutoff tests** + +```python +async def test_contractual_cutoff_promotes_a_post_cutoff_snapshot() -> None: + policy = ReportingContractualCutoffPolicy("two-day-cutoff", "P2D") + producer, store, configuration, obligation, calls, clock = await official_harness(policy) + + clock[0] = obligation.period.end + timedelta(days=2) - timedelta(microseconds=1) + assert (await producer.run_configuration(configuration, now=clock[0])).revisions_committed == [] + assert calls == [] + + clock[0] += timedelta(microseconds=1) + turn = await producer.run_configuration(configuration, now=clock[0]) + revision = await store.get_revision( + account_id=configuration.account_id, + reporting_revision_id=turn.revisions_committed[0], + ) + assert revision is not None + assert calls[0].publication_class == "PROVISIONAL_SNAPSHOT" + assert revision.finality_basis == "contractual_cutoff" + assert revision.finality_policy_id == "two-day-cutoff" + assert revision.finalized_at == revision.created_at +``` + +- [ ] **Step 3: Run focused tests and verify current synthetic behavior fails** + +```bash +uv run pytest tests/test_reporting_official_finality.py -k 'source_final or contractual' -q +``` + +Expected: exact IDs/signals and snapshot promotion fail under current producer behavior. + +- [ ] **Step 4: Implement policy proofs and source/output finality separation** + +```python +@dataclass(frozen=True) +class FinalityProof: + finality_policy_id: str + basis: Literal["source_final", "contractual_cutoff", "stabilized"] + finalized_at: datetime + + +def contractual_cutoff_at( + obligation: ReportingObligationRecord, + policy: ReportingContractualCutoffPolicy, +) -> datetime: + return _utc(obligation.period.end) + timedelta( + milliseconds=iso_duration_milliseconds_v1(policy.duration_after_period_end) + ) +``` + +Split source request class from output finality. Source-final requests `AUTHORITATIVE`; contractual cutoff requests `PROVISIONAL_SNAPSHOT` but passes `finality="official"` and a proof to revision construction. `_automatic_read_ready()` must include the cutoff boundary. + +- [ ] **Step 5: Make official revision construction proof-only** + +```python +if finality == "official" and finality_proof is None: + raise LedgerConflictError("FINALITY_POLICY_UNPROVEN", "official finality requires policy proof") + +finality_basis = finality_proof.basis if finality_proof else None +finality_policy_id = finality_proof.finality_policy_id if finality_proof else None +finalized_at = finality_proof.finalized_at if finality_proof else None +``` + +Validate `period.end <= finalized_at <= created_at`. Keep snapshot fields absent. Preserve replay of already-retained official revisions before applying the new-write requirement. + +- [ ] **Step 6: Run source-final, cutoff, and availability regressions** + +```bash +uv run pytest tests/test_reporting_official_finality.py -k 'source_final or contractual' -q +uv run pytest tests/test_reporting_source_availability.py -q +uv run pytest tests/test_reporting_producer_retry.py -q +uv run ruff check src/adcp/reporting/ledger/finality.py src/adcp/reporting/ledger/producer.py tests/test_reporting_official_finality.py tests/test_reporting_source_availability.py +``` + +Expected: all commands pass. + +- [ ] **Step 7: Commit the two proof flows** + +```bash +git add src/adcp/reporting/ledger/finality.py src/adcp/reporting/ledger/producer.py tests/test_reporting_official_finality.py tests/test_reporting_source_availability.py +git commit -m "feat(reporting): prove official source and cutoff finality" +``` + +--- + +### Task 5: Stabilized finality from retained observation history + +**Files:** +- Modify: `src/adcp/reporting/ledger/finality.py` +- Modify: `src/adcp/reporting/ledger/producer.py:920-1095,1190-1280,1810-1965` +- Modify: `src/adcp/reporting/ledger/store.py:700-750` +- Test: `tests/test_reporting_official_finality.py` +- Test: `tests/test_reporting_provisional_observations.py` + +**Interfaces:** +- Produces: `prove_stabilized_finality(...) -> FinalityProof | None` over a bounded ordered observation suffix. +- Consumes: `list_provisional_observations(..., limit=ceil(unchanged_for/cadence)+2)` from Task 3. +- Produces: a policy-controlled official stabilization loop that commits the qualifying observation before the official revision. + +- [ ] **Step 1: Write failing stable-history tests** + +```python +async def test_stabilized_policy_promotes_unchanged_content_without_duplicate_snapshots() -> None: + policy = ReportingStabilizedPolicy("stable-12h", "P1D", "PT12H") + producer, store, configuration, obligation, fetch, clock = await official_harness(policy) + + await producer.run_configuration(configuration, now=clock[0]) + clock[0] += timedelta(hours=6) + await producer.run_configuration(configuration, now=clock[0]) + clock[0] += timedelta(hours=6) + turn = await restarted_producer(producer, store, clock).run_configuration( + configuration, now=clock[0] + ) + + revisions = await store.list_revisions( + account_id=configuration.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + assert [item.finality for item in revisions].count("snapshot") == 1 + assert [item.finality for item in revisions].count("official") == 1 + official = next(item for item in revisions if item.finality == "official") + assert official.reporting_revision_id in turn.revisions_committed + assert official.finality_policy_id == "stable-12h" +``` + +- [ ] **Step 2: Write failing reset and fail-closed tests** + +```python +async def test_stability_run_restarts_after_content_change() -> None: + await observe_at(hours=0, impressions=10) + await observe_at(hours=6, impressions=11) + await observe_at(hours=12, impressions=11) + assert not await official_revision() + await observe_at(hours=18, impressions=11) + assert await official_revision() + + +async def test_stability_run_restarts_after_missed_deadline() -> None: + await observe_at(hours=0, impressions=10) + await observe_at(hours=13, impressions=10) # cadence is six hours + assert not await official_revision() +``` + +Add an unreadable referenced-revision case: + +```python +async def test_stability_proof_refuses_unreadable_snapshot_history() -> None: + await observe_at(hours=0, impressions=10) + snapshot = await current_snapshot_revision() + await mark_revision_unreadable(snapshot.reporting_revision_id) + await observe_at(hours=6, impressions=10) + + with pytest.raises(LedgerConflictError, match="history.*readable"): + await evaluate_stabilization() + assert not await official_revision() +``` + +- [ ] **Step 3: Run stabilization tests and confirm no proof engine exists** + +```bash +uv run pytest tests/test_reporting_official_finality.py -k stabil -q +``` + +Expected: FAIL because official-required configurations currently perform one authoritative read instead of a retained snapshot loop. + +- [ ] **Step 4: Implement bounded stabilization proof evaluation** + +Walk history backward from the latest observation. For each adjacent pair: + +```python +same_content = current_manifest.content_fingerprint == prior_manifest.content_fingerprint +dispatch_at = current.acquisition.request().period.source_read_cutoff_at +on_time = prior.next_due_at is not None and dispatch_at <= prior.next_due_at +same_revision = current.revision_id == prior.revision_id +``` + +Stop at the first false condition. Validate the latest referenced revision and rows with `validate_retained_provisional_observation`. Return `None` unless the latest checked time reaches `period.end + minimum_age` and `run_start + unchanged_for`. When proven, return `FinalityProof(policy_id, "stabilized", latest.checked_at)`. + +- [ ] **Step 5: Add the policy-controlled stabilization scheduler** + +For a stabilized policy, freeze a provisional acquisition policy whose cadence comes from the snapshot offering and whose window remains open until proof succeeds. Every successful source read commits its observation through the existing observation store. After commit, load bounded history and evaluate it. If unproven, leave durable work pending. + +When proven, reuse the qualifying manifest and rows to commit official. If the manifest is already represented by snapshot, derive a distinct official ID: + +```python +def _derived_official_revision_id(manifest, proof: FinalityProof) -> str: + digest = hashlib.sha256( + canonical_json_utf8_v1( + [manifest.publication_id, "official", proof.finality_policy_id] + ) + ).hexdigest() + return f"rpr_{digest[:40]}" +``` + +Do not supersede the snapshot. Commit the qualifying observation first, then official. Replay must reconstruct the same ID and proof instant. + +- [ ] **Step 6: Run stabilization, observation, and settling suites** + +```bash +uv run pytest tests/test_reporting_official_finality.py -k stabil -q +uv run pytest tests/test_reporting_provisional_observations.py -q +uv run pytest tests/test_reporting_settling.py -q +uv run pytest tests/test_reporting_scheduling_jitter.py -q +uv run ruff check src/adcp/reporting/ledger/finality.py src/adcp/reporting/ledger/producer.py src/adcp/reporting/ledger/store.py tests/test_reporting_official_finality.py tests/test_reporting_provisional_observations.py +``` + +Expected: all commands pass; PostgreSQL cases follow their configured run/skip behavior. + +- [ ] **Step 7: Commit stabilized finality** + +```bash +git add src/adcp/reporting/ledger/finality.py src/adcp/reporting/ledger/producer.py src/adcp/reporting/ledger/store.py tests/test_reporting_official_finality.py tests/test_reporting_provisional_observations.py +git commit -m "feat(reporting): prove stabilized official finality" +``` + +--- + +### Task 6: Legacy handling, explicit close migration, and reporting regressions + +**Files:** +- Modify: `tests/test_reporting_settling.py` +- Modify: `tests/test_reporting_source_availability.py` +- Modify: `tests/test_reporting_producer_retry.py` +- Modify: `tests/test_reporting_scheduling_jitter.py` +- Modify: `tests/conformance/reporting/test_reporting_production_settling.py` +- Modify: `tests/conformance/reporting/_production_support.py` +- Modify: `docs/reporting-source-adapters.md` +- Modify: `docs/reporting-provisional-observations.md` + +**Interfaces:** +- Consumes: all policy, admission, proof, and observation APIs from Tasks 1-5. +- Produces: documented migration for new official work and read-only compatibility for retained legacy evidence. + +- [ ] **Step 1: Add explicit legacy and snapshot-only tests** + +```python +async def test_retained_legacy_official_revision_remains_readable() -> None: + await store.commit_revision(legacy_official_revision(), rows) + result = await ReportingStatusHandler(store).handle({"view": "revision"}, caller) + assert result["revisions"][0]["finality"] == "official" + + +async def test_snapshot_only_settling_never_emits_official_without_selected_policy() -> None: + producer, store, _, clock = await snapshot_package_harness() + await producer.run_worker() + clock[0] += timedelta(days=120) + await producer.run_worker() + assert {item.finality for item in await _revisions(store)} == {"snapshot"} +``` + +- [ ] **Step 2: Run affected suites and inventory fixture failures** + +```bash +uv run pytest tests/test_reporting_settling.py tests/test_reporting_source_availability.py tests/conformance/reporting/test_reporting_production_settling.py -q +rg -n 'required_finality="official"|required_finality=.official.' tests -g '*.py' +``` + +Expected: fixtures that create new official work without a policy fail with the closed error. + +- [ ] **Step 3: Update official fixtures with the policy they actually assert** + +For each new-work fixture, add one explicit policy matching its tested behavior. Do not add a global default helper. Source-final fixtures name the exact manifest `evidence_ref`; cutoff and stabilization fixtures use explicit protocol durations. Leave pure read/projection fixtures unchanged when they intentionally model retained legacy bytes. + +- [ ] **Step 4: Document adopter migration and policy ownership** + +Add this guidance to both reporting guides: + +```text +required_finality: official requires one selected policy copied from the pinned report definition. +The SDK does not choose that policy. source_final uses the authoritative source offering; +contractual_cutoff and stabilized use the snapshot offering. stable_since and +official_close_lag are source evidence/scheduling metadata, not finality policies. +``` + +Document that explicit snapshot-to-official setups pin a policy, while snapshot-only offerings require no policy and never close merely because time elapsed. + +- [ ] **Step 5: Run all reporting-focused suites** + +```bash +uv run pytest tests/test_reporting_*.py tests/conformance/reporting -q +uv run mypy src/adcp/reporting packages/adcp-reporting-storage/src tests/type_checks +uv run pyright tests/type_checks/reporting_finality.py +uv run ruff check src/adcp/reporting packages/adcp-reporting-storage/src tests/test_reporting_*.py tests/conformance/reporting tests/type_checks/reporting_finality.py +``` + +Expected: all commands pass, with only existing environment-dependent skips. + +- [ ] **Step 6: Commit regressions and documentation** + +Stage only task files shown by `git diff --name-only`, adding every path explicitly: + +```bash +git add docs/reporting-source-adapters.md docs/reporting-provisional-observations.md tests/test_reporting_settling.py tests/test_reporting_source_availability.py tests/test_reporting_producer_retry.py tests/test_reporting_scheduling_jitter.py tests/conformance/reporting/test_reporting_production_settling.py tests/conformance/reporting/_production_support.py +git commit -m "docs(reporting): explain official finality policy ownership" +``` + +--- + +### Task 7: Full verification and pull request + +**Files:** +- Verify: all files changed since `origin/main` +- Modify only if verification exposes a defect in implementation or tests + +**Interfaces:** +- Consumes: completed implementation from Tasks 1-6. +- Produces: a conventional-commit PR linked to AI-22659 with verification evidence. + +- [ ] **Step 1: Inspect the complete diff** + +```bash +git status --short +git diff --stat origin/main... +git diff --check origin/main... +git diff origin/main... -- src/adcp/reporting packages/adcp-reporting-storage tests docs/reporting-source-adapters.md docs/reporting-provisional-observations.md +``` + +Expected: only finality-policy, producer, observation-store, tests, and reporting documentation changes; `git diff --check` is silent. + +- [ ] **Step 2: Run the repository verification matrix** + +Confirm CI commands from `pyproject.toml` and workflow files, then run: + +```bash +uv run pytest -q +uv run mypy src packages/adcp-reporting-storage/src tests/type_checks +uv run pyright +uv run ruff check . +uv run ruff format --check . +``` + +Expected: every command exits zero. Record environment-dependent skips; do not report skipped PostgreSQL tests as executed. + +- [ ] **Step 3: Re-run PostgreSQL conformance when configured** + +```bash +uv run pytest tests/test_reporting_provisional_observations.py tests/conformance/reporting/test_pg_reporting_ledger.py -q +``` + +Expected: memory and PostgreSQL pass when `ADCP_PG_TEST_URL` is configured; otherwise record the exact skip reason. + +- [ ] **Step 4: Resolve verification failures through their owning task** + +If verification fails, return to Task 1 for policy/storage defects, Task 2 for admission defects, Task 3 for observation-store defects, Task 4 for source-final/cutoff defects, Task 5 for stabilization defects, or Task 6 for fixture/documentation defects. Add a failing focused test, make the minimal correction, rerun that task's commands, and use that task's explicit commit command before repeating Step 2. + +- [ ] **Step 5: Push and open the PR** + +```bash +git push -u origin rmatani/ai-22659-support-definition-driven-official +gh pr create --base main --title "feat(reporting): support definition-driven official finality" --body-file .context/ai-22659-pr.md +``` + +The PR body summarizes all three flows, states that snapshot-only Reliable Reporting 1.2 still does not require official close, links AI-22659, lists verification commands/results, and identifies skipped PostgreSQL cases. + +- [ ] **Step 6: Start the PR feedback loop** + +Invoke `scope3-developer:pr-feedback-loop`. Monitor checks and inline feedback, fix failures with test-first reproductions, push follow-up commits, and stop only when CI is green and review feedback is addressed. Never merge the PR. From 3c330bf63df3cf00b3ceccd6e209787fc6dfd845 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 11:45:35 +0530 Subject: [PATCH 03/39] feat(reporting): retain selected finality policies --- src/adcp/reporting/ledger/__init__.py | 12 +++ src/adcp/reporting/ledger/_record_rows.py | 9 +- src/adcp/reporting/ledger/finality.py | 121 ++++++++++++++++++++++ src/adcp/reporting/ledger/models.py | 17 ++- tests/fixtures/public_api_snapshot.json | 5 + tests/test_reporting_ledger.py | 113 +++++++++++++++++++- tests/type_checks/reporting_finality.py | 33 ++++++ 7 files changed, 307 insertions(+), 3 deletions(-) create mode 100644 src/adcp/reporting/ledger/finality.py create mode 100644 tests/type_checks/reporting_finality.py diff --git a/src/adcp/reporting/ledger/__init__.py b/src/adcp/reporting/ledger/__init__.py index 52c46d7ef..9c8d7c3e5 100644 --- a/src/adcp/reporting/ledger/__init__.py +++ b/src/adcp/reporting/ledger/__init__.py @@ -123,6 +123,13 @@ ReportingRevisionReceiptRecord, ReportingVerificationRecord, ) +from adcp.reporting.ledger.finality import ( + ReportingContractualCutoffPolicy, + ReportingFinalityPolicy, + ReportingSourceFinalPolicy, + ReportingStabilizedPolicy, + reporting_finality_policy_from_storage, +) from adcp.reporting.ledger.health import ( ObligationProjection, aggregate_reporting_health, @@ -254,6 +261,7 @@ "ReportingCanonicalDigest", "ReportingConfiguration", "ReportingConfigurationGenerationKey", + "ReportingContractualCutoffPolicy", "ReportingControlTotalRecord", "ReportingCurrencyError", "ReportingCurrentRevisionsPageV1", @@ -265,6 +273,7 @@ "ReportingDestinationBinding", "ReportingDestinationStore", "ReportingFinality", + "ReportingFinalityPolicy", "ReportingHealth", "ReportingIssue", "ReportingIssueLifecycle", @@ -306,6 +315,8 @@ "ReportingWorkerRetryPolicy", "ReportingRowStoreError", "ReportingRowStoreErrorCode", + "ReportingSourceFinalPolicy", + "ReportingStabilizedPolicy", "RestatementCheckpoint", "RestatementCheckpointStore", "RetryScheduleEntry", @@ -334,6 +345,7 @@ "receipt_to_wire", "recompute_revision_content_sha256", "reject_reserved_authoritative_party", + "reporting_finality_policy_from_storage", "require_single_currency", "revision_binding_algorithm", "revision_content_sha256", diff --git a/src/adcp/reporting/ledger/_record_rows.py b/src/adcp/reporting/ledger/_record_rows.py index 50d08fe44..9aaea60d4 100644 --- a/src/adcp/reporting/ledger/_record_rows.py +++ b/src/adcp/reporting/ledger/_record_rows.py @@ -17,6 +17,7 @@ from typing import Any from adcp.reporting.evidence import ReportingCanonicalDigest, ReportingControlTotalRecord +from adcp.reporting.ledger.finality import reporting_finality_policy_from_storage from adcp.reporting.ledger.models import ( ConsumerStatusRecord, ReportingAdjustmentRecord, @@ -109,7 +110,13 @@ def _definition_payload( def _definition_from_payload(payload: dict[str, Any] | None) -> ReportingDefinitionBinding | None: - return ReportingDefinitionBinding(**payload) if payload else None + if not payload: + return None + fields = dict(payload) + policy = None + if "finality_policy" in fields: + policy = reporting_finality_policy_from_storage(fields.pop("finality_policy")) + return ReportingDefinitionBinding(**fields, finality_policy=policy) def _schedule_from_payload(payload: dict[str, Any]) -> ReportingScheduleSpec: diff --git a/src/adcp/reporting/ledger/finality.py b/src/adcp/reporting/ledger/finality.py new file mode 100644 index 000000000..46abee686 --- /dev/null +++ b/src/adcp/reporting/ledger/finality.py @@ -0,0 +1,121 @@ +"""Finality policies selected from a content-addressed report definition.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import Any, Literal + +from adcp.reporting.source import iso_duration_milliseconds_v1 + +__all__ = [ + "ReportingContractualCutoffPolicy", + "ReportingFinalityPolicy", + "ReportingSourceFinalPolicy", + "ReportingStabilizedPolicy", + "reporting_finality_policy_from_storage", +] + + +def _validate_policy_id(value: str) -> None: + if not isinstance(value, str) or re.fullmatch(r"[A-Za-z0-9_.:-]{1,255}", value) is None: + raise ValueError("finality_policy_id must match [A-Za-z0-9_.:-]{1,255}") + + +@dataclass(frozen=True) +class ReportingSourceFinalPolicy: + """Close on an authoritative source publication with the declared signal.""" + + finality_policy_id: str + source_signal: str + basis: Literal["source_final"] = field(default="source_final", init=False) + + def __post_init__(self) -> None: + _validate_policy_id(self.finality_policy_id) + if not isinstance(self.source_signal, str) or not 1 <= len(self.source_signal) <= 512: + raise ValueError("source_signal must contain 1 to 512 characters") + + def to_storage(self) -> dict[str, str]: + return { + "finality_policy_id": self.finality_policy_id, + "basis": self.basis, + "source_signal": self.source_signal, + } + + +@dataclass(frozen=True) +class ReportingContractualCutoffPolicy: + """Close on verified content observed after a fixed period-end cutoff.""" + + finality_policy_id: str + duration_after_period_end: str + basis: Literal["contractual_cutoff"] = field(default="contractual_cutoff", init=False) + + def __post_init__(self) -> None: + _validate_policy_id(self.finality_policy_id) + iso_duration_milliseconds_v1(self.duration_after_period_end) + + def to_storage(self) -> dict[str, str]: + return { + "finality_policy_id": self.finality_policy_id, + "basis": self.basis, + "duration_after_period_end": self.duration_after_period_end, + } + + +@dataclass(frozen=True) +class ReportingStabilizedPolicy: + """Close when verified content has remained unchanged for the required interval.""" + + finality_policy_id: str + minimum_age: str + unchanged_for: str + basis: Literal["stabilized"] = field(default="stabilized", init=False) + + def __post_init__(self) -> None: + _validate_policy_id(self.finality_policy_id) + iso_duration_milliseconds_v1(self.minimum_age) + iso_duration_milliseconds_v1(self.unchanged_for) + + def to_storage(self) -> dict[str, str]: + return { + "finality_policy_id": self.finality_policy_id, + "basis": self.basis, + "minimum_age": self.minimum_age, + "unchanged_for": self.unchanged_for, + } + + +ReportingFinalityPolicy = ( + ReportingSourceFinalPolicy | ReportingContractualCutoffPolicy | ReportingStabilizedPolicy +) + + +def reporting_finality_policy_from_storage(value: Mapping[str, Any]) -> ReportingFinalityPolicy: + """Decode the exact stored fields for a supported policy basis.""" + match value: + case { + "finality_policy_id": str(policy_id), + "basis": "source_final", + "source_signal": str(source_signal), + **extra, + } if not extra: + return ReportingSourceFinalPolicy(policy_id, source_signal) + case { + "finality_policy_id": str(policy_id), + "basis": "contractual_cutoff", + "duration_after_period_end": str(duration), + **extra, + } if not extra: + return ReportingContractualCutoffPolicy(policy_id, duration) + case { + "finality_policy_id": str(policy_id), + "basis": "stabilized", + "minimum_age": str(minimum_age), + "unchanged_for": str(unchanged_for), + **extra, + } if not extra: + return ReportingStabilizedPolicy(policy_id, minimum_age, unchanged_for) + case _: + raise ValueError("invalid reporting finality policy storage payload") diff --git a/src/adcp/reporting/ledger/models.py b/src/adcp/reporting/ledger/models.py index eebffbbfb..fee579f88 100644 --- a/src/adcp/reporting/ledger/models.py +++ b/src/adcp/reporting/ledger/models.py @@ -36,6 +36,12 @@ freeze_control_totals, principal_reference, ) +from adcp.reporting.ledger.finality import ( + ReportingContractualCutoffPolicy, + ReportingFinalityPolicy, + ReportingSourceFinalPolicy, + ReportingStabilizedPolicy, +) __all__ = [ "ReportingCaller", @@ -164,6 +170,7 @@ class ReportingDefinitionBinding: # buyer context. Tuples keep these declarations immutable after acceptance. monetary_metric_units: tuple[tuple[str, str], ...] = () monetary_control_total_units: tuple[tuple[str, str], ...] = () + finality_policy: ReportingFinalityPolicy | None = None def __post_init__(self) -> None: for field_name in ("monetary_metric_units", "monetary_control_total_units"): @@ -173,14 +180,22 @@ def __post_init__(self) -> None: if len(dict(units)) != len(units): raise ValueError(f"duplicate names in {field_name}") object.__setattr__(self, field_name, units) + if self.finality_policy is not None and type(self.finality_policy) not in ( + ReportingSourceFinalPolicy, + ReportingContractualCutoffPolicy, + ReportingStabilizedPolicy, + ): + raise ValueError("finality_policy must be a supported reporting finality policy") def to_storage(self) -> dict[str, Any]: - """Retain monetary semantics without changing AdCP wire records or old hashes.""" + """Retain trusted definition metadata alongside the AdCP wire fields.""" payload = self.to_wire() if self.monetary_metric_units: payload["monetary_metric_units"] = list(self.monetary_metric_units) if self.monetary_control_total_units: payload["monetary_control_total_units"] = list(self.monetary_control_total_units) + if self.finality_policy is not None: + payload["finality_policy"] = self.finality_policy.to_storage() return payload def to_wire(self) -> dict[str, Any]: diff --git a/tests/fixtures/public_api_snapshot.json b/tests/fixtures/public_api_snapshot.json index 48f3b62c8..381e66172 100644 --- a/tests/fixtures/public_api_snapshot.json +++ b/tests/fixtures/public_api_snapshot.json @@ -837,6 +837,7 @@ "ReportingCanonicalDigest": "adcp.reporting.evidence", "ReportingConfiguration": "adcp.reporting.ledger.models", "ReportingConfigurationGenerationKey": "adcp.reporting.ledger.models", + "ReportingContractualCutoffPolicy": "adcp.reporting.ledger.finality", "ReportingControlTotalRecord": "adcp.reporting.evidence", "ReportingCurrencyError": "adcp.reporting.currency", "ReportingCurrentRevisionsPageV1": "adcp.reporting.ledger.host_reads", @@ -848,6 +849,7 @@ "ReportingDestinationBinding": "adcp.reporting.ledger.delivery_models", "ReportingDestinationStore": "adcp.reporting.ledger.delivery", "ReportingFinality": "union[value:'official',value:'snapshot']", + "ReportingFinalityPolicy": "union[adcp.reporting.ledger.finality:ReportingContractualCutoffPolicy,adcp.reporting.ledger.finality:ReportingSourceFinalPolicy,adcp.reporting.ledger.finality:ReportingStabilizedPolicy]", "ReportingHealth": "union[value:'action_required',value:'complete',value:'delayed',value:'healthy',value:'waiting']", "ReportingIssue": "adcp.reporting.ledger.models", "ReportingIssueLifecycle": "adcp.reporting.ledger.models", @@ -892,7 +894,9 @@ "ReportingRowStoreError": "adcp.reporting.ledger.row_storage_errors", "ReportingRowStoreErrorCode": "union[value:'ABORTED',value:'CONTENT_CONFLICT',value:'DEADLINE_EXCEEDED',value:'INVALID_INPUT',value:'PROVIDER_UNAVAILABLE',value:'ROWS_EXPIRED',value:'ROWS_INTEGRITY_FAILED',value:'ROWS_UNAVAILABLE',value:'STATE_UNAVAILABLE',value:'UNSAFE_BINDING']", "ReportingScheduleSpec": "adcp.reporting.ledger.models", + "ReportingSourceFinalPolicy": "adcp.reporting.ledger.finality", "ReportingSourcePublicationStore": "adcp.reporting.ledger.store", + "ReportingStabilizedPolicy": "adcp.reporting.ledger.finality", "ReportingStatusCaller": "adcp.reporting.ledger.status", "ReportingStatusCallerResolver": "union[instance:builtins:list,union[adcp.reporting.ledger.status:ReportingStatusCaller]]", "ReportingStatusHandler": "adcp.reporting.ledger.status", @@ -929,6 +933,7 @@ "receipt_to_wire": "adcp.reporting.ledger.delivery", "recompute_revision_content_sha256": "adcp.reporting.ledger.producer", "reject_reserved_authoritative_party": "adcp.reporting.ledger.store", + "reporting_finality_policy_from_storage": "adcp.reporting.ledger.finality", "require_single_currency": "adcp.reporting.currency", "revision_binding_algorithm": "adcp.reporting.ledger.producer", "revision_content_sha256": "adcp.reporting.ledger.producer", diff --git a/tests/test_reporting_ledger.py b/tests/test_reporting_ledger.py index e6dc209e4..e071c1e3b 100644 --- a/tests/test_reporting_ledger.py +++ b/tests/test_reporting_ledger.py @@ -8,6 +8,7 @@ from __future__ import annotations +from dataclasses import replace from datetime import datetime, timedelta, timezone from typing import Any, cast @@ -21,11 +22,16 @@ ProducerOfferings, ReportingAdjustmentRecord, ReportingConfiguration, + ReportingContractualCutoffPolicy, + ReportingDefinitionBinding, ReportingDeliveryEscalation, + ReportingFinalityPolicy, ReportingObligationRecord, ReportingProducer, ReportingRevisionRecord, ReportingScheduleSpec, + ReportingSourceFinalPolicy, + ReportingStabilizedPolicy, ReportingStatusCaller, ReportingStatusHandler, aggregate_reporting_health, @@ -33,9 +39,11 @@ derive_period, iso_duration_to_timedelta, project_obligation_health, + reporting_finality_policy_from_storage, revision_content_sha256, ) from adcp.reporting.ledger import ReportingStatusCaller as OwnershipCaller +from adcp.reporting.ledger._record_rows import _definition_from_payload ACCOUNT = "acct_1" CALLER = ReportingStatusCaller(account_id=ACCOUNT, consumer_id="buyer_1") @@ -62,6 +70,13 @@ } HOURLY = ReportingScheduleSpec(period_duration="PT1H", delivery_sla="PT1H", alignment="utc") +DEFINITION = ReportingDefinitionBinding( + report_definition_uri="https://contracts.example.test/reporting/daily-delivery-v1", + report_definition_sha256="a" * 64, + schema_version="1.0.0", + schema_uri="https://contracts.example.test/reporting/rows-v1.json", + schema_sha256="b" * 64, +) def _configuration(**overrides) -> ReportingConfiguration: @@ -161,6 +176,103 @@ def _revision(obligation: ReportingObligationRecord, **overrides) -> ReportingRe return ReportingRevisionRecord(**defaults), rows # type: ignore[return-value,arg-type] +def test_definition_binding_retains_selected_finality_policy_off_wire() -> None: + policy = ReportingStabilizedPolicy( + finality_policy_id="daily-stable-v1", + minimum_age="P2D", + unchanged_for="PT12H", + ) + binding = replace(DEFINITION, finality_policy=policy) + + assert binding.to_wire() == DEFINITION.to_wire() + assert binding.to_storage()["finality_policy"] == { + "finality_policy_id": "daily-stable-v1", + "basis": "stabilized", + "minimum_age": "P2D", + "unchanged_for": "PT12H", + } + assert _definition_from_payload(binding.to_storage()) == binding + + +@pytest.mark.parametrize("value", ["P1M", "P1Y", "not-a-duration"]) +def test_finality_policy_refuses_unsupported_duration(value: str) -> None: + with pytest.raises(ValueError, match="supported reporting duration"): + ReportingContractualCutoffPolicy("cutoff-v1", value) + + +@pytest.mark.parametrize( + ("policy", "payload"), + [ + ( + ReportingSourceFinalPolicy("daily.source:v1", "source_closed"), + { + "finality_policy_id": "daily.source:v1", + "basis": "source_final", + "source_signal": "source_closed", + }, + ), + ( + ReportingContractualCutoffPolicy("cutoff-v1", "PT4H"), + { + "finality_policy_id": "cutoff-v1", + "basis": "contractual_cutoff", + "duration_after_period_end": "PT4H", + }, + ), + ], +) +def test_finality_policy_storage_round_trip( + policy: ReportingFinalityPolicy, payload: dict[str, str] +) -> None: + assert policy.to_storage() == payload + assert reporting_finality_policy_from_storage(payload) == policy + + +@pytest.mark.parametrize( + "payload", + [ + {"finality_policy_id": "daily-v1", "basis": "unknown"}, + {"finality_policy_id": "daily-v1", "basis": "source_final"}, + {"finality_policy_id": "daily-v1", "basis": "source_final", "source_signal": ["close"]}, + { + "finality_policy_id": "daily-v1", + "basis": "source_final", + "source_signal": "close", + "duration_after_period_end": "PT4H", + }, + None, + ], +) +def test_definition_binding_rejects_invalid_finality_policy_storage(payload: object) -> None: + with pytest.raises(ValueError, match="finality policy"): + _definition_from_payload({**DEFINITION.to_storage(), "finality_policy": payload}) + + +def test_definition_binding_reads_storage_without_a_finality_policy() -> None: + assert DEFINITION.to_storage() == DEFINITION.to_wire() + assert _definition_from_payload(DEFINITION.to_storage()) == DEFINITION + + +def test_definition_binding_rejects_finality_policy_subclass() -> None: + class CustomPolicy(ReportingSourceFinalPolicy): + pass + + with pytest.raises(ValueError, match="finality_policy"): + replace(DEFINITION, finality_policy=CustomPolicy("daily-v1", "close")) + + +@pytest.mark.parametrize("policy_id", ["", "a" * 256, "contains spaces"]) +def test_finality_policy_refuses_invalid_identifier(policy_id: str) -> None: + with pytest.raises(ValueError, match="finality_policy_id"): + ReportingSourceFinalPolicy(policy_id, "close") + + +@pytest.mark.parametrize("source_signal", ["", "a" * 513]) +def test_source_finality_policy_refuses_invalid_signal(source_signal: str) -> None: + with pytest.raises(ValueError, match="source_signal must contain 1 to 512 characters"): + ReportingSourceFinalPolicy("daily-v1", source_signal) + + # -- period derivation ------------------------------------------------------ @@ -331,7 +443,6 @@ async def test_an_unreadable_revision_is_action_required_not_complete() -> None: configuration = _configuration() obligation = _obligation(configuration) revision, _ = _revision(obligation) - from dataclasses import replace projection = project_obligation_health( obligation, diff --git a/tests/type_checks/reporting_finality.py b/tests/type_checks/reporting_finality.py new file mode 100644 index 000000000..27ba2baa5 --- /dev/null +++ b/tests/type_checks/reporting_finality.py @@ -0,0 +1,33 @@ +"""Retaining each official finality policy on a definition binding.""" + +from typing_extensions import assert_type + +from adcp.reporting.ledger import ( + ReportingContractualCutoffPolicy, + ReportingDefinitionBinding, + ReportingFinalityPolicy, + ReportingSourceFinalPolicy, + ReportingStabilizedPolicy, +) + + +def definition_with_policy(policy: ReportingFinalityPolicy) -> ReportingDefinitionBinding: + binding = ReportingDefinitionBinding( + report_definition_uri="https://contracts.example.test/reporting/daily-delivery-v1", + report_definition_sha256="a" * 64, + schema_version="1.0.0", + schema_uri="https://contracts.example.test/reporting/rows-v1.json", + schema_sha256="b" * 64, + finality_policy=policy, + ) + assert_type(binding.finality_policy, ReportingFinalityPolicy | None) + return binding + + +source: ReportingFinalityPolicy = ReportingSourceFinalPolicy("source-v1", "close") +cutoff: ReportingFinalityPolicy = ReportingContractualCutoffPolicy("cutoff-v1", "PT4H") +stabilized: ReportingFinalityPolicy = ReportingStabilizedPolicy("daily-stable-v1", "P2D", "PT12H") + +definition_with_policy(source) +definition_with_policy(cutoff) +definition_with_policy(stabilized) From e488156cab8dec261ed321aaea3f7ff702349727 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 11:58:25 +0530 Subject: [PATCH 04/39] feat(reporting): validate official finality admission --- src/adcp/reporting/ledger/finality.py | 8 + .../reporting/production/configuration.py | 5 + src/adcp/reporting/production/offerings.py | 143 ++++++------ src/adcp/reporting/service.py | 116 +++++++--- ...test_reporting_production_configuration.py | 87 +++++++- tests/test_reliable_reporting_service.py | 208 ++++++++++++++++++ 6 files changed, 473 insertions(+), 94 deletions(-) diff --git a/src/adcp/reporting/ledger/finality.py b/src/adcp/reporting/ledger/finality.py index 46abee686..b644a730e 100644 --- a/src/adcp/reporting/ledger/finality.py +++ b/src/adcp/reporting/ledger/finality.py @@ -14,6 +14,7 @@ "ReportingFinalityPolicy", "ReportingSourceFinalPolicy", "ReportingStabilizedPolicy", + "finality_policy_source_class", "reporting_finality_policy_from_storage", ] @@ -92,6 +93,13 @@ def to_storage(self) -> dict[str, str]: ) +def finality_policy_source_class( + policy: ReportingFinalityPolicy, +) -> Literal["snapshot", "official"]: + """Return the source publication class needed to prove the selected policy.""" + return "official" if isinstance(policy, ReportingSourceFinalPolicy) else "snapshot" + + def reporting_finality_policy_from_storage(value: Mapping[str, Any]) -> ReportingFinalityPolicy: """Decode the exact stored fields for a supported policy basis.""" match value: diff --git a/src/adcp/reporting/production/configuration.py b/src/adcp/reporting/production/configuration.py index 026d498e8..d7f07cc43 100644 --- a/src/adcp/reporting/production/configuration.py +++ b/src/adcp/reporting/production/configuration.py @@ -40,6 +40,11 @@ class ReportingConfigurationAdmission: def __post_init__(self) -> None: from adcp.validation.schema_loader import get_named_validator + definition = self.configuration.definition + if self.configuration.required_finality == "official" and ( + definition is None or definition.finality_policy is None + ): + raise ValueError("official reporting requires a selected finality policy") wire = canonical_json_utf8_v1(dict(self.configuration_wire)) validator = get_named_validator("core/reporting-delivery-config.json") if validator is None or next(validator.iter_errors(json.loads(wire)), None) is not None: diff --git a/src/adcp/reporting/production/offerings.py b/src/adcp/reporting/production/offerings.py index fdd426810..cd3a361d1 100644 --- a/src/adcp/reporting/production/offerings.py +++ b/src/adcp/reporting/production/offerings.py @@ -12,12 +12,14 @@ from adcp.reporting._timestamp import aware_timestamp from adcp.reporting.canonical_json import canonical_json_utf8_v1 from adcp.reporting.ledger.delivery_models import ReportingDestinationBinding +from adcp.reporting.ledger.finality import ReportingStabilizedPolicy, finality_policy_source_class from adcp.reporting.ledger.models import ( ReportingConfiguration, _schedule_clock, iso_duration_to_timedelta, ) from adcp.reporting.ledger.producer import ReportingProducer +from adcp.reporting.ledger.store import LedgerConflictError from adcp.reporting.materializer.contracts import ReportingVerificationKey, failure from adcp.reporting.materializer.verification import _same_definition from adcp.reporting.production.contracts import ( @@ -144,6 +146,49 @@ def reconciled(self) -> bool: def wire(self) -> dict[str, Any]: return dict(json.loads(self._wire)) + def _source_offerings( + self, capabilities: ReportingSourceCapabilitiesV1 + ) -> tuple[ProvisionalSnapshotOfferingV1 | AuthoritativeOfferingV1, ...]: + configured = self.producer._offerings + primary = capabilities.offering(self.source_offering_id) + finalities = self.wire()["supported_finality"] + if "snapshot" in finalities and ( + not isinstance(primary, ProvisionalSnapshotOfferingV1) + or configured.snapshot_offering_id != primary.offering_id + ): + raise failure("UNSUPPORTED_VERIFICATION") + explicit_close = ( + isinstance(primary, ProvisionalSnapshotOfferingV1) + and primary.official_close_lag is not None + ) + if "official" not in finalities and not explicit_close: + return (primary,) + + policy = self.verification_key.definition.finality_policy + if policy is None: + raise failure("UNSUPPORTED_VERIFICATION") + source_class = finality_policy_source_class(policy) + source_id = configured.offering_for(source_class) + if source_id is None or ("official" in finalities and source_id != primary.offering_id): + raise failure("UNSUPPORTED_VERIFICATION") + try: + proof_source = capabilities.offering(source_id) + except KeyError: + raise failure("UNSUPPORTED_VERIFICATION") from None + if source_class == "official": + if not isinstance(proof_source, AuthoritativeOfferingV1) or ( + self.reconciled and proof_source.correction_policy != "immutable_correction" + ): + raise failure("UNSUPPORTED_VERIFICATION") + elif not isinstance(proof_source, ProvisionalSnapshotOfferingV1): + raise failure("UNSUPPORTED_VERIFICATION") + if isinstance(policy, ReportingStabilizedPolicy): + try: + self.producer._observation_store() + except LedgerConflictError: + raise failure("UNSUPPORTED_VERIFICATION") from None + return (primary,) if primary.offering_id == source_id else (primary, proof_source) + def check_source(self, *, effective: bool = False) -> ReportingSourceCapabilitiesV1: source = self.producer._source if ( @@ -158,74 +203,42 @@ def check_source(self, *, effective: bool = False) -> ReportingSourceCapabilitie ) if effective and capabilities.scope != "effective_account": raise failure("BINDING_MISMATCH") - raw = self.wire() - source_offering = capabilities.offering(self.source_offering_id) - contract, key = source_offering.contract, self.verification_key - source_values = contract.model_dump(mode="json") - expected = { + configured = self.producer._offerings + if dict(configured.source_scope) != capabilities.source_scope: + raise failure("UNSUPPORTED_VERIFICATION") + key = self.verification_key + expected_contract = { "report_definition_id": key.report_definition_id, "reporting_profile": key.reporting_profile, + **key.definition.to_wire(), } - expected.update( - { - name: getattr(key.definition, name) - for name in ( - "report_definition_uri", - "report_definition_sha256", - "schema_version", - "schema_uri", - "schema_sha256", - "schema_dialect", - "schema_ref_policy", + schedule = self.wire()["schedule"] + duration = iso_duration_milliseconds_v1(schedule["period_duration"]) + sla = iso_duration_milliseconds_v1(schedule["delivery_sla"]) + for offering in self._source_offerings(capabilities): + if ( + any( + getattr(offering.contract, name) != value + for name, value in expected_contract.items() ) - } - ) - finality = raw["supported_finality"] - configured = self.producer._offerings - if ( - any(source_values.get(name) != value for name, value in expected.items()) - or source_offering.publication_namespace != configured.publication_namespace - or dict(configured.source_scope) != capabilities.source_scope - or not source_offering.provider_execution.supports_cancellation - or ( - "official" in finality - and ( - not isinstance(source_offering, AuthoritativeOfferingV1) - or configured.official_offering_id != self.source_offering_id - or ( - self.reconciled - and source_offering.correction_policy != "immutable_correction" - ) + or offering.publication_namespace != configured.publication_namespace + or not offering.provider_execution.supports_cancellation + or not ( + iso_duration_milliseconds_v1(offering.windowing.minimum_window) + <= duration + <= iso_duration_milliseconds_v1(offering.windowing.maximum_window) ) - ) - or ( - "snapshot" in finality - and ( - not isinstance(source_offering, ProvisionalSnapshotOfferingV1) - or configured.snapshot_offering_id != self.source_offering_id + or sla < iso_duration_milliseconds_v1(offering.worst_case_availability_lag) + or ( + isinstance(offering, ProvisionalSnapshotOfferingV1) + and duration < iso_duration_milliseconds_v1(offering.fastest_safe_cadence) ) - ) - ): - raise failure("UNSUPPORTED_VERIFICATION") - schedule = raw["schedule"] - duration = iso_duration_milliseconds_v1(schedule["period_duration"]) - if not ( - iso_duration_milliseconds_v1(source_offering.windowing.minimum_window) - <= duration - <= iso_duration_milliseconds_v1(source_offering.windowing.maximum_window) - ) or iso_duration_milliseconds_v1(schedule["delivery_sla"]) < iso_duration_milliseconds_v1( - source_offering.worst_case_availability_lag - ): - raise failure("UNSUPPORTED_VERIFICATION") - if isinstance( - source_offering, ProvisionalSnapshotOfferingV1 - ) and duration < iso_duration_milliseconds_v1(source_offering.fastest_safe_cadence): - raise failure("UNSUPPORTED_VERIFICATION") - if ( - schedule["alignment"] == "source_timezone" - and schedule.get("period_timezone") != source_offering.source_timezone - ): - raise failure("UNSUPPORTED_VERIFICATION") + or ( + schedule["alignment"] == "source_timezone" + and schedule.get("period_timezone") != offering.source_timezone + ) + ): + raise failure("UNSUPPORTED_VERIFICATION") return capabilities def source_binding( @@ -245,7 +258,8 @@ def source_binding( raise _SourceAuthorizationRevokedError() if type(binding) is not ReportingProductionSourceBinding: raise failure("BINDING_MISMATCH") - binding.check(configuration, capabilities, self.source_offering_id) + for offering in self._source_offerings(capabilities): + binding.check(configuration, capabilities, offering.offering_id) return binding except _SourceAuthorizationRevokedError: raise @@ -320,14 +334,13 @@ def check_configuration( configuration.report_definition_id, configuration.reporting_profile, configuration.feed_purpose, - configuration.required_finality, ) != ( key.report_definition_id, key.reporting_profile, raw["feed_purpose"], - raw["supported_finality"][0], ) + or configuration.required_finality not in raw["supported_finality"] or not _same_definition(key, configuration.definition) or configuration.authoritative_party != "seller" or binding.reconciliation_mode != raw["reconciliation_mode"] diff --git a/src/adcp/reporting/service.py b/src/adcp/reporting/service.py index 517e0d7fe..eb6b80ad2 100644 --- a/src/adcp/reporting/service.py +++ b/src/adcp/reporting/service.py @@ -49,6 +49,11 @@ ReportingStatusHandler, WorkerTurn, ) +from adcp.reporting.ledger.finality import ( + ReportingFinalityPolicy, + ReportingStabilizedPolicy, + finality_policy_source_class, +) from adcp.reporting.ledger.producer import _advertised_reporting_delivery from adcp.reporting.ledger.store import LedgerConflictError, decode_cursor from adcp.reporting.service_lifecycle import ( @@ -62,6 +67,7 @@ from adcp.reporting.source import ( AuthoritativeOfferingV1, ProvisionalSnapshotOfferingV1, + ReportingContractIdentityV1, ReportingSourceCapabilitiesV1, ReportingSourceExecutor, ReportingSourceSliceRequestV1, @@ -163,6 +169,27 @@ def _wire(value: Any) -> dict[str, Any]: raise TypeError(f"expected a request model or mapping, got {type(value).__name__}") +def _matches_public_contract( + contract: ReportingContractIdentityV1, public: Mapping[str, Any] +) -> bool: + """Compare the public offering's declared definition and profile with the source.""" + profile = public.get("reporting_profile", {}) + return all( + declaration.get(name, expected) == expected + for declaration, name, expected in ( + (public, "report_definition_id", contract.report_definition_id), + (public, "report_definition_uri", contract.report_definition_uri), + (public, "report_definition_sha256", contract.report_definition_sha256), + (profile, "id", contract.reporting_profile), + (profile, "version", contract.schema_version), + (profile, "schema_uri", contract.schema_uri), + (profile, "schema_sha256", contract.schema_sha256), + (profile, "schema_dialect", contract.schema_dialect), + (profile, "schema_ref_policy", contract.schema_ref_policy), + ) + ) + + async def _resolve(value: Any) -> Any: return await value if inspect.isawaitable(value) else value @@ -365,25 +392,14 @@ def register_executor( f"capability offering {offering_id!r} has conflicting declarations" ) declared[offering_id] = public - profile = public["reporting_profile"] for finality in public["supported_finality"]: matching = [ source for source in executor.capabilities.offerings - if ( - "snapshot" - if isinstance(source, ProvisionalSnapshotOfferingV1) - else "official" - ) - == finality - and source.contract.report_definition_id == public["report_definition_id"] - and source.contract.report_definition_uri == public["report_definition_uri"] - and source.contract.report_definition_sha256 - == public["report_definition_sha256"] - and source.contract.reporting_profile == profile["id"] - and source.contract.schema_version == profile["version"] - and source.contract.schema_uri == profile["schema_uri"] - and source.contract.schema_sha256 == profile["schema_sha256"] + # The definition selected at admission determines which + # source class can prove an official delivery. + if (finality == "official" or isinstance(source, ProvisionalSnapshotOfferingV1)) + and _matches_public_contract(source.contract, public) ] if not matching: raise ReliableReportingConfigurationError( @@ -416,6 +432,10 @@ class _Binding: producer: ReportingProducer +class _MissingFinalityPolicyError(ReliableReportingConfigurationError): + """A retained generation may still serve reads while its producer is unbound.""" + + class _AccountContextResolutionError(Exception): """Distinguish resolver failures from invalid static configuration.""" @@ -711,12 +731,12 @@ async def _configure( "registered source capabilities must be scoped to an effective account" ) try: - self._validate_offerings(configuration, context, registration) + policy = self._validate_offerings(configuration, context, registration) offerings = context.producer_offerings() if not isinstance(offerings, ProducerOfferings): raise TypeError("producer_offerings must return ProducerOfferings") except Exception as error: - if retrying: + if retrying or (recovering and isinstance(error, _MissingFinalityPolicyError)): raise _AccountContextResolutionError(error, permanent=True) from error raise producer = self._producer_factory( @@ -733,6 +753,13 @@ async def _configure( else {} ), ) + if isinstance(policy, ReportingStabilizedPolicy): + try: + producer._observation_store() + except LedgerConflictError as error: + raise ReliableReportingConfigurationError( + "stabilized finality requires durable provisional observation storage" + ) from error binding = _Binding(configuration=configuration, context=context, producer=producer) if persist: if self._initialized: @@ -786,24 +813,27 @@ def _validate_offerings( configuration: ReportingConfiguration, context: ReportingAccountContext, registration: AdapterRegistration, - ) -> None: + ) -> ReportingFinalityPolicy | None: capabilities = registration.executor.capabilities if capabilities.source_scope != _thaw(context.source_scope): raise ReliableReportingConfigurationError( "resolved source_scope does not match the registered adapter capabilities" ) selected: list[ProvisionalSnapshotOfferingV1 | AuthoritativeOfferingV1] = [] + snapshot = None + official = None if context.snapshot_offering_id is not None: try: - snapshot = capabilities.offering(context.snapshot_offering_id) + offering = capabilities.offering(context.snapshot_offering_id) except KeyError as error: raise ReliableReportingConfigurationError( f"snapshot offering {context.snapshot_offering_id!r} is not registered" ) from error - if not isinstance(snapshot, ProvisionalSnapshotOfferingV1): + if not isinstance(offering, ProvisionalSnapshotOfferingV1): raise ReliableReportingConfigurationError( "snapshot_offering_id must name a provisional snapshot offering" ) + snapshot = offering selected.append(snapshot) if context.official_offering_id is not None: try: @@ -825,16 +855,45 @@ def _validate_offerings( raise ReliableReportingConfigurationError( "source offering contract does not match the reporting configuration" ) - required = ( - context.official_offering_id - if configuration.required_finality == "official" - else context.snapshot_offering_id - ) - if required is None: + if configuration.required_finality == "snapshot" and snapshot is None: raise ReliableReportingConfigurationError( - f"configuration requires {configuration.required_finality} but its account " - "context declares no matching source offering" + "snapshot configuration requires a matching snapshot source offering" ) + policy = None + if configuration.required_finality == "official" or ( + snapshot is not None and snapshot.official_close_lag is not None + ): + definition = configuration.definition + if definition is None or definition.finality_policy is None: + raise _MissingFinalityPolicyError( + "official reporting requires a selected finality policy" + ) + policy = definition.finality_policy + source_class = finality_policy_source_class(policy) + source = official if source_class == "official" else snapshot + if source is None: + raise ReliableReportingConfigurationError( + f"selected finality policy requires a matching {source_class} source offering" + ) + used = [source] + if ( + configuration.required_finality == "snapshot" + and snapshot is not None + and snapshot is not source + ): + used.append(snapshot) + for used_offering in used: + if any( + getattr(used_offering.contract, name) != value + for name, value in definition.to_wire().items() + ): + raise ReliableReportingConfigurationError( + "source offering contract does not match the pinned report definition" + ) + if not _matches_public_contract(source.contract, context.capability_offering): + raise ReliableReportingConfigurationError( + "selected policy's source contract does not match the public offering" + ) declared_id = context.capability_offering.get("offering_id") if not declared_id: raise ReliableReportingConfigurationError( @@ -864,6 +923,7 @@ def _validate_offerings( raise ReliableReportingConfigurationError( "account capability offering must match a registered public offering" ) + return policy def validate(self) -> None: """Fail fast on tier combinations the configured components cannot honor.""" diff --git a/tests/conformance/reporting/test_reporting_production_configuration.py b/tests/conformance/reporting/test_reporting_production_configuration.py index dcfafb029..9a125bd49 100644 --- a/tests/conformance/reporting/test_reporting_production_configuration.py +++ b/tests/conformance/reporting/test_reporting_production_configuration.py @@ -8,18 +8,90 @@ import pytest +from adcp.reporting.ledger import ( + ProducerOfferings, + ReportingContractualCutoffPolicy, + ReportingProducer, + ReportingSourceFinalPolicy, + ReportingStabilizedPolicy, +) from adcp.reporting.ledger.notification_models import ReportingNotificationError from adcp.reporting.materializer.contracts import ReportingWriterError, binding_fingerprint from adcp.reporting.materializer.work import ReportingMaterializerLease from adcp.reporting.production.configuration import ReportingConfigurationAdmission +from adcp.reporting.production.offerings import ReportingProductionOffering from adcp.validation.schema_loader import get_named_validator from ._feed_support import feed_request, walk from ._generation_support import END -from ._production_support import production_harness +from ._production_support import Source, production_harness from ._production_transport import MountedProduction from ._receipt_transport import error_code from .test_reporting_production_bindings import source_documents +from .test_reporting_production_settling import SettlingSource + +SOURCE_FINAL = ReportingSourceFinalPolicy("source-v1", "delivery.is_final") +CUTOFF = ReportingContractualCutoffPolicy("cutoff-v1", "P2D") +STABILIZED = ReportingStabilizedPolicy("stable-v1", "P1D", "PT6H") + + +@pytest.mark.parametrize( + ("policy", "source_class", "finalities", "close", "official_route", "accepted"), + [ + (SOURCE_FINAL, "official", ["official"], False, True, True), + (CUTOFF, "snapshot", ["official"], False, False, True), + (CUTOFF, "snapshot", ["snapshot", "official"], False, False, True), + (STABILIZED, "snapshot", ["official"], False, False, True), + (None, "official", ["official"], False, True, False), + (SOURCE_FINAL, "snapshot", ["official"], False, False, False), + (CUTOFF, "official", ["official"], False, True, False), + (None, "snapshot", ["snapshot"], True, True, False), + (SOURCE_FINAL, "snapshot", ["snapshot"], True, False, False), + (SOURCE_FINAL, "snapshot", ["snapshot"], True, True, True), + (CUTOFF, "snapshot", ["snapshot"], True, False, True), + ], +) +async def test_production_offering_validates_selected_finality_policy( + policy, source_class, finalities, close, official_route, accepted, tmp_path +): + async with production_harness("memory", tmp_path / "destination.sqlite", count=0) as h: + installed = h.production.offerings[0] + definition = replace(installed.verification_key.definition, finality_policy=policy) + key = replace(installed.verification_key, definition=definition) + if close: + source = SettlingSource(key, tmp_path / "selected") + official_id = source.official_source_id + else: + source = Source(key, tmp_path / "selected", official=source_class == "official") + official_id = source.source_id + config = replace(h.item.config, definition=definition, required_finality=finalities[-1]) + source.bind_generation(config) + producer = ReportingProducer( + source=source, + store=h.store, + object_reader=source.reader, + offerings=ProducerOfferings( + official_offering_id=official_id if official_route else None, + snapshot_offering_id=source.source_id if source_class == "snapshot" else None, + publication_namespace=source.capabilities.offerings[0].publication_namespace, + source_scope=source.capabilities.source_scope, + ), + ) + public = installed.offering.model_copy(update={"supported_finality": finalities}) + if not accepted: + with pytest.raises(ReportingWriterError, match="UNSUPPORTED_VERIFICATION"): + ReportingProductionOffering(public, producer, key, source.source_id) + return + offering = ReportingProductionOffering(public, producer, key, source.source_id) + offering.check_configuration( + config, h.item.binding, retention_days=h.item.writer.resource_retention_days + ) + other = replace(config, definition=replace(definition, finality_policy=None)) + source.bind_generation(other) + with pytest.raises(ReportingWriterError, match="BINDING_MISMATCH"): + offering.check_configuration( + other, h.item.binding, retention_days=h.item.writer.resource_retention_days + ) def wire_configuration(h): @@ -42,6 +114,19 @@ def wire_configuration(h): } +async def test_typed_official_finality_admission_requires_selected_policy(tmp_path): + async with production_harness("memory", tmp_path / "destination.sqlite", count=0) as h: + config = replace(h.item.config, required_finality="official") + wire = {**wire_configuration(h), "required_finality": "official"} + with pytest.raises(ValueError, match="selected finality policy"): + ReportingConfigurationAdmission( + h.production.offerings[0].offering_id, + config, + h.item.binding, + configuration_wire=wire, + ) + + def state_for(h, configuration): ids = list(h.item.config.media_buy_ids) coverage = { diff --git a/tests/test_reliable_reporting_service.py b/tests/test_reliable_reporting_service.py index dae14e5b8..2c4e5bb33 100644 --- a/tests/test_reliable_reporting_service.py +++ b/tests/test_reliable_reporting_service.py @@ -12,6 +12,7 @@ from adcp.reporting.conformance import ReportingSourceConformanceError from adcp.reporting.fixtures import ( + OFFICIAL_OFFERING_ID, SNAPSHOT_OFFERING_ID, redacted_capabilities, redacted_snapshot_request, @@ -20,8 +21,12 @@ InMemoryReportingLedgerStore, ReportingConfiguration, ReportingConfigurationGenerationKey, + ReportingContractualCutoffPolicy, ReportingDefinitionBinding, + ReportingFinalityPolicy, ReportingScheduleSpec, + ReportingSourceFinalPolicy, + ReportingStabilizedPolicy, ReportingStatusCaller, ) from adcp.reporting.ledger.store import LedgerConflictError @@ -31,6 +36,10 @@ ReliableReportingState, ReportingAccountContext, ) +from adcp.reporting.source import ( + ReportingSourceCapabilitiesV1, + reporting_source_capabilities_sha256_v1, +) from adcp.reporting.testing import ( DeterministicReportingClock, ScriptedReportingAdapter, @@ -135,6 +144,205 @@ def _rows(impressions: int) -> list[dict[str, Any]]: ] +def _finality_service( + configuration: ReportingConfiguration, + *, + snapshot: bool, + official: bool, + close: bool = False, + store: InMemoryReportingLedgerStore | None = None, +) -> ReliableReportingService: + public = _capability_offering("gam") + public["supported_finality"] = [configuration.required_finality] + context = replace( + _account_context(configuration), + snapshot_offering_id=SNAPSHOT_OFFERING_ID if snapshot else None, + official_offering_id=OFFICIAL_OFFERING_ID if official else None, + capability_offering=public, + ) + raw = redacted_capabilities().model_dump(mode="json") + if close: + raw["offerings"][0].update(restatement_window="P1D", official_close_lag="P2D") + raw["offerings"] = [ + offering for offering, enabled in zip(raw["offerings"], (snapshot, official)) if enabled + ] + raw["capabilities_sha256"] = reporting_source_capabilities_sha256_v1(raw) + adapter = ScriptedReportingAdapter(ReportingSourceCapabilitiesV1.model_validate(raw), []) + service = ReliableReportingService( + store=store or InMemoryReportingLedgerStore(), account_context=lambda _: context + ) + service.sources.register("gam", adapter, capability_offerings=[public]) + return service + + +@pytest.mark.parametrize( + ("policy", "source_class", "expected_offering"), + [ + ( + ReportingSourceFinalPolicy("source-v1", "delivery.is_final"), + "official", + OFFICIAL_OFFERING_ID, + ), + (ReportingSourceFinalPolicy("source-v1", "s" * 512), "official", OFFICIAL_OFFERING_ID), + (ReportingContractualCutoffPolicy("cutoff-v1", "P2D"), "snapshot", SNAPSHOT_OFFERING_ID), + (ReportingStabilizedPolicy("stable-v1", "P1D", "PT6H"), "snapshot", SNAPSHOT_OFFERING_ID), + ], +) +async def test_official_configuration_routes_by_selected_policy( + policy: ReportingFinalityPolicy, source_class: str, expected_offering: str +) -> None: + config = replace( + _configuration(), + required_finality="official", + definition=replace(DEFINITION, finality_policy=policy), + ) + service = _finality_service( + config, snapshot=source_class == "snapshot", official=source_class == "official" + ) + await service.configure(config) + offerings = service._bindings[config.generation_key].producer._offerings + assert offerings.offering_for(source_class) == expected_offering + + +@pytest.mark.parametrize(("required", "close"), [("official", False), ("snapshot", True)]) +async def test_official_configuration_without_selected_policy_fails_before_obligation( + required: str, close: bool +) -> None: + config = replace(_configuration(), required_finality=required) + service = _finality_service(config, snapshot=True, official=True, close=close) + await service.initialize() + with pytest.raises(ReliableReportingConfigurationError, match="selected finality policy"): + await service.configure(config) + state = await service.store.read_status_snapshot( + caller=ReportingStatusCaller(config.account_id, config.consumer_id) + ) + assert not state.configurations and not state.obligations + + +@pytest.mark.parametrize( + ("policy", "required", "snapshot", "official", "close"), + [ + ( + ReportingSourceFinalPolicy("source-v1", "delivery.is_final"), + "official", + True, + False, + False, + ), + ( + ReportingSourceFinalPolicy("source-v1", "delivery.is_final"), + "snapshot", + True, + False, + True, + ), + (ReportingContractualCutoffPolicy("cutoff-v1", "P2D"), "official", False, True, False), + (ReportingStabilizedPolicy("stable-v1", "P1D", "PT6H"), "official", False, True, False), + ], +) +async def test_selected_policy_rejects_wrong_source_class( + policy, required, snapshot, official, close +) -> None: + config = replace( + _configuration(), + required_finality=required, + definition=replace(DEFINITION, finality_policy=policy), + ) + service = _finality_service(config, snapshot=snapshot, official=official, close=close) + with pytest.raises(ReliableReportingConfigurationError, match="source offering"): + await service.configure(config) + + +async def test_stabilized_official_configuration_requires_observation_storage() -> None: + class SnapshotStore(InMemoryReportingLedgerStore): + reserve_provisional_acquisition = None + + config = replace( + _configuration(), + required_finality="official", + definition=replace( + DEFINITION, finality_policy=ReportingStabilizedPolicy("stable-v1", "P1D", "PT6H") + ), + ) + service = _finality_service(config, snapshot=True, official=False, store=SnapshotStore()) + with pytest.raises(ReliableReportingConfigurationError, match="observation"): + await service.configure(config) + + +async def test_official_configuration_requires_the_source_to_match_the_pinned_definition() -> None: + config = replace( + _configuration(), + required_finality="official", + definition=replace( + DEFINITION, + report_definition_sha256="f" * 64, + finality_policy=ReportingSourceFinalPolicy("source-v1", "delivery.is_final"), + ), + ) + service = _finality_service(config, snapshot=False, official=True) + with pytest.raises(ReliableReportingConfigurationError, match="definition"): + await service.configure(config) + + +async def test_legacy_official_configuration_remains_readable_beside_healthy_generation() -> None: + store = InMemoryReportingLedgerStore() + legacy = replace(_configuration(), required_finality="official") + healthy = _configuration("freewheel") + for config in (legacy, healthy): + await store.put_configuration(config) + service = ReliableReportingService(store=store, account_context=_account_context) + for route in ("gam", "freewheel"): + service.sources.register(route, ScriptedReportingAdapter(redacted_capabilities(), [])) + await service.initialize() + assert list(service._bindings) == [healthy.generation_key] + assert "selected finality policy" in str(service.initialization_errors[legacy.generation_key]) + assert set(await store.list_all_configurations()) == {legacy, healthy} + + +async def test_official_configuration_must_match_the_declared_public_definition() -> None: + definition = replace( + DEFINITION, + report_definition_sha256="f" * 64, + finality_policy=ReportingContractualCutoffPolicy("cutoff-v1", "P2D"), + ) + config = replace(_configuration(), required_finality="official", definition=definition) + raw = redacted_capabilities().model_dump(mode="json") + raw["offerings"][0]["contract"][ + "report_definition_sha256" + ] = definition.report_definition_sha256 + raw["capabilities_sha256"] = reporting_source_capabilities_sha256_v1(raw) + public = {**_capability_offering("gam"), "supported_finality": ["official"]} + context = replace(_account_context(config), capability_offering=public) + service = ReliableReportingService.memory(account_context=lambda _: context) + service.sources.register( + "gam", + ScriptedReportingAdapter(ReportingSourceCapabilitiesV1.model_validate(raw), []), + capability_offerings=[public], + ) + with pytest.raises(ReliableReportingConfigurationError, match="public offering"): + await service.configure(config) + + +async def test_snapshot_close_selected_policy_checks_the_initial_snapshot_definition() -> None: + config = replace( + _configuration(), + definition=replace( + DEFINITION, finality_policy=ReportingSourceFinalPolicy("source-v1", "delivery.is_final") + ), + ) + raw = redacted_capabilities().model_dump(mode="json") + raw["offerings"][0].update(restatement_window="P1D", official_close_lag="P2D") + raw["offerings"][0]["contract"]["report_definition_sha256"] = "f" * 64 + raw["capabilities_sha256"] = reporting_source_capabilities_sha256_v1(raw) + context = replace(_account_context(config), official_offering_id=OFFICIAL_OFFERING_ID) + service = ReliableReportingService.memory(account_context=lambda _: context) + service.sources.register( + "gam", ScriptedReportingAdapter(ReportingSourceCapabilitiesV1.model_validate(raw), []) + ) + with pytest.raises(ReliableReportingConfigurationError, match="definition"): + await service.configure(config) + + async def test_service_routes_two_frozen_generations_and_resolves_each_once() -> None: clock = DeterministicReportingClock(NOW) resolutions: list[str] = [] From 083044b84eed73f2d2fc44e3bfc8a38fe24888cf Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 12:04:29 +0530 Subject: [PATCH 05/39] fix(reporting): validate dual-finality routes and isolate legacy recovery --- src/adcp/reporting/production/offerings.py | 49 +++++++++++-------- src/adcp/reporting/service.py | 11 +++-- ...test_reporting_production_configuration.py | 27 ++++++++++ tests/test_reliable_reporting_service.py | 19 +++++-- 4 files changed, 80 insertions(+), 26 deletions(-) diff --git a/src/adcp/reporting/production/offerings.py b/src/adcp/reporting/production/offerings.py index cd3a361d1..02d491eb5 100644 --- a/src/adcp/reporting/production/offerings.py +++ b/src/adcp/reporting/production/offerings.py @@ -149,32 +149,39 @@ def wire(self) -> dict[str, Any]: def _source_offerings( self, capabilities: ReportingSourceCapabilitiesV1 ) -> tuple[ProvisionalSnapshotOfferingV1 | AuthoritativeOfferingV1, ...]: - configured = self.producer._offerings - primary = capabilities.offering(self.source_offering_id) + def configured_source( + source_class: str, + ) -> ProvisionalSnapshotOfferingV1 | AuthoritativeOfferingV1: + source_id = self.producer._offerings.offering_for(source_class) + if source_id is None: + raise failure("UNSUPPORTED_VERIFICATION") + try: + return capabilities.offering(source_id) + except KeyError: + raise failure("UNSUPPORTED_VERIFICATION") from None + finalities = self.wire()["supported_finality"] - if "snapshot" in finalities and ( - not isinstance(primary, ProvisionalSnapshotOfferingV1) - or configured.snapshot_offering_id != primary.offering_id - ): - raise failure("UNSUPPORTED_VERIFICATION") - explicit_close = ( - isinstance(primary, ProvisionalSnapshotOfferingV1) - and primary.official_close_lag is not None - ) - if "official" not in finalities and not explicit_close: - return (primary,) + snapshot = None + if "snapshot" in finalities: + candidate = configured_source("snapshot") + if not isinstance(candidate, ProvisionalSnapshotOfferingV1): + raise failure("UNSUPPORTED_VERIFICATION") + snapshot = candidate + if "official" not in finalities: + if snapshot is None or self.source_offering_id != snapshot.offering_id: + raise failure("UNSUPPORTED_VERIFICATION") + if snapshot.official_close_lag is None: + return (snapshot,) policy = self.verification_key.definition.finality_policy if policy is None: raise failure("UNSUPPORTED_VERIFICATION") source_class = finality_policy_source_class(policy) - source_id = configured.offering_for(source_class) - if source_id is None or ("official" in finalities and source_id != primary.offering_id): + proof_source = configured_source(source_class) + if self.source_offering_id != proof_source.offering_id and ( + snapshot is None or self.source_offering_id != snapshot.offering_id + ): raise failure("UNSUPPORTED_VERIFICATION") - try: - proof_source = capabilities.offering(source_id) - except KeyError: - raise failure("UNSUPPORTED_VERIFICATION") from None if source_class == "official": if not isinstance(proof_source, AuthoritativeOfferingV1) or ( self.reconciled and proof_source.correction_policy != "immutable_correction" @@ -187,7 +194,9 @@ def _source_offerings( self.producer._observation_store() except LedgerConflictError: raise failure("UNSUPPORTED_VERIFICATION") from None - return (primary,) if primary.offering_id == source_id else (primary, proof_source) + if snapshot is None or snapshot.offering_id == proof_source.offering_id: + return (proof_source,) + return (snapshot, proof_source) def check_source(self, *, effective: bool = False) -> ReportingSourceCapabilitiesV1: source = self.producer._source diff --git a/src/adcp/reporting/service.py b/src/adcp/reporting/service.py index eb6b80ad2..ba651e7cb 100644 --- a/src/adcp/reporting/service.py +++ b/src/adcp/reporting/service.py @@ -435,6 +435,9 @@ class _Binding: class _MissingFinalityPolicyError(ReliableReportingConfigurationError): """A retained generation may still serve reads while its producer is unbound.""" + def __init__(self) -> None: + super().__init__("official reporting requires a selected finality policy") + class _AccountContextResolutionError(Exception): """Distinguish resolver failures from invalid static configuration.""" @@ -702,6 +705,10 @@ async def _configure( ) return try: + if configuration.required_finality == "official" and ( + configuration.definition is None or configuration.definition.finality_policy is None + ): + raise _MissingFinalityPolicyError() context = await _resolve(self._context_resolver(configuration)) except Exception as error: if recovering: @@ -865,9 +872,7 @@ def _validate_offerings( ): definition = configuration.definition if definition is None or definition.finality_policy is None: - raise _MissingFinalityPolicyError( - "official reporting requires a selected finality policy" - ) + raise _MissingFinalityPolicyError() policy = definition.finality_policy source_class = finality_policy_source_class(policy) source = official if source_class == "official" else snapshot diff --git a/tests/conformance/reporting/test_reporting_production_configuration.py b/tests/conformance/reporting/test_reporting_production_configuration.py index 9a125bd49..a75b91b9a 100644 --- a/tests/conformance/reporting/test_reporting_production_configuration.py +++ b/tests/conformance/reporting/test_reporting_production_configuration.py @@ -94,6 +94,33 @@ async def test_production_offering_validates_selected_finality_policy( ) +@pytest.mark.parametrize("primary", ["snapshot", "official"]) +@pytest.mark.parametrize("required", ["snapshot", "official"]) +async def test_dual_finality_source_final_admits_each_configured_route(primary, required, tmp_path): + async with production_harness("memory", tmp_path / "destination.sqlite", count=0) as h: + installed = h.production.offerings[0] + definition = replace(installed.verification_key.definition, finality_policy=SOURCE_FINAL) + key = replace(installed.verification_key, definition=definition) + source = SettlingSource(key, tmp_path / "selected") + routes = replace( + installed.producer._offerings, + snapshot_offering_id=source.source_id, + official_offering_id=source.official_source_id, + ) + producer = ReportingProducer( + source=source, offerings=routes, store=h.store, object_reader=source.reader + ) + config = replace(h.item.config, definition=definition, required_finality=required) + source.bind_generation(config) + public = installed.offering.model_copy( + update={"supported_finality": ["snapshot", "official"]} + ) + offering = ReportingProductionOffering(public, producer, key, routes.offering_for(primary)) + offering.check_configuration( + config, h.item.binding, retention_days=h.item.writer.resource_retention_days + ) + + def wire_configuration(h): config, binding = h.item.config, h.item.binding offering = h.production.offerings[0] diff --git a/tests/test_reliable_reporting_service.py b/tests/test_reliable_reporting_service.py index 2c4e5bb33..e642c7c18 100644 --- a/tests/test_reliable_reporting_service.py +++ b/tests/test_reliable_reporting_service.py @@ -284,16 +284,29 @@ async def test_official_configuration_requires_the_source_to_match_the_pinned_de await service.configure(config) -async def test_legacy_official_configuration_remains_readable_beside_healthy_generation() -> None: +@pytest.mark.parametrize("definition", [None, DEFINITION]) +async def test_legacy_official_configuration_recovers_with_a_retired_source_route( + definition: ReportingDefinitionBinding | None, +) -> None: store = InMemoryReportingLedgerStore() - legacy = replace(_configuration(), required_finality="official") + legacy = replace(_configuration(), required_finality="official", definition=definition) healthy = _configuration("freewheel") for config in (legacy, healthy): await store.put_configuration(config) - service = ReliableReportingService(store=store, account_context=_account_context) + resolved = [] + + def resolve(config: ReportingConfiguration) -> ReportingAccountContext: + resolved.append(config.generation_key) + context = _account_context(config) + if config == legacy: + return replace(context, snapshot_offering_id=None, official_offering_id="retired") + return context + + service = ReliableReportingService(store=store, account_context=resolve) for route in ("gam", "freewheel"): service.sources.register(route, ScriptedReportingAdapter(redacted_capabilities(), [])) await service.initialize() + assert resolved == [healthy.generation_key] assert list(service._bindings) == [healthy.generation_key] assert "selected finality policy" in str(service.initialization_errors[legacy.generation_key]) assert set(await store.list_all_configurations()) == {legacy, healthy} From 5ced54ed3386c493273cdee9dcd32cd6659f014d Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 12:10:04 +0530 Subject: [PATCH 06/39] fix(reporting): preserve verification key identity without finality policy --- src/adcp/reporting/materializer/work.py | 6 +++- .../test_reporting_materializer_contracts.py | 28 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/adcp/reporting/materializer/work.py b/src/adcp/reporting/materializer/work.py index 9bed997e6..35d13e067 100644 --- a/src/adcp/reporting/materializer/work.py +++ b/src/adcp/reporting/materializer/work.py @@ -63,7 +63,11 @@ def scope_id(scope: ReportingDeliveryScope) -> str: def verification_key_id(key: ReportingVerificationKey) -> str: - return hashlib.sha256(canonical_json_utf8_v1(asdict(key))).hexdigest() + payload = asdict(key) + if key.definition.finality_policy is None: + # Preserve persisted materializer references for keys without a selected policy. + del payload["definition"]["finality_policy"] + return hashlib.sha256(canonical_json_utf8_v1(payload)).hexdigest() def key_for( diff --git a/tests/conformance/reporting/test_reporting_materializer_contracts.py b/tests/conformance/reporting/test_reporting_materializer_contracts.py index 2641b3143..ae7440599 100644 --- a/tests/conformance/reporting/test_reporting_materializer_contracts.py +++ b/tests/conformance/reporting/test_reporting_materializer_contracts.py @@ -13,8 +13,10 @@ from adcp.reporting.ledger import ( InMemoryReportingLedgerStore, ProducerOfferings, + ReportingContractualCutoffPolicy, ReportingDeliveryPrincipal, ReportingProducer, + ReportingSourceFinalPolicy, ) from adcp.reporting.ledger.delivery_models import MaterializationFailure from adcp.reporting.materializer import ( @@ -26,6 +28,7 @@ ReportingWriterFailureCode, reference_verifier, ) +from adcp.reporting.materializer.work import verification_key_id from ._generation_support import UncalledSource from ._materializer_support import io_context, materializer_case @@ -158,6 +161,31 @@ def test_readback_page_repr_never_includes_unverified_provider_body(): assert "https://provider" not in str(page) + repr(page) +def test_verification_key_id_without_finality_policy_preserves_stored_identity() -> None: + key = reference_verifier().key + assert ( + verification_key_id(key) + == "d53ff0456ffc9b62c98c8c1d198818956ce11ae4749e81f5118ac1f653ce8b5d" + ) + + +def test_selected_finality_policy_details_distinguish_verification_keys() -> None: + key = reference_verifier().key + policies = ( + None, + ReportingSourceFinalPolicy("close-v1", "delivery.is_final"), + ReportingContractualCutoffPolicy("close-v1", "P1D"), + ReportingContractualCutoffPolicy("close-v1", "P2D"), + ) + ids = { + verification_key_id( + replace(key, definition=replace(key.definition, finality_policy=policy)) + ) + for policy in policies + } + assert len(ids) == len(policies) + + @pytest.mark.parametrize( "field", ["report_definition_uri", "schema_uri", "schema_dialect", "schema_version"] ) From f196ceb9191e7d362fc060220dcca87b0ce0edb2 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 12:17:35 +0530 Subject: [PATCH 07/39] feat(reporting): expose retained observation history --- .../src/adcp_reporting_storage/ledger/pg.py | 21 +++- src/adcp/reporting/ledger/producer.py | 23 +++- src/adcp/reporting/ledger/provisional.py | 17 ++- src/adcp/reporting/ledger/store.py | 32 +++-- src/adcp/reporting/production/offerings.py | 2 +- ..._reporting_prepared_observation_adapter.py | 25 +++- ...test_reporting_production_configuration.py | 38 +++--- ...test_reporting_provisional_observations.py | 118 +++++++++++++++++- 8 files changed, 241 insertions(+), 35 deletions(-) diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py index 6e6c4932e..5c9875ce6 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py @@ -206,6 +206,7 @@ _ROW_MIGRATION_DDL_PATH = Path(__file__).parent / "reporting_row_migration.sql" _CHANGES_DDL_PATH = Path(__file__).parent / "reporting_changes.sql" _RETIREMENT_DDL_PATH = Path(__file__).parent / "reporting_period_retirement.sql" +_BIGINT_MAX = 2**63 - 1 __all__ = ["PG_AVAILABLE", "PgReportingLedgerStore"] @@ -1792,17 +1793,27 @@ async def reserve_provisional_acquisition( async def get_provisional_observation( self, *, account_id: str, reporting_obligation_id: str ) -> ProvisionalObservation | None: + history = await self.list_provisional_observations( + account_id=account_id, reporting_obligation_id=reporting_obligation_id, limit=1 + ) + return history[0] if history else None + + async def list_provisional_observations( + self, *, account_id: str, reporting_obligation_id: str, limit: int + ) -> tuple[ProvisionalObservation, ...]: + if type(limit) is not int or limit <= 0: + raise ValueError("observation history limit must be a positive integer") async with self._connection() as connection: await self._require_provisional_schema(connection) - row = await ( + rows = await ( await connection.execute( "SELECT payload FROM reporting_provisional_observations" " WHERE account_id=%s AND reporting_obligation_id=%s" - " ORDER BY ordinal DESC LIMIT 1", - (account_id, reporting_obligation_id), + " ORDER BY ordinal DESC LIMIT %s", + (account_id, reporting_obligation_id, min(limit, _BIGINT_MAX)), ) - ).fetchone() - return ProvisionalObservation.from_wire(row[0]) if row else None + ).fetchall() + return tuple(ProvisionalObservation.from_wire(row[0]) for row in reversed(rows)) async def commit_provisional_observation( self, diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 3505ec34d..4353c331e 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -56,6 +56,7 @@ ) from adcp.reporting.evidence import ReportingControlTotalRecord, freeze_control_totals from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal +from adcp.reporting.ledger.finality import ReportingStabilizedPolicy from adcp.reporting.ledger.models import ( ReportingConfiguration, ReportingDeliveryEscalation, @@ -68,6 +69,7 @@ PendingProvisionalAcquisitionStore, ProvisionalAcquisition, ProvisionalObservation, + ProvisionalObservationPublicationStore, ProvisionalObservationStore, ProvisionalPolicy, ) @@ -1179,6 +1181,12 @@ async def acquire_obligation( satisfied = current is not None and current.readable if satisfied and not restate: return None + if ( + finality == "official" + and obligation.definition is not None + and isinstance(obligation.definition.finality_policy, ReportingStabilizedPolicy) + ): + self._observation_history_store() # Everything below needs the frozen code: the slice request carries it, # the manifest is checked against it, and the revision is written under # it. Gate here rather than earlier so a settled legacy obligation stays @@ -1625,7 +1633,7 @@ def _restatement_store(self) -> RestatementCheckpointStore: ) return self._store - def _observation_store(self) -> ProvisionalObservationStore: + def _observation_store(self) -> ProvisionalObservationPublicationStore: explicit = all( inspect.getattr_static(self._store, name, None) is not None for name in ( @@ -1634,7 +1642,7 @@ def _observation_store(self) -> ProvisionalObservationStore: "commit_provisional_observation", ) ) - if not explicit or not isinstance(self._store, ProvisionalObservationStore): + if not explicit or not isinstance(self._store, ProvisionalObservationPublicationStore): raise LedgerConflictError( "PROVISIONAL_OBSERVATIONS_NOT_SUPPORTED", "scheduled provisional reads require explicitly implemented atomic " @@ -1642,6 +1650,17 @@ def _observation_store(self) -> ProvisionalObservationStore: ) return self._store + def _observation_history_store(self) -> ProvisionalObservationStore: + store = self._observation_store() + if inspect.getattr_static( + store, "list_provisional_observations", None + ) is None or not isinstance(store, ProvisionalObservationStore): + raise LedgerConflictError( + "PROVISIONAL_OBSERVATION_HISTORY_NOT_SUPPORTED", + "stabilized finality requires explicitly implemented bounded observation history", + ) + return store + async def _pending_acquisition( self, obligation: ReportingObligationRecord, *, ordinal: int ) -> ProvisionalAcquisition | None: diff --git a/src/adcp/reporting/ledger/provisional.py b/src/adcp/reporting/ledger/provisional.py index 027f984d8..90e8ba9ee 100644 --- a/src/adcp/reporting/ledger/provisional.py +++ b/src/adcp/reporting/ledger/provisional.py @@ -195,7 +195,9 @@ async def get_provisional_acquisition( @runtime_checkable -class ProvisionalObservationStore(Protocol): +class ProvisionalObservationPublicationStore(Protocol): + """Reserve, read the latest, and commit scheduled snapshot observations.""" + async def reserve_provisional_acquisition( self, acquisition: ProvisionalAcquisition ) -> ProvisionalAcquisition: @@ -213,3 +215,16 @@ async def commit_provisional_observation( rows: Sequence[dict[str, Any]], ) -> ReportingRevisionRecord: pass + + +@runtime_checkable +class ProvisionalObservationStore(ProvisionalObservationPublicationStore, Protocol): + async def list_provisional_observations( + self, *, account_id: str, reporting_obligation_id: str, limit: int + ) -> tuple[ProvisionalObservation, ...]: + """Return up to ``limit`` latest observations in ascending ordinal order. + + The read is scoped to the requested account and reporting obligation. + ``limit`` must be a positive integer. + """ + pass diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index f55a83a8d..3bbc0ce98 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -41,6 +41,7 @@ from copy import deepcopy from dataclasses import dataclass, field, replace from datetime import datetime, timezone +from itertools import islice from typing import TYPE_CHECKING, Any, Literal, Protocol, runtime_checkable from uuid import uuid4 @@ -839,7 +840,9 @@ def __init__( self._restatement_checkpoints: dict[str, RestatementCheckpoint] = {} self._retry_schedules: dict[str, RetryScheduleEntry] = {} self._provisional_acquisitions: dict[tuple[str, str, int], ProvisionalAcquisition] = {} - self._provisional_observations: dict[tuple[str, str, int], ProvisionalObservation] = {} + self._provisional_observations: dict[tuple[str, str], dict[int, ProvisionalObservation]] = ( + {} + ) self._adjustments: dict[str, ReportingAdjustmentRecord] = {} self._statuses: dict[tuple[str, str, str], ConsumerStatusRecord] = {} self._status_identity: dict[tuple[str, str, str], str] = {} @@ -1320,12 +1323,19 @@ async def reserve_provisional_acquisition( async def get_provisional_observation( self, *, account_id: str, reporting_obligation_id: str ) -> ProvisionalObservation | None: - observations = [ - value - for (account, obligation, _), value in self._provisional_observations.items() - if account == account_id and obligation == reporting_obligation_id - ] - return max(observations, key=lambda item: item.acquisition.ordinal, default=None) + history = await self.list_provisional_observations( + account_id=account_id, reporting_obligation_id=reporting_obligation_id, limit=1 + ) + return history[0] if history else None + + async def list_provisional_observations( + self, *, account_id: str, reporting_obligation_id: str, limit: int + ) -> tuple[ProvisionalObservation, ...]: + if type(limit) is not int or limit <= 0: + raise ValueError("observation history limit must be a positive integer") + observations = self._provisional_observations.get((account_id, reporting_obligation_id), {}) + newest_first = list(islice(reversed(observations.values()), min(limit, len(observations)))) + return tuple(reversed(newest_first)) async def commit_provisional_observation( self, @@ -1334,7 +1344,8 @@ async def commit_provisional_observation( rows: Sequence[dict[str, Any]], ) -> ReportingRevisionRecord: acquisition = observation.acquisition - key = (acquisition.account_id, acquisition.obligation_id, acquisition.ordinal) + owner = (acquisition.account_id, acquisition.obligation_id) + key = (*owner, acquisition.ordinal) reuse_candidate = provisional_observation_reuse_candidate( acquisition, observation, revision ) @@ -1345,7 +1356,7 @@ async def commit_provisional_observation( ): raise LedgerConflictError("OBSERVATION_CONFLICT", "observation identity differs") async with self._mutation(): - existing = self._provisional_observations.get(key) + existing = self._provisional_observations.get(owner, {}).get(acquisition.ordinal) obligation = self._obligations.get(acquisition.obligation_id) if obligation is None or not acquisition.binds(obligation): raise LedgerConflictError( @@ -1432,7 +1443,8 @@ async def commit_provisional_observation( observation.provisional_until, ) ) - self._provisional_observations[key] = observation + # The checkpoint enforces ascending ordinals for distinct observations. + self._provisional_observations.setdefault(owner, {})[acquisition.ordinal] = observation return committed async def get_restatement_checkpoint( diff --git a/src/adcp/reporting/production/offerings.py b/src/adcp/reporting/production/offerings.py index 02d491eb5..67a89748f 100644 --- a/src/adcp/reporting/production/offerings.py +++ b/src/adcp/reporting/production/offerings.py @@ -191,7 +191,7 @@ def configured_source( raise failure("UNSUPPORTED_VERIFICATION") if isinstance(policy, ReportingStabilizedPolicy): try: - self.producer._observation_store() + self.producer._observation_history_store() except LedgerConflictError: raise failure("UNSUPPORTED_VERIFICATION") from None if snapshot is None or snapshot.offering_id == proof_source.offering_id: diff --git a/tests/conformance/reporting/test_reporting_prepared_observation_adapter.py b/tests/conformance/reporting/test_reporting_prepared_observation_adapter.py index 52e6aa044..a4e6fc600 100644 --- a/tests/conformance/reporting/test_reporting_prepared_observation_adapter.py +++ b/tests/conformance/reporting/test_reporting_prepared_observation_adapter.py @@ -6,12 +6,35 @@ import pytest -from adcp.reporting.ledger import LedgerConflictError +from adcp.reporting.ledger import LedgerConflictError, ReportingStabilizedPolicy from ._reliable_support import ScriptedSource, complete_fetch, configuration, reliable_factory from .test_reporting_evidence_currency_integration import frozen_slice +async def test_stabilized_acquisition_requires_publisher_to_expose_history() -> None: + async with reliable_factory("memory") as h: + config = configuration("eur", finality="official") + assert config.definition is not None + config = replace( + config, + definition=replace( + config.definition, + finality_policy=ReportingStabilizedPolicy("stable-v1", "P1D", "PT6H"), + ), + ) + await h.store.put_configuration(config) + script = ScriptedSource(eur=[]) + producer = h.producer(h.source(script.async_fetch)) + (obligation,) = await producer.close_elapsed_periods(config) + + with pytest.raises(LedgerConflictError) as error: + await producer.acquire_obligation(config, obligation) + + assert error.value.code == "PROVISIONAL_OBSERVATION_HISTORY_NOT_SUPPORTED" + assert script.requests == [] + + @pytest.mark.parametrize("backend", ["memory", "postgres"]) @pytest.mark.parametrize("failure_point", ["revision.before", "revision.after"]) async def test_prepared_observation_faults_preserve_complete_atomic_state(backend, failure_point): diff --git a/tests/conformance/reporting/test_reporting_production_configuration.py b/tests/conformance/reporting/test_reporting_production_configuration.py index a75b91b9a..74f750953 100644 --- a/tests/conformance/reporting/test_reporting_production_configuration.py +++ b/tests/conformance/reporting/test_reporting_production_configuration.py @@ -5,6 +5,7 @@ import sqlite3 from dataclasses import replace from datetime import timedelta, timezone +from types import SimpleNamespace import pytest @@ -36,23 +37,24 @@ @pytest.mark.parametrize( - ("policy", "source_class", "finalities", "close", "official_route", "accepted"), + ("policy", "source_class", "finalities", "close", "official_route", "history", "accepted"), [ - (SOURCE_FINAL, "official", ["official"], False, True, True), - (CUTOFF, "snapshot", ["official"], False, False, True), - (CUTOFF, "snapshot", ["snapshot", "official"], False, False, True), - (STABILIZED, "snapshot", ["official"], False, False, True), - (None, "official", ["official"], False, True, False), - (SOURCE_FINAL, "snapshot", ["official"], False, False, False), - (CUTOFF, "official", ["official"], False, True, False), - (None, "snapshot", ["snapshot"], True, True, False), - (SOURCE_FINAL, "snapshot", ["snapshot"], True, False, False), - (SOURCE_FINAL, "snapshot", ["snapshot"], True, True, True), - (CUTOFF, "snapshot", ["snapshot"], True, False, True), + (SOURCE_FINAL, "official", ["official"], False, True, True, True), + (CUTOFF, "snapshot", ["official"], False, False, True, True), + (CUTOFF, "snapshot", ["snapshot", "official"], False, False, True, True), + (STABILIZED, "snapshot", ["official"], False, False, True, True), + (STABILIZED, "snapshot", ["official"], False, False, False, False), + (None, "official", ["official"], False, True, True, False), + (SOURCE_FINAL, "snapshot", ["official"], False, False, True, False), + (CUTOFF, "official", ["official"], False, True, True, False), + (None, "snapshot", ["snapshot"], True, True, True, False), + (SOURCE_FINAL, "snapshot", ["snapshot"], True, False, True, False), + (SOURCE_FINAL, "snapshot", ["snapshot"], True, True, True, True), + (CUTOFF, "snapshot", ["snapshot"], True, False, True, True), ], ) async def test_production_offering_validates_selected_finality_policy( - policy, source_class, finalities, close, official_route, accepted, tmp_path + policy, source_class, finalities, close, official_route, history, accepted, tmp_path ): async with production_harness("memory", tmp_path / "destination.sqlite", count=0) as h: installed = h.production.offerings[0] @@ -66,9 +68,17 @@ async def test_production_offering_validates_selected_finality_policy( official_id = source.source_id config = replace(h.item.config, definition=definition, required_finality=finalities[-1]) source.bind_generation(config) + store = h.store + if not history: + store = SimpleNamespace( + reserve_provisional_acquisition=h.store.reserve_provisional_acquisition, + get_provisional_observation=h.store.get_provisional_observation, + commit_provisional_observation=h.store.commit_provisional_observation, + ) producer = ReportingProducer( source=source, - store=h.store, + store=store, + retry_store=h.store, object_reader=source.reader, offerings=ProducerOfferings( official_offering_id=official_id if official_route else None, diff --git a/tests/test_reporting_provisional_observations.py b/tests/test_reporting_provisional_observations.py index 07c9993be..9933793a7 100644 --- a/tests/test_reporting_provisional_observations.py +++ b/tests/test_reporting_provisional_observations.py @@ -198,7 +198,7 @@ async def _publication_effect_counts(store): return ( len(store._revisions), sum(len(rows) for rows in store._rows.values()), - len(store._provisional_observations), + sum(len(history) for history in store._provisional_observations.values()), len(store._restatement_checkpoints), len(store._changes), len(store._notification_state.events), @@ -223,6 +223,122 @@ async def test_adapter_without_window_rereads_with_sdk_default(make_harness): assert all(item.finality == "snapshot" for item in await _revisions(store)) +async def test_observation_history_is_bounded_ordered_and_account_scoped(make_harness) -> None: + producer, store, _, clock = await make_harness( + _capabilities(restatement_window="P3D", restatement_cadence="PT1H") + ) + await producer.run_worker() + for _ in range(2): + clock[0] += timedelta(hours=1) + await producer.run_worker() + obligation = await _only_obligation(store) + + history = await store.list_provisional_observations( + account_id=ACCOUNT, reporting_obligation_id=obligation.reporting_obligation_id, limit=2 + ) + + assert tuple(item.acquisition.ordinal for item in history) == (1, 2) + assert all(item.acquisition.account_id == ACCOUNT for item in history) + assert all( + item.acquisition.obligation_id == obligation.reporting_obligation_id for item in history + ) + assert await latest(store) == history[-1] + full_history = await store.list_provisional_observations( + account_id=ACCOUNT, reporting_obligation_id=obligation.reporting_obligation_id, limit=2**64 + ) + assert tuple(item.acquisition.ordinal for item in full_history) == (0, 1, 2) + assert ( + await store.list_provisional_observations( + account_id="another-account", + reporting_obligation_id=obligation.reporting_obligation_id, + limit=2, + ) + == () + ) + assert ( + await store.list_provisional_observations( + account_id=ACCOUNT, reporting_obligation_id="another-obligation", limit=2 + ) + == () + ) + + +@pytest.mark.parametrize("limit", [0, -1, True, 1.5]) +async def test_observation_history_requires_a_positive_integer_limit(make_harness, limit) -> None: + _, store, _, _ = await make_harness(_capabilities(restatement_window="P3D")) + + with pytest.raises(ValueError): + await store.list_provisional_observations( + account_id=ACCOUNT, reporting_obligation_id="unknown", limit=limit + ) + + +@pytest.mark.parametrize("make_harness", ["memory"], indirect=True) +async def test_observation_history_looks_up_only_the_requested_owner( + make_harness, monkeypatch +) -> None: + producer, store, _, _ = await make_harness(_capabilities(restatement_window="P3D")) + await producer.run_worker() + first = await latest(store) + + class KeyLookupOnly(dict): + def __iter__(self): + raise AssertionError("observation lookup iterated over the owner map") + + items = values = keys = __iter__ + + monkeypatch.setattr( + store, "_provisional_observations", KeyLookupOnly(store._provisional_observations) + ) + + assert await store.list_provisional_observations( + account_id=ACCOUNT, reporting_obligation_id=first.acquisition.obligation_id, limit=2 + ) == (first,) + assert await latest(store) == first + + +async def test_observation_history_rolls_back_appended_observation( + make_harness, monkeypatch +) -> None: + producer, store, _, clock = await make_harness( + _capabilities(restatement_window="P3D", restatement_cadence="PT1H") + ) + await producer.run_worker() + first = await latest(store) + clock[0] += timedelta(hours=1) + + class AbortTransactionError(Exception): + pass + + commit = store.commit_provisional_observation + + async def abort_after_commit(observation, revision, rows) -> None: + async with store.transaction(): + await commit(observation, revision, rows) + history = await store.list_provisional_observations( + account_id=ACCOUNT, reporting_obligation_id=first.acquisition.obligation_id, limit=2 + ) + assert tuple(item.acquisition.ordinal for item in history) == (0, 1) + raise AbortTransactionError + + with monkeypatch.context() as patch: + patch.setattr(store, "commit_provisional_observation", abort_after_commit) + with pytest.raises(AbortTransactionError): + await producer.run_worker() + + history = await store.list_provisional_observations( + account_id=ACCOUNT, reporting_obligation_id=first.acquisition.obligation_id, limit=2 + ) + assert history == (first,) + assert await latest(store) == first + + await producer.run_worker() + history = await store.list_provisional_observations( + account_id=ACCOUNT, reporting_obligation_id=first.acquisition.obligation_id, limit=2 + ) + assert tuple(item.acquisition.ordinal for item in history) == (0, 1) + + async def test_unchanged_scheduled_read_keeps_snapshot_and_advances_observation(make_harness): producer, store, fetch, clock = await make_harness( _capabilities(restatement_window="P3D", restatement_cadence="PT1H") From c3ca8e60f704cd2ef9c92284e2104280c8453555 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 12:22:24 +0530 Subject: [PATCH 08/39] fix(reporting): require concrete observation history for stabilization --- src/adcp/reporting/ledger/producer.py | 14 +++- src/adcp/reporting/service.py | 4 +- ..._reporting_prepared_observation_adapter.py | 77 ++++++++++++++++++- tests/test_reliable_reporting_service.py | 39 +++++++++- 4 files changed, 125 insertions(+), 9 deletions(-) diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 4353c331e..d2c594a24 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -1652,9 +1652,17 @@ def _observation_store(self) -> ProvisionalObservationPublicationStore: def _observation_history_store(self) -> ProvisionalObservationStore: store = self._observation_store() - if inspect.getattr_static( - store, "list_provisional_observations", None - ) is None or not isinstance(store, ProvisionalObservationStore): + name = "list_provisional_observations" + stub = ProvisionalObservationStore.list_provisional_observations + declared = inspect.getattr_static(store, name, None) + implementation = ( + getattr(store, name, None) if declared is not None and declared is not stub else None + ) + if ( + not callable(implementation) + or getattr(implementation, "__func__", implementation) is stub + or not isinstance(store, ProvisionalObservationStore) + ): raise LedgerConflictError( "PROVISIONAL_OBSERVATION_HISTORY_NOT_SUPPORTED", "stabilized finality requires explicitly implemented bounded observation history", diff --git a/src/adcp/reporting/service.py b/src/adcp/reporting/service.py index ba651e7cb..8c5a7a673 100644 --- a/src/adcp/reporting/service.py +++ b/src/adcp/reporting/service.py @@ -762,10 +762,10 @@ async def _configure( ) if isinstance(policy, ReportingStabilizedPolicy): try: - producer._observation_store() + producer._observation_history_store() except LedgerConflictError as error: raise ReliableReportingConfigurationError( - "stabilized finality requires durable provisional observation storage" + "stabilized finality requires durable provisional observation history" ) from error binding = _Binding(configuration=configuration, context=context, producer=producer) if persist: diff --git a/tests/conformance/reporting/test_reporting_prepared_observation_adapter.py b/tests/conformance/reporting/test_reporting_prepared_observation_adapter.py index a4e6fc600..590a12261 100644 --- a/tests/conformance/reporting/test_reporting_prepared_observation_adapter.py +++ b/tests/conformance/reporting/test_reporting_prepared_observation_adapter.py @@ -7,13 +7,44 @@ import pytest from adcp.reporting.ledger import LedgerConflictError, ReportingStabilizedPolicy +from adcp.reporting.ledger.provisional import ProvisionalObservation, ProvisionalObservationStore -from ._reliable_support import ScriptedSource, complete_fetch, configuration, reliable_factory +from ._reliable_support import ( + ScriptedSource, + _PreparedRevisionStore, + complete_fetch, + configuration, + reliable_factory, +) from .test_reporting_evidence_currency_integration import frozen_slice -async def test_stabilized_acquisition_requires_publisher_to_expose_history() -> None: - async with reliable_factory("memory") as h: +class _PublisherWithProtocolStub(_PreparedRevisionStore, ProvisionalObservationStore): + pass + + +class _PublisherWithNoncallableHistory(_PreparedRevisionStore): + list_provisional_observations = True + + +class _PublisherWithHistory(_PreparedRevisionStore): + async def list_provisional_observations( + self, *, account_id: str, reporting_obligation_id: str, limit: int + ) -> tuple[ProvisionalObservation, ...]: + return await self.harness.store.list_provisional_observations( + account_id=account_id, reporting_obligation_id=reporting_obligation_id, limit=limit + ) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize( + "publisher", + [_PreparedRevisionStore, _PublisherWithProtocolStub, _PublisherWithNoncallableHistory], +) +async def test_stabilized_acquisition_requires_publisher_to_expose_history( + backend, publisher, monkeypatch +) -> None: + async with reliable_factory(backend) as h: config = configuration("eur", finality="official") assert config.definition is not None config = replace( @@ -26,6 +57,7 @@ async def test_stabilized_acquisition_requires_publisher_to_expose_history() -> await h.store.put_configuration(config) script = ScriptedSource(eur=[]) producer = h.producer(h.source(script.async_fetch)) + monkeypatch.setattr(producer, "_store", publisher(h)) (obligation,) = await producer.close_elapsed_periods(config) with pytest.raises(LedgerConflictError) as error: @@ -35,6 +67,45 @@ async def test_stabilized_acquisition_requires_publisher_to_expose_history() -> assert script.requests == [] +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_snapshot_publication_accepts_publisher_inheriting_history_stub( + backend, monkeypatch +) -> None: + async with reliable_factory(backend) as h: + await h.store.put_configuration(configuration("eur")) + script = ScriptedSource(eur=[complete_fetch]) + producer = h.producer(h.source(script.async_fetch)) + monkeypatch.setattr(producer, "_store", _PublisherWithProtocolStub(h)) + + (revision_id,) = (await producer.run_worker()).revisions_committed + + revision = await h.store.get_revision(account_id="eur", reporting_revision_id=revision_id) + assert revision is not None and revision.finality == "snapshot" + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("explicit_wrapper", [False, True]) +async def test_observation_history_accepts_concrete_store_and_explicit_wrapper( + backend, explicit_wrapper, monkeypatch +) -> None: + async with reliable_factory(backend) as h: + await h.store.put_configuration(configuration("eur")) + script = ScriptedSource(eur=[complete_fetch]) + producer = h.producer(h.source(script.async_fetch), managed=explicit_wrapper) + if explicit_wrapper: + monkeypatch.setattr(producer, "_store", _PublisherWithHistory(h)) + + (revision_id,) = (await producer.run_worker()).revisions_committed + (request,) = script.requests + (observation,) = await producer._observation_history_store().list_provisional_observations( + account_id="eur", + reporting_obligation_id=request.identity.reporting_obligation_id, + limit=1, + ) + + assert observation.revision_id == revision_id + + @pytest.mark.parametrize("backend", ["memory", "postgres"]) @pytest.mark.parametrize("failure_point", ["revision.before", "revision.after"]) async def test_prepared_observation_faults_preserve_complete_atomic_state(backend, failure_point): diff --git a/tests/test_reliable_reporting_service.py b/tests/test_reliable_reporting_service.py index e642c7c18..aaf038c8d 100644 --- a/tests/test_reliable_reporting_service.py +++ b/tests/test_reliable_reporting_service.py @@ -24,6 +24,7 @@ ReportingContractualCutoffPolicy, ReportingDefinitionBinding, ReportingFinalityPolicy, + ReportingProducer, ReportingScheduleSpec, ReportingSourceFinalPolicy, ReportingStabilizedPolicy, @@ -31,6 +32,7 @@ ) from adcp.reporting.ledger.store import LedgerConflictError from adcp.reporting.service import ( + ProducerFactory, ReliableReportingConfigurationError, ReliableReportingService, ReliableReportingState, @@ -151,6 +153,7 @@ def _finality_service( official: bool, close: bool = False, store: InMemoryReportingLedgerStore | None = None, + producer_factory: ProducerFactory = ReportingProducer, ) -> ReliableReportingService: public = _capability_offering("gam") public["supported_finality"] = [configuration.required_finality] @@ -169,7 +172,9 @@ def _finality_service( raw["capabilities_sha256"] = reporting_source_capabilities_sha256_v1(raw) adapter = ScriptedReportingAdapter(ReportingSourceCapabilitiesV1.model_validate(raw), []) service = ReliableReportingService( - store=store or InMemoryReportingLedgerStore(), account_context=lambda _: context + store=store or InMemoryReportingLedgerStore(), + account_context=lambda _: context, + producer_factory=producer_factory, ) service.sources.register("gam", adapter, capability_offerings=[public]) return service @@ -269,6 +274,38 @@ class SnapshotStore(InMemoryReportingLedgerStore): await service.configure(config) +async def test_service_rejects_stabilized_configuration_without_history_before_persisting() -> None: + def producer_factory(**kwargs: Any) -> ReportingProducer: + store = kwargs.pop("store") + publisher = SimpleNamespace( + reserve_provisional_acquisition=store.reserve_provisional_acquisition, + get_provisional_observation=store.get_provisional_observation, + commit_provisional_observation=store.commit_provisional_observation, + ) + return ReportingProducer(store=publisher, retry_store=store, **kwargs) + + config = replace( + _configuration(), + required_finality="official", + definition=replace( + DEFINITION, finality_policy=ReportingStabilizedPolicy("stable-v1", "P1D", "PT6H") + ), + ) + service = _finality_service( + config, snapshot=True, official=False, producer_factory=producer_factory + ) + await service.initialize() + + with pytest.raises(ReliableReportingConfigurationError, match="observation"): + await service.configure(config) + + state = await service.store.read_status_snapshot( + caller=ReportingStatusCaller(config.account_id, config.consumer_id) + ) + assert not state.configurations and not state.obligations + assert config.generation_key not in service._bindings + + async def test_official_configuration_requires_the_source_to_match_the_pinned_definition() -> None: config = replace( _configuration(), From 8a5313758ee2d02b20ee74c1eaa07b4120b9e37c Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 12:35:30 +0530 Subject: [PATCH 09/39] feat(reporting): prove official source and cutoff finality --- src/adcp/reporting/ledger/finality.py | 99 ++++- src/adcp/reporting/ledger/producer.py | 146 ++++--- src/adcp/reporting/ledger/provisional.py | 7 +- src/adcp/reporting/source.py | 3 +- tests/test_reporting_official_finality.py | 428 ++++++++++++++++++++ tests/test_reporting_source_availability.py | 103 ++++- 6 files changed, 732 insertions(+), 54 deletions(-) create mode 100644 tests/test_reporting_official_finality.py diff --git a/src/adcp/reporting/ledger/finality.py b/src/adcp/reporting/ledger/finality.py index b644a730e..1bf86587c 100644 --- a/src/adcp/reporting/ledger/finality.py +++ b/src/adcp/reporting/ledger/finality.py @@ -5,17 +5,26 @@ import re from collections.abc import Mapping from dataclasses import dataclass, field -from typing import Any, Literal +from datetime import datetime, timedelta, timezone +from typing import TYPE_CHECKING, Any, Literal from adcp.reporting.source import iso_duration_milliseconds_v1 +if TYPE_CHECKING: + from adcp.reporting.ledger.models import ReportingObligationRecord + from adcp.reporting.source import SourceBatchManifestV1 + __all__ = [ + "FinalityProof", "ReportingContractualCutoffPolicy", "ReportingFinalityPolicy", "ReportingSourceFinalPolicy", "ReportingStabilizedPolicy", + "contractual_cutoff_at", "finality_policy_source_class", + "manifest_finality_proof", "reporting_finality_policy_from_storage", + "source_finality_proof", ] @@ -93,6 +102,94 @@ def to_storage(self) -> dict[str, str]: ) +@dataclass(frozen=True) +class FinalityProof: + """The selected policy and the instant its publication requirements were met.""" + + finality_policy_id: str + basis: Literal["source_final", "contractual_cutoff", "stabilized"] + finalized_at: datetime + + def validate( + self, + obligation: ReportingObligationRecord, + manifest: SourceBatchManifestV1, + created_at: datetime, + ) -> None: + from adcp.reporting.ledger.store import LedgerConflictError + + if self.finalized_at.utcoffset() is None or not ( + obligation.period.end.astimezone(timezone.utc) + <= self.finalized_at.astimezone(timezone.utc) + <= created_at.astimezone(timezone.utc) + ): + raise LedgerConflictError( + "PUBLICATION_TIME_INVALID", "proven finality is outside the publication time bounds" + ) + if self != manifest_finality_proof(obligation, manifest, created_at): + raise LedgerConflictError( + "FINALITY_POLICY_UNPROVEN", "proof differs from the selected policy or its evidence" + ) + + +def contractual_cutoff_at( + obligation: ReportingObligationRecord, policy: ReportingContractualCutoffPolicy +) -> datetime: + return obligation.period.end.astimezone(timezone.utc) + timedelta( + milliseconds=iso_duration_milliseconds_v1(policy.duration_after_period_end) + ) + + +def source_finality_proof( + obligation: ReportingObligationRecord, + manifest: SourceBatchManifestV1, + policy: ReportingSourceFinalPolicy, +) -> FinalityProof: + from adcp.reporting.ledger.store import LedgerConflictError + + evidence = manifest.finality_evidence + if ( + manifest.publication_class != "AUTHORITATIVE" + or evidence.basis not in {"provider_declared", "provider_job_terminal"} + or evidence.evidence_ref != policy.source_signal + or evidence.observed_at.astimezone(timezone.utc) + < obligation.period.end.astimezone(timezone.utc) + ): + raise LedgerConflictError( + "MANIFEST_MISMATCH", "authoritative evidence must prove the selected source signal" + ) + return FinalityProof(policy.finality_policy_id, policy.basis, evidence.observed_at) + + +def manifest_finality_proof( + obligation: ReportingObligationRecord, manifest: SourceBatchManifestV1, published_at: datetime +) -> FinalityProof: + """Verify the policy that a single complete source publication can prove.""" + from adcp.reporting.ledger.store import LedgerConflictError + + if ( + manifest.coverage.status != "full" + or manifest.period.start != obligation.period.start + or manifest.period.end != obligation.period.end + ): + raise LedgerConflictError("MANIFEST_MISMATCH", "official proof requires the full period") + policy = obligation.definition.finality_policy if obligation.definition else None + if isinstance(policy, ReportingSourceFinalPolicy): + return source_finality_proof(obligation, manifest, policy) + if isinstance(policy, ReportingContractualCutoffPolicy): + cutoff = contractual_cutoff_at(obligation, policy) + if ( + manifest.publication_class != "PROVISIONAL_SNAPSHOT" + or not cutoff <= manifest.period.source_read_cutoff_at <= published_at + or manifest.acquired_at < cutoff + ): + raise LedgerConflictError( + "FINALITY_POLICY_UNPROVEN", "contractual cutoff requires a post-cutoff snapshot" + ) + return FinalityProof(policy.finality_policy_id, policy.basis, published_at) + raise LedgerConflictError("FINALITY_POLICY_UNPROVEN", "official finality requires policy proof") + + def finality_policy_source_class( policy: ReportingFinalityPolicy, ) -> Literal["snapshot", "official"]: diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index d2c594a24..4ca3e75c2 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -56,7 +56,14 @@ ) from adcp.reporting.evidence import ReportingControlTotalRecord, freeze_control_totals from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal -from adcp.reporting.ledger.finality import ReportingStabilizedPolicy +from adcp.reporting.ledger.finality import ( + FinalityProof, + ReportingContractualCutoffPolicy, + ReportingStabilizedPolicy, + contractual_cutoff_at, + finality_policy_source_class, + manifest_finality_proof, +) from adcp.reporting.ledger.models import ( ReportingConfiguration, ReportingDeliveryEscalation, @@ -163,6 +170,13 @@ def _utc(value: datetime) -> datetime: return value.astimezone(timezone.utc) +def _source_finality(obligation: ReportingObligationRecord, output_finality: str) -> str: + policy = obligation.definition.finality_policy if obligation.definition else None + if output_finality == "official" and policy is not None: + return finality_policy_source_class(policy) + return output_finality + + def revision_content_sha256( *, reporting_revision_id: str, @@ -863,7 +877,9 @@ def _scheduled_source_ready_at( obligation.report_definition_id, _utc(obligation.period.start).isoformat(), _utc(obligation.period.end).isoformat(), - self._offerings.offering_for(obligation.required_finality), + self._offerings.offering_for( + _source_finality(obligation, obligation.required_finality) + ), ] ) offset = int.from_bytes(hashlib.sha256(seed).digest(), "big") % (microseconds + 1) @@ -878,12 +894,20 @@ def _source_available_at( worst_case: bool = False, ) -> datetime: end = _utc(obligation.period.end) - offering_id = offering_id or self._offerings.offering_for( - target_finality or obligation.required_finality - ) + output_finality = target_finality or obligation.required_finality + policy = obligation.definition.finality_policy if obligation.definition else None + cutoff = end + if offering_id is None: + offering_id = self._offerings.offering_for( + _source_finality(obligation, output_finality) + ) + if output_finality == "official" and isinstance( + policy, ReportingContractualCutoffPolicy + ): + cutoff = contractual_cutoff_at(obligation, policy) if offering_id is None: # The acquisition path reports the existing missing-offering error. - return end + return cutoff offering = self._source.capabilities.offering(offering_id) lag = ( offering.worst_case_availability_lag @@ -905,7 +929,7 @@ def _source_available_at( _utc(local_ready.replace(fold=0)), _utc(local_ready.replace(fold=1)), ) - return ready_at + return max(ready_at, cutoff) def _settling_policy( self, @@ -966,7 +990,7 @@ async def _acquire_with_settling_policy( policy = _SettlingPolicy(frozen.window, frozen.cadence, frozen.official_close_lag) explicit_close = ( policy.official_close_lag is not None - and self._offerings.official_offering_id is not None + and self._offerings.offering_for(_source_finality(obligation, "official")) is not None ) revisions = await self._store.list_revisions( account_id=obligation.account_id, @@ -993,7 +1017,8 @@ async def _acquire_with_settling_policy( restate=bool(revisions), turn=turn, now=now, - target_finality=( + target_finality=pending.output_finality + or ( "snapshot" if request.publication_class == "PROVISIONAL_SNAPSHOT" else "official" @@ -1068,8 +1093,7 @@ async def _acquire_with_settling_policy( return True return False - if policy.official_close_lag is None or self._offerings.official_offering_id is None: - return True + assert policy.official_close_lag is not None closes_at = max( settles_at, _utc(obligation.period.end) + policy.official_close_lag, @@ -1181,19 +1205,22 @@ async def acquire_obligation( satisfied = current is not None and current.readable if satisfied and not restate: return None - if ( - finality == "official" - and obligation.definition is not None - and isinstance(obligation.definition.finality_policy, ReportingStabilizedPolicy) - ): - self._observation_history_store() + selected_policy = obligation.definition.finality_policy if obligation.definition else None + if finality == "official": + if isinstance(selected_policy, ReportingStabilizedPolicy): + self._observation_history_store() + if isinstance(selected_policy, ReportingContractualCutoffPolicy) and _utc( + now + ) < contractual_cutoff_at(obligation, selected_policy): + return None # Everything below needs the frozen code: the slice request carries it, # the manifest is checked against it, and the revision is written under # it. Gate here rather than earlier so a settled legacy obligation stays # the no-op it already was instead of becoming an error on every turn. require_frozen_currency(obligation.currency) - offering_id = self._offerings.offering_for(finality) + source_finality = _source_finality(obligation, finality) + offering_id = self._offerings.offering_for(source_finality) if offering_id is None: raise LedgerConflictError( "NO_OFFERING_FOR_FINALITY", @@ -1203,6 +1230,11 @@ async def acquire_obligation( constituents = await self._admitted_source_constituents(configuration, obligation) checkpoint_store = self._restatement_store() if track_settling else None + derived_close = finality == "official" and source_finality == "snapshot" + if derived_close and revisions and isinstance(self._store, RestatementCheckpointStore): + # A manual close and a scheduled close must replay the same acquisition, + # even after unchanged observations have advanced beyond the revision count. + checkpoint_store = self._store checkpoint = ( await checkpoint_store.get_restatement_checkpoint( account_id=obligation.account_id, @@ -1245,6 +1277,7 @@ async def acquire_obligation( frozen_policy, leaf.reporting_revision_id if leaf is not None else None, checkpoint.provisional_until if checkpoint is not None else None, + output_finality="official" if derived_close else None, ) ) if not acquisition.binds(obligation): @@ -1256,9 +1289,13 @@ async def acquire_obligation( request = acquisition.request(deadline_at=_utc(now) + self._offerings.slice_timeout) if manual_replay: request = request.model_copy(update={"trigger": "manual_replay"}) - finality = ( + finality = acquisition.output_finality or ( "snapshot" if request.publication_class == "PROVISIONAL_SNAPSHOT" else "official" ) + if acquisition.output_finality and _utc(now) < _utc( + request.period.source_read_cutoff_at + ): + return None cancel = asyncio.Event() execution = asyncio.create_task( self._execute_source( @@ -1348,6 +1385,11 @@ async def acquire_obligation( manifest, rows=rows, finality=finality, + finality_proof=( + manifest_finality_proof(obligation, manifest, published_at) + if finality == "official" + else None + ), now=published_at, turn=turn, acquisition=acquisition, @@ -1757,6 +1799,7 @@ async def commit_revision_from_manifest( *, rows: Sequence[dict[str, Any]], finality: str, + finality_proof: FinalityProof | None = None, now: datetime | None = None, turn: WorkerTurn | None = None, acquisition: ProvisionalAcquisition | None = None, @@ -1767,7 +1810,8 @@ async def commit_revision_from_manifest( edit path. An official close is terminal, so a later source correction must arrive as an adjustment instead. ``now`` is a trusted publication instant, unlike the dispatch instant accepted by ``acquire_obligation``. - Replaying a publication retains its original creation time and parent. + A new official revision requires proof of the obligation's selected policy. + Replaying a publication retains its original times, policy fields and parent. """ obligation = await self._stored_obligation(obligation) self._validate_manifest_currency(obligation, manifest) @@ -1836,16 +1880,31 @@ async def commit_revision_from_manifest( _utc(manifest.acquired_at) > _utc(now) or _utc(manifest.observed_at) > _utc(created_at) or _utc(manifest.finality_evidence.observed_at) > _utc(created_at) - or ( - finality == "official" - and _utc(manifest.finality_evidence.observed_at) < _utc(obligation.period.end) - ) ): raise LedgerConflictError( "PUBLICATION_TIME_INVALID", "source observation or finality is outside the publication time bounds; " "check source evidence and the producer clock before retrying", ) + if finality != "official": + finality_proof = None + elif prior is not None and prior.finality == "official": + assert prior.finality_policy_id and prior.finality_basis and prior.finalized_at + finality_proof = FinalityProof( + prior.finality_policy_id, prior.finality_basis, prior.finalized_at + ) + elif finality_proof is None: + raise LedgerConflictError( + "FINALITY_POLICY_UNPROVEN", "official finality requires policy proof" + ) + else: + if manifest.offering_id != self._offerings.offering_for( + _source_finality(obligation, "official") + ): + raise LedgerConflictError( + "MANIFEST_MISMATCH", "official proof must use the selected source offering" + ) + finality_proof.validate(obligation, manifest, created_at) revision = ReportingRevisionRecord( reporting_revision_id=revision_id, account_id=obligation.account_id, @@ -1865,21 +1924,9 @@ async def commit_revision_from_manifest( data_through=manifest.data_through, created_at=created_at, supersedes_reporting_revision_id=supersedes, - finality_basis=( - ( - "stabilized" - if manifest.finality_evidence.basis == "elapsed_settlement_window" - else "source_final" - ) - if finality == "official" - else None - ), - finality_policy_id=( - f"{obligation.report_definition_id}:{manifest.offering_id}" - if finality == "official" - else None - ), - finalized_at=manifest.finality_evidence.observed_at if finality == "official" else None, + finality_basis=finality_proof.basis if finality_proof else None, + finality_policy_id=finality_proof.finality_policy_id if finality_proof else None, + finalized_at=finality_proof.finalized_at if finality_proof else None, source_publication_id=manifest.publication_id, source_manifest_sha256=manifest.content_fingerprint.split(":", 1)[-1], ) @@ -2127,17 +2174,22 @@ def _build_slice( "an obligation with no media buys has no source work; it is a platform-owned " "no-op, not a slice", ) + resolved_finality = finality or obligation.required_finality + source_finality = _source_finality(obligation, resolved_finality) + execution_identity: list[str | int] = [ + obligation.reporting_obligation_id, + offering_id, + observation, + ] + if resolved_finality == "official" and source_finality == "snapshot": + # A policy close requires its own post-cutoff read, even when the + # same snapshot offering already sealed an observation at this ordinal. + execution_identity.append("official") source_execution_key = ( - "rse-" - + hashlib.sha256( - canonical_json_utf8_v1( - [obligation.reporting_obligation_id, offering_id, observation] - ) - ).hexdigest()[:40] + "rse-" + hashlib.sha256(canonical_json_utf8_v1(execution_identity)).hexdigest()[:40] ) - resolved_finality = finality or obligation.required_finality publication_class: ReportingPublicationClass = ( - "AUTHORITATIVE" if resolved_finality == "official" else "PROVISIONAL_SNAPSHOT" + "AUTHORITATIVE" if source_finality == "official" else "PROVISIONAL_SNAPSHOT" ) return ReportingSourceSliceRequestV1( identity=ReportingSourceIdentityV1( diff --git a/src/adcp/reporting/ledger/provisional.py b/src/adcp/reporting/ledger/provisional.py index 90e8ba9ee..8f7dcfc13 100644 --- a/src/adcp/reporting/ledger/provisional.py +++ b/src/adcp/reporting/ledger/provisional.py @@ -10,7 +10,7 @@ from collections.abc import Sequence from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable +from typing import TYPE_CHECKING, Any, Literal, Protocol, runtime_checkable from adcp.reporting.source import ReportingSourceSliceRequestV1 @@ -64,11 +64,14 @@ class ProvisionalAcquisition: policy: ProvisionalPolicy predecessor_revision_id: str | None = None previous_boundary: datetime | None = None + output_finality: Literal["official"] | None = None def __post_init__(self) -> None: if type(self.ordinal) is not int or self.ordinal < 0: raise ValueError("observation ordinal must be a nonnegative integer") self.request() + if self.output_finality not in {None, "official"}: + raise ValueError("invalid provisional acquisition output finality") if self.previous_boundary is not None: utc(self.previous_boundary) @@ -109,6 +112,7 @@ def binds(self, obligation: ReportingObligationRecord) -> bool: def to_wire(self) -> dict[str, Any]: return { "request": json.loads(self.request_json), + **({"output_finality": self.output_finality} if self.output_finality else {}), "ordinal": self.ordinal, "policy": self.policy.to_wire(), "predecessor_revision_id": self.predecessor_revision_id, @@ -133,6 +137,7 @@ def from_wire(cls, value: dict[str, Any]) -> ProvisionalAcquisition: if value["previous_boundary"] is not None else None ), + value.get("output_finality"), ) diff --git a/src/adcp/reporting/source.py b/src/adcp/reporting/source.py index 3963c500d..a7cd91a99 100644 --- a/src/adcp/reporting/source.py +++ b/src/adcp/reporting/source.py @@ -183,6 +183,7 @@ OpaqueReference = Annotated[str, StringConstraints(pattern=_OPAQUE_REFERENCE_PATTERN)] Identifier = Annotated[str, StringConstraints(pattern=_IDENTIFIER_PATTERN)] ExternalId = Annotated[str, StringConstraints(min_length=1, max_length=255)] +EvidenceRef = Annotated[str, StringConstraints(min_length=1, max_length=512)] MetricName = Annotated[str, StringConstraints(pattern=_METRIC_NAME_PATTERN)] Grain = Annotated[str, StringConstraints(pattern=_GRAIN_PATTERN)] EvidenceReason = Annotated[str, StringConstraints(pattern=_EVIDENCE_REASON_PATTERN)] @@ -1005,7 +1006,7 @@ class ReportingFinalityEvidenceV1(_Frozen): "elapsed_settlement_window", ] observed_at: datetime - evidence_ref: ExternalId | None = None + evidence_ref: EvidenceRef | None = None stable_since: datetime | None = Field( default=None, description="Source evidence timestamp preserved without deriving SDK finality policy", diff --git a/tests/test_reporting_official_finality.py b/tests/test_reporting_official_finality.py new file mode 100644 index 000000000..8ccf2e23e --- /dev/null +++ b/tests/test_reporting_official_finality.py @@ -0,0 +1,428 @@ +"""Official publication binds an exact policy to its qualifying source acquisition.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from dataclasses import replace +from datetime import timedelta +from functools import partial +from typing import Any + +import pytest +from adcp_reporting_storage import PgReportingLedgerStore +from pydantic import ValidationError + +from adcp.reporting.evidence import ReportingControlTotalRecord +from adcp.reporting.fixtures import ( + OFFICIAL_OFFERING_ID, + SNAPSHOT_OFFERING_ID, + redacted_completed_result, +) +from adcp.reporting.inline_source import ( + InlineReportingSource, + InMemorySealStore, + InMemoryStagingStore, +) +from adcp.reporting.ledger import ( + InMemoryReportingLedgerStore, + LedgerConflictError, + ReportingContractualCutoffPolicy, + ReportingProducer, + ReportingRevisionRecord, + ReportingSourceFinalPolicy, + revision_content_sha256, +) +from adcp.reporting.ledger.finality import FinalityProof, source_finality_proof +from adcp.reporting.ledger.provisional import ProvisionalAcquisition +from adcp.reporting.source import ReportingFinalityEvidenceV1 +from tests.conformance.reporting._generation_support import isolated_reporting_pool +from tests.test_reporting_source_availability import ( + END, + PROVIDER_ROW, + _closing_capabilities, + _harness, +) + +ROWS = [{"campaign_id": "campaign-redacted-1", "impressions": 10}] + + +@pytest.fixture(params=["memory", "postgres"]) +async def store_factory(request: pytest.FixtureRequest) -> AsyncIterator[Any]: + if request.param == "memory": + yield InMemoryReportingLedgerStore + else: + async with isolated_reporting_pool() as pool: + yield partial(PgReportingLedgerStore, pool=pool) + + +def test_source_final_evidence_ref_accepts_512_characters() -> None: + evidence = ReportingFinalityEvidenceV1( + basis="provider_declared", observed_at=END, evidence_ref="s" * 512 + ) + assert evidence.evidence_ref == "s" * 512 + + +@pytest.mark.parametrize("signal", ["", "s" * 513]) +def test_source_final_evidence_ref_rejects_out_of_bounds(signal: str) -> None: + with pytest.raises(ValidationError): + ReportingFinalityEvidenceV1(basis="provider_declared", observed_at=END, evidence_ref=signal) + + +@pytest.mark.parametrize( + ("basis", "signal"), + [("provider_declared", "delivery.is_final"), ("provider_job_terminal", "s" * 512)], +) +async def test_source_final_emits_exact_selected_policy(basis: str, signal: str) -> None: + producer, _, config, obligation, _, clock = await _harness( + official=True, finality_policy=ReportingSourceFinalPolicy("delivery-final-v1", signal) + ) + provider = producer._source + provider.basis, provider.signal = basis, signal + provider.read_delay = timedelta(seconds=3) + observed = clock[0] + + revision = await producer.acquire_obligation(config, obligation) + + assert revision is not None and revision.finality == "official" + assert revision.finality_policy_id == "delivery-final-v1" + assert revision.finality_basis == "source_final" + assert revision.finalized_at == observed < revision.created_at + assert provider.requests[0].publication_class == "AUTHORITATIVE" + assert provider.requests[0].offering_id == OFFICIAL_OFFERING_ID + + +@pytest.mark.parametrize( + ("basis", "signal"), + [ + ("provider_declared", "Delivery.is_final"), + ("provider_declared", None), + ("elapsed_settlement_window", "delivery.is_final"), + ], +) +async def test_source_final_rejects_unproven_signal(basis: str, signal: str | None) -> None: + producer, store, config, obligation, _, _ = await _harness( + official=True, + finality_policy=ReportingSourceFinalPolicy("delivery-v1", "delivery.is_final"), + ) + producer._source.basis, producer._source.signal = basis, signal + with pytest.raises(LedgerConflictError) as conflict: + await producer.acquire_obligation(config, obligation) + assert conflict.value.code == "MANIFEST_MISMATCH" + assert not await store.list_revisions( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + + +async def test_source_final_refuses_evidence_before_obligation_period_end() -> None: + policy = ReportingSourceFinalPolicy("selected", "source-evidence-redacted") + producer, _, config, obligation, _, clock = await _harness( + official=True, finality_policy=policy + ) + request = producer._build_slice(config, obligation, OFFICIAL_OFFERING_ID, now=clock[0]) + _, manifest, _ = redacted_completed_result(request, page_bodies=[PROVIDER_ROW]) + later_end = END + timedelta(hours=1) + later = replace( + obligation, period=replace(obligation.period, end=later_end), scope_resolved_at=later_end + ) + with pytest.raises(LedgerConflictError, match="source signal"): + source_finality_proof(later, manifest, policy) + + +async def test_source_final_refuses_a_manifest_from_another_offering() -> None: + policy = ReportingSourceFinalPolicy("selected", "source-evidence-redacted") + producer, _, config, obligation, _, clock = await _harness( + official=True, finality_policy=policy + ) + request = producer._build_slice(config, obligation, OFFICIAL_OFFERING_ID, now=clock[0]) + _, manifest, _ = redacted_completed_result(request, page_bodies=[PROVIDER_ROW]) + producer._offerings = replace(producer._offerings, official_offering_id="other-authority") + with pytest.raises(LedgerConflictError) as conflict: + await producer.commit_revision_from_manifest( + obligation, + manifest, + rows=ROWS, + finality="official", + finality_proof=source_finality_proof(obligation, manifest, policy), + ) + assert conflict.value.code == "MANIFEST_MISMATCH" + + +async def test_contractual_cutoff_promotes_a_post_cutoff_snapshot() -> None: + producer, store, config, obligation, calls, clock = await _harness( + official=True, + finality_policy=ReportingContractualCutoffPolicy("two-hour-cutoff", "PT2H"), + capabilities=_closing_capabilities(), + ) + producer._offerings = replace(producer._offerings, official_offering_id=None) + producer._source.read_delay = timedelta(seconds=3) + cutoff = END + timedelta(hours=2) + clock[0] = cutoff - timedelta(microseconds=1) + assert not (await producer.run_configuration(config)).revisions_committed + assert await producer.acquire_obligation(config, obligation, manual_replay=True) is None + assert not calls + clock[0] = cutoff + turn = await producer.run_configuration(config) + + assert len(turn.revisions_committed) == 1 + revision = await store.get_revision( + account_id=config.account_id, reporting_revision_id=turn.revisions_committed[0] + ) + assert revision is not None and revision.finality == "official" + assert revision.finality_policy_id == "two-hour-cutoff" + assert revision.finality_basis == "contractual_cutoff" + assert revision.observed_at == cutoff + assert revision.finalized_at == revision.created_at == cutoff + timedelta(seconds=3) + request = producer._source.requests[0] + assert (request.offering_id, request.publication_class) == ( + SNAPSHOT_OFFERING_ID, + "PROVISIONAL_SNAPSHOT", + ) + assert request.coverage.expected == "full" + + +async def test_manual_contractual_cutoff_acquires_fresh_content_after_unchanged_reads( + monkeypatch: pytest.MonkeyPatch, +) -> None: + scheduled, store, config, obligation, _, clock = await _harness( + capabilities=_closing_capabilities(), + finality_policy=ReportingContractualCutoffPolicy("three-hours", "PT3H"), + ) + staging = InMemoryStagingStore() + calls = [] + + def fetch(request: Any) -> list[dict[str, Any]]: + calls.append(request) + return [{"media_buy_id": "media-buy-redacted", "impressions": 10, "spend": "1.25"}] + + source = InlineReportingSource( + capabilities=scheduled._source.capabilities, + fetch=fetch, + clock=lambda: clock[0], + staging=staging, + seals=InMemorySealStore(), + ) + producer = ReportingProducer( + source=source, + store=store, + offerings=scheduled._offerings, + object_reader=staging, + clock=lambda: clock[0], + read_jitter_window=timedelta(0), + ) + clock[0] = END + timedelta(hours=1) + first = await producer.acquire_obligation(config, obligation, track_settling=True) + assert first is not None and first.finality == "snapshot" + clock[0] += timedelta(hours=1) + assert ( + await producer.acquire_obligation(config, obligation, restate=True, track_settling=True) + == first + ) + clock[0] += timedelta(hours=1) + + async def interrupted(**kwargs: Any) -> bytes: + raise OSError("interrupted staged read") + + with monkeypatch.context() as patch: + patch.setattr(staging, "read", interrupted) + with pytest.raises(OSError, match="interrupted staged read"): + await producer.acquire_obligation( + config, obligation, restate=True, target_finality="official", manual_replay=True + ) + assert len(calls) == 3 + + turn = await producer.run_configuration(config) + assert len(turn.revisions_committed) == 1 + revision = await store.get_revision( + account_id=config.account_id, reporting_revision_id=turn.revisions_committed[0] + ) + assert revision is not None and revision.finality == "official" + assert len(calls) == 3 + assert calls[-1].period.source_read_cutoff_at == revision.finalized_at == clock[0] + + +@pytest.mark.parametrize("pending_close", [False, True]) +async def test_contractual_cutoff_restores_frozen_output_on_restart( + monkeypatch: pytest.MonkeyPatch, pending_close: bool, store_factory: Any +) -> None: + producer, store, config, obligation, _, clock = await _harness( + store_factory, + capabilities=_closing_capabilities(), + finality_policy=ReportingContractualCutoffPolicy("scheduled-cutoff", "PT3H"), + ) + producer._offerings = replace(producer._offerings, official_offering_id=None) + clock[0] = END + timedelta(hours=1) + if pending_close: + assert len((await producer.run_configuration(config)).revisions_committed) == 1 + clock[0] = END + timedelta(hours=3) + + async def interrupted(**kwargs: Any) -> bytes: + raise OSError("interrupted staged read") + + with monkeypatch.context() as patch: + patch.setattr(producer._source, "read", interrupted) + with pytest.raises(OSError, match="interrupted staged read"): + await producer.run_configuration(config) + request = producer._source.requests[-1] + reserved = await store.get_provisional_acquisition( + account_id=config.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ordinal=int(pending_close), + ) + assert reserved is not None + wire = reserved.to_wire() + if pending_close: + assert wire["output_finality"] == "official" + clock[0] = END + timedelta(hours=2) + calls_before_replay = len(producer._source.requests) + assert ( + await producer.acquire_obligation( + config, obligation, restate=True, target_finality="snapshot", track_settling=True + ) + is None + ) + assert len(producer._source.requests) == calls_before_replay + else: + assert "output_finality" not in wire + assert ProvisionalAcquisition.from_wire(wire) == reserved + restarted = ReportingProducer( + source=producer._source, + object_reader=producer._source, + store=store, + offerings=producer._offerings, + clock=lambda: clock[0], + read_jitter_window=timedelta(0), + ) + clock[0] = END + timedelta(hours=4) + turn = await restarted.run_configuration(config) + assert producer._source.requests[-1].period == request.period + assert len(turn.revisions_committed) == 1 + revision = await store.get_revision( + account_id=config.account_id, reporting_revision_id=turn.revisions_committed[0] + ) + assert revision is not None + assert revision.finality == ("official" if pending_close else "snapshot") + assert revision.finality_policy_id == ("scheduled-cutoff" if pending_close else None) + if not pending_close: + closed = await restarted.run_configuration(config) + assert len(closed.revisions_committed) == 1 + official = await store.get_revision( + account_id=config.account_id, reporting_revision_id=closed.revisions_committed[0] + ) + assert official is not None and official.finality == "official" + + +async def test_official_construction_requires_proof_on_new_writes() -> None: + producer, _, config, obligation, _, clock = await _harness( + official=True, + finality_policy=ReportingSourceFinalPolicy("delivery-v1", "source-evidence-redacted"), + ) + request = producer._build_slice(config, obligation, OFFICIAL_OFFERING_ID, now=clock[0]) + _, manifest, _ = redacted_completed_result(request, page_bodies=[PROVIDER_ROW]) + with pytest.raises(LedgerConflictError) as conflict: + await producer.commit_revision_from_manifest( + obligation, manifest, rows=ROWS, finality="official" + ) + assert conflict.value.code == "FINALITY_POLICY_UNPROVEN" + + +@pytest.mark.parametrize("offset", [timedelta(microseconds=-1), timedelta(seconds=31)]) +async def test_official_proof_time_must_be_within_publication_bounds(offset: timedelta) -> None: + producer, _, config, obligation, _, clock = await _harness( + official=True, + finality_policy=ReportingSourceFinalPolicy("delivery-v1", "source-evidence-redacted"), + ) + request = producer._build_slice(config, obligation, OFFICIAL_OFFERING_ID, now=clock[0]) + _, manifest, _ = redacted_completed_result(request, page_bodies=[PROVIDER_ROW]) + with pytest.raises(LedgerConflictError) as conflict: + await producer.commit_revision_from_manifest( + obligation, + manifest, + rows=ROWS, + finality="official", + finality_proof=FinalityProof("delivery-v1", "source_final", END + offset), + ) + assert conflict.value.code == "PUBLICATION_TIME_INVALID" + + +async def test_contractual_cutoff_refuses_pre_cutoff_source_publication() -> None: + producer, _, config, obligation, _, clock = await _harness( + official=True, finality_policy=ReportingContractualCutoffPolicy("cutoff-v1", "PT2H") + ) + request = producer._build_slice( + config, obligation, SNAPSHOT_OFFERING_ID, finality="snapshot", now=clock[0] + ) + _, manifest, _ = redacted_completed_result(request, page_bodies=[PROVIDER_ROW]) + clock[0] = END + timedelta(hours=3) + with pytest.raises(LedgerConflictError) as conflict: + await producer.commit_revision_from_manifest( + obligation, + manifest, + rows=ROWS, + finality="official", + finality_proof=FinalityProof("cutoff-v1", "contractual_cutoff", clock[0]), + ) + assert conflict.value.code == "FINALITY_POLICY_UNPROVEN" + + +async def test_official_construction_rejects_proof_for_another_policy() -> None: + producer, _, config, obligation, _, clock = await _harness( + official=True, + finality_policy=ReportingSourceFinalPolicy("selected", "source-evidence-redacted"), + ) + request = producer._build_slice(config, obligation, OFFICIAL_OFFERING_ID, now=clock[0]) + _, manifest, _ = redacted_completed_result(request, page_bodies=[PROVIDER_ROW]) + with pytest.raises(LedgerConflictError) as conflict: + await producer.commit_revision_from_manifest( + obligation, + manifest, + rows=ROWS, + finality="official", + finality_proof=FinalityProof("unselected", "source_final", clock[0]), + ) + assert conflict.value.code == "FINALITY_POLICY_UNPROVEN" + + +async def test_retained_official_replay_preserves_original_policy_fields() -> None: + producer, store, config, obligation, _, clock = await _harness(official=True) + request = producer._build_slice(config, obligation, OFFICIAL_OFFERING_ID, now=clock[0]) + _, manifest, _ = redacted_completed_result(request, page_bodies=[PROVIDER_ROW]) + revision_id = f"rpr_{manifest.publication_id[4:44]}" + totals = (("impressions", "10"),) + managed_totals = (ReportingControlTotalRecord("impressions", "10", "integer"),) + historical = ReportingRevisionRecord( + reporting_revision_id=revision_id, + account_id=config.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + finality="official", + revision_content_sha256=revision_content_sha256( + reporting_revision_id=revision_id, + row_count=1, + control_totals=totals, + reporting_rows=ROWS, + control_total_evidence=managed_totals, + ), + row_count=1, + control_totals=totals, + managed_control_totals=managed_totals, + observed_at=manifest.observed_at, + data_through=manifest.data_through, + created_at=clock[0], + finality_policy_id="archived-policy", + finality_basis="stabilized", + finalized_at=END, + source_publication_id=manifest.publication_id, + source_manifest_sha256=manifest.content_fingerprint.split(":", 1)[1], + ) + historical = await store.commit_revision(historical, ROWS) + clock[0] += timedelta(hours=1) + assert ( + await producer.commit_revision_from_manifest( + obligation, manifest, rows=ROWS, finality="official" + ) + == historical + ) + with pytest.raises(LedgerConflictError) as conflict: + await producer.commit_revision_from_manifest( + obligation, manifest, rows=[dict(ROWS[0], campaign_id="other")], finality="official" + ) + assert conflict.value.code == "REVISION_IMMUTABLE" diff --git a/tests/test_reporting_source_availability.py b/tests/test_reporting_source_availability.py index b998ccf58..33fe81822 100644 --- a/tests/test_reporting_source_availability.py +++ b/tests/test_reporting_source_availability.py @@ -12,6 +12,8 @@ OFFICIAL_OFFERING_ID, SNAPSHOT_OFFERING_ID, redacted_capabilities, + redacted_completed_result, + redacted_contract_identity, ) from adcp.reporting.inline_source import ( InlineFetchResult, @@ -23,19 +25,31 @@ InMemoryReportingLedgerStore, ProducerOfferings, ReportingConfiguration, + ReportingContractualCutoffPolicy, + ReportingDefinitionBinding, + ReportingFinalityPolicy, ReportingObligationRecord, ReportingProducer, ReportingScheduleSpec, + ReportingSourceFinalPolicy, WorkerTurn, ) from adcp.reporting.source import ( MediaBuyConstituentV1, + ReportingFinalityEvidenceV1, ReportingSourceCapabilitiesV1, + ReportingSourceExecutorResult, + ReportingSourceSliceRequestV1, + ReportingSourceStagedObjectReader, + SourceBatchManifestV1, + encode_source_batch_manifest_v1, reporting_source_capabilities_sha256_v1, + source_batch_manifest_reference_v1, ) from tests.test_reporting_producer_retry import FailingSource END = datetime(2026, 11, 2, tzinfo=timezone.utc) +PROVIDER_ROW = '{"campaign_id":"campaign-redacted-1","impressions":10}\n' def _capabilities(*, official: bool = False, **changes: Any) -> ReportingSourceCapabilitiesV1: @@ -101,12 +115,59 @@ async def finish_producer_acquisition( self.finished.append(reporting_obligation_id) +class EvidenceSource: + """A provider with an explicit authoritative signal and generation-pinned staged rows.""" + + def __init__( + self, + capabilities: ReportingSourceCapabilitiesV1, + clock: list[datetime], + calls: list[datetime], + ) -> None: + self.capabilities = capabilities + self.clock = clock + self.calls = calls + self.requests: list[ReportingSourceSliceRequestV1] = [] + self.signal: str | None = "delivery.is_final" + self.basis = "provider_declared" + self.read_delay = timedelta(0) + self.manifest: SourceBatchManifestV1 + self.reader: ReportingSourceStagedObjectReader + + async def execute( + self, request: ReportingSourceSliceRequestV1, **kwargs: Any + ) -> ReportingSourceExecutorResult: + self.calls.append(self.clock[0]) + self.requests.append(request) + result, self.manifest, self.reader = redacted_completed_result( + request, observed_at=self.clock[0], page_bodies=[PROVIDER_ROW] + ) + if request.publication_class == "AUTHORITATIVE": + evidence = ReportingFinalityEvidenceV1.model_validate( + {"basis": self.basis, "evidence_ref": self.signal, "observed_at": self.clock[0]} + ) + self.manifest = self.manifest.model_copy(update={"finality_evidence": evidence}) + raw = encode_source_batch_manifest_v1(self.manifest) + return ReportingSourceExecutorResult.completed( + request=request, + manifest=source_batch_manifest_reference_v1("manifest-redacted-v1", raw), + manifest_bytes=raw, + ) + return result + + async def read(self, **kwargs: Any) -> bytes: + rows = await self.reader.read(**kwargs) + self.clock[0] += self.read_delay + return rows + + async def _harness( store_class: Any = InMemoryReportingLedgerStore, *, official: bool = False, delivery_sla: str = "PT6H", capabilities: ReportingSourceCapabilitiesV1 | None = None, + finality_policy: ReportingFinalityPolicy | None = None, read_jitter_window: timedelta = timedelta(0), **changes: Any, ) -> tuple[Any, ...]: @@ -143,6 +204,16 @@ def fetch(request: Any) -> InlineFetchResult: period_duration="P1D", delivery_sla=delivery_sla, alignment="utc" ), required_finality="official" if official else "snapshot", + definition=( + ReportingDefinitionBinding( + **redacted_contract_identity.model_dump( + exclude={"report_definition_id", "reporting_profile"} + ), + finality_policy=finality_policy, + ) + if finality_policy is not None + else None + ), automated_recovery_window=timedelta(0), activated_at=END - timedelta(days=1, minutes=10), media_buy_ids=("media-buy-redacted",), @@ -154,11 +225,12 @@ def fetch(request: Any) -> InlineFetchResult: source_scope=dict(capabilities.source_scope), publication_namespace="reporting-source:fixture", ) + provider = EvidenceSource(capabilities, clock, calls) if finality_policy is not None else None producer = ReportingProducer( - source=source, + source=provider or source, offerings=offerings, store=store, - object_reader=staging, + object_reader=provider or staging, clock=lambda: clock[0], read_jitter_window=read_jitter_window, ) @@ -191,7 +263,9 @@ async def test_snapshot_official_close_waits_for_the_authoritative_offering( store_class: Any, pending_lookup: bool ) -> None: producer, store, configuration, obligation, calls, clock = await _harness( - store_class, capabilities=_closing_capabilities() + store_class, + capabilities=_closing_capabilities(), + finality_policy=ReportingSourceFinalPolicy("delivery-v1", "delivery.is_final"), ) if not pending_lookup: # Custom stores can keep the original observation contract. @@ -227,7 +301,9 @@ async def test_snapshot_official_close_waits_for_the_authoritative_offering( async def test_official_close_uses_its_own_worst_case_window(retryable: bool) -> None: capabilities = _closing_capabilities() producer, _, configuration, obligation, _, clock = await _harness( - delivery_sla="PT10M", capabilities=capabilities + delivery_sla="PT10M", + capabilities=capabilities, + finality_policy=ReportingSourceFinalPolicy("delivery-v1", "delivery.is_final"), ) clock[0] = END + timedelta(hours=1) await producer.run_configuration(configuration, now=clock[0]) @@ -295,6 +371,25 @@ async def test_zero_lag_keeps_immediate_acquisition() -> None: assert len(turn.revisions_committed) == 1 +@pytest.mark.parametrize("store_class", [InMemoryReportingLedgerStore, ProgressStore]) +async def test_contractual_cutoff_waits_for_snapshot_availability(store_class: Any) -> None: + producer, store, config, obligation, calls, clock = await _harness( + store_class, + official=True, + capabilities=_closing_capabilities(), + finality_policy=ReportingContractualCutoffPolicy("half-hour", "PT30M"), + ) + clock[0] = END + timedelta(minutes=59) + assert not (await producer.run_configuration(config)).revisions_committed + assert not calls + if isinstance(store, ProgressStore): + assert obligation.reporting_obligation_id in store.pending + clock[0] = END + timedelta(hours=1) + assert len((await producer.run_configuration(config)).revisions_committed) == 1 + assert calls == [clock[0]] + assert producer._source.requests[0].offering_id == SNAPSHOT_OFFERING_ID + + @pytest.mark.parametrize( ("period_end", "days_after", "ready_time", "expected"), [ From ae1ab9c40e2521fa9718f8732b0c3545773d75a9 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 13:03:53 +0530 Subject: [PATCH 10/39] fix(reporting): bind official proof to verified source acquisitions --- src/adcp/reporting/inline_source.py | 8 +- src/adcp/reporting/ledger/finality.py | 6 +- src/adcp/reporting/ledger/producer.py | 384 ++++++++++++++++++---- src/adcp/reporting/source.py | 12 +- tests/test_reliable_reporting_service.py | 53 ++- tests/test_reporting_official_finality.py | 310 ++++++++++++++++- tests/test_reporting_producer_retry.py | 44 ++- 7 files changed, 731 insertions(+), 86 deletions(-) diff --git a/src/adcp/reporting/inline_source.py b/src/adcp/reporting/inline_source.py index 87c4721a7..3b84df635 100644 --- a/src/adcp/reporting/inline_source.py +++ b/src/adcp/reporting/inline_source.py @@ -284,6 +284,9 @@ class InlineFetchResult: scheduling correctly after a restart. """ + finality_evidence: ReportingFinalityEvidenceV1 | None = field(default=None, kw_only=True) + """Explicit provider evidence for this publication, including its observed time and signal.""" + cell_availability: Mapping[str, Mapping[str, MetricEvidence]] | None = field( default=None, kw_only=True ) @@ -1021,6 +1024,7 @@ async def _publish( control_totals=control_totals, warnings=warnings, provisional_until=result.provisional_until, + finality_evidence=result.finality_evidence, ) manifest_bytes = encode_source_batch_manifest_v1(manifest) @@ -1196,6 +1200,7 @@ def _seal_manifest( control_totals: list[SourceControlTotalV1], warnings: list[str], provisional_until: datetime | None, + finality_evidence: ReportingFinalityEvidenceV1 | None, ) -> SourceBatchManifestV1: coverage = SourceBatchCoverageV1( denominator_fingerprint=request.coverage.denominator_fingerprint, @@ -1226,7 +1231,8 @@ def _seal_manifest( "period": request.period, "revision_kind": request.revision_kind, "supersedes_publication_id": request.supersedes_publication_id, - "finality_evidence": ReportingFinalityEvidenceV1( + "finality_evidence": finality_evidence + or ReportingFinalityEvidenceV1( basis=( "provisional_observation" if request.publication_class == "PROVISIONAL_SNAPSHOT" diff --git a/src/adcp/reporting/ledger/finality.py b/src/adcp/reporting/ledger/finality.py index 1bf86587c..3545610bf 100644 --- a/src/adcp/reporting/ledger/finality.py +++ b/src/adcp/reporting/ledger/finality.py @@ -181,11 +181,15 @@ def manifest_finality_proof( if ( manifest.publication_class != "PROVISIONAL_SNAPSHOT" or not cutoff <= manifest.period.source_read_cutoff_at <= published_at - or manifest.acquired_at < cutoff + or manifest.observed_at < cutoff ): raise LedgerConflictError( "FINALITY_POLICY_UNPROVEN", "contractual cutoff requires a post-cutoff snapshot" ) + try: + manifest.require_full_period_watermarks() + except ValueError as error: + raise LedgerConflictError("FINALITY_POLICY_UNPROVEN", str(error)) from error return FinalityProof(policy.finality_policy_id, policy.basis, published_at) raise LedgerConflictError("FINALITY_POLICY_UNPROVEN", "official finality requires policy proof") diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 4ca3e75c2..0b8e9a7ef 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -935,10 +935,14 @@ def _settling_policy( self, configuration: ReportingConfiguration, obligation: ReportingObligationRecord, + *, + cutoff_reservation: bool = False, ) -> _SettlingPolicy | None: """Resolve the source policy or SDK fallback for a snapshot obligation.""" offering_id = self._offerings.snapshot_offering_id - if obligation.required_finality != "snapshot" or offering_id is None: + if ( + obligation.required_finality != "snapshot" and not cutoff_reservation + ) or offering_id is None: return None offering = self._source.capabilities.offering(offering_id) if not isinstance(offering, ProvisionalSnapshotOfferingV1): @@ -1207,12 +1211,12 @@ async def acquire_obligation( return None selected_policy = obligation.definition.finality_policy if obligation.definition else None if finality == "official": + if selected_policy is None: + raise LedgerConflictError( + "FINALITY_POLICY_UNPROVEN", "official finality requires a selected policy" + ) if isinstance(selected_policy, ReportingStabilizedPolicy): self._observation_history_store() - if isinstance(selected_policy, ReportingContractualCutoffPolicy) and _utc( - now - ) < contractual_cutoff_at(obligation, selected_policy): - return None # Everything below needs the frozen code: the slice request carries it, # the manifest is checked against it, and the revision is written under # it. Gate here rather than earlier so a settled legacy obligation stays @@ -1229,12 +1233,17 @@ async def acquire_obligation( constituents = await self._admitted_source_constituents(configuration, obligation) - checkpoint_store = self._restatement_store() if track_settling else None derived_close = finality == "official" and source_finality == "snapshot" - if derived_close and revisions and isinstance(self._store, RestatementCheckpointStore): - # A manual close and a scheduled close must replay the same acquisition, - # even after unchanged observations have advanced beyond the revision count. - checkpoint_store = self._store + if isinstance(selected_policy, ReportingContractualCutoffPolicy) and derived_close: + track_settling = ( + track_settling + or manual_replay + or ( + self._optional_observation_store() is not None + and isinstance(self._store, RestatementCheckpointStore) + ) + ) + checkpoint_store = self._restatement_store() if track_settling else None checkpoint = ( await checkpoint_store.get_restatement_checkpoint( account_id=obligation.account_id, @@ -1244,6 +1253,11 @@ async def acquire_obligation( else None ) observation = checkpoint.next_observation if checkpoint is not None else len(revisions) + if derived_close and isinstance(selected_policy, ReportingContractualCutoffPolicy): + if _utc(now) < contractual_cutoff_at(obligation, selected_policy): + pending = await self._pending_acquisition(obligation, ordinal=observation) + if pending is None or pending.output_finality == "official": + return None request = self._build_slice( configuration, obligation, @@ -1256,36 +1270,15 @@ async def acquire_obligation( ) acquisition = None if track_settling: - policy = self._settling_policy(configuration, obligation) - assert policy is not None - latest = await self._observation_store().get_provisional_observation( - account_id=obligation.account_id, - reporting_obligation_id=obligation.reporting_obligation_id, - ) - frozen_policy = ( - latest.acquisition.policy - if latest is not None - else ProvisionalPolicy( - policy.restatement_window, policy.restatement_cadence, policy.official_close_lag - ) - ) - leaf = self._current_snapshot(revisions) - acquisition = await self._observation_store().reserve_provisional_acquisition( - ProvisionalAcquisition( - request.model_dump_json(), - observation, - frozen_policy, - leaf.reporting_revision_id if leaf is not None else None, - checkpoint.provisional_until if checkpoint is not None else None, - output_finality="official" if derived_close else None, - ) + acquisition = await self._reserve_observation( + configuration, + obligation, + request, + ordinal=observation, + revisions=revisions, + previous_boundary=checkpoint.provisional_until if checkpoint else None, + derived_close=derived_close, ) - if not acquisition.binds(obligation): - raise LedgerConflictError( - "OBSERVATION_CONFLICT", - "frozen source acquisition does not bind the obligation's consumer; " - "operator reconciliation and a new configuration generation are required", - ) request = acquisition.request(deadline_at=_utc(now) + self._offerings.slice_timeout) if manual_replay: request = request.model_copy(update={"trigger": "manual_replay"}) @@ -1370,7 +1363,17 @@ async def acquire_obligation( "a full-coverage request cannot complete with partial or missing coverage", ) self._validate_manifest_currency(obligation, manifest) - rows = await self._read_rows(request, manifest) + if finality == "official": + self._official_source_request( + configuration, + obligation, + manifest, + constituents=constituents, + ordinal=observation, + cutoff_ceiling=now, + acquisition=acquisition, + ) + rows = await self._read_rows(request, manifest, require_staged=finality == "official") # ``now`` freezes dispatch/lease/cutoff decisions, not publication. # A conforming source can observe finality while acquisition is running. # Every successful scheduled read reaches the atomic observation/checkpoint commit. @@ -1380,20 +1383,28 @@ async def acquire_obligation( "PUBLICATION_TIME_INVALID", "producer clock regressed during acquisition; correct the clock before retrying", ) - committed = await self.commit_revision_from_manifest( - obligation, - manifest, - rows=rows, - finality=finality, - finality_proof=( - manifest_finality_proof(obligation, manifest, published_at) - if finality == "official" - else None - ), - now=published_at, - turn=turn, - acquisition=acquisition, - ) + if finality == "official": + committed = await self._commit_revision_from_manifest( + obligation, + manifest, + rows=rows, + finality=finality, + finality_proof=manifest_finality_proof(obligation, manifest, published_at), + now=published_at, + turn=turn, + acquisition=acquisition, + verified_source_rows=True, + ) + else: + committed = await self.commit_revision_from_manifest( + obligation, + manifest, + rows=rows, + finality=finality, + now=published_at, + turn=turn, + acquisition=acquisition, + ) await self._clear_retry(obligation, turn=turn, now=published_at) return committed @@ -1638,6 +1649,19 @@ async def _revision_publication( if not isinstance(self._source, ReportingProductionSource): yield return + configuration = await self._retained_source_configuration(obligation) + # Read immutable admission scope before taking the publication lock. It + # is not an authorization grant: the callback is checked again inside + # the lock against this exact mapping, including on replay. + constituents = await self._admitted_source_constituents(configuration, obligation) + async with self._source_publication( + configuration, offering_id, admitted_constituents=constituents + ): + yield + + async def _retained_source_configuration( + self, obligation: ReportingObligationRecord + ) -> ReportingConfiguration: configuration = next( ( candidate @@ -1653,14 +1677,7 @@ async def _revision_publication( ) if configuration is None: raise LedgerConflictError("HISTORY_UNAVAILABLE", "source generation is unavailable") - # Read immutable admission scope before taking the publication lock. It - # is not an authorization grant: the callback is checked again inside - # the lock against this exact mapping, including on replay. - constituents = await self._admitted_source_constituents(configuration, obligation) - async with self._source_publication( - configuration, offering_id, admitted_constituents=constituents - ): - yield + return configuration def _restatement_store(self) -> RestatementCheckpointStore: explicit = all( @@ -1675,7 +1692,7 @@ def _restatement_store(self) -> RestatementCheckpointStore: ) return self._store - def _observation_store(self) -> ProvisionalObservationPublicationStore: + def _optional_observation_store(self) -> ProvisionalObservationPublicationStore | None: explicit = all( inspect.getattr_static(self._store, name, None) is not None for name in ( @@ -1684,13 +1701,19 @@ def _observation_store(self) -> ProvisionalObservationPublicationStore: "commit_provisional_observation", ) ) - if not explicit or not isinstance(self._store, ProvisionalObservationPublicationStore): + if explicit and isinstance(self._store, ProvisionalObservationPublicationStore): + return self._store + return None + + def _observation_store(self) -> ProvisionalObservationPublicationStore: + store = self._optional_observation_store() + if store is None: raise LedgerConflictError( "PROVISIONAL_OBSERVATIONS_NOT_SUPPORTED", "scheduled provisional reads require explicitly implemented atomic " "observation methods, including any decorating publisher's preparation", ) - return self._store + return store def _observation_history_store(self) -> ProvisionalObservationStore: store = self._observation_store() @@ -1711,6 +1734,50 @@ def _observation_history_store(self) -> ProvisionalObservationStore: ) return store + async def _reserve_observation( + self, + configuration: ReportingConfiguration, + obligation: ReportingObligationRecord, + request: ReportingSourceSliceRequestV1, + *, + ordinal: int, + revisions: Sequence[ReportingRevisionRecord], + previous_boundary: datetime | None, + derived_close: bool, + ) -> ProvisionalAcquisition: + store = self._observation_store() + latest = await store.get_provisional_observation( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + if latest is not None: + frozen_policy = latest.acquisition.policy + else: + policy = self._settling_policy( + configuration, obligation, cutoff_reservation=derived_close + ) + assert policy is not None + frozen_policy = ProvisionalPolicy( + policy.restatement_window, policy.restatement_cadence, policy.official_close_lag + ) + acquisition = await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + request.model_dump_json(), + ordinal, + frozen_policy, + self._current_snapshot_leaf(revisions), + previous_boundary, + output_finality="official" if derived_close else None, + ) + ) + if not acquisition.binds(obligation): + raise LedgerConflictError( + "OBSERVATION_CONFLICT", + "frozen source acquisition does not bind the obligation's consumer; " + "operator reconciliation and a new configuration generation are required", + ) + return acquisition + async def _pending_acquisition( self, obligation: ReportingObligationRecord, *, ordinal: int ) -> ProvisionalAcquisition | None: @@ -1765,10 +1832,19 @@ def _verified_manifest(self, result: ReportingSourceExecutorResult) -> SourceBat return parse_verified_source_batch_manifest_v1(response.manifest, manifest_bytes) async def _read_rows( - self, request: ReportingSourceSliceRequestV1, manifest: SourceBatchManifestV1 + self, + request: ReportingSourceSliceRequestV1, + manifest: SourceBatchManifestV1, + *, + require_staged: bool = False, ) -> list[dict[str, Any]]: """Materialize the manifest's staged rows into the revision's frozen rows.""" if self._object_reader is None: + if require_staged: + raise LedgerConflictError( + "STAGED_OBJECT_UNAVAILABLE", + "official publication requires a staged object reader", + ) return [] import json @@ -1782,16 +1858,83 @@ async def _read_rows( source_scope=request.identity.source_scope, cancel=cancel, ) - if hashlib.sha256(payload).hexdigest() != staged.sha256: + if hashlib.sha256(payload).hexdigest() != staged.sha256 or ( + require_staged and len(payload) != staged.byte_count + ): raise LedgerConflictError( "STAGED_OBJECT_MISMATCH", - f"staged object {staged.ordinal} does not match its manifest digest", + f"staged object {staged.ordinal} does not match its manifest evidence", ) + start = len(rows) for line in payload.decode("utf-8").splitlines(): if line.strip(): rows.append(json.loads(line)) + if require_staged and len(rows) - start != staged.row_count: + raise LedgerConflictError( + "STAGED_OBJECT_MISMATCH", "staged object row count differs from its manifest" + ) return rows + def _official_source_request( + self, + configuration: ReportingConfiguration, + obligation: ReportingObligationRecord, + manifest: SourceBatchManifestV1, + *, + constituents: tuple[ReportingConstituent, ...] | None, + ordinal: int, + cutoff_ceiling: datetime, + acquisition: ProvisionalAcquisition | None, + ) -> ReportingSourceSliceRequestV1: + from adcp.reporting.conformance import ( + ReportingSourceConformanceError, + _validate_manifest_against_request, + ) + + offering_id = self._offerings.offering_for(_source_finality(obligation, "official")) + if offering_id is None or offering_id != manifest.offering_id: + raise LedgerConflictError( + "MANIFEST_MISMATCH", "official proof must use the selected source offering" + ) + cutoff = manifest.period.source_read_cutoff_at + if _utc(cutoff) > _utc(cutoff_ceiling): + raise LedgerConflictError( + "MANIFEST_MISMATCH", "source cutoff exceeds the requested read" + ) + # A retry can receive an earlier seal for its immutable execution key. + # Reconstruct the authorized source and denominator, using the original dispatch + # identity. A durable reservation also binds its exact cutoff and run. + request = self._build_slice( + configuration, + obligation, + offering_id, + finality="official", + now=cutoff, + observation=ordinal, + constituents=constituents, + ) + if acquisition is not None and not self._matches_reservation(request, acquisition): + raise LedgerConflictError( + "MANIFEST_MISMATCH", "official source manifest differs from its frozen request" + ) + try: + _validate_manifest_against_request( + self._source.capabilities.offering(offering_id), request, manifest + ) + except ReportingSourceConformanceError as error: + raise LedgerConflictError( + "MANIFEST_MISMATCH", "official manifest does not bind the configured source request" + ) from error + return request + + @staticmethod + def _matches_reservation( + request: ReportingSourceSliceRequestV1, acquisition: ProvisionalAcquisition + ) -> bool: + return request == acquisition.request().model_copy( + update={"deadline_at": request.deadline_at, "trigger": request.trigger} + ) + async def commit_revision_from_manifest( self, obligation: ReportingObligationRecord, @@ -1813,6 +1956,31 @@ async def commit_revision_from_manifest( A new official revision requires proof of the obligation's selected policy. Replaying a publication retains its original times, policy fields and parent. """ + return await self._commit_revision_from_manifest( + obligation, + manifest, + rows=rows, + finality=finality, + finality_proof=finality_proof, + now=now, + turn=turn, + acquisition=acquisition, + verified_source_rows=False, + ) + + async def _commit_revision_from_manifest( + self, + obligation: ReportingObligationRecord, + manifest: SourceBatchManifestV1, + *, + rows: Sequence[dict[str, Any]], + finality: str, + finality_proof: FinalityProof | None, + now: datetime | None, + turn: WorkerTurn | None, + acquisition: ProvisionalAcquisition | None, + verified_source_rows: bool, + ) -> ReportingRevisionRecord: obligation = await self._stored_obligation(obligation) self._validate_manifest_currency(obligation, manifest) if any(cell.status == "unsupported" for cell in manifest.metric_availability): @@ -1905,6 +2073,15 @@ async def commit_revision_from_manifest( "MANIFEST_MISMATCH", "official proof must use the selected source offering" ) finality_proof.validate(obligation, manifest, created_at) + if not verified_source_rows: + acquisition = await self._verify_direct_official_publication( + obligation, + manifest, + rows, + acquisition, + revisions=existing, + created_at=created_at, + ) revision = ReportingRevisionRecord( reporting_revision_id=revision_id, account_id=obligation.account_id, @@ -1990,6 +2167,77 @@ async def commit_revision_from_manifest( turn.revisions_committed.append(committed.reporting_revision_id) return committed + async def _verify_direct_official_publication( + self, + obligation: ReportingObligationRecord, + manifest: SourceBatchManifestV1, + rows: Sequence[dict[str, Any]], + acquisition: ProvisionalAcquisition | None, + *, + revisions: Sequence[ReportingRevisionRecord], + created_at: datetime, + ) -> ProvisionalAcquisition | None: + configuration = await self._retained_source_configuration(obligation) + constituents = await self._admitted_source_constituents(configuration, obligation) + ordinal = len(revisions) + checkpoint = None + if isinstance(self._store, RestatementCheckpointStore): + checkpoint = await self._store.get_restatement_checkpoint( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + if checkpoint is not None: + ordinal = checkpoint.next_observation + pending = await self._pending_acquisition(obligation, ordinal=ordinal) + if pending is not None: + if (acquisition is not None and acquisition != pending) or ( + pending.output_finality is None + and pending.request().publication_class == "PROVISIONAL_SNAPSHOT" + ): + raise LedgerConflictError( + "OBSERVATION_CONFLICT", + "finish the reserved acquisition before official publication", + ) + acquisition = pending + request = self._official_source_request( + configuration, + obligation, + manifest, + constituents=constituents, + ordinal=ordinal, + cutoff_ceiling=created_at, + acquisition=acquisition, + ) + staged_rows = await self._read_rows(request, manifest, require_staged=True) + if rfc8785.dumps(staged_rows) != rfc8785.dumps(list(rows)): + raise LedgerConflictError( + "STAGED_OBJECT_MISMATCH", "supplied rows differ from verified staged objects" + ) + if ( + pending is None + and request.publication_class == "PROVISIONAL_SNAPSHOT" + and self._optional_observation_store() is not None + and isinstance(self._store, RestatementCheckpointStore) + ): + # Staged reads release the ledger lock. Reserve the verified request before + # publication so an ordinary snapshot that won meanwhile is never skipped. + acquisition = await self._reserve_observation( + configuration, + obligation, + request, + ordinal=ordinal, + revisions=revisions, + previous_boundary=checkpoint.provisional_until if checkpoint else None, + derived_close=True, + ) + if acquisition.output_finality != "official" or not self._matches_reservation( + request, acquisition + ): + raise LedgerConflictError( + "OBSERVATION_CONFLICT", "another source acquisition won the reservation" + ) + return acquisition + async def _commit_publication( self, obligation: ReportingObligationRecord, diff --git a/src/adcp/reporting/source.py b/src/adcp/reporting/source.py index a7cd91a99..4e280bd32 100644 --- a/src/adcp/reporting/source.py +++ b/src/adcp/reporting/source.py @@ -1462,7 +1462,8 @@ def _validate_time_evidence(self) -> None: for cell in self.metric_availability: if cell.data_through is not None: self._require_bounded_watermark(cell.data_through, "metric") - self._validate_authoritative_full_window(end) + if self.publication_class == "AUTHORITATIVE" and self.coverage.status == "full": + self.require_full_period_watermarks() def _require_bounded_watermark(self, value: datetime, label: str) -> None: moment = _aware(value, f"{label} data_through") @@ -1480,12 +1481,9 @@ def _require_bounded_watermark(self, value: datetime, label: str) -> None: "the batch watermark or the requested source cutoff" ) - def _validate_authoritative_full_window(self, end: datetime) -> None: - # Full official coverage is the strongest claim in the contract: it - # says the period is closed and complete. Data that never reached the - # period boundary stays delayed/stale/partial instead. - if self.publication_class != "AUTHORITATIVE" or self.coverage.status != "full": - return + def require_full_period_watermarks(self) -> None: + """Verify completeness of the batch and each available constituent and metric.""" + end = _aware(self.period.end, "period.end") if _aware(self.data_through, "data_through") != end: raise ValueError("full authoritative coverage must prove the complete reporting window") for item in self.coverage.constituents: diff --git a/tests/test_reliable_reporting_service.py b/tests/test_reliable_reporting_service.py index aaf038c8d..330dfddb3 100644 --- a/tests/test_reliable_reporting_service.py +++ b/tests/test_reliable_reporting_service.py @@ -17,6 +17,7 @@ redacted_capabilities, redacted_snapshot_request, ) +from adcp.reporting.inline_source import InlineFetchResult from adcp.reporting.ledger import ( InMemoryReportingLedgerStore, ReportingConfiguration, @@ -39,6 +40,7 @@ ReportingAccountContext, ) from adcp.reporting.source import ( + ReportingFinalityEvidenceV1, ReportingSourceCapabilitiesV1, reporting_source_capabilities_sha256_v1, ) @@ -154,6 +156,8 @@ def _finality_service( close: bool = False, store: InMemoryReportingLedgerStore | None = None, producer_factory: ProducerFactory = ReportingProducer, + results: tuple[InlineFetchResult, ...] = (), + clock: DeterministicReportingClock | None = None, ) -> ReliableReportingService: public = _capability_offering("gam") public["supported_finality"] = [configuration.required_finality] @@ -170,16 +174,63 @@ def _finality_service( offering for offering, enabled in zip(raw["offerings"], (snapshot, official)) if enabled ] raw["capabilities_sha256"] = reporting_source_capabilities_sha256_v1(raw) - adapter = ScriptedReportingAdapter(ReportingSourceCapabilitiesV1.model_validate(raw), []) + adapter = ScriptedReportingAdapter(ReportingSourceCapabilitiesV1.model_validate(raw), results) service = ReliableReportingService( store=store or InMemoryReportingLedgerStore(), account_context=lambda _: context, producer_factory=producer_factory, + clock=clock, ) service.sources.register("gam", adapter, capability_offerings=[public]) return service +@pytest.mark.parametrize("signal", [None, "delivery.other", "delivery.is_final"]) +async def test_registered_inline_adapter_supplies_explicit_source_final_evidence( + signal: str | None, +) -> None: + config = replace( + _configuration(), + required_finality="official", + deactivated_at=NOW, + definition=replace( + DEFINITION, + finality_policy=ReportingSourceFinalPolicy("selected-v1", "delivery.is_final"), + ), + ) + evidence_time = NOW + timedelta(days=1) + answer = InlineFetchResult(rows=_rows(12)) + if signal is not None: + answer = replace( + answer, + finality_evidence=ReportingFinalityEvidenceV1( + basis="provider_job_terminal", + observed_at=evidence_time, + evidence_ref=signal, + ), + ) + service = _finality_service( + config, + snapshot=False, + official=True, + results=(answer,), + clock=DeterministicReportingClock(NOW + timedelta(days=3)), + ) + await service.configure(config) + turn = await service.run_worker(now=NOW + timedelta(days=3)) + if signal != "delivery.is_final": + error = turn.configuration_errors[config.generation_key] + assert isinstance(error, LedgerConflictError) and error.code == "MANIFEST_MISMATCH" + else: + assert not turn.configuration_errors + identifier = turn.configurations[config.generation_key].revisions_committed[0] + revision = await service.store.get_revision( + account_id=config.account_id, reporting_revision_id=identifier + ) + assert revision is not None and revision.finality_policy_id == "selected-v1" + assert revision.finalized_at == evidence_time + + @pytest.mark.parametrize( ("policy", "source_class", "expected_offering"), [ diff --git a/tests/test_reporting_official_finality.py b/tests/test_reporting_official_finality.py index 8ccf2e23e..2b5853c70 100644 --- a/tests/test_reporting_official_finality.py +++ b/tests/test_reporting_official_finality.py @@ -32,9 +32,20 @@ ReportingSourceFinalPolicy, revision_content_sha256, ) -from adcp.reporting.ledger.finality import FinalityProof, source_finality_proof -from adcp.reporting.ledger.provisional import ProvisionalAcquisition -from adcp.reporting.source import ReportingFinalityEvidenceV1 +from adcp.reporting.ledger.finality import ( + FinalityProof, + manifest_finality_proof, + source_finality_proof, +) +from adcp.reporting.ledger.provisional import ProvisionalAcquisition, ProvisionalPolicy +from adcp.reporting.source import ( + ReportingFinalityEvidenceV1, + ReportingSourceExecutorResult, + SourceBatchManifestV1, + coverage_denominator_fingerprint_v1, + publication_content_fingerprint_v1, + source_batch_object_set_sha256_v1, +) from tests.conformance.reporting._generation_support import isolated_reporting_pool from tests.test_reporting_source_availability import ( END, @@ -182,8 +193,10 @@ async def test_contractual_cutoff_promotes_a_post_cutoff_snapshot() -> None: async def test_manual_contractual_cutoff_acquires_fresh_content_after_unchanged_reads( monkeypatch: pytest.MonkeyPatch, + store_factory: Any, ) -> None: scheduled, store, config, obligation, _, clock = await _harness( + store_factory, capabilities=_closing_capabilities(), finality_policy=ReportingContractualCutoffPolicy("three-hours", "PT3H"), ) @@ -230,6 +243,7 @@ async def interrupted(**kwargs: Any) -> bytes: ) assert len(calls) == 3 + clock[0] += timedelta(hours=1) turn = await producer.run_configuration(config) assert len(turn.revisions_committed) == 1 revision = await store.get_revision( @@ -237,7 +251,294 @@ async def interrupted(**kwargs: Any) -> bytes: ) assert revision is not None and revision.finality == "official" assert len(calls) == 3 - assert calls[-1].period.source_read_cutoff_at == revision.finalized_at == clock[0] + observation = await store.get_provisional_observation( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + assert observation is not None + persisted = observation.acquisition.request() + sealed = SourceBatchManifestV1.model_validate_json(observation.manifest_json) + assert persisted.identity == sealed.identity == calls[-1].identity + assert persisted.period == sealed.period == calls[-1].period + assert revision.finalized_at == clock[0] + + +@pytest.mark.parametrize("stale", ["observed_at", "batch", "constituent", "metric"]) +async def test_contractual_proof_requires_post_cutoff_observation_and_full_watermarks( + stale: str, +) -> None: + policy = ReportingContractualCutoffPolicy("cutoff", "PT2H") + producer, _, config, obligation, _, clock = await _harness( + official=True, finality_policy=policy + ) + clock[0] = END + timedelta(hours=2) + request = producer._build_slice(config, obligation, SNAPSHOT_OFFERING_ID, now=clock[0]) + _, manifest, _ = redacted_completed_result( + request, + observed_at=clock[0] - timedelta(seconds=1) if stale == "observed_at" else clock[0], + data_through=END - timedelta(seconds=1) if stale == "batch" else END, + ) + raw = manifest.model_dump() + raw["acquired_at"] = clock[0] + if stale == "constituent": + raw["coverage"]["constituents"][0]["data_through"] = END - timedelta(seconds=1) + if stale == "metric": + raw["metric_availability"][0]["data_through"] = END - timedelta(seconds=1) + manifest = SourceBatchManifestV1.model_validate(raw) + with pytest.raises(LedgerConflictError) as error: + manifest_finality_proof(obligation, manifest, clock[0]) + assert error.value.code == "FINALITY_POLICY_UNPROVEN" + + +@pytest.mark.parametrize("entry", ["acquire", "commit"]) +@pytest.mark.parametrize("different", ["namespace", "scope", "build", "constituents", "metrics"]) +async def test_new_official_binds_provider_and_denominator( + entry: str, different: str, monkeypatch: pytest.MonkeyPatch +) -> None: + policy = ReportingSourceFinalPolicy("selected", "source-evidence-redacted") + producer, store, config, obligation, _, clock = await _harness( + official=True, finality_policy=policy + ) + request = producer._build_slice(config, obligation, OFFICIAL_OFFERING_ID, now=clock[0]) + other = request + if different == "namespace": + other = request.model_copy(update={"publication_namespace": "reporting-source:alternate"}) + elif different == "scope": + other = request.model_copy( + update={ + "identity": request.identity.model_copy( + update={"source_scope": {"provider": "other"}} + ) + } + ) + elif different == "build": + other = request.model_copy( + update={ + "adapter_build": request.adapter_build.model_copy( + update={"adapter_version": "other"} + ) + } + ) + elif different == "constituents": + members = [request.coverage.constituents[0].model_copy(update={"product_id": "other"})] + other = request.model_copy( + update={ + "coverage": request.coverage.model_copy( + update={ + "constituents": members, + "denominator_fingerprint": coverage_denominator_fingerprint_v1(members), + } + ) + } + ) + else: + other = request.model_copy(update={"requested_metrics": ["impressions"]}) + result, manifest, _ = redacted_completed_result(other, page_bodies=[PROVIDER_ROW]) + _, _, producer._source.reader = redacted_completed_result(request, page_bodies=[PROVIDER_ROW]) + + async def execute(*args: Any, **kwargs: Any) -> ReportingSourceExecutorResult: + return result + + monkeypatch.setattr(producer._source, "execute", execute) + with pytest.raises(LedgerConflictError) as error: + if entry == "acquire": + await producer.acquire_obligation(config, obligation) + else: + await producer.commit_revision_from_manifest( + obligation, + manifest, + rows=ROWS, + finality="official", + finality_proof=source_finality_proof(obligation, manifest, policy), + ) + assert error.value.code == "MANIFEST_MISMATCH" + assert not await store.list_revisions( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + + +@pytest.mark.parametrize("after_reservation", [timedelta(hours=1), timedelta(hours=3)]) +async def test_manual_cutoff_finishes_reserved_snapshot_first( + monkeypatch: pytest.MonkeyPatch, after_reservation: timedelta +) -> None: + producer, store, config, obligation, _, clock = await _harness( + capabilities=_closing_capabilities(), + finality_policy=ReportingContractualCutoffPolicy("three-hours", "PT3H"), + ) + clock[0] = END + timedelta(hours=1) + + async def interrupted(**kwargs: Any) -> bytes: + raise OSError("interrupted snapshot") + + with monkeypatch.context() as patch: + patch.setattr(producer._source, "read", interrupted) + with pytest.raises(OSError, match="interrupted snapshot"): + await producer.run_configuration(config) + frozen = producer._source.requests[-1] + clock[0] += after_reservation + if after_reservation == timedelta(hours=3): + direct = producer._build_slice( + config, obligation, SNAPSHOT_OFFERING_ID, finality="official", now=clock[0] + ) + _, manifest, producer._source.reader = redacted_completed_result( + direct, page_bodies=[PROVIDER_ROW] + ) + with pytest.raises(LedgerConflictError) as conflict: + await producer.commit_revision_from_manifest( + obligation, + manifest, + rows=ROWS, + finality="official", + finality_proof=manifest_finality_proof(obligation, manifest, clock[0]), + ) + assert conflict.value.code == "OBSERVATION_CONFLICT" + revision = await producer.acquire_obligation( + config, obligation, restate=True, target_finality="official", manual_replay=True + ) + assert revision is not None and revision.finality == "snapshot" + latest = await store.get_provisional_observation( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + assert latest is not None and latest.acquisition.request().period == frozen.period + + +@pytest.mark.parametrize("entry", ["acquire", "commit"]) +@pytest.mark.parametrize("reader", ["absent", "mismatched", "verified"]) +async def test_official_zero_rows_require_staged_verification( + entry: str, reader: str, monkeypatch: pytest.MonkeyPatch +) -> None: + policy = ReportingSourceFinalPolicy("selected", "source-evidence-redacted") + producer, _, config, obligation, _, clock = await _harness( + official=True, finality_policy=policy + ) + request = producer._build_slice(config, obligation, OFFICIAL_OFFERING_ID, now=clock[0]) + result, manifest, _ = redacted_completed_result(request, page_bodies=[""], row_counts=[0]) + + async def execute(*args: Any, **kwargs: Any) -> ReportingSourceExecutorResult: + return result + + monkeypatch.setattr(producer._source, "execute", execute) + reads = [] + + async def read(**kwargs: Any) -> bytes: + reads.append(kwargs["object_ref"]) + return b"" if reader == "verified" else b"other" + + if reader == "absent": + producer._object_reader = None + else: + monkeypatch.setattr(producer._source, "read", read) + publish = ( + producer.acquire_obligation(config, obligation) + if entry == "acquire" + else producer.commit_revision_from_manifest( + obligation, + manifest, + rows=[], + finality="official", + finality_proof=source_finality_proof(obligation, manifest, policy), + ) + ) + if reader == "verified": + revision = await publish + assert revision is not None and revision.row_count == 0 and revision.finality == "official" + else: + with pytest.raises(LedgerConflictError, match="staged"): + await publish + assert reads == ([] if reader == "absent" else [manifest.objects[0].object_ref]) + + +@pytest.mark.parametrize("competing_snapshot", [False, True]) +async def test_direct_cutoff_honors_the_winning_reservation_during_staged_verification( + monkeypatch: pytest.MonkeyPatch, + competing_snapshot: bool, +) -> None: + policy = ReportingContractualCutoffPolicy("contract", "PT2H") + producer, store, config, obligation, _, clock = await _harness( + capabilities=_closing_capabilities(), finality_policy=policy + ) + clock[0] = END + timedelta(hours=3) + snapshot = producer._build_slice(config, obligation, SNAPSHOT_OFFERING_ID, now=clock[0]) + close = producer._build_slice( + config, obligation, SNAPSHOT_OFFERING_ID, now=clock[0], finality="official" + ) + _, manifest, producer._source.reader = redacted_completed_result( + close, page_bodies=[PROVIDER_ROW] + ) + original = producer._source.read + + async def competing_reservation(**kwargs: Any) -> bytes: + winner = await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + snapshot.model_dump_json(), + 0, + ProvisionalPolicy(timedelta(hours=12), timedelta(hours=1), timedelta(hours=2)), + ) + ) + assert winner.request().identity == snapshot.identity + return await original(**kwargs) + + if competing_snapshot: + monkeypatch.setattr(producer._source, "read", competing_reservation) + publication = producer.commit_revision_from_manifest( + obligation, + manifest, + rows=ROWS, + finality="official", + finality_proof=manifest_finality_proof(obligation, manifest, clock[0]), + ) + if competing_snapshot: + with pytest.raises(LedgerConflictError) as conflict: + await publication + assert conflict.value.code == "OBSERVATION_CONFLICT" + assert not await store.list_revisions( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + else: + official = await publication + observation = await store.get_provisional_observation( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + assert observation is not None and observation.revision_id == official.reporting_revision_id + assert observation.acquisition.request() == close + + +async def test_direct_official_commit_checks_staged_size() -> None: + policy = ReportingSourceFinalPolicy("selected", "source-evidence-redacted") + producer, _, config, obligation, _, clock = await _harness( + official=True, finality_policy=policy + ) + request = producer._build_slice(config, obligation, OFFICIAL_OFFERING_ID, now=clock[0]) + _, manifest, producer._source.reader = redacted_completed_result( + request, page_bodies=[""], row_counts=[0] + ) + objects = [manifest.objects[0].model_copy(update={"byte_count": 1})] + draft = manifest.model_copy( + update={ + "byte_count": 1, + "objects": objects, + "object_set_sha256": source_batch_object_set_sha256_v1(objects), + } + ) + manifest = SourceBatchManifestV1.model_validate( + {**draft.model_dump(), "content_fingerprint": publication_content_fingerprint_v1(draft)} + ) + with pytest.raises(LedgerConflictError) as conflict: + await producer.commit_revision_from_manifest( + obligation, + manifest, + rows=[], + finality="official", + finality_proof=source_finality_proof(obligation, manifest, policy), + ) + assert conflict.value.code == "STAGED_OBJECT_MISMATCH" + + +async def test_policyless_official_does_not_dispatch() -> None: + producer, _, config, obligation, calls, _ = await _harness(official=True) + with pytest.raises(LedgerConflictError) as error: + await producer.acquire_obligation(config, obligation) + assert error.value.code == "FINALITY_POLICY_UNPROVEN" + assert not calls @pytest.mark.parametrize("pending_close", [False, True]) @@ -415,6 +716,7 @@ async def test_retained_official_replay_preserves_original_policy_fields() -> No ) historical = await store.commit_revision(historical, ROWS) clock[0] += timedelta(hours=1) + assert await producer.acquire_obligation(config, obligation) is None assert ( await producer.commit_revision_from_manifest( obligation, manifest, rows=ROWS, finality="official" diff --git a/tests/test_reporting_producer_retry.py b/tests/test_reporting_producer_retry.py index 9f99df520..a06e2bae8 100644 --- a/tests/test_reporting_producer_retry.py +++ b/tests/test_reporting_producer_retry.py @@ -9,8 +9,10 @@ OFFICIAL_OFFERING_ID, SNAPSHOT_OFFERING_ID, redacted_capabilities, + redacted_contract_identity, ) from adcp.reporting.inline_source import ( + InlineFetchResult, InlineReportingSource, InMemorySealStore, InMemoryStagingStore, @@ -19,12 +21,15 @@ InMemoryReportingLedgerStore, ProducerOfferings, ReportingConfiguration, + ReportingDefinitionBinding, ReportingProducer, ReportingScheduleSpec, + ReportingSourceFinalPolicy, RetryScheduleEntry, WorkerTurn, ) from adcp.reporting.source import ( + ReportingFinalityEvidenceV1, ReportingSourceCapabilitiesV1, ReportingSourceErrorV1, ReportingSourceExecutorResult, @@ -90,6 +95,14 @@ def configuration(account: str = "account-1", config: str = "config-1") -> Repor period_duration="PT1H", delivery_sla="PT10M", alignment="utc" ), required_finality="official", + definition=ReportingDefinitionBinding( + **redacted_contract_identity.model_dump( + exclude={"report_definition_id", "reporting_profile"} + ), + finality_policy=ReportingSourceFinalPolicy( + "fixture-authority", "source-evidence-redacted" + ), + ), activated_at=datetime(2026, 11, 1, 0, 20, tzinfo=timezone.utc), media_buy_ids=("media-buy-1",), ) @@ -187,23 +200,35 @@ async def test_post_deadline_failure_uses_configured_slow_cadence(): assert skipped.earliest_retry_at == turn.earliest_retry_at -async def test_unanswered_slice_is_deferred_on_worker_turn(): +async def test_unanswered_slice_is_deferred_until_a_retry_supplies_provider_evidence(): store = InMemoryReportingLedgerStore() + staging = InMemoryStagingStore() + clock = [NOW] calls = [] def unanswered(request): calls.append(request) - return None + if len(calls) == 1: + return None + return InlineFetchResult( + rows=[], + finality_evidence=ReportingFinalityEvidenceV1( + basis="provider_declared", + evidence_ref="source-evidence-redacted", + observed_at=clock[0], + ), + ) source = InlineReportingSource( capabilities=_ready_capabilities(), fetch=unanswered, - staging=InMemoryStagingStore(), + staging=staging, seals=InMemorySealStore(), + clock=lambda: clock[0], ) config = configuration() await store.put_configuration(config) - worker = producer(source, store) + worker = producer(source, store, clock=lambda: clock[0], object_reader=staging) first = await worker.run_configuration(config, now=NOW) assert len(calls) == 1 @@ -214,6 +239,17 @@ def unanswered(request): assert len(calls) == 1 assert second.earliest_retry_at == first.earliest_retry_at + clock[0] = first.earliest_retry_at + recovered = await worker.run_configuration(config, now=clock[0]) + assert len(calls) == 2 + (revision_id,) = recovered.revisions_committed + revision = await store.get_revision( + account_id=config.account_id, reporting_revision_id=revision_id + ) + assert revision.finality == "official" + assert revision.finality_basis == "source_final" + assert revision.finalized_at == clock[0] + async def test_retry_starts_when_a_slow_source_finishes(): clock = [NOW] From 1779114b6d725cdfa45ab0150fbc5f7b2977b9c7 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 13:26:38 +0530 Subject: [PATCH 11/39] fix(reporting): recover ineligible cutoff reads and bind verified publications --- src/adcp/reporting/conformance.py | 12 +- src/adcp/reporting/ledger/producer.py | 352 ++++++++++---- src/adcp/reporting/production/offerings.py | 4 +- src/adcp/reporting/service.py | 9 +- ...test_reporting_production_configuration.py | 14 + tests/test_reliable_reporting_service.py | 20 + tests/test_reporting_official_acquisition.py | 438 ++++++++++++++++++ tests/test_reporting_producer_retry.py | 5 +- 8 files changed, 762 insertions(+), 92 deletions(-) create mode 100644 tests/test_reporting_official_acquisition.py diff --git a/src/adcp/reporting/conformance.py b/src/adcp/reporting/conformance.py index b240f8ebe..6ec5a1c50 100644 --- a/src/adcp/reporting/conformance.py +++ b/src/adcp/reporting/conformance.py @@ -151,10 +151,10 @@ async def _with_deadline( # -------------------------------------------------------------------------- -def _validate_request_against_capabilities( +def _validate_request_authority( capabilities: ReportingSourceCapabilitiesV1, request: ReportingSourceSliceRequestV1 ) -> ProvisionalSnapshotOfferingV1 | AuthoritativeOfferingV1: - """Is this frozen slice an exact atomic offering of this executor build?""" + """Verify the source account, executor build, and offering named by the request.""" if capabilities.scope != "effective_account": raise _fail( "CAPABILITY_MISMATCH", @@ -190,6 +190,14 @@ def _validate_request_against_capabilities( "CAPABILITY_MISMATCH", "the frozen slice is not an exact atomic offering of this executor build", ) + return offering + + +def _validate_request_against_capabilities( + capabilities: ReportingSourceCapabilitiesV1, request: ReportingSourceSliceRequestV1 +) -> ProvisionalSnapshotOfferingV1 | AuthoritativeOfferingV1: + """Is this frozen slice an exact atomic offering of this executor build?""" + offering = _validate_request_authority(capabilities, request) if ( offering.source_timezone is not None and offering.source_timezone != request.period.source_timezone diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 0b8e9a7ef..4aefd266e 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -29,9 +29,12 @@ from __future__ import annotations import asyncio +import gzip import hashlib import inspect +import json import logging +import zlib from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence from contextlib import asynccontextmanager from dataclasses import dataclass, field, replace @@ -87,6 +90,7 @@ ReportingRowStagingStore, ReportingSourcePublicationStore, ReportingWorkerRetryPolicy, + RestatementCheckpoint, RestatementCheckpointStore, RetryScheduleEntry, RetryScheduleStore, @@ -108,6 +112,7 @@ ReportingSourceStagedObjectReader, ReportingTriggerKind, SourceBatchManifestV1, + SourceBatchObjectV1, _validate_metric_applicability, coverage_denominator_fingerprint_v1, iso_duration_milliseconds_v1, @@ -170,6 +175,43 @@ def _utc(value: datetime) -> datetime: return value.astimezone(timezone.utc) +def _source_run_id(cutoff: datetime) -> str: + return f"run-{_utc(cutoff).strftime('%Y%m%dT%H%M%S')}" + + +def _supports_official_source_formats( + offering: ProvisionalSnapshotOfferingV1 | AuthoritativeOfferingV1, +) -> bool: + """An official source may emit any encoding it declares for the selected offering.""" + return all( + item.media_type in {"application/json", "application/x-ndjson"} + and item.compression in {"none", "gzip"} + for item in offering.formats + ) + + +def _official_object_rows(staged: SourceBatchObjectV1, payload: bytes) -> list[dict[str, Any]]: + try: + if staged.compression == "gzip": + payload = gzip.decompress(payload) + if staged.media_type == "application/json": + document = json.loads(payload) + rows = document if isinstance(document, list) else [document] + else: + rows = [ + json.loads(line) for line in payload.decode("utf-8").splitlines() if line.strip() + ] + except (ValueError, OSError, EOFError, zlib.error) as error: + raise LedgerConflictError( + "STAGED_OBJECT_MISMATCH", "staged object does not decode with its declared format" + ) from error + if len(rows) != staged.row_count or not all(isinstance(row, dict) for row in rows): + raise LedgerConflictError( + "STAGED_OBJECT_MISMATCH", "staged object does not contain the declared reporting rows" + ) + return rows + + def _source_finality(obligation: ReportingObligationRecord, output_finality: str) -> str: policy = obligation.definition.finality_policy if obligation.definition else None if output_finality == "official" and policy is not None: @@ -796,6 +838,10 @@ async def _acquire_progress( try: if policy is None: await self.acquire_obligation(configuration, obligation, turn=turn, now=now) + if obligation.definition and isinstance( + obligation.definition.finality_policy, ReportingContractualCutoffPolicy + ): + finished = identifier not in turn._retry_keys_by_obligation else: finished = await self._acquire_with_settling_policy( configuration, obligation, policy=policy, turn=turn, now=now @@ -1206,10 +1252,32 @@ async def acquire_obligation( # An official close is terminal. A later source correction is an # adjustment, never another acquisition. return None - satisfied = current is not None and current.readable - if satisfied and not restate: - return None selected_policy = obligation.definition.finality_policy if obligation.definition else None + checkpoint_store = ( + self._restatement_store() if track_settling else self._observation_checkpoint_store() + ) + checkpoint = ( + await checkpoint_store.get_restatement_checkpoint( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + if checkpoint_store is not None + else None + ) + observation = checkpoint.next_observation if checkpoint else len(revisions) + pending = ( + await self._pending_acquisition(obligation, ordinal=observation) + if checkpoint_store is not None + else None + ) + if pending is not None: + finality = pending.output_finality or ( + "snapshot" + if pending.request().publication_class == "PROVISIONAL_SNAPSHOT" + else "official" + ) + if current is not None and current.readable and not restate and pending is None: + return None if finality == "official": if selected_policy is None: raise LedgerConflictError( @@ -1234,30 +1302,13 @@ async def acquire_obligation( constituents = await self._admitted_source_constituents(configuration, obligation) derived_close = finality == "official" and source_finality == "snapshot" - if isinstance(selected_policy, ReportingContractualCutoffPolicy) and derived_close: - track_settling = ( - track_settling - or manual_replay - or ( - self._optional_observation_store() is not None - and isinstance(self._store, RestatementCheckpointStore) - ) - ) - checkpoint_store = self._restatement_store() if track_settling else None - checkpoint = ( - await checkpoint_store.get_restatement_checkpoint( - account_id=obligation.account_id, - reporting_obligation_id=obligation.reporting_obligation_id, - ) - if checkpoint_store is not None - else None - ) - observation = checkpoint.next_observation if checkpoint is not None else len(revisions) if derived_close and isinstance(selected_policy, ReportingContractualCutoffPolicy): if _utc(now) < contractual_cutoff_at(obligation, selected_policy): - pending = await self._pending_acquisition(obligation, ordinal=observation) - if pending is None or pending.output_finality == "official": - return None + return None + if checkpoint_store is None: + observation = await self._cutoff_observation_ordinal(obligation, observation) + else: + track_settling = True request = self._build_slice( configuration, obligation, @@ -1268,8 +1319,8 @@ async def acquire_obligation( constituents=constituents, trigger="manual_replay" if manual_replay else "scheduled_poll", ) - acquisition = None - if track_settling: + acquisition = pending + if track_settling and acquisition is None: acquisition = await self._reserve_observation( configuration, obligation, @@ -1279,6 +1330,7 @@ async def acquire_obligation( previous_boundary=checkpoint.provisional_until if checkpoint else None, derived_close=derived_close, ) + if acquisition is not None: request = acquisition.request(deadline_at=_utc(now) + self._offerings.slice_timeout) if manual_replay: request = request.model_copy(update={"trigger": "manual_replay"}) @@ -1289,6 +1341,8 @@ async def acquire_obligation( request.period.source_read_cutoff_at ): return None + if finality == "official": + self._official_source_authority(request) cancel = asyncio.Event() execution = asyncio.create_task( self._execute_source( @@ -1364,14 +1418,13 @@ async def acquire_obligation( ) self._validate_manifest_currency(obligation, manifest) if finality == "official": - self._official_source_request( - configuration, - obligation, - manifest, - constituents=constituents, - ordinal=observation, - cutoff_ceiling=now, - acquisition=acquisition, + assert result.response is not None + if result.response.identity != request.identity: + raise LedgerConflictError( + "MANIFEST_MISMATCH", "source response identity differs from its request" + ) + self._validate_official_manifest( + request, manifest, cutoff_ceiling=now, acquisition=acquisition ) rows = await self._read_rows(request, manifest, require_staged=finality == "official") # ``now`` freezes dispatch/lease/cutoff decisions, not publication. @@ -1384,12 +1437,46 @@ async def acquire_obligation( "producer clock regressed during acquisition; correct the clock before retrying", ) if finality == "official": + if _utc(manifest.acquired_at) > _utc(published_at): + raise LedgerConflictError( + "PUBLICATION_TIME_INVALID", + "source acquisition is later than the publication clock", + ) + try: + proof = manifest_finality_proof(obligation, manifest, published_at) + except LedgerConflictError as error: + if ( + not isinstance(selected_policy, ReportingContractualCutoffPolicy) + or error.code != "FINALITY_POLICY_UNPROVEN" + ): + raise + # A valid, stale seal cannot become a qualifying cutoff observation. + # Retire its execution identity only after staged verification. + turn.slices_failed.append(obligation.reporting_obligation_id) + await self._schedule_retry( + obligation, turn, now=published_at, replayed=manual_replay + ) + await self._retire_cutoff_attempt( + obligation, + ordinal=observation, + checked_at=published_at, + checkpoint_store=checkpoint_store, + checkpoint=checkpoint, + ) + self._note_escalation( + obligation, + turn, + now=published_at, + availability_retry=True, + offering_id=request.offering_id, + ) + return None committed = await self._commit_revision_from_manifest( obligation, manifest, rows=rows, finality=finality, - finality_proof=manifest_finality_proof(obligation, manifest, published_at), + finality_proof=proof, now=published_at, turn=turn, acquisition=acquisition, @@ -1408,6 +1495,48 @@ async def acquire_obligation( await self._clear_retry(obligation, turn=turn, now=published_at) return committed + @staticmethod + def _cutoff_ordinal_key(obligation: ReportingObligationRecord) -> str: + return f"cutoff:{obligation.account_id}:{obligation.reporting_obligation_id}" + + async def _retire_cutoff_attempt( + self, + obligation: ReportingObligationRecord, + *, + ordinal: int, + checked_at: datetime, + checkpoint_store: RestatementCheckpointStore | None, + checkpoint: RestatementCheckpoint | None, + ) -> None: + """Advance the acquisition cursor without publishing a revision or observation.""" + if checkpoint_store is not None: + await checkpoint_store.record_restatement_checkpoint( + RestatementCheckpoint( + obligation.account_id, + obligation.reporting_obligation_id, + checked_at, + ordinal + 1, + checkpoint.provisional_until if checkpoint else None, + ) + ) + else: + # Core ledgers have a durable retry schedule but no observation cursor. + # Keep the attempt number independent from transient source backoff. + await self._record_retry( + RetryScheduleEntry( + self._cutoff_ordinal_key(obligation), + checked_at, + ordinal + 1, + recorded_at=checked_at, + ) + ) + + async def _cutoff_observation_ordinal( + self, obligation: ReportingObligationRecord, ordinal: int + ) -> int: + retired = await self._get_retry(self._cutoff_ordinal_key(obligation)) + return max(ordinal, retired.attempt) if retired else ordinal + def _retry_keys(self, obligation: ReportingObligationRecord) -> tuple[str, str, str]: source = hashlib.sha256( canonical_json_utf8_v1( @@ -1679,18 +1808,29 @@ async def _retained_source_configuration( raise LedgerConflictError("HISTORY_UNAVAILABLE", "source generation is unavailable") return configuration - def _restatement_store(self) -> RestatementCheckpointStore: + def _optional_restatement_store(self) -> RestatementCheckpointStore | None: explicit = all( inspect.getattr_static(self._store, name, None) is not None for name in ("get_restatement_checkpoint", "record_restatement_checkpoint") ) - if not explicit or not isinstance(self._store, RestatementCheckpointStore): + if explicit and isinstance(self._store, RestatementCheckpointStore): + return self._store + return None + + def _observation_checkpoint_store(self) -> RestatementCheckpointStore | None: + if self._optional_observation_store() is None: + return None + return self._optional_restatement_store() + + def _restatement_store(self) -> RestatementCheckpointStore: + store = self._optional_restatement_store() + if store is None: raise LedgerConflictError( "RESTATEMENT_CHECKPOINTS_NOT_SUPPORTED", "a source settling window requires explicitly implemented durable " "restatement checkpoint methods", ) - return self._store + return store def _optional_observation_store(self) -> ProvisionalObservationPublicationStore | None: explicit = all( @@ -1846,8 +1986,6 @@ async def _read_rows( "official publication requires a staged object reader", ) return [] - import json - rows: list[dict[str, Any]] = [] cancel = asyncio.Event() for staged in manifest.objects: @@ -1865,67 +2003,78 @@ async def _read_rows( "STAGED_OBJECT_MISMATCH", f"staged object {staged.ordinal} does not match its manifest evidence", ) - start = len(rows) - for line in payload.decode("utf-8").splitlines(): - if line.strip(): - rows.append(json.loads(line)) - if require_staged and len(rows) - start != staged.row_count: - raise LedgerConflictError( - "STAGED_OBJECT_MISMATCH", "staged object row count differs from its manifest" + if require_staged: + rows.extend(_official_object_rows(staged, payload)) + else: + rows.extend( + json.loads(line) + for line in payload.decode("utf-8").splitlines() + if line.strip() ) return rows - def _official_source_request( + def _official_source_authority( + self, request: ReportingSourceSliceRequestV1 + ) -> ProvisionalSnapshotOfferingV1 | AuthoritativeOfferingV1: + from adcp.reporting.conformance import ( + ReportingSourceConformanceError, + _validate_request_authority, + ) + + try: + offering = _validate_request_authority(self._source.capabilities, request) + except ReportingSourceConformanceError as error: + raise LedgerConflictError( + "MANIFEST_MISMATCH", "official request contradicts its declared source authority" + ) from error + if not _supports_official_source_formats(offering): + raise LedgerConflictError( + "UNSUPPORTED_SOURCE_FORMAT", + "official producer requires JSON or NDJSON with gzip or no compression", + ) + return offering + + def _validate_official_manifest( self, - configuration: ReportingConfiguration, - obligation: ReportingObligationRecord, + request: ReportingSourceSliceRequestV1, manifest: SourceBatchManifestV1, *, - constituents: tuple[ReportingConstituent, ...] | None, - ordinal: int, cutoff_ceiling: datetime, acquisition: ProvisionalAcquisition | None, - ) -> ReportingSourceSliceRequestV1: + ) -> None: from adcp.reporting.conformance import ( ReportingSourceConformanceError, _validate_manifest_against_request, ) - offering_id = self._offerings.offering_for(_source_finality(obligation, "official")) - if offering_id is None or offering_id != manifest.offering_id: - raise LedgerConflictError( - "MANIFEST_MISMATCH", "official proof must use the selected source offering" - ) cutoff = manifest.period.source_read_cutoff_at if _utc(cutoff) > _utc(cutoff_ceiling): raise LedgerConflictError( "MANIFEST_MISMATCH", "source cutoff exceeds the requested read" ) - # A retry can receive an earlier seal for its immutable execution key. - # Reconstruct the authorized source and denominator, using the original dispatch - # identity. A durable reservation also binds its exact cutoff and run. - request = self._build_slice( - configuration, - obligation, - offering_id, - finality="official", - now=cutoff, - observation=ordinal, - constituents=constituents, - ) + # A core-store retry can receive an earlier seal for the same execution key. + # Only its dispatch time can differ. A durable reservation also fixes that time. + if cutoff != request.period.source_read_cutoff_at: + request = request.model_copy( + update={ + "identity": request.identity.model_copy( + update={"run_id": _source_run_id(cutoff)} + ), + "period": request.period.model_copy(update={"source_read_cutoff_at": cutoff}), + } + ) if acquisition is not None and not self._matches_reservation(request, acquisition): raise LedgerConflictError( "MANIFEST_MISMATCH", "official source manifest differs from its frozen request" ) try: _validate_manifest_against_request( - self._source.capabilities.offering(offering_id), request, manifest + self._official_source_authority(request), request, manifest ) except ReportingSourceConformanceError as error: raise LedgerConflictError( "MANIFEST_MISMATCH", "official manifest does not bind the configured source request" ) from error - return request @staticmethod def _matches_reservation( @@ -1992,6 +2141,7 @@ async def _commit_revision_from_manifest( "MANIFEST_MISMATCH", "unsupported cells require partial metric support with a reason", ) from None + sample_after_staging = now is None now = now or self._clock() turn = turn or WorkerTurn() existing = await self._store.list_revisions( @@ -2057,6 +2207,9 @@ async def _commit_revision_from_manifest( if finality != "official": finality_proof = None elif prior is not None and prior.finality == "official": + # A retained immutable publication may be checked or replayed while + # another acquisition is reserved. Its replay cannot advance that work. + acquisition = None assert prior.finality_policy_id and prior.finality_basis and prior.finalized_at finality_proof = FinalityProof( prior.finality_policy_id, prior.finality_basis, prior.finalized_at @@ -2072,9 +2225,18 @@ async def _commit_revision_from_manifest( raise LedgerConflictError( "MANIFEST_MISMATCH", "official proof must use the selected source offering" ) - finality_proof.validate(obligation, manifest, created_at) + proof_anchor = ( + finality_proof.finalized_at + if sample_after_staging and finality_proof.basis == "contractual_cutoff" + else created_at + ) + if _utc(proof_anchor) > _utc(created_at): + raise LedgerConflictError( + "PUBLICATION_TIME_INVALID", "proof is later than the publication clock" + ) + finality_proof.validate(obligation, manifest, proof_anchor) if not verified_source_rows: - acquisition = await self._verify_direct_official_publication( + acquisition, rows = await self._verify_direct_official_publication( obligation, manifest, rows, @@ -2082,6 +2244,15 @@ async def _commit_revision_from_manifest( revisions=existing, created_at=created_at, ) + if sample_after_staging: + completed_at = self._clock() + if _utc(completed_at) < _utc(now): + raise LedgerConflictError( + "PUBLICATION_TIME_INVALID", + "producer clock regressed during staged verification", + ) + created_at = now = completed_at + finality_proof = manifest_finality_proof(obligation, manifest, created_at) revision = ReportingRevisionRecord( reporting_revision_id=revision_id, account_id=obligation.account_id, @@ -2176,18 +2347,23 @@ async def _verify_direct_official_publication( *, revisions: Sequence[ReportingRevisionRecord], created_at: datetime, - ) -> ProvisionalAcquisition | None: + ) -> tuple[ProvisionalAcquisition | None, list[dict[str, Any]]]: configuration = await self._retained_source_configuration(obligation) constituents = await self._admitted_source_constituents(configuration, obligation) ordinal = len(revisions) checkpoint = None - if isinstance(self._store, RestatementCheckpointStore): - checkpoint = await self._store.get_restatement_checkpoint( + checkpoint_store = self._observation_checkpoint_store() + if checkpoint_store is not None: + checkpoint = await checkpoint_store.get_restatement_checkpoint( account_id=obligation.account_id, reporting_obligation_id=obligation.reporting_obligation_id, ) if checkpoint is not None: ordinal = checkpoint.next_observation + elif obligation.definition and isinstance( + obligation.definition.finality_policy, ReportingContractualCutoffPolicy + ): + ordinal = await self._cutoff_observation_ordinal(obligation, ordinal) pending = await self._pending_acquisition(obligation, ordinal=ordinal) if pending is not None: if (acquisition is not None and acquisition != pending) or ( @@ -2199,14 +2375,17 @@ async def _verify_direct_official_publication( "finish the reserved acquisition before official publication", ) acquisition = pending - request = self._official_source_request( + request = self._build_slice( configuration, obligation, - manifest, + manifest.offering_id, + finality="official", + now=manifest.period.source_read_cutoff_at, constituents=constituents, - ordinal=ordinal, - cutoff_ceiling=created_at, - acquisition=acquisition, + observation=ordinal, + ) + self._validate_official_manifest( + request, manifest, cutoff_ceiling=created_at, acquisition=acquisition ) staged_rows = await self._read_rows(request, manifest, require_staged=True) if rfc8785.dumps(staged_rows) != rfc8785.dumps(list(rows)): @@ -2216,8 +2395,7 @@ async def _verify_direct_official_publication( if ( pending is None and request.publication_class == "PROVISIONAL_SNAPSHOT" - and self._optional_observation_store() is not None - and isinstance(self._store, RestatementCheckpointStore) + and checkpoint_store is not None ): # Staged reads release the ledger lock. Reserve the verified request before # publication so an ordinary snapshot that won meanwhile is never skipped. @@ -2236,7 +2414,7 @@ async def _verify_direct_official_publication( raise LedgerConflictError( "OBSERVATION_CONFLICT", "another source acquisition won the reservation" ) - return acquisition + return acquisition, staged_rows async def _commit_publication( self, @@ -2449,7 +2627,7 @@ def _build_slice( reporting_obligation_id=obligation.reporting_obligation_id, period_key=obligation.period.period_key, source_execution_key=source_execution_key, - run_id=f"run-{_utc(now).strftime('%Y%m%dT%H%M%S')}", + run_id=_source_run_id(now), logical_slice_fingerprint=_logical_slice_fingerprint(obligation, offering_id), source_scope=dict(self._offerings.source_scope), ), diff --git a/src/adcp/reporting/production/offerings.py b/src/adcp/reporting/production/offerings.py index 67a89748f..485487c72 100644 --- a/src/adcp/reporting/production/offerings.py +++ b/src/adcp/reporting/production/offerings.py @@ -18,7 +18,7 @@ _schedule_clock, iso_duration_to_timedelta, ) -from adcp.reporting.ledger.producer import ReportingProducer +from adcp.reporting.ledger.producer import ReportingProducer, _supports_official_source_formats from adcp.reporting.ledger.store import LedgerConflictError from adcp.reporting.materializer.contracts import ReportingVerificationKey, failure from adcp.reporting.materializer.verification import _same_definition @@ -178,6 +178,8 @@ def configured_source( raise failure("UNSUPPORTED_VERIFICATION") source_class = finality_policy_source_class(policy) proof_source = configured_source(source_class) + if not _supports_official_source_formats(proof_source): + raise failure("UNSUPPORTED_VERIFICATION") if self.source_offering_id != proof_source.offering_id and ( snapshot is None or self.source_offering_id != snapshot.offering_id ): diff --git a/src/adcp/reporting/service.py b/src/adcp/reporting/service.py index 8c5a7a673..199c724bb 100644 --- a/src/adcp/reporting/service.py +++ b/src/adcp/reporting/service.py @@ -54,7 +54,10 @@ ReportingStabilizedPolicy, finality_policy_source_class, ) -from adcp.reporting.ledger.producer import _advertised_reporting_delivery +from adcp.reporting.ledger.producer import ( + _advertised_reporting_delivery, + _supports_official_source_formats, +) from adcp.reporting.ledger.store import LedgerConflictError, decode_cursor from adcp.reporting.service_lifecycle import ( ReliableReportingServiceError, @@ -880,6 +883,10 @@ def _validate_offerings( raise ReliableReportingConfigurationError( f"selected finality policy requires a matching {source_class} source offering" ) + if not _supports_official_source_formats(source): + raise ReliableReportingConfigurationError( + "official source formats must be JSON or NDJSON with gzip or no compression" + ) used = [source] if ( configuration.required_finality == "snapshot" diff --git a/tests/conformance/reporting/test_reporting_production_configuration.py b/tests/conformance/reporting/test_reporting_production_configuration.py index 74f750953..0300cf479 100644 --- a/tests/conformance/reporting/test_reporting_production_configuration.py +++ b/tests/conformance/reporting/test_reporting_production_configuration.py @@ -21,6 +21,10 @@ from adcp.reporting.materializer.work import ReportingMaterializerLease from adcp.reporting.production.configuration import ReportingConfigurationAdmission from adcp.reporting.production.offerings import ReportingProductionOffering +from adcp.reporting.source import ( + ReportingSourceCapabilitiesV1, + reporting_source_capabilities_sha256_v1, +) from adcp.validation.schema_loader import get_named_validator from ._feed_support import feed_request, walk @@ -129,6 +133,16 @@ async def test_dual_finality_source_final_admits_each_configured_route(primary, offering.check_configuration( config, h.item.binding, retention_days=h.item.writer.resource_retention_days ) + capabilities = source.capabilities.model_dump(mode="json") + for item in capabilities["offerings"]: + if item["offering_id"] == source.official_source_id: + item["formats"] = [ + {"media_type": "application/vnd.apache.parquet", "compression": "none"} + ] + capabilities["capabilities_sha256"] = reporting_source_capabilities_sha256_v1(capabilities) + source.capabilities = ReportingSourceCapabilitiesV1.model_validate(capabilities) + with pytest.raises(ReportingWriterError, match="UNSUPPORTED_VERIFICATION"): + offering.check_source(effective=True) def wire_configuration(h): diff --git a/tests/test_reliable_reporting_service.py b/tests/test_reliable_reporting_service.py index 330dfddb3..8cc2a48d5 100644 --- a/tests/test_reliable_reporting_service.py +++ b/tests/test_reliable_reporting_service.py @@ -158,6 +158,7 @@ def _finality_service( producer_factory: ProducerFactory = ReportingProducer, results: tuple[InlineFetchResult, ...] = (), clock: DeterministicReportingClock | None = None, + source_format: str | None = None, ) -> ReliableReportingService: public = _capability_offering("gam") public["supported_finality"] = [configuration.required_finality] @@ -173,6 +174,9 @@ def _finality_service( raw["offerings"] = [ offering for offering, enabled in zip(raw["offerings"], (snapshot, official)) if enabled ] + if source_format: + for offering in raw["offerings"]: + offering["formats"] = [{"media_type": source_format, "compression": "none"}] raw["capabilities_sha256"] = reporting_source_capabilities_sha256_v1(raw) adapter = ScriptedReportingAdapter(ReportingSourceCapabilitiesV1.model_validate(raw), results) service = ReliableReportingService( @@ -325,6 +329,22 @@ class SnapshotStore(InMemoryReportingLedgerStore): await service.configure(config) +async def test_service_refuses_an_unreadable_official_source_format_before_admission() -> None: + config = replace( + _configuration(), + required_finality="official", + definition=replace( + DEFINITION, finality_policy=ReportingSourceFinalPolicy("source", "closed") + ), + ) + service = _finality_service( + config, snapshot=False, official=True, source_format="application/vnd.apache.parquet" + ) + with pytest.raises(ReliableReportingConfigurationError, match="source format"): + await service.configure(config) + assert config.generation_key not in service._bindings + + async def test_service_rejects_stabilized_configuration_without_history_before_persisting() -> None: def producer_factory(**kwargs: Any) -> ReportingProducer: store = kwargs.pop("store") diff --git a/tests/test_reporting_official_acquisition.py b/tests/test_reporting_official_acquisition.py new file mode 100644 index 000000000..de7c076f5 --- /dev/null +++ b/tests/test_reporting_official_acquisition.py @@ -0,0 +1,438 @@ +"""Recoverable acquisition, source authority, and immutable official publication.""" + +from __future__ import annotations + +import gzip +import hashlib +from collections.abc import AsyncIterator +from dataclasses import replace +from datetime import timedelta +from functools import partial +from typing import Any + +import pytest +from adcp_reporting_storage import PgReportingLedgerStore + +from adcp.reporting.fixtures import ( + OFFICIAL_OFFERING_ID, + SNAPSHOT_OFFERING_ID, + redacted_completed_result, +) +from adcp.reporting.inline_source import ( + InlineFetchResult, + InlineReportingSource, + InMemorySealStore, + InMemoryStagingStore, +) +from adcp.reporting.ledger import ( + InMemoryReportingLedgerStore, + LedgerConflictError, + ReportingContractualCutoffPolicy, + ReportingProducer, + ReportingSourceFinalPolicy, + ReportingStabilizedPolicy, +) +from adcp.reporting.ledger.finality import manifest_finality_proof +from adcp.reporting.ledger.provisional import ProvisionalAcquisition, ProvisionalPolicy +from adcp.reporting.source import ( + SourceBatchManifestV1, + SourceFormatV1, + publication_content_fingerprint_v1, + source_batch_object_set_sha256_v1, +) +from tests.conformance.reporting._generation_support import isolated_reporting_pool +from tests.test_reporting_official_finality import ROWS +from tests.test_reporting_source_availability import ( + END, + PROVIDER_ROW, + ProgressStore, + _closing_capabilities, + _harness, +) + + +class CoreStore: + """Expose the base ledger and retry schedule without optional observation extensions.""" + + def __init__(self, store: Any) -> None: + self.store = store + + def __getattr__(self, name: str) -> Any: + if "provisional" in name or "restatement" in name: + raise AttributeError(name) + return getattr(self.store, name) + + +@pytest.fixture(params=["memory", "postgres", "core", "progress"]) +async def cutoff_backend(request: Any) -> AsyncIterator[Any]: + if request.param == "postgres": + async with isolated_reporting_pool() as pool: + yield partial(PgReportingLedgerStore, pool=pool), False + else: + yield ( + ProgressStore if request.param == "progress" else InMemoryReportingLedgerStore + ), request.param == "core" + + +async def test_stale_cutoff_seal_retries_with_fresh_source_work( + cutoff_backend: Any, monkeypatch: Any +) -> None: + factory, core = cutoff_backend + base, store, config, obligation, _, clock = await _harness( + factory, + official=True, + finality_policy=ReportingContractualCutoffPolicy("cutoff", "PT2H"), + ) + if core: + store = CoreStore(store) + calls = [] + staging = InMemoryStagingStore() + + def fetch(request: Any) -> InlineFetchResult: + calls.append(request) + return InlineFetchResult( + rows=[{"media_buy_id": "media-buy-redacted", "impressions": 10, "spend": "1.25"}], + data_through=END - timedelta(seconds=1) if len(calls) == 1 else END, + ) + + source = InlineReportingSource( + capabilities=base._source.capabilities, + fetch=fetch, + staging=staging, + seals=InMemorySealStore(), + clock=lambda: clock[0], + ) + + def restart() -> ReportingProducer: + return ReportingProducer( + store=store, + retry_store=base._retry_store, + source=source, + offerings=base._offerings, + object_reader=staging, + clock=lambda: clock[0], + read_jitter_window=timedelta(0), + ) + + clock[0] = END + timedelta(hours=2) + first = await restart().run_configuration(config) + assert first.earliest_retry_at is not None and first.revisions_committed == [] + assert len(calls) == 1 + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + assert not await store.list_revisions(**owner) + if not core: + assert not await store.list_provisional_observations(**owner, limit=2) + checkpoint = await store.get_restatement_checkpoint(**owner) + assert checkpoint.next_observation == 1 + clock[0] = first.earliest_retry_at - timedelta(microseconds=1) + assert not (await restart().run_configuration(config)).revisions_committed + assert len(calls) == 1 + clock[0] = first.earliest_retry_at + + async def interrupted(**kwargs: Any) -> bytes: + raise OSError("interrupted verified read") + + with monkeypatch.context() as patch: + patch.setattr(staging, "read", interrupted) + with pytest.raises(OSError, match="interrupted verified read"): + await restart().run_configuration(config) + assert len(calls) == 2 + assert calls[0].identity.source_execution_key != calls[1].identity.source_execution_key + if not core: + reserved = await store.get_provisional_acquisition(**owner, ordinal=1) + assert reserved.request().identity == calls[1].identity + clock[0] += timedelta(seconds=1) + turn = await restart().run_configuration(config) + assert len(calls) == 2 and len(turn.revisions_committed) == 1 + official = await store.get_revision( + account_id=config.account_id, reporting_revision_id=turn.revisions_committed[0] + ) + assert official.finality == "official" and official.finalized_at == clock[0] + + +async def test_unproven_stability_does_not_retire_a_cutoff_attempt() -> None: + producer, store, config, obligation, _, _ = await _harness( + official=True, finality_policy=ReportingStabilizedPolicy("stable", "PT2H", "PT1H") + ) + try: + await producer.acquire_obligation(config, obligation) + except LedgerConflictError as conflict: + assert conflict.code == "FINALITY_POLICY_UNPROVEN" + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + history = await store.list_provisional_observations(**owner, limit=1) + checkpoint = await store.get_restatement_checkpoint(**owner) + if history: + assert checkpoint.next_observation == history[-1].acquisition.ordinal + 1 + else: + assert checkpoint is None + assert ( + await store.get_retry_schedule( + scope_key=f"cutoff:{config.account_id}:{obligation.reporting_obligation_id}" + ) + is None + ) + + +async def test_stale_cutoff_with_a_future_acquisition_retains_its_reservation( + monkeypatch: Any, +) -> None: + producer, store, config, obligation, _, clock = await _harness( + official=True, finality_policy=ReportingContractualCutoffPolicy("cutoff", "PT2H") + ) + clock[0] = END + timedelta(hours=2) + + async def future_source(request: Any, **kwargs: Any) -> Any: + result, _, producer._source.reader = redacted_completed_result( + request, + page_bodies=[PROVIDER_ROW], + observed_at=clock[0] + timedelta(hours=1), + data_through=END - timedelta(seconds=1), + ) + return result + + monkeypatch.setattr(producer._source, "execute", future_source) + with pytest.raises(LedgerConflictError) as error: + await producer.acquire_obligation(config, obligation) + assert error.value.code == "PUBLICATION_TIME_INVALID" + assert ( + await store.get_restatement_checkpoint( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + is None + ) + + +@pytest.mark.parametrize( + "different", + [ + "capability_class", + "capability_scope", + "capability_namespace", + "response", + "build_during_fetch", + ], +) +async def test_contradictory_official_source_authority_is_rejected( + different: str, monkeypatch: Any +) -> None: + producer, _, config, obligation, _, _ = await _harness( + official=True, finality_policy=ReportingSourceFinalPolicy("source", "delivery.is_final") + ) + if different == "capability_class": + producer._offerings = replace( + producer._offerings, official_offering_id=SNAPSHOT_OFFERING_ID + ) + elif different == "capability_scope": + producer._offerings = replace(producer._offerings, source_scope={"another": "account"}) + elif different == "capability_namespace": + producer._offerings = replace(producer._offerings, publication_namespace="another-source") + else: + original = producer._source.execute + + async def wrong_response(request: Any, **kwargs: Any) -> Any: + if different == "build_during_fetch": + build = request.adapter_build.model_copy(update={"adapter_version": "other"}) + producer._source.capabilities = producer._source.capabilities.model_copy( + update={"adapter_build": build} + ) + return await original(request.model_copy(update={"adapter_build": build}), **kwargs) + result = await original(request, **kwargs) + wrong = request.identity.model_copy(update={"account_id": "other"}) + return result.model_copy( + update={"response": result.response.model_copy(update={"identity": wrong})} + ) + + monkeypatch.setattr(producer._source, "execute", wrong_response) + with pytest.raises(LedgerConflictError) as conflict: + await producer.acquire_obligation(config, obligation) + assert conflict.value.code == "MANIFEST_MISMATCH" + + +async def test_explicit_snapshot_resumes_a_reserved_cutoff(monkeypatch: Any) -> None: + producer, store, config, obligation, _, clock = await _harness( + capabilities=_closing_capabilities(), + finality_policy=ReportingContractualCutoffPolicy("cutoff", "PT3H"), + ) + clock[0] = END + timedelta(hours=3) + + async def interrupted(**kwargs: Any) -> bytes: + raise OSError("interrupted close") + + with monkeypatch.context() as patch: + patch.setattr(producer._source, "read", interrupted) + with pytest.raises(OSError): + await producer.acquire_obligation( + config, obligation, restate=True, target_finality="official" + ) + frozen = producer._source.requests[-1] + clock[0] += timedelta(minutes=5) + resumed = await producer.acquire_obligation( + config, obligation, target_finality="snapshot", manual_replay=True + ) + assert resumed.finality == "official" + assert producer._source.requests[-1].identity == frozen.identity + latest = await store.get_provisional_observation( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + assert latest.acquisition.request().identity == frozen.identity + + +async def test_manual_cutoff_on_core_store() -> None: + producer, store, config, obligation, _, clock = await _harness( + official=True, finality_policy=ReportingContractualCutoffPolicy("cutoff", "PT2H") + ) + producer._store = CoreStore(store) + clock[0] = END + timedelta(hours=3) + revision = await producer.acquire_obligation(config, obligation, manual_replay=True) + assert revision.finality == "official" + + +async def test_retained_official_replay_leaves_an_unrelated_reservation_alone() -> None: + producer, store, config, obligation, _, clock = await _harness( + official=True, finality_policy=ReportingSourceFinalPolicy("source", "delivery.is_final") + ) + original = await producer.acquire_obligation(config, obligation) + manifest = producer._source.manifest + request = producer._build_slice( + config, obligation, SNAPSHOT_OFFERING_ID, finality="snapshot", now=clock[0], observation=1 + ) + reserved = await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + request.model_dump_json(), + 1, + ProvisionalPolicy(timedelta(hours=2), timedelta(hours=1), None), + ) + ) + replayed = await producer.commit_revision_from_manifest( + obligation, manifest, rows=ROWS, finality="official", acquisition=reserved + ) + assert replayed == original + assert ( + await store.get_provisional_observation( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + is None + ) + assert ( + await store.get_restatement_checkpoint( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + is None + ) + + +@pytest.mark.parametrize("change", ["caller_rows", "publication_clock", "explicit_clock"]) +async def test_direct_cutoff_publishes_its_verified_rows_and_completion_time( + change: str, monkeypatch: Any +) -> None: + producer, store, config, obligation, _, clock = await _harness( + official=True, finality_policy=ReportingContractualCutoffPolicy("cutoff", "PT2H") + ) + start = clock[0] = END + timedelta(hours=2) + request = producer._build_slice(config, obligation, SNAPSHOT_OFFERING_ID, now=start) + _, manifest, producer._source.reader = redacted_completed_result( + request, page_bodies=[PROVIDER_ROW] + ) + caller = [dict(row) for row in ROWS] + original = store.reserve_provisional_acquisition + + async def reserve(acquisition: Any) -> Any: + if change == "caller_rows": + caller[0]["campaign_id"] = "caller-mutated" + return await original(acquisition) + + monkeypatch.setattr(store, "reserve_provisional_acquisition", reserve) + producer._source.read_delay = timedelta(minutes=5) + revision = await producer.commit_revision_from_manifest( + obligation, + manifest, + rows=caller, + finality="official", + finality_proof=manifest_finality_proof(obligation, manifest, start), + now=start if change == "explicit_clock" else None, + ) + page = await store.read_revision_rows( + account_id=config.account_id, reporting_revision_id=revision.reporting_revision_id + ) + assert list(page.rows) == ROWS + anchor = start if change == "explicit_clock" else clock[0] + assert revision.finalized_at == revision.created_at == anchor + + +@pytest.mark.parametrize("encoding", ["gzip", "json", "corrupt_gzip", "unsupported"]) +async def test_official_source_format_is_decoded_or_refused_before_fetch( + encoding: str, monkeypatch: Any +) -> None: + producer, _, config, obligation, calls, clock = await _harness( + official=True, + finality_policy=ReportingSourceFinalPolicy("source", "source-evidence-redacted"), + ) + caps = producer._source.capabilities + offering = caps.offering(OFFICIAL_OFFERING_ID) + fmt = SourceFormatV1(media_type="application/x-ndjson", compression="gzip") + if encoding == "json": + fmt = SourceFormatV1(media_type="application/json") + elif encoding == "unsupported": + fmt = SourceFormatV1(media_type="application/vnd.apache.parquet") + producer._source.capabilities = caps.model_copy( + update={ + "offerings": [ + item.model_copy(update={"formats": [fmt]}) if item == offering else item + for item in caps.offerings + ] + } + ) + if encoding == "unsupported": + with pytest.raises(LedgerConflictError) as conflict: + await producer.acquire_obligation(config, obligation) + assert conflict.value.code == "UNSUPPORTED_SOURCE_FORMAT" and not calls + return + request = producer._build_slice(config, obligation, OFFICIAL_OFFERING_ID, now=clock[0]) + _, manifest, _ = redacted_completed_result(request, page_bodies=[PROVIDER_ROW]) + payload = gzip.compress(PROVIDER_ROW.encode()) + if encoding == "json": + payload = f"[{PROVIDER_ROW}]".encode() + elif encoding == "corrupt_gzip": + payload = b"not a gzip object" + objects = [ + manifest.objects[0].model_copy( + update={ + **fmt.model_dump(), + "sha256": hashlib.sha256(payload).hexdigest(), + "byte_count": len(payload), + } + ) + ] + draft = manifest.model_copy( + update={ + "objects": objects, + "byte_count": len(payload), + "object_set_sha256": source_batch_object_set_sha256_v1(objects), + } + ) + manifest = SourceBatchManifestV1.model_validate( + {**draft.model_dump(), "content_fingerprint": publication_content_fingerprint_v1(draft)} + ) + + async def read(**kwargs: Any) -> bytes: + return payload + + monkeypatch.setattr(producer._source, "read", read) + publication = producer.commit_revision_from_manifest( + obligation, + manifest, + rows=ROWS, + finality="official", + finality_proof=manifest_finality_proof(obligation, manifest, clock[0]), + ) + if encoding == "corrupt_gzip": + with pytest.raises(LedgerConflictError) as conflict: + await publication + assert conflict.value.code == "STAGED_OBJECT_MISMATCH" + return + official = await publication + assert official.finality == "official" and official.row_count == 1 diff --git a/tests/test_reporting_producer_retry.py b/tests/test_reporting_producer_retry.py index a06e2bae8..3a735100b 100644 --- a/tests/test_reporting_producer_retry.py +++ b/tests/test_reporting_producer_retry.py @@ -114,7 +114,10 @@ def producer(source, store, *, clock=None, **kwargs): store=store, offerings=ProducerOfferings( official_offering_id=OFFICIAL_OFFERING_ID, - publication_namespace="reporting-source:retry-test", + publication_namespace=source.capabilities.offering( + OFFICIAL_OFFERING_ID + ).publication_namespace, + source_scope=dict(source.capabilities.source_scope), ), clock=clock or (lambda: NOW), **kwargs, From d63fd841e093da0b1ed4a8e32f1f07096da49480 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 13:59:44 +0530 Subject: [PATCH 12/39] fix(reporting): freeze source authority and serialize observation work --- .../src/adcp_reporting_storage/ledger/pg.py | 34 +- src/adcp/reporting/conformance.py | 12 +- src/adcp/reporting/inline_source.py | 12 + src/adcp/reporting/ledger/producer.py | 154 +++++---- src/adcp/reporting/ledger/provisional.py | 6 + src/adcp/reporting/ledger/store.py | 32 +- src/adcp/reporting/production/offerings.py | 9 +- src/adcp/reporting/service.py | 14 +- .../reporting/test_reporting_currency.py | 5 +- ...test_reporting_production_configuration.py | 47 ++- .../test_reporting_publication_time.py | 1 + tests/test_reliable_reporting_service.py | 15 +- tests/test_reporting_official_acquisition.py | 314 +++++++++++++++++- tests/test_reporting_official_finality.py | 17 +- tests/test_reporting_producer_retry.py | 2 + tests/test_reporting_source_availability.py | 4 + 16 files changed, 571 insertions(+), 107 deletions(-) diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py index 5c9875ce6..596fa1d4b 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py @@ -1499,6 +1499,20 @@ async def commit_revision( "this period is being retired; no revision may be added to it", ) validate_revision_currency(_obligation_from_row(obligation), revision, rows) + pending = await ( + await connection.execute( + "SELECT 1 FROM reporting_provisional_acquisitions a" + " LEFT JOIN reporting_restatement_checkpoints c" + " USING (account_id, reporting_obligation_id)" + " WHERE a.account_id=%s AND a.reporting_obligation_id=%s" + " AND a.ordinal >= COALESCE(c.next_observation, 0) LIMIT 1", + (revision.account_id, revision.reporting_obligation_id), + ) + ).fetchone() + if pending is not None: + raise LedgerConflictError( + "OBSERVATION_CONFLICT", "a source acquisition is reserved" + ) if revision.supersedes_reporting_revision_id: await self._require_current_leaf(connection, revision) write_error = None @@ -1894,6 +1908,17 @@ async def commit_provisional_observation( ) if checkpoint is not None and checkpoint.next_observation != acquisition.ordinal: raise LedgerConflictError("OBSERVATION_CONFLICT", "observation ordinal changed") + # Consume the reservation under the account lock. Failure of the + # ordinary revision write also rolls this checkpoint back. + await self.record_restatement_checkpoint( + RestatementCheckpoint( + acquisition.account_id, + acquisition.obligation_id, + observation.checked_at, + acquisition.ordinal + 1, + observation.provisional_until, + ) + ) if revision is None or reuse_candidate: if acquisition.predecessor_revision_id != observation.revision_id: raise LedgerConflictError( @@ -1936,15 +1961,6 @@ async def commit_provisional_observation( observation = replace(observation, revision_id=committed.reporting_revision_id) else: committed = await self.commit_revision(revision, rows) - await self.record_restatement_checkpoint( - RestatementCheckpoint( - acquisition.account_id, - acquisition.obligation_id, - observation.checked_at, - acquisition.ordinal + 1, - observation.provisional_until, - ) - ) await connection.execute( "INSERT INTO reporting_provisional_observations" " (account_id,reporting_obligation_id,ordinal,reporting_revision_id,payload)" diff --git a/src/adcp/reporting/conformance.py b/src/adcp/reporting/conformance.py index 6ec5a1c50..b240f8ebe 100644 --- a/src/adcp/reporting/conformance.py +++ b/src/adcp/reporting/conformance.py @@ -151,10 +151,10 @@ async def _with_deadline( # -------------------------------------------------------------------------- -def _validate_request_authority( +def _validate_request_against_capabilities( capabilities: ReportingSourceCapabilitiesV1, request: ReportingSourceSliceRequestV1 ) -> ProvisionalSnapshotOfferingV1 | AuthoritativeOfferingV1: - """Verify the source account, executor build, and offering named by the request.""" + """Is this frozen slice an exact atomic offering of this executor build?""" if capabilities.scope != "effective_account": raise _fail( "CAPABILITY_MISMATCH", @@ -190,14 +190,6 @@ def _validate_request_authority( "CAPABILITY_MISMATCH", "the frozen slice is not an exact atomic offering of this executor build", ) - return offering - - -def _validate_request_against_capabilities( - capabilities: ReportingSourceCapabilitiesV1, request: ReportingSourceSliceRequestV1 -) -> ProvisionalSnapshotOfferingV1 | AuthoritativeOfferingV1: - """Is this frozen slice an exact atomic offering of this executor build?""" - offering = _validate_request_authority(capabilities, request) if ( offering.source_timezone is not None and offering.source_timezone != request.period.source_timezone diff --git a/src/adcp/reporting/inline_source.py b/src/adcp/reporting/inline_source.py index 3b84df635..7349d8185 100644 --- a/src/adcp/reporting/inline_source.py +++ b/src/adcp/reporting/inline_source.py @@ -102,6 +102,7 @@ SourceBatchManifestV1, SourceBatchObjectV1, SourceControlTotalV1, + SourceFormatV1, _validate_metric_applicability, deterministic_source_publication_id_v1, encode_source_batch_manifest_v1, @@ -717,6 +718,9 @@ def __init__( f"{sorted(unsupported)} declare evidenced manifests, which would require " "per-call provider evidence this adapter cannot truthfully produce" ) + for offering in capabilities.offerings: + if not self.supports_output_format(offering.formats): + raise ValueError("inline source formats must include uncompressed NDJSON") self._capabilities = capabilities self._fetch = fetch self._staging = staging or InMemoryStagingStore() @@ -736,6 +740,14 @@ def __init__( def capabilities(self) -> ReportingSourceCapabilitiesV1: return self._capabilities + @staticmethod + def supports_output_format(formats: Sequence[SourceFormatV1]) -> bool: + """The inline writer stages one uncompressed NDJSON object.""" + return any( + fmt.media_type == "application/x-ndjson" and fmt.compression == "none" + for fmt in formats + ) + @property def staging(self) -> ReportingStagingStore: """The staging store, which also serves as the staged-object reader.""" diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 4aefd266e..96d7f3068 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -40,6 +40,7 @@ from dataclasses import dataclass, field, replace from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any, Literal, TypeAlias +from uuid import uuid4 from zoneinfo import ZoneInfo import rfc8785 @@ -190,7 +191,7 @@ def _supports_official_source_formats( ) -def _official_object_rows(staged: SourceBatchObjectV1, payload: bytes) -> list[dict[str, Any]]: +def _staged_object_rows(staged: SourceBatchObjectV1, payload: bytes) -> list[dict[str, Any]]: try: if staged.compression == "gzip": payload = gzip.decompress(payload) @@ -1067,12 +1068,7 @@ async def _acquire_with_settling_policy( restate=bool(revisions), turn=turn, now=now, - target_finality=pending.output_finality - or ( - "snapshot" - if request.publication_class == "PROVISIONAL_SNAPSHOT" - else "official" - ), + target_finality=pending.finality, track_settling=True, ) revisions = await self._store.list_revisions( @@ -1271,12 +1267,13 @@ async def acquire_obligation( else None ) if pending is not None: - finality = pending.output_finality or ( - "snapshot" - if pending.request().publication_class == "PROVISIONAL_SNAPSHOT" - else "official" - ) - if current is not None and current.readable and not restate and pending is None: + finality = pending.finality + if ( + current is not None + and current.readable + and not (restate or manual_replay) + and pending is None + ): return None if finality == "official": if selected_policy is None: @@ -1302,13 +1299,12 @@ async def acquire_obligation( constituents = await self._admitted_source_constituents(configuration, obligation) derived_close = finality == "official" and source_finality == "snapshot" + manual_observation = manual_replay or (restate and not track_settling) if derived_close and isinstance(selected_policy, ReportingContractualCutoffPolicy): if _utc(now) < contractual_cutoff_at(obligation, selected_policy): return None if checkpoint_store is None: observation = await self._cutoff_observation_ordinal(obligation, observation) - else: - track_settling = True request = self._build_slice( configuration, obligation, @@ -1317,10 +1313,15 @@ async def acquire_obligation( now=now, observation=observation, constituents=constituents, - trigger="manual_replay" if manual_replay else "scheduled_poll", + trigger="manual_replay" if manual_observation else "scheduled_poll", + invocation_id=( + uuid4().hex + if manual_observation and checkpoint_store is None and finality == "snapshot" + else None + ), ) acquisition = pending - if track_settling and acquisition is None: + if checkpoint_store is not None and acquisition is None: acquisition = await self._reserve_observation( configuration, obligation, @@ -1334,19 +1335,21 @@ async def acquire_obligation( request = acquisition.request(deadline_at=_utc(now) + self._offerings.slice_timeout) if manual_replay: request = request.model_copy(update={"trigger": "manual_replay"}) - finality = acquisition.output_finality or ( - "snapshot" if request.publication_class == "PROVISIONAL_SNAPSHOT" else "official" - ) + finality = acquisition.finality if acquisition.output_finality and _utc(now) < _utc( request.period.source_read_cutoff_at ): return None + request = request.model_copy(deep=True) if finality == "official": self._official_source_authority(request) cancel = asyncio.Event() execution = asyncio.create_task( self._execute_source( - configuration, request, admitted_constituents=constituents, cancel=cancel + configuration, + request.model_copy(deep=True), + admitted_constituents=constituents, + cancel=cancel, ) ) try: @@ -1436,6 +1439,7 @@ async def acquire_obligation( "PUBLICATION_TIME_INVALID", "producer clock regressed during acquisition; correct the clock before retrying", ) + proof = None if finality == "official": if _utc(manifest.acquired_at) > _utc(published_at): raise LedgerConflictError( @@ -1471,6 +1475,10 @@ async def acquire_obligation( offering_id=request.offering_id, ) return None + untracked_restatement = ( + finality == "snapshot" and manual_observation and acquisition is None + ) + if finality == "official" or untracked_restatement: committed = await self._commit_revision_from_manifest( obligation, manifest, @@ -1481,6 +1489,7 @@ async def acquire_obligation( turn=turn, acquisition=acquisition, verified_source_rows=True, + suppress_untracked_unchanged=untracked_restatement, ) else: committed = await self.commit_revision_from_manifest( @@ -1893,12 +1902,16 @@ async def _reserve_observation( if latest is not None: frozen_policy = latest.acquisition.policy else: - policy = self._settling_policy( - configuration, obligation, cutoff_reservation=derived_close - ) - assert policy is not None - frozen_policy = ProvisionalPolicy( - policy.restatement_window, policy.restatement_cadence, policy.official_close_lag + policy = self._settling_policy(configuration, obligation, cutoff_reservation=True) + # An authoritative-only acquisition has no further due observation. + frozen_policy = ( + ProvisionalPolicy( + policy.restatement_window, policy.restatement_cadence, policy.official_close_lag + ) + if policy is not None + else ProvisionalPolicy( + timedelta(0), iso_duration_to_timedelta(configuration.schedule.period_duration) + ) ) acquisition = await store.reserve_provisional_acquisition( ProvisionalAcquisition( @@ -2003,14 +2016,7 @@ async def _read_rows( "STAGED_OBJECT_MISMATCH", f"staged object {staged.ordinal} does not match its manifest evidence", ) - if require_staged: - rows.extend(_official_object_rows(staged, payload)) - else: - rows.extend( - json.loads(line) - for line in payload.decode("utf-8").splitlines() - if line.strip() - ) + rows.extend(_staged_object_rows(staged, payload)) return rows def _official_source_authority( @@ -2018,11 +2024,11 @@ def _official_source_authority( ) -> ProvisionalSnapshotOfferingV1 | AuthoritativeOfferingV1: from adcp.reporting.conformance import ( ReportingSourceConformanceError, - _validate_request_authority, + _validate_request_against_capabilities, ) try: - offering = _validate_request_authority(self._source.capabilities, request) + offering = _validate_request_against_capabilities(self._source.capabilities, request) except ReportingSourceConformanceError as error: raise LedgerConflictError( "MANIFEST_MISMATCH", "official request contradicts its declared source authority" @@ -2105,6 +2111,7 @@ async def commit_revision_from_manifest( A new official revision requires proof of the obligation's selected policy. Replaying a publication retains its original times, policy fields and parent. """ + manifest = SourceBatchManifestV1.model_validate_json(manifest.model_dump_json()) return await self._commit_revision_from_manifest( obligation, manifest, @@ -2129,6 +2136,7 @@ async def _commit_revision_from_manifest( turn: WorkerTurn | None, acquisition: ProvisionalAcquisition | None, verified_source_rows: bool, + suppress_untracked_unchanged: bool = False, ) -> ReportingRevisionRecord: obligation = await self._stored_obligation(obligation) self._validate_manifest_currency(obligation, manifest) @@ -2206,6 +2214,23 @@ async def _commit_revision_from_manifest( ) if finality != "official": finality_proof = None + if manifest.publication_class != "PROVISIONAL_SNAPSHOT": + raise LedgerConflictError( + "OBSERVATION_CONFLICT", "a snapshot requires a provisional source manifest" + ) + if acquisition is not None: + request = acquisition.request() + if ( + acquisition.finality != finality + or manifest.identity != request.identity + or manifest.publication_class != request.publication_class + or manifest.offering_id != request.offering_id + or manifest.period != request.period + ): + raise LedgerConflictError( + "OBSERVATION_CONFLICT", + "snapshot differs from the reserved source acquisition", + ) elif prior is not None and prior.finality == "official": # A retained immutable publication may be checked or replayed while # another acquisition is reserved. Its replay cannot advance that work. @@ -2289,13 +2314,13 @@ async def _commit_revision_from_manifest( staging = self._store if isinstance(self._store, ReportingRowStagingStore) else None retained = None if ( - acquisition is not None - and finality == "snapshot" - and acquisition.predecessor_revision_id + finality == "snapshot" + and supersedes + and (acquisition is not None or suppress_untracked_unchanged) ): predecessor = await self._store.get_revision( account_id=obligation.account_id, - reporting_revision_id=acquisition.predecessor_revision_id, + reporting_revision_id=supersedes, ) if predecessor is None: raise LedgerConflictError("HISTORY_UNAVAILABLE", "snapshot predecessor is missing") @@ -2317,6 +2342,8 @@ async def _commit_revision_from_manifest( and predecessor.revision_content_sha256 == predecessor_content_sha256 ): retained = predecessor + if acquisition is None: + return retained staged = staging is not None and bool(await staging.stage_revision_rows(revision, rows)) try: committed = await self._commit_publication( @@ -2327,14 +2354,16 @@ async def _commit_revision_from_manifest( finality, now, acquisition, - retained_revision_id=retained.reporting_revision_id - if retained is not None - else None, + retained_revision_id=( + retained.reporting_revision_id if retained is not None else None + ), ) finally: if staged and staging is not None: staging.discard_staged_revision_rows(revision) - if retained is None or committed.reporting_revision_id != retained.reporting_revision_id: + if prior is None and ( + retained is None or committed.reporting_revision_id != retained.reporting_revision_id + ): turn.revisions_committed.append(committed.reporting_revision_id) return committed @@ -2366,13 +2395,12 @@ async def _verify_direct_official_publication( ordinal = await self._cutoff_observation_ordinal(obligation, ordinal) pending = await self._pending_acquisition(obligation, ordinal=ordinal) if pending is not None: - if (acquisition is not None and acquisition != pending) or ( - pending.output_finality is None - and pending.request().publication_class == "PROVISIONAL_SNAPSHOT" - ): + if ( + acquisition is not None and acquisition != pending + ) or pending.finality != "official": raise LedgerConflictError( "OBSERVATION_CONFLICT", - "finish the reserved acquisition before official publication", + "finish the reserved acquisition before publishing this manifest", ) acquisition = pending request = self._build_slice( @@ -2383,7 +2411,10 @@ async def _verify_direct_official_publication( now=manifest.period.source_read_cutoff_at, constituents=constituents, observation=ordinal, + trigger="manual_replay", ) + if acquisition is not None and acquisition.finality != "official": + raise LedgerConflictError("OBSERVATION_CONFLICT", "reserved output finality differs") self._validate_official_manifest( request, manifest, cutoff_ceiling=created_at, acquisition=acquisition ) @@ -2392,28 +2423,27 @@ async def _verify_direct_official_publication( raise LedgerConflictError( "STAGED_OBJECT_MISMATCH", "supplied rows differ from verified staged objects" ) - if ( - pending is None - and request.publication_class == "PROVISIONAL_SNAPSHOT" - and checkpoint_store is not None - ): + if checkpoint_store is not None: # Staged reads release the ledger lock. Reserve the verified request before # publication so an ordinary snapshot that won meanwhile is never skipped. - acquisition = await self._reserve_observation( + winner = await self._reserve_observation( configuration, obligation, request, ordinal=ordinal, revisions=revisions, previous_boundary=checkpoint.provisional_until if checkpoint else None, - derived_close=True, + derived_close=request.publication_class == "PROVISIONAL_SNAPSHOT", ) - if acquisition.output_finality != "official" or not self._matches_reservation( - request, acquisition + if ( + (acquisition is not None and acquisition != winner) + or winner.finality != "official" + or not self._matches_reservation(request, winner) ): raise LedgerConflictError( "OBSERVATION_CONFLICT", "another source acquisition won the reservation" ) + acquisition = winner return acquisition, staged_rows async def _commit_publication( @@ -2564,6 +2594,7 @@ def _build_slice( observation: int = 0, constituents: tuple[ReportingConstituent, ...] | None = None, trigger: ReportingTriggerKind = "scheduled_poll", + invocation_id: str | None = None, ) -> ReportingSourceSliceRequestV1: """Freeze one slice request from the obligation. @@ -2580,6 +2611,9 @@ def _build_slice( and the source is genuinely re-read as a new immutable observation. The ordinal comes from durable state, not from the clock, so neither behavior depends on wall time. + + A manual snapshot without durable observation support uses a fresh + invocation ID so an interrupted earlier invocation cannot replay stale data. """ offering = self._source.capabilities.offering(offering_id) resolved_constituents: list[ReportingConstituent] = ( @@ -2611,6 +2645,8 @@ def _build_slice( # A policy close requires its own post-cutoff read, even when the # same snapshot offering already sealed an observation at this ordinal. execution_identity.append("official") + if invocation_id is not None: + execution_identity.append(invocation_id) source_execution_key = ( "rse-" + hashlib.sha256(canonical_json_utf8_v1(execution_identity)).hexdigest()[:40] ) diff --git a/src/adcp/reporting/ledger/provisional.py b/src/adcp/reporting/ledger/provisional.py index 8f7dcfc13..3d2eb19e7 100644 --- a/src/adcp/reporting/ledger/provisional.py +++ b/src/adcp/reporting/ledger/provisional.py @@ -94,6 +94,12 @@ def obligation_id(self) -> str: def execution_key(self) -> str: return self.request().identity.source_execution_key + @property + def finality(self) -> Literal["snapshot", "official"]: + if self.output_finality or self.request().publication_class == "AUTHORITATIVE": + return "official" + return "snapshot" + def binds(self, obligation: ReportingObligationRecord) -> bool: request = self.request() identity = request.identity diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index 3bbc0ce98..d13c23888 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -422,6 +422,8 @@ async def commit_revision( Enforces terminal officials, exact supersession, and idempotent replay of an identical ``reporting_revision_id``. + Stores supporting durable observations must atomically reject a new + ordinary revision while the obligation has a pending acquisition. """ ... @@ -1201,6 +1203,16 @@ async def commit_revision( for item in self._revisions.values() if item.reporting_obligation_id == revision.reporting_obligation_id ] + checkpoint = self._restatement_checkpoints.get(revision.reporting_obligation_id) + ordinal = checkpoint.next_observation if checkpoint else len(siblings) + if ( + revision.account_id, + revision.reporting_obligation_id, + ordinal, + ) in self._provisional_acquisitions: + raise LedgerConflictError( + "OBSERVATION_CONFLICT", "a source acquisition is reserved" + ) if revision.finality == "official" and any( item.finality == "official" for item in siblings ): @@ -1397,6 +1409,17 @@ async def commit_provisional_observation( checkpoint = self._restatement_checkpoints.get(acquisition.obligation_id) if checkpoint is not None and checkpoint.next_observation != acquisition.ordinal: raise LedgerConflictError("OBSERVATION_CONFLICT", "observation ordinal changed") + # Consume the reservation inside this transaction before its revision + # uses the ordinary commit path. Any publication failure rolls back both. + await self.record_restatement_checkpoint( + RestatementCheckpoint( + acquisition.account_id, + acquisition.obligation_id, + observation.checked_at, + acquisition.ordinal + 1, + observation.provisional_until, + ) + ) if revision is None or reuse_candidate: if acquisition.predecessor_revision_id != observation.revision_id: raise LedgerConflictError( @@ -1434,15 +1457,6 @@ async def commit_provisional_observation( observation = replace(observation, revision_id=committed.reporting_revision_id) else: committed = await self.commit_revision(revision, rows) - await self.record_restatement_checkpoint( - RestatementCheckpoint( - acquisition.account_id, - acquisition.obligation_id, - observation.checked_at, - acquisition.ordinal + 1, - observation.provisional_until, - ) - ) # The checkpoint enforces ascending ordinals for distinct observations. self._provisional_observations.setdefault(owner, {})[acquisition.ordinal] = observation return committed diff --git a/src/adcp/reporting/production/offerings.py b/src/adcp/reporting/production/offerings.py index 485487c72..d012f5229 100644 --- a/src/adcp/reporting/production/offerings.py +++ b/src/adcp/reporting/production/offerings.py @@ -11,6 +11,7 @@ from adcp.reporting._source_authorization import require_account_work from adcp.reporting._timestamp import aware_timestamp from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.inline_source import InlineReportingSource from adcp.reporting.ledger.delivery_models import ReportingDestinationBinding from adcp.reporting.ledger.finality import ReportingStabilizedPolicy, finality_policy_source_class from adcp.reporting.ledger.models import ( @@ -178,7 +179,9 @@ def configured_source( raise failure("UNSUPPORTED_VERIFICATION") source_class = finality_policy_source_class(policy) proof_source = configured_source(source_class) - if not _supports_official_source_formats(proof_source): + if not _supports_official_source_formats(proof_source) or ( + snapshot is not None and not _supports_official_source_formats(snapshot) + ): raise failure("UNSUPPORTED_VERIFICATION") if self.source_offering_id != proof_source.offering_id and ( snapshot is None or self.source_offering_id != snapshot.offering_id @@ -227,6 +230,10 @@ def check_source(self, *, effective: bool = False) -> ReportingSourceCapabilitie duration = iso_duration_milliseconds_v1(schedule["period_duration"]) sla = iso_duration_milliseconds_v1(schedule["delivery_sla"]) for offering in self._source_offerings(capabilities): + if isinstance(source, InlineReportingSource) and not source.supports_output_format( + offering.formats + ): + raise failure("UNSUPPORTED_VERIFICATION") if ( any( getattr(offering.contract, name) != value diff --git a/src/adcp/reporting/service.py b/src/adcp/reporting/service.py index 199c724bb..30afed3f1 100644 --- a/src/adcp/reporting/service.py +++ b/src/adcp/reporting/service.py @@ -858,6 +858,12 @@ def _validate_offerings( ) selected.append(official) for offering in selected: + if isinstance( + registration.executor, InlineReportingSource + ) and not registration.executor.supports_output_format(offering.formats): + raise ReliableReportingConfigurationError( + "inline source formats must include uncompressed NDJSON" + ) if ( offering.contract.report_definition_id != configuration.report_definition_id or offering.contract.reporting_profile != configuration.reporting_profile @@ -883,10 +889,6 @@ def _validate_offerings( raise ReliableReportingConfigurationError( f"selected finality policy requires a matching {source_class} source offering" ) - if not _supports_official_source_formats(source): - raise ReliableReportingConfigurationError( - "official source formats must be JSON or NDJSON with gzip or no compression" - ) used = [source] if ( configuration.required_finality == "snapshot" @@ -895,6 +897,10 @@ def _validate_offerings( ): used.append(snapshot) for used_offering in used: + if not _supports_official_source_formats(used_offering): + raise ReliableReportingConfigurationError( + "official source formats must be JSON or NDJSON with gzip or no compression" + ) if any( getattr(used_offering.contract, name) != value for name, value in definition.to_wire().items() diff --git a/tests/conformance/reporting/test_reporting_currency.py b/tests/conformance/reporting/test_reporting_currency.py index 18bdec28c..6591b8c8e 100644 --- a/tests/conformance/reporting/test_reporting_currency.py +++ b/tests/conformance/reporting/test_reporting_currency.py @@ -231,7 +231,10 @@ async def fetch(request: ReportingSourceSliceRequestV1) -> InlineFetchResult | N assert stored is not None and stored.currency == request.currency if not ready: return None - return InlineFetchResult(rows=money_rows(request), currency=request.currency) + rows = money_rows(request) + if request.trigger == "manual_replay": + rows[0]["impressions"] += 1 + return InlineFetchResult(rows=rows, currency=request.currency) source, staging = source_for(fetch) producer = producer_for(store, source, staging, resolver=resolver) diff --git a/tests/conformance/reporting/test_reporting_production_configuration.py b/tests/conformance/reporting/test_reporting_production_configuration.py index 0300cf479..6cddb3052 100644 --- a/tests/conformance/reporting/test_reporting_production_configuration.py +++ b/tests/conformance/reporting/test_reporting_production_configuration.py @@ -21,6 +21,7 @@ from adcp.reporting.materializer.work import ReportingMaterializerLease from adcp.reporting.production.configuration import ReportingConfigurationAdmission from adcp.reporting.production.offerings import ReportingProductionOffering +from adcp.reporting.service import ReportingAdapterRegistry from adcp.reporting.source import ( ReportingSourceCapabilitiesV1, reporting_source_capabilities_sha256_v1, @@ -110,7 +111,10 @@ async def test_production_offering_validates_selected_finality_policy( @pytest.mark.parametrize("primary", ["snapshot", "official"]) @pytest.mark.parametrize("required", ["snapshot", "official"]) -async def test_dual_finality_source_final_admits_each_configured_route(primary, required, tmp_path): +@pytest.mark.parametrize("unsupported", ["snapshot", "official"]) +async def test_dual_finality_source_final_admits_each_configured_route( + primary, required, unsupported, tmp_path +): async with production_harness("memory", tmp_path / "destination.sqlite", count=0) as h: installed = h.production.offerings[0] definition = replace(installed.verification_key.definition, finality_policy=SOURCE_FINAL) @@ -135,7 +139,7 @@ async def test_dual_finality_source_final_admits_each_configured_route(primary, ) capabilities = source.capabilities.model_dump(mode="json") for item in capabilities["offerings"]: - if item["offering_id"] == source.official_source_id: + if item["offering_id"] == routes.offering_for(unsupported): item["formats"] = [ {"media_type": "application/vnd.apache.parquet", "compression": "none"} ] @@ -145,6 +149,45 @@ async def test_dual_finality_source_final_admits_each_configured_route(primary, offering.check_source(effective=True) +@pytest.mark.parametrize("policy", [CUTOFF, SOURCE_FINAL]) +async def test_production_rechecks_inline_output_before_admission(policy, tmp_path): + async with production_harness("memory", tmp_path / "destination.sqlite", count=0) as h: + installed = h.production.offerings[0] + key = replace( + installed.verification_key, + definition=replace(installed.verification_key.definition, finality_policy=policy), + ) + binding = Source(key, tmp_path / "selected", official=policy == SOURCE_FINAL) + adapter = SimpleNamespace( + capabilities=binding.capabilities, + fetch_slice=binding.fetch, + configuration_binding=binding.configuration_binding, + ) + registered = ReportingAdapterRegistry(clock=lambda: END).register("adapter", adapter) + routes = replace( + installed.producer._offerings, + official_offering_id=binding.source_id if policy == SOURCE_FINAL else None, + snapshot_offering_id=binding.source_id if policy == CUTOFF else None, + ) + producer = ReportingProducer( + source=registered.executor, + offerings=routes, + store=h.store, + object_reader=registered.object_reader, + ) + public = installed.offering.model_copy(update={"supported_finality": ["official"]}) + offering = ReportingProductionOffering(public, producer, key, binding.source_id) + raw = registered.executor.capabilities.model_dump(mode="json") + raw["offerings"][0]["formats"] = [ + {"media_type": "application/x-ndjson", "compression": "gzip"} + ] + raw["capabilities_sha256"] = reporting_source_capabilities_sha256_v1(raw) + registered.executor._capabilities = ReportingSourceCapabilitiesV1.model_validate(raw) + with pytest.raises(ReportingWriterError, match="UNSUPPORTED_VERIFICATION"): + offering.check_source(effective=True) + assert not binding.requests + + def wire_configuration(h): config, binding = h.item.config, h.item.binding offering = h.production.offerings[0] diff --git a/tests/conformance/reporting/test_reporting_publication_time.py b/tests/conformance/reporting/test_reporting_publication_time.py index 96bbba85b..74fbc1da8 100644 --- a/tests/conformance/reporting/test_reporting_publication_time.py +++ b/tests/conformance/reporting/test_reporting_publication_time.py @@ -304,6 +304,7 @@ async def test_same_publication_replay_preserves_committed_time_and_source_evide if finality == "snapshot": clock.now = PUBLISHED + timedelta(minutes=1) reader.complete = lambda: None + source.rows = [dict(ROWS[0], impressions=6)] restated = await producer.acquire_obligation(config, obligation, restate=True) assert restated is not None assert restated.supersedes_reporting_revision_id == first.reporting_revision_id diff --git a/tests/test_reliable_reporting_service.py b/tests/test_reliable_reporting_service.py index 8cc2a48d5..c209f8a6e 100644 --- a/tests/test_reliable_reporting_service.py +++ b/tests/test_reliable_reporting_service.py @@ -169,6 +169,9 @@ def _finality_service( capability_offering=public, ) raw = redacted_capabilities().model_dump(mode="json") + for offering in raw["offerings"]: + offering.update(source_timezone="UTC", product_ids=[configuration.report_definition_id]) + offering["windowing"]["minimum_window"] = "PT1H" if close: raw["offerings"][0].update(restatement_window="P1D", official_close_lag="P2D") raw["offerings"] = [ @@ -329,7 +332,10 @@ class SnapshotStore(InMemoryReportingLedgerStore): await service.configure(config) -async def test_service_refuses_an_unreadable_official_source_format_before_admission() -> None: +@pytest.mark.parametrize("media_type", ["application/json", "application/vnd.apache.parquet"]) +async def test_service_refuses_an_unreadable_official_source_format_before_admission( + media_type: str, +) -> None: config = replace( _configuration(), required_finality="official", @@ -337,12 +343,9 @@ async def test_service_refuses_an_unreadable_official_source_format_before_admis DEFINITION, finality_policy=ReportingSourceFinalPolicy("source", "closed") ), ) - service = _finality_service( - config, snapshot=False, official=True, source_format="application/vnd.apache.parquet" - ) - with pytest.raises(ReliableReportingConfigurationError, match="source format"): + with pytest.raises(ValueError, match="source format"): + service = _finality_service(config, snapshot=False, official=True, source_format=media_type) await service.configure(config) - assert config.generation_key not in service._bindings async def test_service_rejects_stabilized_configuration_without_history_before_persisting() -> None: diff --git a/tests/test_reporting_official_acquisition.py b/tests/test_reporting_official_acquisition.py index de7c076f5..401e893c3 100644 --- a/tests/test_reporting_official_acquisition.py +++ b/tests/test_reporting_official_acquisition.py @@ -29,15 +29,20 @@ LedgerConflictError, ReportingContractualCutoffPolicy, ReportingProducer, + ReportingRevisionRecord, ReportingSourceFinalPolicy, ReportingStabilizedPolicy, + WorkerTurn, ) from adcp.reporting.ledger.finality import manifest_finality_proof from adcp.reporting.ledger.provisional import ProvisionalAcquisition, ProvisionalPolicy from adcp.reporting.source import ( + ReportingSourceExecutorResult, SourceBatchManifestV1, SourceFormatV1, + encode_source_batch_manifest_v1, publication_content_fingerprint_v1, + source_batch_manifest_reference_v1, source_batch_object_set_sha256_v1, ) from tests.conformance.reporting._generation_support import isolated_reporting_pool @@ -296,6 +301,11 @@ async def test_retained_official_replay_leaves_an_unrelated_reservation_alone() official=True, finality_policy=ReportingSourceFinalPolicy("source", "delivery.is_final") ) original = await producer.acquire_obligation(config, obligation) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + previous_observation = await store.get_provisional_observation(**owner) + previous_checkpoint = await store.get_restatement_checkpoint(**owner) manifest = producer._source.manifest request = producer._build_slice( config, obligation, SNAPSHOT_OFFERING_ID, finality="snapshot", now=clock[0], observation=1 @@ -307,25 +317,29 @@ async def test_retained_official_replay_leaves_an_unrelated_reservation_alone() ProvisionalPolicy(timedelta(hours=2), timedelta(hours=1), None), ) ) + turn = WorkerTurn() replayed = await producer.commit_revision_from_manifest( - obligation, manifest, rows=ROWS, finality="official", acquisition=reserved + obligation, manifest, rows=ROWS, finality="official", acquisition=reserved, turn=turn ) assert replayed == original assert ( await store.get_provisional_observation( account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id ) - is None + == previous_observation ) assert ( await store.get_restatement_checkpoint( account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id ) - is None + == previous_checkpoint ) + assert not turn.did_work and not turn.revisions_committed -@pytest.mark.parametrize("change", ["caller_rows", "publication_clock", "explicit_clock"]) +@pytest.mark.parametrize( + "change", ["caller_rows", "caller_manifest", "publication_clock", "explicit_clock"] +) async def test_direct_cutoff_publishes_its_verified_rows_and_completion_time( change: str, monkeypatch: Any ) -> None: @@ -343,6 +357,8 @@ async def test_direct_cutoff_publishes_its_verified_rows_and_completion_time( async def reserve(acquisition: Any) -> Any: if change == "caller_rows": caller[0]["campaign_id"] = "caller-mutated" + if change == "caller_manifest": + manifest.objects.clear() return await original(acquisition) monkeypatch.setattr(store, "reserve_provisional_acquisition", reserve) @@ -359,6 +375,10 @@ async def reserve(acquisition: Any) -> Any: account_id=config.account_id, reporting_revision_id=revision.reporting_revision_id ) assert list(page.rows) == ROWS + stored = await store.get_provisional_observation( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + assert len(SourceBatchManifestV1.model_validate_json(stored.manifest_json).objects) == 1 anchor = start if change == "explicit_clock" else clock[0] assert revision.finalized_at == revision.created_at == anchor @@ -436,3 +456,289 @@ async def read(**kwargs: Any) -> bytes: return official = await publication assert official.finality == "official" and official.row_count == 1 + + +@pytest.mark.parametrize( + "mismatch", ["product", "kind", "timezone", "window", "metric", "mutated_request"] +) +async def test_official_acquisition_verifies_full_frozen_capability( + mismatch: str, monkeypatch: Any +) -> None: + producer, _, config, obligation, calls, _ = await _harness( + official=True, finality_policy=ReportingSourceFinalPolicy("source", "delivery.is_final") + ) + caps = producer._source.capabilities + offering = caps.offering(OFFICIAL_OFFERING_ID) + if mismatch == "product": + offering = offering.model_copy(update={"product_ids": ["forbidden"]}) + elif mismatch == "kind": + offering = offering.model_copy(update={"constituent_kinds": ["package_item"]}) + elif mismatch == "timezone": + offering = offering.model_copy(update={"source_timezone": "Asia/Tokyo"}) + elif mismatch == "window": + offering = offering.model_copy( + update={"windowing": offering.windowing.model_copy(update={"minimum_window": "P2D"})} + ) + elif mismatch == "metric": + offering = offering.model_copy( + update={"metrics": [row for row in offering.metrics if row.name != "spend"]} + ) + else: + original = producer._source.execute + + async def remove_metric(request: Any, **kwargs: Any) -> Any: + request.requested_metrics.remove("spend") + return await original(request, **kwargs) + + monkeypatch.setattr(producer._source, "execute", remove_metric) + producer._source.capabilities = caps.model_copy( + update={ + "offerings": [ + offering if item.offering_id == offering.offering_id else item + for item in caps.offerings + ] + } + ) + with pytest.raises(LedgerConflictError) as conflict: + await producer.acquire_obligation(config, obligation) + assert conflict.value.code == "MANIFEST_MISMATCH" + if mismatch != "mutated_request": + assert not calls + + +@pytest.mark.parametrize("collision", ["concurrent", "lookup_unavailable"]) +async def test_manual_snapshot_obeys_winning_close(collision: str, monkeypatch: Any) -> None: + producer, store, config, obligation, _, clock = await _harness( + capabilities=_closing_capabilities(), + finality_policy=ReportingContractualCutoffPolicy("cutoff", "PT2H"), + ) + clock[0] = END + timedelta(hours=3) + close = producer._build_slice( + config, obligation, SNAPSHOT_OFFERING_ID, now=clock[0], finality="official" + ) + reservation = ProvisionalAcquisition( + close.model_dump_json(), + 0, + ProvisionalPolicy(timedelta(hours=2), timedelta(hours=1)), + output_finality="official", + ) + if collision == "lookup_unavailable": + await store.reserve_provisional_acquisition(reservation) + monkeypatch.setattr(store, "get_provisional_acquisition", None) + else: + + async def raced_lookup(**kwargs: Any) -> None: + await store.reserve_provisional_acquisition(reservation) + + monkeypatch.setattr(store, "get_provisional_acquisition", raced_lookup) + revision = await producer.acquire_obligation( + config, obligation, target_finality="snapshot", restate=True, manual_replay=True + ) + assert revision.finality == "official" + assert producer._source.requests[-1].identity == close.identity + + +@pytest.mark.parametrize("supply_reservation", [False, True]) +async def test_direct_snapshot_cannot_consume_official_reservation( + supply_reservation: bool, monkeypatch: Any +) -> None: + producer, store, config, obligation, _, clock = await _harness( + capabilities=_closing_capabilities(), + finality_policy=ReportingContractualCutoffPolicy("cutoff", "PT2H"), + ) + clock[0] = END + timedelta(hours=3) + close = producer._build_slice( + config, obligation, SNAPSHOT_OFFERING_ID, now=clock[0], finality="official" + ) + reservation = await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + close.model_dump_json(), + 0, + ProvisionalPolicy(timedelta(hours=2), timedelta(hours=1)), + output_finality="official", + ) + ) + _, manifest, _ = redacted_completed_result(close, page_bodies=[PROVIDER_ROW]) + monkeypatch.setattr(store, "get_provisional_acquisition", None) + with pytest.raises(LedgerConflictError) as conflict: + await producer.commit_revision_from_manifest( + obligation, + manifest, + rows=ROWS, + finality="snapshot", + acquisition=reservation if supply_reservation else None, + ) + assert conflict.value.code == "OBSERVATION_CONFLICT" + assert not await store.list_provisional_observations( + account_id=config.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + limit=1, + ) + + +async def test_satisfied_manual_snapshot_recovers_close_without_lookup(monkeypatch: Any) -> None: + producer, store, config, obligation, _, clock = await _harness( + capabilities=_closing_capabilities(), + finality_policy=ReportingContractualCutoffPolicy("cutoff", "PT2H"), + ) + await producer.acquire_obligation( + config, obligation, target_finality="snapshot", track_settling=True + ) + clock[0] = END + timedelta(hours=3) + close = producer._build_slice( + config, obligation, SNAPSHOT_OFFERING_ID, now=clock[0], finality="official", observation=1 + ) + reservation = await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + close.model_dump_json(), + 1, + ProvisionalPolicy(timedelta(hours=2), timedelta(hours=1)), + output_finality="official", + ) + ) + monkeypatch.setattr(store, "get_provisional_acquisition", None) + revision = await producer.acquire_obligation( + config, obligation, target_finality="snapshot", manual_replay=True + ) + assert revision is not None and revision.finality == "official" + assert producer._source.requests[-1].identity == reservation.request().identity + + +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +@pytest.mark.parametrize("reserved_close", [False, True]) +async def test_bare_revision_cannot_overtake_an_observation( + cutoff_backend: Any, reserved_close: bool +) -> None: + factory, _ = cutoff_backend + producer, store, config, obligation, _, clock = await _harness(factory) + request = producer._build_slice(config, obligation, SNAPSHOT_OFFERING_ID, now=clock[0]) + acquisition = ProvisionalAcquisition( + request.model_dump_json(), + 0, + ProvisionalPolicy(timedelta(hours=2), timedelta(hours=1)), + output_finality="official" if reserved_close else None, + ) + await store.reserve_provisional_acquisition(acquisition) + revision = ReportingRevisionRecord( + reporting_revision_id="rpr_competing", + reporting_obligation_id=obligation.reporting_obligation_id, + account_id=config.account_id, + finality="snapshot", + revision_content_sha256="a" * 64, + row_count=0, + control_totals=(), + observed_at=clock[0], + data_through=END, + created_at=clock[0], + ) + with pytest.raises(LedgerConflictError) as conflict: + await store.commit_revision(revision, ()) + assert conflict.value.code == "OBSERVATION_CONFLICT" + assert not await store.list_revisions( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + + +@pytest.mark.parametrize("core", [False, True]) +async def test_manual_restatements_fetch_current_content(core: bool, monkeypatch: Any) -> None: + producer, store, config, obligation, _, clock = await _harness() + if core: + producer._store = CoreStore(store) + calls = [] + values = iter([1, 99, 1, 2, 3] if core else [1, 1, 2, 3]) + + def fetch(request: Any) -> list[dict[str, Any]]: + calls.append(request) + return [{"media_buy_id": "media-buy-redacted", "impressions": next(values), "spend": "0"}] + + monkeypatch.setattr(producer._source, "_fetch", fetch) + first = await producer.acquire_obligation(config, obligation, track_settling=not core) + if core: + + async def interrupted(**kwargs: Any) -> bytes: + raise OSError("interrupted manual read") + + with monkeypatch.context() as patch: + patch.setattr(producer._object_reader, "read", interrupted) + with pytest.raises(OSError): + await producer.acquire_obligation(config, obligation, restate=True) + winners = [ + await producer.acquire_obligation(config, obligation, restate=True) for _ in range(3) + ] + assert len(calls) == len(set(req.identity.source_execution_key for req in calls)) == 4 + core + assert all(req.trigger == "manual_replay" for req in calls[1:]) + assert len({r.reporting_revision_id for r in winners}) == 3 + assert winners[0] == first + if not core: + history = await store.list_provisional_observations( + account_id=config.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + limit=5, + ) + assert [o.acquisition.request().trigger for o in history] == ["scheduled_poll"] + [ + "manual_replay" + ] * 3 + + +async def test_contractual_workflow_reads_gzipped_prerequisite_then_closes( + monkeypatch: Any, +) -> None: + producer, store, config, obligation, _, clock = await _harness( + capabilities=_closing_capabilities(), + finality_policy=ReportingContractualCutoffPolicy("cutoff", "PT2H"), + ) + caps = producer._source.capabilities + snapshot = caps.offering(SNAPSHOT_OFFERING_ID) + fmt = SourceFormatV1(media_type="application/x-ndjson", compression="gzip") + producer._source.capabilities = caps.model_copy( + update={ + "offerings": [ + snapshot.model_copy(update={"formats": [fmt]}) if item == snapshot else item + for item in caps.offerings + ] + } + ) + payload = gzip.compress(PROVIDER_ROW.encode()) + + async def fetch(request: Any, **kwargs: Any) -> Any: + _, old, _ = redacted_completed_result(request, page_bodies=[PROVIDER_ROW]) + objects = [ + old.objects[0].model_copy( + update={ + **fmt.model_dump(), + "sha256": hashlib.sha256(payload).hexdigest(), + "byte_count": len(payload), + } + ) + ] + draft = old.model_copy( + update={ + "objects": objects, + "byte_count": len(payload), + "object_set_sha256": source_batch_object_set_sha256_v1(objects), + } + ) + raw = encode_source_batch_manifest_v1( + draft.model_copy( + update={"content_fingerprint": publication_content_fingerprint_v1(draft)} + ) + ) + return ReportingSourceExecutorResult.completed( + request=request, + manifest=source_batch_manifest_reference_v1("sealed", raw), + manifest_bytes=raw, + ) + + async def read(**kwargs: Any) -> bytes: + return payload + + monkeypatch.setattr(producer._source, "execute", fetch) + monkeypatch.setattr(producer._source, "read", read) + clock[0] = END + timedelta(hours=1) + assert len((await producer.run_configuration(config)).revisions_committed) == 1 + clock[0] += timedelta(hours=2) + assert len((await producer.run_configuration(config)).revisions_committed) == 1 + revisions = await store.list_revisions( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + assert {r.finality for r in revisions} == {"snapshot", "official"} diff --git a/tests/test_reporting_official_finality.py b/tests/test_reporting_official_finality.py index 2b5853c70..d09a9b090 100644 --- a/tests/test_reporting_official_finality.py +++ b/tests/test_reporting_official_finality.py @@ -448,18 +448,29 @@ async def read(**kwargs: Any) -> bytes: @pytest.mark.parametrize("competing_snapshot", [False, True]) +@pytest.mark.parametrize("authoritative", [False, True]) async def test_direct_cutoff_honors_the_winning_reservation_during_staged_verification( monkeypatch: pytest.MonkeyPatch, competing_snapshot: bool, + authoritative: bool, ) -> None: - policy = ReportingContractualCutoffPolicy("contract", "PT2H") + policy = ( + ReportingSourceFinalPolicy("provider", "source-evidence-redacted") + if authoritative + else ReportingContractualCutoffPolicy("contract", "PT2H") + ) producer, store, config, obligation, _, clock = await _harness( capabilities=_closing_capabilities(), finality_policy=policy ) clock[0] = END + timedelta(hours=3) snapshot = producer._build_slice(config, obligation, SNAPSHOT_OFFERING_ID, now=clock[0]) close = producer._build_slice( - config, obligation, SNAPSHOT_OFFERING_ID, now=clock[0], finality="official" + config, + obligation, + OFFICIAL_OFFERING_ID if authoritative else SNAPSHOT_OFFERING_ID, + now=clock[0], + finality="official", + trigger="manual_replay", ) _, manifest, producer._source.reader = redacted_completed_result( close, page_bodies=[PROVIDER_ROW] @@ -500,6 +511,8 @@ async def competing_reservation(**kwargs: Any) -> bytes: ) assert observation is not None and observation.revision_id == official.reporting_revision_id assert observation.acquisition.request() == close + if authoritative: + assert observation.acquisition.request().publication_class == "AUTHORITATIVE" async def test_direct_official_commit_checks_staged_size() -> None: diff --git a/tests/test_reporting_producer_retry.py b/tests/test_reporting_producer_retry.py index 3a735100b..121550a42 100644 --- a/tests/test_reporting_producer_retry.py +++ b/tests/test_reporting_producer_retry.py @@ -43,9 +43,11 @@ def _ready_capabilities(): """Retry tests begin after data readiness, without an availability delay.""" payload = redacted_capabilities().model_dump(mode="json") for offering in payload["offerings"]: + offering.update(source_timezone="UTC", product_ids=["PAID_MEDIA_DAILY_V1"]) offering.update(expected_availability_lag="PT0S", worst_case_availability_lag="PT0S") if offering["offering_id"] == OFFICIAL_OFFERING_ID: offering.update(source_local_ready_time="00:00", days_after_period_end=0) + offering["windowing"]["minimum_window"] = "PT1H" payload["capabilities_sha256"] = reporting_source_capabilities_sha256_v1(payload) return ReportingSourceCapabilitiesV1.model_validate(payload) diff --git a/tests/test_reporting_source_availability.py b/tests/test_reporting_source_availability.py index 33fe81822..f778056e3 100644 --- a/tests/test_reporting_source_availability.py +++ b/tests/test_reporting_source_availability.py @@ -56,6 +56,9 @@ def _capabilities(*, official: bool = False, **changes: Any) -> ReportingSourceC payload = redacted_capabilities().model_dump(mode="json") identifier = OFFICIAL_OFFERING_ID if official else SNAPSHOT_OFFERING_ID for offering in payload["offerings"]: + offering.update( + source_timezone="UTC", product_ids=["fixture-product", "PAID_MEDIA_DAILY_V1"] + ) if offering["offering_id"] == identifier: offering.update(expected_availability_lag="PT1H", worst_case_availability_lag="PT6H") offering.update(changes) @@ -414,6 +417,7 @@ async def test_authoritative_local_readiness_is_conservative_across_clock_change producer, _, _, obligation, _, _ = await _harness( official=True, expected_availability_lag="PT0S", + source_timezone="America/New_York", days_after_period_end=days_after, source_local_ready_time=ready_time, ) From d569ed60c233bf04a49b2a1949f16c1f9fe2c1bf Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 14:36:35 +0530 Subject: [PATCH 13/39] fix(reporting): bind source authority and isolate manual restatements --- .../src/adcp_reporting_storage/ledger/pg.py | 12 +- src/adcp/reporting/ledger/producer.py | 269 ++++++---- src/adcp/reporting/ledger/store.py | 36 +- .../reporting/_production_support.py | 2 +- tests/test_reliable_reporting_service.py | 73 ++- tests/test_reporting_official_acquisition.py | 500 +++++++++++++++++- tests/test_reporting_producer_retry.py | 12 +- tests/test_reporting_typed_control_totals.py | 24 +- 8 files changed, 784 insertions(+), 144 deletions(-) diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py index 596fa1d4b..6455c5d8a 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py @@ -148,10 +148,10 @@ configuration_lifecycle, encode_cursor, issue_is_retirable, - provisional_observation_reuse_candidate, reject_reserved_authoritative_party, validate_adjustment_currency, validate_managed_revision_rows, + validate_provisional_revision, validate_retained_provisional_observation, validate_revision_currency, ) @@ -1837,15 +1837,7 @@ async def commit_provisional_observation( ) -> ReportingRevisionRecord: acquisition = observation.acquisition key = (acquisition.account_id, acquisition.obligation_id, acquisition.ordinal) - reuse_candidate = provisional_observation_reuse_candidate( - acquisition, observation, revision - ) - if revision is not None and ( - revision.account_id != acquisition.account_id - or revision.reporting_obligation_id != acquisition.obligation_id - or (revision.reporting_revision_id != observation.revision_id and not reuse_candidate) - ): - raise LedgerConflictError("OBSERVATION_CONFLICT", "observation identity differs") + reuse_candidate = validate_provisional_revision(observation, revision) async with self.transaction(), self._connection() as connection: await self._lock_account(connection, acquisition.account_id) await self._require_provisional_schema(connection) diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 96d7f3068..860c19a48 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -1105,16 +1105,13 @@ async def _acquire_with_settling_policy( if checkpoint is not None and checkpoint.provisional_until is not None else declared_until ) - last_checked = ( - checkpoint.checked_at - if checkpoint is not None - else max(revisions, key=lambda item: _utc(item.created_at)).created_at - ) - next_due = ( - latest.next_due_at - if latest is not None - else min(_utc(last_checked) + policy.restatement_cadence, settles_at) - ) + if latest is not None: + next_due = latest.next_due_at + elif checkpoint is not None: + next_due = min(_utc(checkpoint.checked_at) + policy.restatement_cadence, settles_at) + else: + # A direct revision has no scheduled observation time. + next_due = self._scheduled_source_ready_at(configuration, obligation) # Preserve explicitly configured official-close boundary precedence. # Snapshot-only policies retain their final inclusive due read through # downtime; expiry alone is never an official publication. @@ -1268,12 +1265,7 @@ async def acquire_obligation( ) if pending is not None: finality = pending.finality - if ( - current is not None - and current.readable - and not (restate or manual_replay) - and pending is None - ): + if current is not None and current.readable and not restate and pending is None: return None if finality == "official": if selected_policy is None: @@ -1297,9 +1289,13 @@ async def acquire_obligation( ) constituents = await self._admitted_source_constituents(configuration, obligation) + constituents = self._check_source_authorization(configuration, offering_id, constituents) derived_close = finality == "official" and source_finality == "snapshot" manual_observation = manual_replay or (restate and not track_settling) + manual_snapshot = ( + restate and manual_observation and finality == "snapshot" and pending is None + ) if derived_close and isinstance(selected_policy, ReportingContractualCutoffPolicy): if _utc(now) < contractual_cutoff_at(obligation, selected_policy): return None @@ -1314,14 +1310,10 @@ async def acquire_obligation( observation=observation, constituents=constituents, trigger="manual_replay" if manual_observation else "scheduled_poll", - invocation_id=( - uuid4().hex - if manual_observation and checkpoint_store is None and finality == "snapshot" - else None - ), + invocation_id=uuid4().hex if manual_snapshot else None, ) acquisition = pending - if checkpoint_store is not None and acquisition is None: + if checkpoint_store is not None and acquisition is None and not manual_snapshot: acquisition = await self._reserve_observation( configuration, obligation, @@ -1333,22 +1325,21 @@ async def acquire_obligation( ) if acquisition is not None: request = acquisition.request(deadline_at=_utc(now) + self._offerings.slice_timeout) - if manual_replay: - request = request.model_copy(update={"trigger": "manual_replay"}) finality = acquisition.finality if acquisition.output_finality and _utc(now) < _utc( request.period.source_read_cutoff_at ): return None request = request.model_copy(deep=True) - if finality == "official": - self._official_source_authority(request) + full_authority = self._requires_source_authority(obligation) + if full_authority: + self._source_authority(obligation, request) cancel = asyncio.Event() execution = asyncio.create_task( self._execute_source( configuration, request.model_copy(deep=True), - admitted_constituents=constituents, + admitted_constituents=tuple(request.coverage.constituents), cancel=cancel, ) ) @@ -1420,16 +1411,20 @@ async def acquire_obligation( "a full-coverage request cannot complete with partial or missing coverage", ) self._validate_manifest_currency(obligation, manifest) - if finality == "official": + if full_authority: assert result.response is not None if result.response.identity != request.identity: raise LedgerConflictError( "MANIFEST_MISMATCH", "source response identity differs from its request" ) - self._validate_official_manifest( - request, manifest, cutoff_ceiling=now, acquisition=acquisition + self._validate_source_manifest( + obligation, + request, + manifest, + cutoff_ceiling=now, + acquisition=acquisition, ) - rows = await self._read_rows(request, manifest, require_staged=finality == "official") + rows = await self._read_rows(request, manifest, require_staged=full_authority) # ``now`` freezes dispatch/lease/cutoff decisions, not publication. # A conforming source can observe finality while acquisition is running. # Every successful scheduled read reaches the atomic observation/checkpoint commit. @@ -1478,29 +1473,18 @@ async def acquire_obligation( untracked_restatement = ( finality == "snapshot" and manual_observation and acquisition is None ) - if finality == "official" or untracked_restatement: - committed = await self._commit_revision_from_manifest( - obligation, - manifest, - rows=rows, - finality=finality, - finality_proof=proof, - now=published_at, - turn=turn, - acquisition=acquisition, - verified_source_rows=True, - suppress_untracked_unchanged=untracked_restatement, - ) - else: - committed = await self.commit_revision_from_manifest( - obligation, - manifest, - rows=rows, - finality=finality, - now=published_at, - turn=turn, - acquisition=acquisition, - ) + committed = await self._commit_revision_from_manifest( + obligation, + manifest, + rows=rows, + finality=finality, + finality_proof=proof, + now=published_at, + turn=turn, + acquisition=acquisition, + verified_source_rows=True, + suppress_untracked_unchanged=untracked_restatement, + ) await self._clear_retry(obligation, turn=turn, now=published_at) return committed @@ -1700,7 +1684,7 @@ def _check_source_authorization( configuration: ReportingConfiguration, offering_id: str, admitted_constituents: tuple[ReportingConstituent, ...] | None, - ) -> None: + ) -> tuple[ReportingConstituent, ...] | None: from adcp.reporting.materializer.contracts import failure from adcp.reporting.production.contracts import ( ReportingProductionSource, @@ -1717,15 +1701,17 @@ def _check_source_authorization( if type(binding) is not ReportingProductionSourceBinding: raise failure("BINDING_MISMATCH") binding.check(configuration, self._source.capabilities, offering_id) - if ( - admitted_constituents is not None - and binding.constituents() != admitted_constituents - ): + authorized = binding.constituents() + if admitted_constituents is not None and coverage_denominator_fingerprint_v1( + authorized + ) != coverage_denominator_fingerprint_v1(admitted_constituents): raise failure("BINDING_MISMATCH") + return authorized except _SourceAuthorizationRevokedError: raise except Exception: raise failure("BINDING_MISMATCH") from None + return admitted_constituents @asynccontextmanager async def _source_publication( @@ -1780,20 +1766,37 @@ async def _execute_source( @asynccontextmanager async def _revision_publication( - self, obligation: ReportingObligationRecord, offering_id: str + self, + obligation: ReportingObligationRecord, + manifest: SourceBatchManifestV1, + *, + official: bool, ) -> AsyncIterator[None]: from adcp.reporting.production.contracts import ReportingProductionSource if not isinstance(self._source, ReportingProductionSource): - yield + if official and isinstance(self._store, ReportingSourcePublicationStore): + async with self._store.source_publication(obligation.account_id): + yield + else: + yield return configuration = await self._retained_source_configuration(obligation) # Read immutable admission scope before taking the publication lock. It # is not an authorization grant: the callback is checked again inside # the lock against this exact mapping, including on replay. constituents = await self._admitted_source_constituents(configuration, obligation) + published = tuple(part.constituent for part in manifest.coverage.constituents) + if ( + constituents is not None + and coverage_denominator_fingerprint_v1(constituents) + != manifest.coverage.denominator_fingerprint + ): + raise LedgerConflictError( + "MANIFEST_MISMATCH", "source denominator differs from admission" + ) async with self._source_publication( - configuration, offering_id, admitted_constituents=constituents + configuration, manifest.offering_id, admitted_constituents=published ): yield @@ -1913,6 +1916,11 @@ async def _reserve_observation( timedelta(0), iso_duration_to_timedelta(configuration.schedule.period_duration) ) ) + if self._requires_source_authority(obligation): + self._source_authority(obligation, request) + self._check_source_authorization( + configuration, request.offering_id, tuple(request.coverage.constituents) + ) acquisition = await store.reserve_provisional_acquisition( ProvisionalAcquisition( request.model_dump_json(), @@ -1996,7 +2004,7 @@ async def _read_rows( if require_staged: raise LedgerConflictError( "STAGED_OBJECT_UNAVAILABLE", - "official publication requires a staged object reader", + "verified publication requires a staged object reader", ) return [] rows: list[dict[str, Any]] = [] @@ -2019,29 +2027,42 @@ async def _read_rows( rows.extend(_staged_object_rows(staged, payload)) return rows - def _official_source_authority( - self, request: ReportingSourceSliceRequestV1 + def _requires_source_authority(self, obligation: ReportingObligationRecord) -> bool: + from adcp.reporting.production.contracts import ReportingProductionSource + + return bool(obligation.definition and obligation.definition.finality_policy) or isinstance( + self._source, ReportingProductionSource + ) + + def _source_authority( + self, obligation: ReportingObligationRecord, request: ReportingSourceSliceRequestV1 ) -> ProvisionalSnapshotOfferingV1 | AuthoritativeOfferingV1: from adcp.reporting.conformance import ( ReportingSourceConformanceError, _validate_request_against_capabilities, ) + self._validate_definition_binding(obligation, request.contract) try: offering = _validate_request_against_capabilities(self._source.capabilities, request) except ReportingSourceConformanceError as error: raise LedgerConflictError( - "MANIFEST_MISMATCH", "official request contradicts its declared source authority" + "MANIFEST_MISMATCH", "request contradicts its declared source authority" ) from error - if not _supports_official_source_formats(offering): + if ( + obligation.definition + and obligation.definition.finality_policy + and not _supports_official_source_formats(offering) + ): raise LedgerConflictError( "UNSUPPORTED_SOURCE_FORMAT", "official producer requires JSON or NDJSON with gzip or no compression", ) return offering - def _validate_official_manifest( + def _validate_source_manifest( self, + obligation: ReportingObligationRecord, request: ReportingSourceSliceRequestV1, manifest: SourceBatchManifestV1, *, @@ -2058,9 +2079,19 @@ def _validate_official_manifest( raise LedgerConflictError( "MANIFEST_MISMATCH", "source cutoff exceeds the requested read" ) - # A core-store retry can receive an earlier seal for the same execution key. - # Only its dispatch time can differ. A durable reservation also fixes that time. + if acquisition is not None and not self._matches_reservation(request, acquisition): + raise LedgerConflictError( + "MANIFEST_MISMATCH", "source acquisition differs from its frozen request" + ) + # An immutable authoritative seal can predate the reservation for this + # execution key. A reserved provisional read fixes its exact cutoff. if cutoff != request.period.source_read_cutoff_at: + if _utc(cutoff) > _utc(request.period.source_read_cutoff_at) or ( + acquisition is not None and request.publication_class != "AUTHORITATIVE" + ): + raise LedgerConflictError( + "MANIFEST_MISMATCH", "source cutoff differs from its frozen request" + ) request = request.model_copy( update={ "identity": request.identity.model_copy( @@ -2069,17 +2100,16 @@ def _validate_official_manifest( "period": request.period.model_copy(update={"source_read_cutoff_at": cutoff}), } ) - if acquisition is not None and not self._matches_reservation(request, acquisition): - raise LedgerConflictError( - "MANIFEST_MISMATCH", "official source manifest differs from its frozen request" - ) try: - _validate_manifest_against_request( - self._official_source_authority(request), request, manifest + offering = ( + self._source_authority(obligation, request) + if self._requires_source_authority(obligation) + else self._source.capabilities.offering(request.offering_id) ) + _validate_manifest_against_request(offering, request, manifest) except ReportingSourceConformanceError as error: raise LedgerConflictError( - "MANIFEST_MISMATCH", "official manifest does not bind the configured source request" + "MANIFEST_MISMATCH", "manifest does not bind the configured source request" ) from error @staticmethod @@ -2087,7 +2117,7 @@ def _matches_reservation( request: ReportingSourceSliceRequestV1, acquisition: ProvisionalAcquisition ) -> bool: return request == acquisition.request().model_copy( - update={"deadline_at": request.deadline_at, "trigger": request.trigger} + update={"deadline_at": request.deadline_at} ) async def commit_revision_from_manifest( @@ -2231,6 +2261,17 @@ async def _commit_revision_from_manifest( "OBSERVATION_CONFLICT", "snapshot differs from the reserved source acquisition", ) + if not verified_source_rows: + self._validate_source_manifest( + obligation, + request, + manifest, + cutoff_ceiling=now, + acquisition=acquisition, + ) + rows = await self._verify_supplied_rows( + request, manifest, rows, retained=prior is not None + ) elif prior is not None and prior.finality == "official": # A retained immutable publication may be checked or replayed while # another acquisition is reserved. Its replay cannot advance that work. @@ -2346,7 +2387,7 @@ async def _commit_revision_from_manifest( return retained staged = staging is not None and bool(await staging.stage_revision_rows(revision, rows)) try: - committed = await self._commit_publication( + committed, replayed = await self._commit_publication( obligation, manifest, revision, @@ -2361,8 +2402,13 @@ async def _commit_revision_from_manifest( finally: if staged and staging is not None: staging.discard_staged_revision_rows(revision) - if prior is None and ( - retained is None or committed.reporting_revision_id != retained.reporting_revision_id + if ( + prior is None + and not replayed + and ( + retained is None + or committed.reporting_revision_id != retained.reporting_revision_id + ) ): turn.revisions_committed.append(committed.reporting_revision_id) return committed @@ -2379,6 +2425,9 @@ async def _verify_direct_official_publication( ) -> tuple[ProvisionalAcquisition | None, list[dict[str, Any]]]: configuration = await self._retained_source_configuration(obligation) constituents = await self._admitted_source_constituents(configuration, obligation) + constituents = self._check_source_authorization( + configuration, manifest.offering_id, constituents + ) ordinal = len(revisions) checkpoint = None checkpoint_store = self._observation_checkpoint_store() @@ -2403,26 +2452,27 @@ async def _verify_direct_official_publication( "finish the reserved acquisition before publishing this manifest", ) acquisition = pending + reserved = acquisition.request() if acquisition is not None else None request = self._build_slice( configuration, obligation, manifest.offering_id, finality="official", - now=manifest.period.source_read_cutoff_at, + now=( + reserved.period.source_read_cutoff_at + if reserved + else manifest.period.source_read_cutoff_at + ), constituents=constituents, observation=ordinal, - trigger="manual_replay", + trigger=reserved.trigger if reserved else "manual_replay", ) if acquisition is not None and acquisition.finality != "official": raise LedgerConflictError("OBSERVATION_CONFLICT", "reserved output finality differs") - self._validate_official_manifest( - request, manifest, cutoff_ceiling=created_at, acquisition=acquisition + self._validate_source_manifest( + obligation, request, manifest, cutoff_ceiling=created_at, acquisition=acquisition ) - staged_rows = await self._read_rows(request, manifest, require_staged=True) - if rfc8785.dumps(staged_rows) != rfc8785.dumps(list(rows)): - raise LedgerConflictError( - "STAGED_OBJECT_MISMATCH", "supplied rows differ from verified staged objects" - ) + staged_rows = await self._verify_supplied_rows(request, manifest, rows) if checkpoint_store is not None: # Staged reads release the ledger lock. Reserve the verified request before # publication so an ordinary snapshot that won meanwhile is never skipped. @@ -2446,6 +2496,22 @@ async def _verify_direct_official_publication( acquisition = winner return acquisition, staged_rows + async def _verify_supplied_rows( + self, + request: ReportingSourceSliceRequestV1, + manifest: SourceBatchManifestV1, + rows: Sequence[dict[str, Any]], + *, + retained: bool = False, + ) -> list[dict[str, Any]]: + staged_rows = await self._read_rows(request, manifest, require_staged=True) + if rfc8785.dumps(staged_rows) != rfc8785.dumps(list(rows)): + raise LedgerConflictError( + "REVISION_IMMUTABLE" if retained else "STAGED_OBJECT_MISMATCH", + "supplied rows differ from verified staged objects", + ) + return staged_rows + async def _commit_publication( self, obligation: ReportingObligationRecord, @@ -2457,8 +2523,21 @@ async def _commit_publication( acquisition: ProvisionalAcquisition | None, *, retained_revision_id: str | None = None, - ) -> ReportingRevisionRecord: - async with self._revision_publication(obligation, manifest.offering_id): + ) -> tuple[ReportingRevisionRecord, bool]: + async with self._revision_publication( + obligation, manifest, official=finality == "official" + ): + replay = None + if finality == "official" and revision is not None: + replay = await self._store.get_revision( + account_id=obligation.account_id, + reporting_revision_id=revision.reporting_revision_id, + ) + if replay is not None and replay.finality == "official": + acquisition = None + revision = replace( + revision, created_at=replay.created_at, finalized_at=replay.finalized_at + ) if acquisition is None: assert revision is not None committed = await self._store.commit_revision(revision, rows) @@ -2491,7 +2570,7 @@ async def _commit_publication( revision, rows, ) - return committed + return committed, replay is not None async def _stored_obligation( self, obligation: ReportingObligationRecord @@ -2612,8 +2691,8 @@ def _build_slice( The ordinal comes from durable state, not from the clock, so neither behavior depends on wall time. - A manual snapshot without durable observation support uses a fresh - invocation ID so an interrupted earlier invocation cannot replay stale data. + A direct manual snapshot uses a fresh invocation ID so an interrupted + invocation cannot replay stale data or affect the scheduled cadence. """ offering = self._source.capabilities.offering(offering_id) resolved_constituents: list[ReportingConstituent] = ( diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index d13c23888..6840afa78 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -731,18 +731,34 @@ def validate_retained_provisional_observation( raise LedgerConflictError("HISTORY_UNAVAILABLE", "snapshot predecessor is not readable") -def provisional_observation_reuse_candidate( - acquisition: ProvisionalAcquisition, +def validate_provisional_revision( observation: ProvisionalObservation, revision: ReportingRevisionRecord | None, ) -> bool: - """Whether a candidate can replace its unchanged predecessor if reuse becomes unsafe.""" - return ( - revision is not None + """Check the reserved output and identify a candidate for unchanged snapshot reuse.""" + acquisition = observation.acquisition + if revision is None: + if acquisition.finality != "snapshot": + raise LedgerConflictError( + "OBSERVATION_CONFLICT", "official acquisition requires an official revision" + ) + return False + reuse_candidate = ( + revision.finality == "snapshot" and revision.reporting_revision_id != observation.revision_id and acquisition.predecessor_revision_id == observation.revision_id and revision.supersedes_reporting_revision_id == observation.revision_id ) + if ( + revision.finality != acquisition.finality + or revision.account_id != acquisition.account_id + or revision.reporting_obligation_id != acquisition.obligation_id + or (revision.reporting_revision_id != observation.revision_id and not reuse_candidate) + ): + raise LedgerConflictError( + "OBSERVATION_CONFLICT", "observation revision differs from the reserved output" + ) + return reuse_candidate def _consumer_status_identity(status: ConsumerStatusRecord) -> str: @@ -1358,15 +1374,7 @@ async def commit_provisional_observation( acquisition = observation.acquisition owner = (acquisition.account_id, acquisition.obligation_id) key = (*owner, acquisition.ordinal) - reuse_candidate = provisional_observation_reuse_candidate( - acquisition, observation, revision - ) - if revision is not None and ( - revision.account_id != acquisition.account_id - or revision.reporting_obligation_id != acquisition.obligation_id - or (revision.reporting_revision_id != observation.revision_id and not reuse_candidate) - ): - raise LedgerConflictError("OBSERVATION_CONFLICT", "observation identity differs") + reuse_candidate = validate_provisional_revision(observation, revision) async with self._mutation(): existing = self._provisional_observations.get(owner, {}).get(acquisition.ordinal) obligation = self._obligations.get(acquisition.obligation_id) diff --git a/tests/conformance/reporting/_production_support.py b/tests/conformance/reporting/_production_support.py index 28bbae998..a759a8e61 100644 --- a/tests/conformance/reporting/_production_support.py +++ b/tests/conformance/reporting/_production_support.py @@ -293,11 +293,11 @@ def __init__( ): raw = redacted_capabilities().model_dump(mode="json") raw["offerings"] = [raw["offerings"][int(official)]] + raw["offerings"][0]["source_timezone"] = "UTC" self.source_id = raw["offerings"][0]["offering_id"] if official: raw["offerings"][0].update( grain="fixed_window", - source_timezone="UTC", source_local_ready_time="00:00", days_after_period_end=0, expected_availability_lag="PT0S", diff --git a/tests/test_reliable_reporting_service.py b/tests/test_reliable_reporting_service.py index c209f8a6e..46134b41c 100644 --- a/tests/test_reliable_reporting_service.py +++ b/tests/test_reliable_reporting_service.py @@ -32,6 +32,7 @@ ReportingStatusCaller, ) from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.production.contracts import ReportingProductionSourceBinding from adcp.reporting.service import ( ProducerFactory, ReliableReportingConfigurationError, @@ -159,6 +160,8 @@ def _finality_service( results: tuple[InlineFetchResult, ...] = (), clock: DeterministicReportingClock | None = None, source_format: str | None = None, + products: tuple[str, ...] = (), + adapter_type: type[ScriptedReportingAdapter] = ScriptedReportingAdapter, ) -> ReliableReportingService: public = _capability_offering("gam") public["supported_finality"] = [configuration.required_finality] @@ -170,7 +173,9 @@ def _finality_service( ) raw = redacted_capabilities().model_dump(mode="json") for offering in raw["offerings"]: - offering.update(source_timezone="UTC", product_ids=[configuration.report_definition_id]) + offering.update( + source_timezone="UTC", product_ids=products or [configuration.report_definition_id] + ) offering["windowing"]["minimum_window"] = "PT1H" if close: raw["offerings"][0].update(restatement_window="P1D", official_close_lag="P2D") @@ -181,7 +186,7 @@ def _finality_service( for offering in raw["offerings"]: offering["formats"] = [{"media_type": source_format, "compression": "none"}] raw["capabilities_sha256"] = reporting_source_capabilities_sha256_v1(raw) - adapter = ScriptedReportingAdapter(ReportingSourceCapabilitiesV1.model_validate(raw), results) + adapter = adapter_type(ReportingSourceCapabilitiesV1.model_validate(raw), results) service = ReliableReportingService( store=store or InMemoryReportingLedgerStore(), account_context=lambda _: context, @@ -192,6 +197,70 @@ def _finality_service( return service +@pytest.mark.parametrize("also_offers_definition", [False, True]) +async def test_registered_source_publishes_only_its_bound_product( + also_offers_definition: bool, +) -> None: + config = replace( + _configuration(), + required_finality="official", + deactivated_at=NOW, + definition=replace( + DEFINITION, finality_policy=ReportingSourceFinalPolicy("selected", "closed") + ), + ) + + class BoundAdapter(ScriptedReportingAdapter): + def configuration_binding( + self, configuration: ReportingConfiguration + ) -> ReportingProductionSourceBinding: + return ReportingProductionSourceBinding.for_configuration( + configuration, + capabilities_sha256=self.capabilities.capabilities_sha256, + media_buy_products=(("media-buy-redacted", "real-product"),), + ) + + def fetch_slice(self, request: Any) -> InlineFetchResult: + self.calls.append(request) + product = request.coverage.constituents[0].product_id + return InlineFetchResult( + rows=[{**_rows(12)[0], "product_id": product}], + finality_evidence=ReportingFinalityEvidenceV1( + basis="provider_declared", + evidence_ref="closed", + observed_at=NOW + timedelta(days=1), + ), + ) + + service = _finality_service( + config, + snapshot=False, + official=True, + products=( + ("real-product", config.report_definition_id) + if also_offers_definition + else ("real-product",) + ), + adapter_type=BoundAdapter, + clock=DeterministicReportingClock(NOW + timedelta(days=3)), + ) + await service.configure(config) + turn = await service.run_worker() + assert not turn.configuration_errors + adapter = service.sources.get("gam").adapter + assert isinstance(adapter, BoundAdapter) + assert {c.product_id for req in adapter.calls for c in req.coverage.constituents} == { + "real-product" + } + revisions = turn.configurations[config.generation_key].revisions_committed + assert len(revisions) == len(adapter.calls) + for revision_id in revisions: + page = await service.store.read_revision_rows( + account_id=config.account_id, reporting_revision_id=revision_id + ) + assert [row["product_id"] for row in page.rows] == ["real-product"] + + @pytest.mark.parametrize("signal", [None, "delivery.other", "delivery.is_final"]) async def test_registered_inline_adapter_supplies_explicit_source_final_evidence( signal: str | None, diff --git a/tests/test_reporting_official_acquisition.py b/tests/test_reporting_official_acquisition.py index 401e893c3..d266971a3 100644 --- a/tests/test_reporting_official_acquisition.py +++ b/tests/test_reporting_official_acquisition.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import gzip import hashlib from collections.abc import AsyncIterator @@ -35,11 +36,17 @@ WorkerTurn, ) from adcp.reporting.ledger.finality import manifest_finality_proof -from adcp.reporting.ledger.provisional import ProvisionalAcquisition, ProvisionalPolicy +from adcp.reporting.ledger.provisional import ( + ProvisionalAcquisition, + ProvisionalObservation, + ProvisionalPolicy, +) from adcp.reporting.source import ( + ReportingFinalityEvidenceV1, ReportingSourceExecutorResult, SourceBatchManifestV1, SourceFormatV1, + coverage_denominator_fingerprint_v1, encode_source_batch_manifest_v1, publication_content_fingerprint_v1, source_batch_manifest_reference_v1, @@ -507,7 +514,9 @@ async def remove_metric(request: Any, **kwargs: Any) -> Any: @pytest.mark.parametrize("collision", ["concurrent", "lookup_unavailable"]) -async def test_manual_snapshot_obeys_winning_close(collision: str, monkeypatch: Any) -> None: +async def test_manual_snapshot_cannot_overtake_a_hidden_winning_close( + collision: str, monkeypatch: Any +) -> None: producer, store, config, obligation, _, clock = await _harness( capabilities=_closing_capabilities(), finality_policy=ReportingContractualCutoffPolicy("cutoff", "PT2H"), @@ -531,9 +540,15 @@ async def raced_lookup(**kwargs: Any) -> None: await store.reserve_provisional_acquisition(reservation) monkeypatch.setattr(store, "get_provisional_acquisition", raced_lookup) - revision = await producer.acquire_obligation( - config, obligation, target_finality="snapshot", restate=True, manual_replay=True + with pytest.raises(LedgerConflictError) as conflict: + await producer.acquire_obligation( + config, obligation, target_finality="snapshot", restate=True, manual_replay=True + ) + assert conflict.value.code == "OBSERVATION_CONFLICT" + assert not await store.list_revisions( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id ) + revision = await producer.acquire_obligation(config, obligation, track_settling=True) assert revision.finality == "official" assert producer._source.requests[-1].identity == close.identity @@ -576,7 +591,9 @@ async def test_direct_snapshot_cannot_consume_official_reservation( ) -async def test_satisfied_manual_snapshot_recovers_close_without_lookup(monkeypatch: Any) -> None: +async def test_satisfied_manual_snapshot_preserves_a_hidden_close_for_scheduled_recovery( + monkeypatch: Any, +) -> None: producer, store, config, obligation, _, clock = await _harness( capabilities=_closing_capabilities(), finality_policy=ReportingContractualCutoffPolicy("cutoff", "PT2H"), @@ -597,8 +614,14 @@ async def test_satisfied_manual_snapshot_recovers_close_without_lookup(monkeypat ) ) monkeypatch.setattr(store, "get_provisional_acquisition", None) + assert ( + await producer.acquire_obligation( + config, obligation, target_finality="snapshot", manual_replay=True + ) + is None + ) revision = await producer.acquire_obligation( - config, obligation, target_finality="snapshot", manual_replay=True + config, obligation, restate=True, track_settling=True ) assert revision is not None and revision.finality == "official" assert producer._source.requests[-1].identity == reservation.request().identity @@ -675,9 +698,7 @@ async def interrupted(**kwargs: Any) -> bytes: reporting_obligation_id=obligation.reporting_obligation_id, limit=5, ) - assert [o.acquisition.request().trigger for o in history] == ["scheduled_poll"] + [ - "manual_replay" - ] * 3 + assert [o.acquisition.request().trigger for o in history] == ["scheduled_poll"] async def test_contractual_workflow_reads_gzipped_prerequisite_then_closes( @@ -742,3 +763,464 @@ async def read(**kwargs: Any) -> bytes: account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id ) assert {r.finality for r in revisions} == {"snapshot", "official"} + + +@pytest.mark.parametrize("route", ["source_final", "cutoff_prerequisite", "bound_snapshot"]) +async def test_invalid_source_namespace_leaves_first_reservation_available(route: str) -> None: + producer, store, config, obligation, calls, _ = await _harness( + official=route == "source_final", + finality_policy=( + ReportingSourceFinalPolicy("source", "delivery.is_final") + if route == "source_final" + else ( + ReportingContractualCutoffPolicy("cutoff", "PT3H") + if route == "cutoff_prerequisite" + else None + ) + ), + ) + if route == "bound_snapshot": + from adcp.reporting.production.contracts import ReportingProductionSourceBinding + + producer._source.configuration_binding = ( + lambda configuration: ReportingProductionSourceBinding.for_configuration( + configuration, + capabilities_sha256=producer._source.capabilities.capabilities_sha256, + media_buy_products=(("media-buy-redacted", "fixture-product"),), + ) + ) + correct = producer._offerings + producer._offerings = replace(correct, publication_namespace="other-namespace") + with pytest.raises(LedgerConflictError) as conflict: + await producer.acquire_obligation(config, obligation) + assert conflict.value.code == "MANIFEST_MISMATCH" and not calls + assert ( + await store.get_provisional_acquisition( + account_id=config.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ordinal=0, + ) + is None + ) + producer._offerings = correct + revision = await producer.acquire_obligation(config, obligation) + assert revision.finality == obligation.required_finality and len(calls) == 1 + + +async def test_mismatched_definition_is_rejected_before_reservation_and_dispatch() -> None: + producer, store, config, obligation, calls, _ = await _harness( + official=True, finality_policy=ReportingSourceFinalPolicy("source", "delivery.is_final") + ) + original = producer._source.capabilities + official = original.offering(OFFICIAL_OFFERING_ID) + incompatible = official.model_copy( + update={ + "contract": official.contract.model_copy(update={"report_definition_sha256": "f" * 64}) + } + ) + producer._source.capabilities = original.model_copy( + update={ + "offerings": [incompatible if item == official else item for item in original.offerings] + } + ) + with pytest.raises(LedgerConflictError) as conflict: + await producer.acquire_obligation(config, obligation) + assert conflict.value.code == "REPORT_DEFINITION_MISMATCH" + assert not calls + assert ( + await store.get_provisional_acquisition( + account_id=config.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ordinal=0, + ) + is None + ) + producer._source.capabilities = original + assert (await producer.acquire_obligation(config, obligation)).finality == "official" + + +async def test_scheduled_policy_snapshot_requires_verified_empty_staging(monkeypatch: Any) -> None: + producer, store, config, obligation, _, _ = await _harness( + finality_policy=ReportingContractualCutoffPolicy("cutoff", "PT2H") + ) + + async def empty(request: Any, **kwargs: Any) -> ReportingSourceExecutorResult: + result, _, _ = redacted_completed_result(request, page_bodies=[""], row_counts=[0]) + return result + + monkeypatch.setattr(producer._source, "execute", empty) + producer._object_reader = None + with pytest.raises(LedgerConflictError) as conflict: + await producer.acquire_obligation(config, obligation, track_settling=True) + assert conflict.value.code == "STAGED_OBJECT_UNAVAILABLE" + assert ( + await store.get_provisional_observation( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + is None + ) + + +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +async def test_source_final_restart_resumes_authoritative_seal_before_first_reservation( + cutoff_backend: Any, + monkeypatch: Any, +) -> None: + factory, _ = cutoff_backend + base, store, config, obligation, _, clock = await _harness( + factory, + official=True, + finality_policy=ReportingSourceFinalPolicy("source", "delivery.is_final"), + ) + staging = InMemoryStagingStore() + fetched = [] + + def fetch(request: Any) -> InlineFetchResult: + fetched.append(request) + return InlineFetchResult( + rows=[{"media_buy_id": "media-buy-redacted", "impressions": 10, "spend": "1.25"}], + finality_evidence=ReportingFinalityEvidenceV1( + basis="provider_declared", + evidence_ref="delivery.is_final", + observed_at=clock[0], + ), + ) + + source = InlineReportingSource( + capabilities=base._source.capabilities, + staging=staging, + seals=InMemorySealStore(), + fetch=fetch, + clock=lambda: clock[0], + ) + + def restart(ledger: Any) -> ReportingProducer: + return ReportingProducer( + store=ledger, + retry_store=base._retry_store, + source=source, + object_reader=staging, + offerings=base._offerings, + clock=lambda: clock[0], + ) + + async def interrupted(**kwargs: Any) -> bytes: + raise OSError("interrupted staged read") + + with monkeypatch.context() as patch: + patch.setattr(staging, "read", interrupted) + with pytest.raises(OSError, match="interrupted staged read"): + await restart(CoreStore(store)).acquire_obligation(config, obligation) + first_dispatch = clock[0] + clock[0] += timedelta(minutes=5) + revision = await restart(store).acquire_obligation(config, obligation) + assert revision is not None and revision.finality == "official" + assert revision.finalized_at == first_dispatch and len(fetched) == 1 + + +async def test_manual_restatement_preserves_scheduled_due_time_and_evidence( + monkeypatch: Any, +) -> None: + producer, store, config, obligation, _, clock = await _harness( + restatement_window="PT6H", restatement_cadence="PT1H" + ) + requests = [] + + def fetch(request: Any) -> list[dict[str, Any]]: + requests.append(request) + return [{"media_buy_id": "media-buy-redacted", "impressions": len(requests), "spend": "0"}] + + monkeypatch.setattr(producer._source, "_fetch", fetch) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + clock[0] = END + timedelta(hours=1) + assert len((await producer.run_configuration(config)).revisions_committed) == 1 + first = await store.get_provisional_observation(**owner) + assert first.next_due_at == END + timedelta(hours=2) + clock[0] = first.next_due_at - timedelta(minutes=1) + for _ in range(2): + assert await producer.acquire_obligation(config, obligation, restate=True) is not None + assert await store.get_provisional_observation(**owner) == first + assert await producer.acquire_obligation(config, obligation, manual_replay=True) is None + clock[0] = first.next_due_at + assert len((await producer.run_configuration(config)).revisions_committed) == 1 + assert [req.trigger for req in requests] == [ + "scheduled_poll", + "manual_replay", + "manual_replay", + "scheduled_poll", + ] + assert len({req.identity.source_execution_key for req in requests}) == len(requests) + history = await store.list_provisional_observations(**owner, limit=5) + assert [o.acquisition.ordinal for o in history] == [0, 1] + assert [o.acquisition.request().trigger for o in history] == [ + "scheduled_poll", + "scheduled_poll", + ] + + +async def test_manual_replay_of_satisfied_snapshot_skips_source() -> None: + producer, _, config, obligation, calls, _ = await _harness() + assert await producer.acquire_obligation(config, obligation) is not None + assert ( + await producer.acquire_obligation(config, obligation, manual_replay=True, restate=False) + is None + ) + assert len(calls) == 1 + + +async def test_manual_snapshot_before_first_poll_preserves_the_scheduled_source_window() -> None: + producer, store, config, obligation, calls, clock = await _harness( + restatement_window="PT6H", restatement_cadence="PT1H" + ) + clock[0] = END + timedelta(minutes=59) + await producer.acquire_obligation(config, obligation, restate=True) + clock[0] = END + timedelta(hours=1) + await producer.run_configuration(config) + assert calls == [END + timedelta(minutes=59), END + timedelta(hours=1)] + observation = await store.get_provisional_observation( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + assert observation.acquisition.request().trigger == "scheduled_poll" + assert observation.next_due_at == END + timedelta(hours=2) + + +async def test_manual_completion_dispatches_the_scheduled_request_in_history() -> None: + producer, store, config, obligation, _, clock = await _harness( + finality_policy=ReportingContractualCutoffPolicy("cutoff", "P2D") + ) + request = producer._build_slice( + config, obligation, SNAPSHOT_OFFERING_ID, now=clock[0], finality="snapshot" + ) + await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + request.model_dump_json(), 0, ProvisionalPolicy(timedelta(days=1), timedelta(hours=1)) + ) + ) + clock[0] += timedelta(minutes=10) + revision = await producer.acquire_obligation( + config, obligation, restate=True, manual_replay=True + ) + assert revision.finality == "snapshot" + history = await store.get_provisional_observation( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + (dispatched,) = producer._source.requests + assert dispatched == history.acquisition.request(deadline_at=dispatched.deadline_at) + assert dispatched.trigger == "scheduled_poll" + + +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +async def test_concurrent_scheduled_reservation_blocks_manual_snapshot( + cutoff_backend: Any, monkeypatch: Any +) -> None: + factory, _ = cutoff_backend + producer, store, config, obligation, _, clock = await _harness(factory) + first = await producer.acquire_obligation(config, obligation) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + initial = await store.get_provisional_observation(**owner) + clock[0] += timedelta(hours=1) + request = producer._build_slice( + config, obligation, SNAPSHOT_OFFERING_ID, now=clock[0], observation=1 + ) + scheduled = ProvisionalAcquisition( + request.model_dump_json(), + 1, + initial.acquisition.policy, + first.reporting_revision_id, + initial.provisional_until, + ) + read = producer._object_reader.read + + async def reserve_during_read(**kwargs: Any) -> bytes: + await store.reserve_provisional_acquisition(scheduled) + return await read(**kwargs) + + def changed(request: Any) -> list[dict[str, Any]]: + return [{"media_buy_id": "media-buy-redacted", "impressions": 99, "spend": "1.25"}] + + monkeypatch.setattr(producer._object_reader, "read", reserve_during_read) + monkeypatch.setattr(producer._source, "_fetch", changed) + with pytest.raises(LedgerConflictError) as conflict: + await producer.acquire_obligation(config, obligation, restate=True) + assert conflict.value.code == "OBSERVATION_CONFLICT" + assert await store.get_provisional_observation(**owner) == initial + assert (await store.get_provisional_acquisition(**owner, ordinal=1)).request() == request + assert [r.reporting_revision_id for r in await store.list_revisions(**owner)] == [ + first.reporting_revision_id + ] + + +@pytest.mark.parametrize( + "change", ["namespace", "metric", "constituent", "rows", "caller_mutation"] +) +async def test_direct_reserved_snapshot_verifies_authority_and_staged_content( + change: str, monkeypatch: Any +) -> None: + producer, store, config, obligation, _, clock = await _harness( + finality_policy=ReportingContractualCutoffPolicy("cutoff", "P2D") + ) + request = producer._build_slice( + config, obligation, SNAPSHOT_OFFERING_ID, now=clock[0], finality="snapshot" + ) + reserved = await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + request.model_dump_json(), 0, ProvisionalPolicy(timedelta(days=1), timedelta(hours=1)) + ) + ) + if change == "namespace": + request = request.model_copy(update={"publication_namespace": "another-source"}) + elif change == "metric": + request = request.model_copy(update={"requested_metrics": ["impressions"]}) + elif change == "constituent": + constituents = [ + request.coverage.constituents[0].model_copy(update={"product_id": "other-product"}) + ] + request = request.model_copy( + update={ + "coverage": request.coverage.model_copy( + update={ + "constituents": constituents, + "denominator_fingerprint": coverage_denominator_fingerprint_v1( + constituents + ), + } + ) + } + ) + _, manifest, producer._source.reader = redacted_completed_result( + request, page_bodies=[PROVIDER_ROW] + ) + caller = [dict(row) for row in ROWS] + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + if change == "caller_mutation": + commit = store.commit_provisional_observation + + async def alter_caller(*args: Any) -> Any: + caller[0]["campaign_id"] = "caller-mutated" + return await commit(*args) + + monkeypatch.setattr(store, "commit_provisional_observation", alter_caller) + revision = await producer.commit_revision_from_manifest( + obligation, + manifest, + rows=caller, + acquisition=reserved, + finality="snapshot", + ) + page = await store.read_revision_rows( + account_id=config.account_id, reporting_revision_id=revision.reporting_revision_id + ) + assert list(page.rows) == ROWS + else: + if change == "rows": + caller[0]["campaign_id"] = "other-campaign" + with pytest.raises(LedgerConflictError) as conflict: + await producer.commit_revision_from_manifest( + obligation, + manifest, + rows=caller, + acquisition=reserved, + finality="snapshot", + ) + assert conflict.value.code == ( + "STAGED_OBJECT_MISMATCH" if change == "rows" else "MANIFEST_MISMATCH" + ) + assert not await store.list_revisions(**owner) + assert await store.get_provisional_acquisition(**owner, ordinal=0) == reserved + + +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +@pytest.mark.parametrize("route", ["snapshot", "authoritative", "cutoff"]) +async def test_store_rejects_revision_outside_reserved_output_finality( + cutoff_backend: Any, route: str +) -> None: + factory, _ = cutoff_backend + producer, store, config, obligation, _, clock = await _harness(factory) + source_final = route == "authoritative" + request = producer._build_slice( + config, + obligation, + OFFICIAL_OFFERING_ID if source_final else SNAPSHOT_OFFERING_ID, + now=clock[0], + finality="official" if source_final else "snapshot", + ) + acquisition = await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + request.model_dump_json(), + 0, + ProvisionalPolicy(timedelta(days=1), timedelta(hours=1)), + output_finality="official" if route == "cutoff" else None, + ) + ) + _, manifest, _ = redacted_completed_result(request, page_bodies=[PROVIDER_ROW]) + wrong_official = route == "snapshot" + revision = ReportingRevisionRecord( + reporting_revision_id="rpr_wrong_finality", + account_id=config.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + finality="official" if wrong_official else "snapshot", + finality_policy_id="selected" if wrong_official else None, + finality_basis="source_final" if wrong_official else None, + finalized_at=clock[0] if wrong_official else None, + revision_content_sha256="a" * 64, + row_count=1, + control_totals=(), + observed_at=clock[0], + data_through=END, + created_at=clock[0], + ) + observation = ProvisionalObservation( + acquisition, + revision.reporting_revision_id, + clock[0], + END + timedelta(days=1), + None, + manifest.model_dump_json(), + ) + with pytest.raises(LedgerConflictError) as conflict: + await store.commit_provisional_observation(observation, revision, ROWS) + assert conflict.value.code == "OBSERVATION_CONFLICT" + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + assert not await store.list_revisions(**owner) + assert await store.get_provisional_observation(**owner) is None + assert await store.get_restatement_checkpoint(**owner) is None + assert await store.get_provisional_acquisition(**owner, ordinal=0) == acquisition + + +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +async def test_concurrent_official_acquisitions_report_one_new_commit( + cutoff_backend: Any, monkeypatch: Any +) -> None: + factory, _ = cutoff_backend + producer, _, config, obligation, _, _ = await _harness( + factory, + official=True, + finality_policy=ReportingSourceFinalPolicy("source", "delivery.is_final"), + ) + turns = [WorkerTurn(), WorkerTurn()] + ready = asyncio.Event() + arrivals = 0 + commit = producer._commit_publication + + async def both_staged(*args: Any, **kwargs: Any) -> Any: + nonlocal arrivals + arrivals += 1 + if arrivals == 2: + ready.set() + await asyncio.wait_for(ready.wait(), timeout=5) + return await commit(*args, **kwargs) + + monkeypatch.setattr(producer, "_commit_publication", both_staged) + revisions = await asyncio.gather( + *(producer.acquire_obligation(config, obligation, turn=turn) for turn in turns) + ) + assert revisions[0] == revisions[1] + assert sum(len(turn.revisions_committed) for turn in turns) == 1 diff --git a/tests/test_reporting_producer_retry.py b/tests/test_reporting_producer_retry.py index 121550a42..a2af7bd5c 100644 --- a/tests/test_reporting_producer_retry.py +++ b/tests/test_reporting_producer_retry.py @@ -401,14 +401,20 @@ async def execute(self, request, *, cancel): assert turn.earliest_retry_at == first_due -async def test_provisional_manual_replay_marks_restored_request(): +async def test_provisional_manual_replay_preserves_the_reserved_trigger(): store = InMemoryReportingLedgerStore() source = FailingSource(code="AUTHENTICATION_FAILED", retry="terminal") config = replace(configuration(), required_finality="snapshot") worker = ReportingProducer( source=source, store=store, - offerings=ProducerOfferings(snapshot_offering_id=SNAPSHOT_OFFERING_ID), + offerings=ProducerOfferings( + snapshot_offering_id=SNAPSHOT_OFFERING_ID, + publication_namespace=source.capabilities.offering( + SNAPSHOT_OFFERING_ID + ).publication_namespace, + source_scope=dict(source.capabilities.source_scope), + ), clock=lambda: NOW, ) obligation = (await close_admitted_periods(worker, config, now=NOW))[0] @@ -421,4 +427,4 @@ async def test_provisional_manual_replay_marks_restored_request(): manual_replay=True, ) assert source.calls == 2 - assert source.requests[-1].trigger == "manual_replay" + assert source.requests[-1].trigger == source.requests[0].trigger == "scheduled_poll" diff --git a/tests/test_reporting_typed_control_totals.py b/tests/test_reporting_typed_control_totals.py index 792b67f45..6dbb9e92c 100644 --- a/tests/test_reporting_typed_control_totals.py +++ b/tests/test_reporting_typed_control_totals.py @@ -26,6 +26,7 @@ from adcp.reporting.ledger.producer import revision_binding_algorithm from adcp.reporting.ledger.status import _revision_to_wire from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.source import SourceBatchManifestV1 from tests.conformance.reporting._generation_support import isolated_reporting_pool from tests.test_reporting_settling import ACCOUNT, _capabilities, _harness, _revisions @@ -123,16 +124,8 @@ async def test_replaying_a_retained_pair_hashed_revision_keeps_its_original_bind make_harness: Any, monkeypatch: pytest.MonkeyPatch ) -> None: producer, store, _, _ = await make_harness(_capabilities(restatement_window="P3D")) - calls: list[tuple[tuple[Any, ...], dict[str, Any]]] = [] - original = producer.commit_revision_from_manifest - - async def spy(*args: Any, **kwargs: Any) -> Any: - calls.append((args, kwargs)) - return await original(*args, **kwargs) - # Seed the history an earlier SDK would have written: pair-hashed totals. with monkeypatch.context() as legacy: - legacy.setattr(producer, "commit_revision_from_manifest", spy) legacy.setattr(producer_module, "_typed_control_totals", lambda *_: None) await producer.run_worker() (retained,) = await _revisions(store) @@ -141,8 +134,19 @@ async def spy(*args: Any, **kwargs: Any) -> Any: # The same publication replayed by the new code reproduces the stored # evidence instead of conflicting with it or rehashing it. - args, kwargs = calls[0] - replayed = await original(*args, **kwargs) + owner = dict(account_id=ACCOUNT, reporting_obligation_id=retained.reporting_obligation_id) + obligation = await store.get_obligation(**owner) + observation = await store.get_provisional_observation(**owner) + rows = await store.read_revision_rows( + account_id=ACCOUNT, reporting_revision_id=retained.reporting_revision_id + ) + replayed = await producer.commit_revision_from_manifest( + obligation, + SourceBatchManifestV1.model_validate_json(observation.manifest_json), + rows=rows.rows, + finality="snapshot", + acquisition=observation.acquisition, + ) assert replayed == retained assert await _revisions(store) == (retained,) From cefd387b13c95adc1aa7b714fab03d6faf1e568a Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 15:22:31 +0530 Subject: [PATCH 14/39] fix(reporting): preserve poll cadence and verify replay evidence --- src/adcp/reporting/ledger/producer.py | 227 +++++++++-- src/adcp/reporting/ledger/provisional.py | 27 ++ tests/test_reporting_official_acquisition.py | 385 ++++++++++++++++++- tests/test_reporting_scheduling_jitter.py | 26 +- 4 files changed, 601 insertions(+), 64 deletions(-) diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 860c19a48..25f1ea2af 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -95,6 +95,8 @@ RestatementCheckpointStore, RetryScheduleEntry, RetryScheduleStore, + managed_revision_metadata, + validate_retained_provisional_observation, ) from adcp.reporting.revision_selection import select_reporting_revision from adcp.reporting.source import ( @@ -1036,8 +1038,9 @@ async def _acquire_with_settling_policy( account_id=obligation.account_id, reporting_obligation_id=obligation.reporting_obligation_id, ) - if latest is not None: - frozen = latest.acquisition.policy + scheduled = latest if latest is not None and latest.is_scheduled else None + if scheduled is not None: + frozen = scheduled.acquisition.policy policy = _SettlingPolicy(frozen.window, frozen.cadence, frozen.official_close_lag) explicit_close = ( policy.official_close_lag is not None @@ -1084,6 +1087,7 @@ async def _acquire_with_settling_policy( ) return ( published is not None + and published.is_scheduled and published.acquisition.ordinal == ordinal and published.next_due_at is None ) @@ -1099,14 +1103,18 @@ async def _acquire_with_settling_policy( ) return False + if latest is not None and scheduled is None: + # A tracked manual retry advances the durable ordinal only. The + # first automatic source read keeps its configured phase and window. + checkpoint = None declared_until = _utc(obligation.period.end) + policy.restatement_window settles_at = ( _utc(checkpoint.provisional_until) if checkpoint is not None and checkpoint.provisional_until is not None else declared_until ) - if latest is not None: - next_due = latest.next_due_at + if scheduled is not None: + next_due = scheduled.next_due_at elif checkpoint is not None: next_due = min(_utc(checkpoint.checked_at) + policy.restatement_cadence, settles_at) else: @@ -1132,7 +1140,12 @@ async def _acquire_with_settling_policy( account_id=obligation.account_id, reporting_obligation_id=obligation.reporting_obligation_id, ) - if not explicit_close and published is not None and published.next_due_at is None: + if ( + not explicit_close + and published is not None + and published.is_scheduled + and published.next_due_at is None + ): return True return False @@ -1293,9 +1306,7 @@ async def acquire_obligation( derived_close = finality == "official" and source_finality == "snapshot" manual_observation = manual_replay or (restate and not track_settling) - manual_snapshot = ( - restate and manual_observation and finality == "snapshot" and pending is None - ) + manual_snapshot = manual_observation and finality == "snapshot" and pending is None if derived_close and isinstance(selected_policy, ReportingContractualCutoffPolicy): if _utc(now) < contractual_cutoff_at(obligation, selected_policy): return None @@ -1313,7 +1324,13 @@ async def acquire_obligation( invocation_id=uuid4().hex if manual_snapshot else None, ) acquisition = pending - if checkpoint_store is not None and acquisition is None and not manual_snapshot: + # An explicitly tracked first invocation must survive a failed fetch and restart. + track_first_manual = manual_snapshot and track_settling and not restate and not revisions + if ( + checkpoint_store is not None + and acquisition is None + and (not manual_snapshot or track_first_manual) + ): acquisition = await self._reserve_observation( configuration, obligation, @@ -1902,9 +1919,11 @@ async def _reserve_observation( account_id=obligation.account_id, reporting_obligation_id=obligation.reporting_obligation_id, ) - if latest is not None: + if latest is not None and latest.is_scheduled: frozen_policy = latest.acquisition.policy else: + if latest is not None: + previous_boundary = None policy = self._settling_policy(configuration, obligation, cutoff_reservation=True) # An authoritative-only acquisition has no further due observation. frozen_policy = ( @@ -2138,6 +2157,7 @@ async def commit_revision_from_manifest( edit path. An official close is terminal, so a later source correction must arrive as an adjustment instead. ``now`` is a trusted publication instant, unlike the dispatch instant accepted by ``acquire_obligation``. + Without it, a reserved publication samples the clock after staged verification. A new official revision requires proof of the obligation's selected policy. Replaying a publication retains its original times, policy fields and parent. """ @@ -2225,7 +2245,13 @@ async def _commit_revision_from_manifest( if control_totals and (prior is None or prior.managed_control_totals is not None) else None ) - created_at = prior.created_at if prior is not None else now + retained_official = prior is not None and prior.finality == finality == "official" + proved_candidate = finality == "official" and finality_proof is not None + created_at = ( + now + if prior is None or (retained_official and finality_proof is not None) + else prior.created_at + ) if prior is not None: # Still reconstruct and verify the supplied content below. Merely # finding the ID must not bypass immutable-content validation. @@ -2262,6 +2288,11 @@ async def _commit_revision_from_manifest( "snapshot differs from the reserved source acquisition", ) if not verified_source_rows: + replay = await self._retained_snapshot_replay( + obligation, manifest, rows, acquisition, prior, now + ) + if replay is not None: + return replay self._validate_source_manifest( obligation, request, @@ -2272,7 +2303,7 @@ async def _commit_revision_from_manifest( rows = await self._verify_supplied_rows( request, manifest, rows, retained=prior is not None ) - elif prior is not None and prior.finality == "official": + elif prior is not None and retained_official and finality_proof is None: # A retained immutable publication may be checked or replayed while # another acquisition is reserved. Its replay cannot advance that work. acquisition = None @@ -2301,7 +2332,9 @@ async def _commit_revision_from_manifest( "PUBLICATION_TIME_INVALID", "proof is later than the publication clock" ) finality_proof.validate(obligation, manifest, proof_anchor) - if not verified_source_rows: + if retained_official and not verified_source_rows: + acquisition = None + elif not verified_source_rows: acquisition, rows = await self._verify_direct_official_publication( obligation, manifest, @@ -2310,15 +2343,21 @@ async def _commit_revision_from_manifest( revisions=existing, created_at=created_at, ) - if sample_after_staging: - completed_at = self._clock() - if _utc(completed_at) < _utc(now): - raise LedgerConflictError( - "PUBLICATION_TIME_INVALID", - "producer clock regressed during staged verification", - ) - created_at = now = completed_at - finality_proof = manifest_finality_proof(obligation, manifest, created_at) + if ( + sample_after_staging + and not verified_source_rows + and prior is None + and (acquisition is not None or finality == "official") + ): + completed_at = self._clock() + if _utc(completed_at) < _utc(now): + raise LedgerConflictError( + "PUBLICATION_TIME_INVALID", + "producer clock regressed during staged verification", + ) + created_at = now = completed_at + if finality == "official": + finality_proof = manifest_finality_proof(obligation, manifest, created_at) revision = ReportingRevisionRecord( reporting_revision_id=revision_id, account_id=obligation.account_id, @@ -2395,6 +2434,7 @@ async def _commit_revision_from_manifest( finality, now, acquisition, + proved_candidate=proved_candidate, retained_revision_id=( retained.reporting_revision_id if retained is not None else None ), @@ -2413,6 +2453,81 @@ async def _commit_revision_from_manifest( turn.revisions_committed.append(committed.reporting_revision_id) return committed + async def _retained_snapshot_replay( + self, + obligation: ReportingObligationRecord, + manifest: SourceBatchManifestV1, + rows: Sequence[dict[str, Any]], + acquisition: ProvisionalAcquisition, + prior: ReportingRevisionRecord | None, + now: datetime, + ) -> ReportingRevisionRecord | None: + store = self._observation_store() + latest = await store.get_provisional_observation( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + if latest is None or latest.acquisition.ordinal < acquisition.ordinal: + return None + revision_id = ( + prior.reporting_revision_id + if prior is not None + else acquisition.predecessor_revision_id + ) + if revision_id is None: + return None + observation = ProvisionalObservation.from_manifest(acquisition, revision_id, now, manifest) + is_latest = latest.acquisition.ordinal == acquisition.ordinal + if is_latest and ( + latest.acquisition != acquisition + or latest.manifest_json != observation.manifest_json + or latest.revision_id != revision_id + ): + return None + if not acquisition.binds(obligation): + raise LedgerConflictError("OBSERVATION_CONFLICT", "retained snapshot owner differs") + async with self._revision_publication(obligation, manifest, official=False): + revision = await self._store.get_revision( + account_id=obligation.account_id, reporting_revision_id=revision_id + ) + if ( + revision is None + or revision.finality != "snapshot" + or revision.reporting_obligation_id != obligation.reporting_obligation_id + ): + raise LedgerConflictError("HISTORY_UNAVAILABLE", "retained snapshot is missing") + if prior is not None and ( + revision.source_publication_id != manifest.publication_id + or revision.observed_at != manifest.observed_at + or revision.data_through != manifest.data_through + or revision.supersedes_reporting_revision_id != acquisition.predecessor_revision_id + ): + raise LedgerConflictError( + "REVISION_IMMUTABLE", "retained source publication differs" + ) + if ( + revision.row_count != manifest.row_count + or revision.control_totals + != tuple((total.name, total.value) for total in manifest.control_totals) + or ( + revision.managed_control_totals is not None + and revision.managed_control_totals + != _typed_control_totals(obligation, manifest) + ) + ): + raise LedgerConflictError("REVISION_IMMUTABLE", "retained source totals differ") + validate_retained_provisional_observation(revision, observation, rows) + if is_latest: + return revision + # A later observation closes this ordinal to new writes. The store + # atomically checks the historical acquisition, manifest and revision. + try: + return await store.commit_provisional_observation(observation, revision, rows) + except LedgerConflictError as error: + if error.code != "OBSERVATION_CONFLICT": + raise + return None + async def _verify_direct_official_publication( self, obligation: ReportingObligationRecord, @@ -2522,6 +2637,7 @@ async def _commit_publication( now: datetime, acquisition: ProvisionalAcquisition | None, *, + proved_candidate: bool, retained_revision_id: str | None = None, ) -> tuple[ReportingRevisionRecord, bool]: async with self._revision_publication( @@ -2534,44 +2650,73 @@ async def _commit_publication( reporting_revision_id=revision.reporting_revision_id, ) if replay is not None and replay.finality == "official": - acquisition = None - revision = replace( - revision, created_at=replay.created_at, finalized_at=replay.finalized_at + revision = await self._official_replay_revision( + revision, replay, manifest, acquisition, proved_candidate=proved_candidate ) + acquisition = None if acquisition is None: assert revision is not None committed = await self._store.commit_revision(revision, rows) else: # ``now`` is the publication anchor, sampled after row acquisition. # The bounds check above already rejects acquired_at after now. - checked_at = _utc(now) - boundary = manifest.finality_evidence.provisional_until or ( - _utc(obligation.period.end) + acquisition.policy.window - ) - next_due = ( - min(checked_at + acquisition.policy.cadence, _utc(boundary)) - if finality == "snapshot" and checked_at < _utc(boundary) - else None - ) if retained_revision_id is not None: observation_revision_id = retained_revision_id else: assert revision is not None observation_revision_id = revision.reporting_revision_id committed = await self._observation_store().commit_provisional_observation( - ProvisionalObservation( - acquisition, - observation_revision_id, - checked_at, - _utc(boundary), - next_due, - manifest.model_dump_json(), + ProvisionalObservation.from_manifest( + acquisition, observation_revision_id, now, manifest ), revision, rows, ) return committed, replay is not None + async def _official_replay_revision( + self, + candidate: ReportingRevisionRecord, + retained: ReportingRevisionRecord, + manifest: SourceBatchManifestV1, + acquisition: ProvisionalAcquisition | None, + *, + proved_candidate: bool, + ) -> ReportingRevisionRecord: + """Compare official source evidence inside the account publication lock.""" + store = self._optional_observation_store() + observation = ( + await store.get_provisional_observation( + account_id=retained.account_id, + reporting_obligation_id=retained.reporting_obligation_id, + ) + if store is not None + else None + ) + if observation is None or observation.revision_id != retained.reporting_revision_id: + if proved_candidate: + raise LedgerConflictError( + "REVISION_IMMUTABLE", "the winning official source evidence is unavailable" + ) + elif json.loads(observation.manifest_json) != manifest.model_dump(mode="json") or ( + acquisition is not None and observation.acquisition != acquisition + ): + raise LedgerConflictError("REVISION_IMMUTABLE", "official source observation differs") + # Creation and contractual finality are sampled by each producer. A + # source-final timestamp belongs to the provider's immutable evidence. + replay = replace( + candidate, + created_at=retained.created_at, + finalized_at=( + retained.finalized_at + if candidate.finality_basis == "contractual_cutoff" + else candidate.finalized_at + ), + ) + if managed_revision_metadata(replay) != managed_revision_metadata(retained): + raise LedgerConflictError("REVISION_IMMUTABLE", "official revision evidence differs") + return replay + async def _stored_obligation( self, obligation: ReportingObligationRecord ) -> ReportingObligationRecord: diff --git a/src/adcp/reporting/ledger/provisional.py b/src/adcp/reporting/ledger/provisional.py index 3d2eb19e7..90f073ca3 100644 --- a/src/adcp/reporting/ledger/provisional.py +++ b/src/adcp/reporting/ledger/provisional.py @@ -16,6 +16,7 @@ if TYPE_CHECKING: from adcp.reporting.ledger.models import ReportingObligationRecord, ReportingRevisionRecord + from adcp.reporting.source import SourceBatchManifestV1 def utc(value: datetime) -> datetime: @@ -156,6 +157,32 @@ class ProvisionalObservation: next_due_at: datetime | None manifest_json: str + @property + def is_scheduled(self) -> bool: + return self.acquisition.request().trigger == "scheduled_poll" + + @classmethod + def from_manifest( + cls, + acquisition: ProvisionalAcquisition, + revision_id: str, + checked_at: datetime, + manifest: SourceBatchManifestV1, + ) -> ProvisionalObservation: + checked_at = utc(checked_at) + boundary = utc( + manifest.finality_evidence.provisional_until + or (utc(manifest.period.end) + acquisition.policy.window) + ) + next_due = ( + min(checked_at + acquisition.policy.cadence, boundary) + if acquisition.finality == "snapshot" and checked_at < boundary + else None + ) + return cls( + acquisition, revision_id, checked_at, boundary, next_due, manifest.model_dump_json() + ) + def __post_init__(self) -> None: utc(self.checked_at) utc(self.provisional_until) diff --git a/tests/test_reporting_official_acquisition.py b/tests/test_reporting_official_acquisition.py index d266971a3..c9a4e2a37 100644 --- a/tests/test_reporting_official_acquisition.py +++ b/tests/test_reporting_official_acquisition.py @@ -9,6 +9,7 @@ from dataclasses import replace from datetime import timedelta from functools import partial +from types import SimpleNamespace from typing import Any import pytest @@ -34,6 +35,7 @@ ReportingSourceFinalPolicy, ReportingStabilizedPolicy, WorkerTurn, + revision_content_sha256, ) from adcp.reporting.ledger.finality import manifest_finality_proof from adcp.reporting.ledger.provisional import ( @@ -57,6 +59,7 @@ from tests.test_reporting_source_availability import ( END, PROVIDER_ROW, + EvidenceSource, ProgressStore, _closing_capabilities, _harness, @@ -986,6 +989,46 @@ async def test_manual_snapshot_before_first_poll_preserves_the_scheduled_source_ assert observation.next_due_at == END + timedelta(hours=2) +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +async def test_initial_manual_replay_keeps_the_first_scheduled_poll( + cutoff_backend: Any, monkeypatch: Any +) -> None: + factory, _ = cutoff_backend + producer, store, config, obligation, calls, clock = await _harness( + factory, + finality_policy=ReportingContractualCutoffPolicy("cutoff", "P2D"), + restatement_window="PT6H", + restatement_cadence="PT1H", + ) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + clock[0] = END + timedelta(minutes=59) + + async def interrupted(**kwargs: Any) -> bytes: + raise OSError("interrupted manual read") + + with monkeypatch.context() as patch: + patch.setattr(producer._object_reader, "read", interrupted) + with pytest.raises(OSError, match="interrupted manual read"): + await producer.acquire_obligation(config, obligation, manual_replay=True) + manual = await producer.acquire_obligation(config, obligation, manual_replay=True) + assert manual is not None and manual.finality == "snapshot" + assert await store.get_restatement_checkpoint(**owner) is None + assert not await store.list_provisional_observations(**owner, limit=5) + assert await producer.acquire_obligation(config, obligation, manual_replay=True) is None + await producer.run_configuration(config) + clock[0] = END + timedelta(hours=1) + await producer.run_configuration(config) + assert calls == [END + timedelta(minutes=59)] * 2 + [END + timedelta(hours=1)] + requests = producer._source.requests + assert [r.trigger for r in requests] == ["manual_replay", "manual_replay", "scheduled_poll"] + assert len({r.identity.source_execution_key for r in requests}) == 3 + (scheduled,) = await store.list_provisional_observations(**owner, limit=5) + assert scheduled.checked_at == END + timedelta(hours=1) + assert scheduled.next_due_at == END + timedelta(hours=2) + + async def test_manual_completion_dispatches_the_scheduled_request_in_history() -> None: producer, store, config, obligation, _, clock = await _harness( finality_policy=ReportingContractualCutoffPolicy("cutoff", "P2D") @@ -1135,6 +1178,206 @@ async def alter_caller(*args: Any) -> Any: assert await store.get_provisional_acquisition(**owner, ordinal=0) == reserved +@pytest.mark.parametrize("explicit", [False, True]) +async def test_direct_reserved_snapshot_anchors_the_next_poll_after_staging(explicit: bool) -> None: + producer, store, config, obligation, _, clock = await _harness( + finality_policy=ReportingContractualCutoffPolicy("cutoff", "P2D") + ) + start = clock[0] = END + timedelta(hours=2) + request = producer._build_slice(config, obligation, SNAPSHOT_OFFERING_ID, now=start) + acquisition = await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + request.model_dump_json(), 0, ProvisionalPolicy(timedelta(days=1), timedelta(hours=1)) + ) + ) + _, manifest, producer._source.reader = redacted_completed_result( + request, page_bodies=[PROVIDER_ROW] + ) + producer._source.read_delay = timedelta(minutes=15) + revision = await producer.commit_revision_from_manifest( + obligation, + manifest, + rows=ROWS, + finality="snapshot", + acquisition=acquisition, + now=start if explicit else None, + ) + observation = await store.get_provisional_observation( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + assert clock[0] == END + timedelta(hours=2, minutes=15) + anchor = start if explicit else END + timedelta(hours=2, minutes=15) + assert revision.created_at == observation.checked_at == anchor + assert observation.next_due_at == anchor + timedelta(hours=1) + + +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +@pytest.mark.parametrize("unavailable", ["reader", "object", "publication_store"]) +async def test_retained_snapshot_replay_survives_source_staging_expiry( + cutoff_backend: Any, unavailable: str +) -> None: + factory, _ = cutoff_backend + producer, store, config, obligation, _, clock = await _harness( + factory, finality_policy=ReportingContractualCutoffPolicy("cutoff", "P2D") + ) + first = await producer.acquire_obligation(config, obligation, track_settling=True) + clock[0] += timedelta(hours=1) + assert first == await producer.acquire_obligation( + config, obligation, restate=True, track_settling=True + ) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + history = await store.list_provisional_observations(**owner, limit=5) + assert len(history) == 2 + checkpoint = await store.get_restatement_checkpoint(**owner) + page = await store.read_revision_rows( + account_id=config.account_id, reporting_revision_id=first.reporting_revision_id + ) + if unavailable == "object": + producer._source.reader._objects.clear() + else: + producer._object_reader = None + if unavailable == "publication_store": + producer._store = SimpleNamespace( + get_obligation=store.get_obligation, + list_revisions=store.list_revisions, + get_revision=store.get_revision, + reserve_provisional_acquisition=store.reserve_provisional_acquisition, + get_provisional_observation=store.get_provisional_observation, + commit_provisional_observation=store.commit_provisional_observation, + ) + turn = WorkerTurn() + for observation in history: + manifest = SourceBatchManifestV1.model_validate_json(observation.manifest_json) + replayed = await producer.commit_revision_from_manifest( + obligation, + manifest, + rows=page.rows, + finality="snapshot", + acquisition=observation.acquisition, + turn=turn, + ) + assert replayed == first + assert not turn.did_work + assert await store.list_provisional_observations(**owner, limit=5) == history + assert await store.get_restatement_checkpoint(**owner) == checkpoint + + +@pytest.mark.parametrize("change", ["rows", "manifest", "acquisition", "new_acquisition"]) +async def test_retained_snapshot_replay_requires_the_exact_ledger_binding(change: str) -> None: + producer, store, config, obligation, _, clock = await _harness( + finality_policy=ReportingContractualCutoffPolicy("cutoff", "P2D") + ) + first = await producer.acquire_obligation(config, obligation, track_settling=True) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + stored = await store.get_provisional_observation(**owner) + checkpoint = await store.get_restatement_checkpoint(**owner) + acquisition = stored.acquisition + manifest = SourceBatchManifestV1.model_validate_json(stored.manifest_json) + rows = [dict(row) for row in ROWS] + if change == "rows": + rows[0]["impressions"] = 11 + elif change == "manifest": + evidence = manifest.finality_evidence.model_copy(update={"evidence_ref": "another"}) + manifest = manifest.model_copy(update={"finality_evidence": evidence}) + elif change == "acquisition": + acquisition = replace( + acquisition, policy=ProvisionalPolicy(timedelta(days=2), timedelta(hours=2)) + ) + else: + request = producer._build_slice( + config, obligation, SNAPSHOT_OFFERING_ID, now=clock[0], observation=1 + ) + acquisition = await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + request.model_dump_json(), + 1, + stored.acquisition.policy, + first.reporting_revision_id, + stored.provisional_until, + ) + ) + _, manifest, _ = redacted_completed_result(request, page_bodies=[PROVIDER_ROW]) + producer._object_reader = None + with pytest.raises(LedgerConflictError) as conflict: + await producer.commit_revision_from_manifest( + obligation, manifest, rows=rows, acquisition=acquisition, finality="snapshot" + ) + if change == "rows": + assert conflict.value.code == "REVISION_IMMUTABLE" + else: + assert conflict.value.code == "STAGED_OBJECT_UNAVAILABLE" + if change in {"manifest", "acquisition"}: + producer._object_reader = producer._source + with pytest.raises(LedgerConflictError) as mismatch: + await producer.commit_revision_from_manifest( + obligation, manifest, rows=rows, acquisition=acquisition, finality="snapshot" + ) + assert mismatch.value.code == "OBSERVATION_CONFLICT" + assert await store.list_provisional_observations(**owner, limit=5) == (stored,) + assert await store.get_restatement_checkpoint(**owner) == checkpoint + assert await store.list_revisions(**owner) == (first,) + + +@pytest.mark.parametrize("different", ["source_publication", "control_totals", "typed_totals"]) +async def test_retained_snapshot_replay_checks_the_revision_source_publication( + different: str, + monkeypatch: Any, +) -> None: + producer, store, config, obligation, _, _ = await _harness( + finality_policy=ReportingContractualCutoffPolicy("cutoff", "P2D") + ) + commit = store.commit_provisional_observation + + async def inconsistent(observation: Any, revision: Any, rows: Any) -> Any: + if different == "source_publication": + revision = replace(revision, source_publication_id="unrelated") + else: + totals = ( + (("impressions", "11"),) + if different == "control_totals" + else revision.control_totals + ) + typed = ( + (replace(revision.managed_control_totals[0], unit="other"),) + if different == "typed_totals" + else None + ) + revision = replace( + revision, + control_totals=totals, + managed_control_totals=typed, + revision_content_sha256=revision_content_sha256( + reporting_revision_id=revision.reporting_revision_id, + row_count=revision.row_count, + control_totals=totals, + reporting_rows=rows, + control_total_evidence=typed, + ), + ) + return await commit(observation, revision, rows) + + with monkeypatch.context() as patch: + patch.setattr(store, "commit_provisional_observation", inconsistent) + await producer.acquire_obligation(config, obligation, track_settling=True) + retained = await store.get_provisional_observation( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + producer._object_reader = None + with pytest.raises(LedgerConflictError) as conflict: + await producer.commit_revision_from_manifest( + obligation, + SourceBatchManifestV1.model_validate_json(retained.manifest_json), + rows=ROWS, + finality="snapshot", + acquisition=retained.acquisition, + ) + assert conflict.value.code == "REVISION_IMMUTABLE" + + @pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) @pytest.mark.parametrize("route", ["snapshot", "authoritative", "cutoff"]) async def test_store_rejects_revision_outside_reserved_output_finality( @@ -1196,31 +1439,131 @@ async def test_store_rejects_revision_outside_reserved_output_finality( @pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) -async def test_concurrent_official_acquisitions_report_one_new_commit( - cutoff_backend: Any, monkeypatch: Any +@pytest.mark.parametrize("prior_visible", [False, True]) +@pytest.mark.parametrize( + ("basis", "change", "core_only"), + [ + ("source_final", "identical", False), + ("source_final", "finalized_at", False), + ("source_final", "provider_basis", False), + ("source_final", "provider_basis", True), + ("contractual_cutoff", "identical", False), + ("contractual_cutoff", "evidence_ref", False), + ], +) +async def test_concurrent_official_acquisitions_require_the_same_source_evidence( + cutoff_backend: Any, + prior_visible: bool, + basis: str, + change: str, + core_only: bool, + monkeypatch: Any, ) -> None: factory, _ = cutoff_backend - producer, _, config, obligation, _, _ = await _harness( + producer, store, config, obligation, _, clock = await _harness( factory, official=True, - finality_policy=ReportingSourceFinalPolicy("source", "delivery.is_final"), + finality_policy=( + ReportingSourceFinalPolicy("source", "delivery.is_final") + if basis == "source_final" + else ReportingContractualCutoffPolicy("cutoff", "PT2H") + ), + ) + clock[0] = END + timedelta(hours=2) + if core_only: + producer._store = CoreStore(store) + other_source = EvidenceSource(producer._source.capabilities, clock, []) + if change == "provider_basis": + other_source.basis = "provider_job_terminal" + elif change in {"finalized_at", "evidence_ref"}: + execute = other_source.execute + + async def changed_evidence(request: Any, **kwargs: Any) -> Any: + await execute(request, **kwargs) + manifest = other_source.manifest + evidence = manifest.finality_evidence.model_copy( + update=( + {"observed_at": clock[0] - timedelta(seconds=1)} + if change == "finalized_at" + else {"evidence_ref": "other-observation"} + ) + ) + raw = encode_source_batch_manifest_v1( + manifest.model_copy(update={"finality_evidence": evidence}) + ) + return ReportingSourceExecutorResult.completed( + request=request, + manifest=source_batch_manifest_reference_v1("manifest-redacted-v1", raw), + manifest_bytes=raw, + ) + + monkeypatch.setattr(other_source, "execute", changed_evidence) + other = ReportingProducer( + store=producer._store, + retry_store=producer._retry_store, + source=other_source, + object_reader=other_source, + offerings=producer._offerings, + clock=lambda: clock[0] + timedelta(minutes=1), ) turns = [WorkerTurn(), WorkerTurn()] - ready = asyncio.Event() - arrivals = 0 - commit = producer._commit_publication - - async def both_staged(*args: Any, **kwargs: Any) -> Any: - nonlocal arrivals - arrivals += 1 - if arrivals == 2: - ready.set() - await asyncio.wait_for(ready.wait(), timeout=5) - return await commit(*args, **kwargs) - - monkeypatch.setattr(producer, "_commit_publication", both_staged) - revisions = await asyncio.gather( - *(producer.acquire_obligation(config, obligation, turn=turn) for turn in turns) - ) - assert revisions[0] == revisions[1] + both_staged, first_published = asyncio.Event(), asyncio.Event() + hook = "_commit_revision_from_manifest" if prior_visible else "_commit_publication" + first_commit, second_commit = getattr(producer, hook), getattr(other, hook) + + async def publish_first(*args: Any, **kwargs: Any) -> Any: + await asyncio.wait_for(both_staged.wait(), timeout=5) + try: + return await first_commit(*args, **kwargs) + finally: + first_published.set() + + async def publish_second(*args: Any, **kwargs: Any) -> Any: + both_staged.set() + await asyncio.wait_for(first_published.wait(), timeout=5) + return await second_commit(*args, **kwargs) + + monkeypatch.setattr(producer, hook, publish_first) + monkeypatch.setattr(other, hook, publish_second) + winner, contender = await asyncio.gather( + producer.acquire_obligation(config, obligation, turn=turns[0], now=clock[0]), + other.acquire_obligation(config, obligation, turn=turns[1], now=clock[0]), + return_exceptions=True, + ) + assert isinstance(winner, ReportingRevisionRecord) + assert other_source.requests[0].identity == producer._source.requests[0].identity + if change == "identical": + assert contender == winner + else: + assert isinstance(contender, LedgerConflictError) + assert contender.code in {"REVISION_IMMUTABLE", "OBSERVATION_CONFLICT"} + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + assert await store.list_revisions(**owner) == (winner,) + history = await store.list_provisional_observations(**owner, limit=5) + if core_only: + assert not history + assert await producer.acquire_obligation(config, obligation) is None + assert ( + await producer.commit_revision_from_manifest( + obligation, producer._source.manifest, rows=ROWS, finality="official" + ) + == winner + ) + with pytest.raises(LedgerConflictError) as direct_conflict: + await producer.commit_revision_from_manifest( + obligation, + other_source.manifest, + rows=ROWS, + finality="official", + finality_proof=manifest_finality_proof(obligation, other_source.manifest, clock[0]), + ) + assert direct_conflict.value.code == "REVISION_IMMUTABLE" + else: + (observation,) = history + assert ( + SourceBatchManifestV1.model_validate_json(observation.manifest_json) + == producer._source.manifest + ) assert sum(len(turn.revisions_committed) for turn in turns) == 1 diff --git a/tests/test_reporting_scheduling_jitter.py b/tests/test_reporting_scheduling_jitter.py index 537b16496..e6fb14a67 100644 --- a/tests/test_reporting_scheduling_jitter.py +++ b/tests/test_reporting_scheduling_jitter.py @@ -169,8 +169,11 @@ async def test_jitter_keeps_explicit_official_boundary_and_manual_acquisition(): assert observation.acquisition.request().period.source_read_cutoff_at == clock[0] -async def test_manual_pending_retry_keeps_its_clock_before_first_automatic_phase(): - producer, store, configuration, obligation, _, clock = await _harness(read_jitter_window=WINDOW) +@pytest.mark.parametrize("store_class", [InMemoryReportingLedgerStore, ProgressStore]) +async def test_manual_pending_retry_keeps_its_clock_before_first_automatic_phase(store_class): + producer, store, configuration, obligation, _, clock = await _harness( + store_class, read_jitter_window=WINDOW, restatement_window="PT1H" + ) source = producer._source original_fetch = source._fetch requests = [] @@ -198,6 +201,7 @@ def unavailable_once(request): store=store, object_reader=producer._object_reader, clock=lambda: clock[0], + read_jitter_window=WINDOW, ) clock[0] = retry.retry_not_before - timedelta(microseconds=1) await restarted.run_configuration(configuration, now=clock[0]) @@ -206,6 +210,24 @@ def unavailable_once(request): await restarted.run_configuration(configuration, now=clock[0]) assert len(requests) == 2 and requests[1].identity == requests[0].identity assert requests[1].period.source_read_cutoff_at == requests[0].period.source_read_cutoff_at + owner = dict( + account_id=configuration.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + recovered = await store.get_provisional_observation(**owner) + assert recovered.acquisition.request().trigger == "manual_replay" + scheduled_due = producer._scheduled_source_ready_at(configuration, obligation) + clock[0] = scheduled_due - timedelta(microseconds=1) + await restarted.run_configuration(configuration, now=clock[0]) + assert len(requests) == 2 + clock[0] = scheduled_due + await restarted.run_configuration(configuration, now=clock[0]) + assert [r.trigger for r in requests] == ["manual_replay", "manual_replay", "scheduled_poll"] + assert requests[2].identity.source_execution_key != requests[0].identity.source_execution_key + history = await store.list_provisional_observations(**owner, limit=3) + assert [o.acquisition.request().trigger for o in history] == ["manual_replay", "scheduled_poll"] + assert [o.is_scheduled for o in history] == [False, True] + assert history[-1].checked_at == scheduled_due async def test_successful_manual_snapshot_keeps_immediate_automatic_official_close(): From 4a0856fce8e3bb90353d2b484d1819797b3414e5 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 15:52:56 +0530 Subject: [PATCH 15/39] feat(reporting): prove stabilized official finality --- .../src/adcp_reporting_storage/ledger/pg.py | 24 +- src/adcp/reporting/ledger/finality.py | 72 ++- src/adcp/reporting/ledger/producer.py | 495 ++++++++++++++++-- src/adcp/reporting/ledger/provisional.py | 69 ++- src/adcp/reporting/ledger/store.py | 76 ++- src/adcp/reporting/production/offerings.py | 19 +- src/adcp/reporting/service.py | 14 + ...test_reporting_production_configuration.py | 9 + tests/test_reliable_reporting_service.py | 18 + tests/test_reporting_official_finality.py | 385 ++++++++++++++ ...test_reporting_provisional_observations.py | 81 +++ 11 files changed, 1190 insertions(+), 72 deletions(-) diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py index 6455c5d8a..98bbf1059 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py @@ -1813,7 +1813,12 @@ async def get_provisional_observation( return history[0] if history else None async def list_provisional_observations( - self, *, account_id: str, reporting_obligation_id: str, limit: int + self, + *, + account_id: str, + reporting_obligation_id: str, + limit: int, + scheduled_snapshots_only: bool = False, ) -> tuple[ProvisionalObservation, ...]: if type(limit) is not int or limit <= 0: raise ValueError("observation history limit must be a positive integer") @@ -1823,8 +1828,18 @@ async def list_provisional_observations( await connection.execute( "SELECT payload FROM reporting_provisional_observations" " WHERE account_id=%s AND reporting_obligation_id=%s" + " AND (NOT %s OR (" + " payload #>> '{acquisition,request,trigger}' = 'scheduled_poll'" + " AND payload #>> '{acquisition,request,publication_class}'" + " = 'PROVISIONAL_SNAPSHOT'" + " AND payload #>> '{acquisition,output_finality}' IS NULL))" " ORDER BY ordinal DESC LIMIT %s", - (account_id, reporting_obligation_id, min(limit, _BIGINT_MAX)), + ( + account_id, + reporting_obligation_id, + scheduled_snapshots_only, + min(limit, _BIGINT_MAX), + ), ) ).fetchall() return tuple(ProvisionalObservation.from_wire(row[0]) for row in reversed(rows)) @@ -1882,7 +1897,9 @@ async def commit_provisional_observation( # Reuse the immutable publication replay checks while holding # the observation's account lock and transaction. if revision is None or (reuse_candidate and existing[0] == observation.revision_id): - validate_retained_provisional_observation(retained_revision, observation, rows) + validate_retained_provisional_observation( + retained_revision, retained_observation, rows + ) return retained_revision return await self.commit_revision(revision, rows) reserved = await ( @@ -1953,6 +1970,7 @@ async def commit_provisional_observation( observation = replace(observation, revision_id=committed.reporting_revision_id) else: committed = await self.commit_revision(revision, rows) + observation = observation.bind_revision(committed) await connection.execute( "INSERT INTO reporting_provisional_observations" " (account_id,reporting_obligation_id,ordinal,reporting_revision_id,payload)" diff --git a/src/adcp/reporting/ledger/finality.py b/src/adcp/reporting/ledger/finality.py index 3545610bf..b71edd221 100644 --- a/src/adcp/reporting/ledger/finality.py +++ b/src/adcp/reporting/ledger/finality.py @@ -3,7 +3,7 @@ from __future__ import annotations import re -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any, Literal @@ -12,10 +12,12 @@ if TYPE_CHECKING: from adcp.reporting.ledger.models import ReportingObligationRecord + from adcp.reporting.ledger.provisional import ProvisionalObservation from adcp.reporting.source import SourceBatchManifestV1 __all__ = [ "FinalityProof", + "MAX_STABILIZATION_HISTORY_OBSERVATIONS", "ReportingContractualCutoffPolicy", "ReportingFinalityPolicy", "ReportingSourceFinalPolicy", @@ -23,10 +25,14 @@ "contractual_cutoff_at", "finality_policy_source_class", "manifest_finality_proof", + "prove_stabilized_finality", "reporting_finality_policy_from_storage", "source_finality_proof", + "stabilization_history_limit", ] +MAX_STABILIZATION_HISTORY_OBSERVATIONS = 256 + def _validate_policy_id(value: str) -> None: if not isinstance(value, str) or re.fullmatch(r"[A-Za-z0-9_.:-]{1,255}", value) is None: @@ -110,12 +116,7 @@ class FinalityProof: basis: Literal["source_final", "contractual_cutoff", "stabilized"] finalized_at: datetime - def validate( - self, - obligation: ReportingObligationRecord, - manifest: SourceBatchManifestV1, - created_at: datetime, - ) -> None: + def validate_time(self, obligation: ReportingObligationRecord, created_at: datetime) -> None: from adcp.reporting.ledger.store import LedgerConflictError if self.finalized_at.utcoffset() is None or not ( @@ -126,12 +127,69 @@ def validate( raise LedgerConflictError( "PUBLICATION_TIME_INVALID", "proven finality is outside the publication time bounds" ) + + def validate( + self, + obligation: ReportingObligationRecord, + manifest: SourceBatchManifestV1, + created_at: datetime, + ) -> None: + from adcp.reporting.ledger.store import LedgerConflictError + + self.validate_time(obligation, created_at) if self != manifest_finality_proof(obligation, manifest, created_at): raise LedgerConflictError( "FINALITY_POLICY_UNPROVEN", "proof differs from the selected policy or its evidence" ) +def stabilization_history_limit(policy: ReportingStabilizedPolicy, cadence: timedelta) -> int: + """Admit only policy intervals that can be proved with a bounded ledger read.""" + from adcp.reporting.ledger.store import LedgerConflictError + + cadence_us = cadence // timedelta(microseconds=1) + duration_us = iso_duration_milliseconds_v1(policy.unchanged_for) * 1000 + limit = (duration_us + cadence_us - 1) // cadence_us + 2 + if limit > MAX_STABILIZATION_HISTORY_OBSERVATIONS: + raise LedgerConflictError( + "STABILIZATION_HISTORY_LIMIT", + f"stabilization history requires {limit} observations, exceeding the " + f"maximum of {MAX_STABILIZATION_HISTORY_OBSERVATIONS}", + ) + return limit + + +def prove_stabilized_finality( + obligation: ReportingObligationRecord, + policy: ReportingStabilizedPolicy, + history: Sequence[tuple[ProvisionalObservation, str]], +) -> FinalityProof | None: + """Evaluate a bounded, verified sequence of scheduled snapshot observations.""" + if not history: + return None + latest, _ = history[-1] + run_start = latest.checked_at + for (prior, prior_fingerprint), (current, fingerprint) in zip( + reversed(history[:-1]), reversed(history[1:]) + ): + dispatch_at = current.acquisition.request().period.source_read_cutoff_at + if ( + prior.next_due_at is None + or dispatch_at > prior.next_due_at + or prior.revision_id != current.revision_id + or prior_fingerprint != fingerprint + ): + break + run_start = prior.checked_at + age_ms = (latest.checked_at - obligation.period.end) // timedelta(milliseconds=1) + stable_ms = (latest.checked_at - run_start) // timedelta(milliseconds=1) + if age_ms < iso_duration_milliseconds_v1(policy.minimum_age) or ( + stable_ms < iso_duration_milliseconds_v1(policy.unchanged_for) + ): + return None + return FinalityProof(policy.finality_policy_id, policy.basis, latest.checked_at) + + def contractual_cutoff_at( obligation: ReportingObligationRecord, policy: ReportingContractualCutoffPolicy ) -> datetime: diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 25f1ea2af..512973947 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -61,12 +61,15 @@ from adcp.reporting.evidence import ReportingControlTotalRecord, freeze_control_totals from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal from adcp.reporting.ledger.finality import ( + MAX_STABILIZATION_HISTORY_OBSERVATIONS, FinalityProof, ReportingContractualCutoffPolicy, ReportingStabilizedPolicy, contractual_cutoff_at, finality_policy_source_class, manifest_finality_proof, + prove_stabilized_finality, + stabilization_history_limit, ) from adcp.reporting.ledger.models import ( ReportingConfiguration, @@ -97,6 +100,8 @@ RetryScheduleStore, managed_revision_metadata, validate_retained_provisional_observation, + validate_snapshot_observation_binding, + validate_snapshot_rows, ) from adcp.reporting.revision_selection import select_reporting_revision from adcp.reporting.source import ( @@ -222,6 +227,17 @@ def _source_finality(obligation: ReportingObligationRecord, output_finality: str return output_finality +def snapshot_restatement_cadence( + offering: ProvisionalSnapshotOfferingV1, period_duration: str +) -> timedelta: + if offering.restatement_cadence is not None: + return timedelta(milliseconds=iso_duration_milliseconds_v1(offering.restatement_cadence)) + return max( + iso_duration_to_timedelta(period_duration), + timedelta(milliseconds=iso_duration_milliseconds_v1(offering.fastest_safe_cadence)), + ) + + def revision_content_sha256( *, reporting_revision_id: str, @@ -399,6 +415,13 @@ class _SettlingPolicy: official_close_lag: timedelta | None +@dataclass(frozen=True) +class _StabilizedPublication: + proof: FinalityProof + manifest: SourceBatchManifestV1 + rows: Sequence[dict[str, Any]] + + class ReportingProducer: """Closes periods, drives the source, and commits immutable revisions.""" @@ -674,6 +697,24 @@ async def _close_elapsed_periods( ) after = boundary.end continue + selected = ( + configuration.definition.finality_policy if configuration.definition else None + ) + if ( + isinstance(selected, ReportingStabilizedPolicy) + and self._offerings.snapshot_offering_id + ): + offering = self._source.capabilities.offering(self._offerings.snapshot_offering_id) + if isinstance(offering, ProvisionalSnapshotOfferingV1) and ( + configuration.required_finality == "official" + or offering.official_close_lag is not None + ): + stabilization_history_limit( + selected, + snapshot_restatement_cadence( + offering, configuration.schedule.period_duration + ), + ) obligation = ReportingObligationRecord( reporting_obligation_id=self._obligation_id(configuration, boundary), account_id=configuration.account_id, @@ -863,6 +904,10 @@ async def _acquire_progress( finished = error.code != "OBSERVATION_CONFLICT" and ( policy is None or isinstance(error, LedgerConflictError) ) + if obligation.definition and isinstance( + obligation.definition.finality_policy, ReportingStabilizedPolicy + ): + finished = False if finished: await progress.finish_producer_acquisition( configuration, reporting_obligation_id=identifier @@ -989,22 +1034,17 @@ def _settling_policy( ) -> _SettlingPolicy | None: """Resolve the source policy or SDK fallback for a snapshot obligation.""" offering_id = self._offerings.snapshot_offering_id + selected = obligation.definition.finality_policy if obligation.definition else None if ( - obligation.required_finality != "snapshot" and not cutoff_reservation + obligation.required_finality != "snapshot" + and not cutoff_reservation + and not isinstance(selected, ReportingStabilizedPolicy) ) or offering_id is None: return None offering = self._source.capabilities.offering(offering_id) if not isinstance(offering, ProvisionalSnapshotOfferingV1): return None - if offering.restatement_cadence is not None: - cadence = timedelta( - milliseconds=iso_duration_milliseconds_v1(offering.restatement_cadence) - ) - else: - cadence = max( - iso_duration_to_timedelta(configuration.schedule.period_duration), - timedelta(milliseconds=iso_duration_milliseconds_v1(offering.fastest_safe_cadence)), - ) + cadence = snapshot_restatement_cadence(offering, configuration.schedule.period_duration) close_lag = ( timedelta(milliseconds=iso_duration_milliseconds_v1(offering.official_close_lag)) if offering.official_close_lag is not None @@ -1032,13 +1072,29 @@ async def _acquire_with_settling_policy( A readable snapshot completes one acquisition, not the settling policy. The progress store retains its rotating work item until the policy ends. """ + revisions = await self._store.list_revisions( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + if any(item.finality == "official" for item in revisions): + return True + selected = obligation.definition.finality_policy if obligation.definition else None + if isinstance(selected, ReportingStabilizedPolicy) and ( + obligation.required_finality == "official" or policy.official_close_lag is not None + ): + await self._acquire_stabilized(configuration, obligation, selected, turn, now) + revisions = await self._store.list_revisions( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + return any(r.finality == "official" for r in revisions) checkpoint_store = self._restatement_store() observation_store = self._observation_store() latest = await observation_store.get_provisional_observation( account_id=obligation.account_id, reporting_obligation_id=obligation.reporting_obligation_id, ) - scheduled = latest if latest is not None and latest.is_scheduled else None + scheduled = await self._latest_scheduled_snapshot(obligation, latest) if scheduled is not None: frozen = scheduled.acquisition.policy policy = _SettlingPolicy(frozen.window, frozen.cadence, frozen.official_close_lag) @@ -1046,12 +1102,6 @@ async def _acquire_with_settling_policy( policy.official_close_lag is not None and self._offerings.offering_for(_source_finality(obligation, "official")) is not None ) - revisions = await self._store.list_revisions( - account_id=obligation.account_id, - reporting_obligation_id=obligation.reporting_obligation_id, - ) - if any(item.finality == "official" for item in revisions): - return True checkpoint = await checkpoint_store.get_restatement_checkpoint( account_id=obligation.account_id, reporting_obligation_id=obligation.reporting_obligation_id, @@ -1107,12 +1157,11 @@ async def _acquire_with_settling_policy( # A tracked manual retry advances the durable ordinal only. The # first automatic source read keeps its configured phase and window. checkpoint = None - declared_until = _utc(obligation.period.end) + policy.restatement_window - settles_at = ( - _utc(checkpoint.provisional_until) - if checkpoint is not None and checkpoint.provisional_until is not None - else declared_until - ) + settles_at = _utc(obligation.period.end) + policy.restatement_window + if scheduled is not None: + settles_at = _utc(scheduled.provisional_until) + elif checkpoint is not None and checkpoint.provisional_until is not None: + settles_at = _utc(checkpoint.provisional_until) if scheduled is not None: next_due = scheduled.next_due_at elif checkpoint is not None: @@ -1174,6 +1223,274 @@ async def _acquire_with_settling_policy( return any(item.finality == "official" for item in revisions) return False + async def _scheduled_snapshot_history( + self, obligation: ReportingObligationRecord, limit: int + ) -> tuple[ProvisionalObservation, ...]: + store = self._observation_history_store() + read = store.list_provisional_observations + parameters = inspect.signature(read).parameters.values() + if any(p.name == "scheduled_snapshots_only" or p.kind == p.VAR_KEYWORD for p in parameters): + return await read( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + limit=limit, + scheduled_snapshots_only=True, + ) + # Custom publishers with the original history interface can still prove a + # run. Widen a mixed suffix only as far as the same finite evidence budget. + count = limit + while True: + history = await read( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + limit=count, + ) + scheduled = tuple(o for o in history if o.is_scheduled_snapshot) + if ( + len(scheduled) >= limit + or len(history) < count + or history[0].acquisition.ordinal == 0 + ): + return scheduled[-limit:] + if count == MAX_STABILIZATION_HISTORY_OBSERVATIONS: + raise LedgerConflictError( + "STABILIZATION_HISTORY_LIMIT", + "mixed observation history requires a scheduled snapshot filter", + ) + count = min(2 * count, MAX_STABILIZATION_HISTORY_OBSERVATIONS) + + async def _latest_scheduled_snapshot( + self, obligation: ReportingObligationRecord, latest: ProvisionalObservation | None + ) -> ProvisionalObservation | None: + if latest is None or latest.is_scheduled_snapshot: + return latest + if inspect.getattr_static(self._store, "list_provisional_observations", None) is None: + return None + try: + history = await self._scheduled_snapshot_history(obligation, 1) + except LedgerConflictError as error: + if error.code == "PROVISIONAL_OBSERVATION_HISTORY_NOT_SUPPORTED": + return None + raise + return history[-1] if history else None + + async def _acquire_stabilized( + self, + configuration: ReportingConfiguration, + obligation: ReportingObligationRecord, + policy: ReportingStabilizedPolicy, + turn: WorkerTurn, + now: datetime, + *, + manual: bool = False, + track_manual: bool = False, + ) -> ReportingRevisionRecord | None: + owner = dict( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + revisions = await self._store.list_revisions(**owner) + if any(r.finality == "official" for r in revisions): + return None + store = self._observation_history_store() + checkpoint = await self._restatement_store().get_restatement_checkpoint(**owner) + ordinal = checkpoint.next_observation if checkpoint else len(revisions) + pending = await self._pending_acquisition(obligation, ordinal=ordinal) + if pending is not None and pending.finality != "snapshot": + raise LedgerConflictError( + "OBSERVATION_CONFLICT", "stabilized finality requires a reserved snapshot output" + ) + latest = await self._latest_scheduled_snapshot( + obligation, await store.get_provisional_observation(**owner) + ) + if latest is not None and pending is None: + official = await self._promote_stabilized(obligation, policy, latest, turn) + if official is not None: + return official + if not manual and _utc(now) < _utc(latest.next_stabilization_due_at): + return None + elif latest is None and not manual and pending is None: + if _utc(now) < self._scheduled_source_ready_at(configuration, obligation): + return None + snapshot = await self.acquire_obligation( + configuration, + obligation, + target_finality="snapshot", + turn=turn, + now=now, + track_settling=not manual or track_manual, + restate=bool(revisions), + manual_replay=manual, + ) + if snapshot is not None: + observed = await store.get_provisional_observation(**owner) + if ( + observed is not None + and observed.is_scheduled_snapshot + and observed.acquisition.ordinal == ordinal + ): + official = await self._promote_stabilized(obligation, policy, observed, turn) + if official is not None: + return official + return snapshot + + async def _promote_stabilized( + self, + obligation: ReportingObligationRecord, + policy: ReportingStabilizedPolicy, + latest: ProvisionalObservation, + turn: WorkerTurn, + ) -> ReportingRevisionRecord | None: + evidence = await self._stabilized_publication(obligation, policy, latest) + if evidence is None: + return None + return await self._commit_revision_from_manifest( + obligation, + evidence.manifest, + rows=evidence.rows, + finality="official", + finality_proof=evidence.proof, + now=self._clock(), + turn=turn, + acquisition=None, + verified_source_rows=True, + ) + + async def _stabilized_publication( + self, + obligation: ReportingObligationRecord, + policy: ReportingStabilizedPolicy, + latest: ProvisionalObservation, + ) -> _StabilizedPublication | None: + limit = stabilization_history_limit(policy, latest.acquisition.policy.cadence) + history = await self._scheduled_snapshot_history(obligation, limit) + if not history or history[-1] != latest or len(history) > limit: + raise LedgerConflictError( + "OBSERVATION_CONFLICT", "stabilization history changed during proof" + ) + verified: list[tuple[ProvisionalObservation, str]] = [] + revision = None + rows: list[dict[str, Any]] = [] + for observation in history: + acquisition = observation.acquisition + if ( + not observation.is_scheduled_snapshot + or not acquisition.binds(obligation) + or ( + verified + and ( + acquisition.ordinal <= verified[-1][0].acquisition.ordinal + or acquisition.request().period.source_read_cutoff_at + < verified[-1][0].checked_at + ) + ) + ): + raise LedgerConflictError( + "HISTORY_UNAVAILABLE", "stabilization history acquisition differs" + ) + if revision is None or revision.reporting_revision_id != observation.revision_id: + revision = await self._store.get_revision( + account_id=obligation.account_id, reporting_revision_id=observation.revision_id + ) + if ( + revision is None + or not revision.readable + or revision.finality != "snapshot" + or revision.account_id != obligation.account_id + or revision.reporting_obligation_id != obligation.reporting_obligation_id + ): + raise LedgerConflictError( + "HISTORY_UNAVAILABLE", "stabilization history requires a readable snapshot" + ) + rows = await self._retained_snapshot_rows(revision) + try: + manifest = SourceBatchManifestV1.model_validate_json(observation.manifest_json) + self._validate_source_manifest( + obligation, + acquisition.request(), + manifest, + cutoff_ceiling=observation.checked_at, + acquisition=acquisition, + ) + if manifest.acquired_at > observation.checked_at: + raise ValueError("acquisition is later than its observation") + if acquisition.observation_deadlines(observation.checked_at, manifest) != ( + observation.provisional_until, + observation.next_due_at, + ): + raise ValueError("observation deadline differs from its frozen schedule") + if revision.source_publication_id == manifest.publication_id and ( + revision.observed_at != manifest.observed_at + or revision.data_through != manifest.data_through + or revision.supersedes_reporting_revision_id + != acquisition.predecessor_revision_id + ): + raise ValueError("originating snapshot differs from its publication") + validate_snapshot_observation_binding(revision, observation) + if ( + revision.managed_control_totals is not None + and revision.managed_control_totals + != _typed_control_totals(obligation, manifest) + ): + raise ValueError("snapshot control totals differ") + except (ValueError, LedgerConflictError) as error: + raise LedgerConflictError( + "HISTORY_UNAVAILABLE", + "stabilization history differs from its snapshot publication", + ) from error + verified.append((observation, manifest.content_fingerprint)) + proof = prove_stabilized_finality(obligation, policy, verified) + if proof is None: + return None + revisions = await self._store.list_revisions( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + if self._current_snapshot_leaf(revisions) != latest.revision_id: + return None + return _StabilizedPublication(proof, manifest, rows) + + async def _retained_snapshot_rows( + self, revision: ReportingRevisionRecord + ) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + cursor: str | None = None + while True: + page = await self._store.read_revision_rows( + account_id=revision.account_id, + reporting_revision_id=revision.reporting_revision_id, + cursor=cursor, + ) + if ( + page.reporting_revision_id not in (None, revision.reporting_revision_id) + or page.total_count != revision.row_count + ): + raise LedgerConflictError( + "HISTORY_UNAVAILABLE", + "stabilization history row page differs from its snapshot", + ) + rows.extend(page.rows) + if not page.has_more: + break + if ( + not page.rows + or page.cursor is None + or page.cursor == cursor + or len(rows) >= revision.row_count + ): + raise LedgerConflictError( + "HISTORY_UNAVAILABLE", "stabilization history rows are incomplete" + ) + cursor = page.cursor + try: + validate_snapshot_rows(revision, rows) + except LedgerConflictError as error: + raise LedgerConflictError( + "HISTORY_UNAVAILABLE", + "stabilization history rows differ from their snapshot binding", + ) from error + return rows + async def acquire_obligation( self, configuration: ReportingConfiguration, @@ -1208,6 +1525,17 @@ async def acquire_obligation( turn = turn or WorkerTurn() now = now or self._clock() finality = target_finality or obligation.required_finality + selected_policy = obligation.definition.finality_policy if obligation.definition else None + if finality == "official" and isinstance(selected_policy, ReportingStabilizedPolicy): + return await self._acquire_stabilized( + configuration, + obligation, + selected_policy, + turn, + now, + manual=manual_replay or (restate and not track_settling), + track_manual=track_settling, + ) if not manual_replay and await self._retry_not_before(obligation, turn, now=now): retry_offering_id: str | None = None if track_settling: @@ -1258,7 +1586,6 @@ async def acquire_obligation( # An official close is terminal. A later source correction is an # adjustment, never another acquisition. return None - selected_policy = obligation.definition.finality_policy if obligation.definition else None checkpoint_store = ( self._restatement_store() if track_settling else self._observation_checkpoint_store() ) @@ -1278,6 +1605,21 @@ async def acquire_obligation( ) if pending is not None: finality = pending.finality + elif ( + finality == "snapshot" + and track_settling + and not manual_replay + and isinstance(selected_policy, ReportingStabilizedPolicy) + ): + # Another worker can finish the due observation between selecting this + # period and reading its checkpoint. Its successor has a fresh deadline. + latest = await self._observation_store().get_provisional_observation( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + scheduled = await self._latest_scheduled_snapshot(obligation, latest) + if scheduled is not None and _utc(now) < _utc(scheduled.next_stabilization_due_at): + return None if current is not None and current.readable and not restate and pending is None: return None if finality == "official": @@ -1285,8 +1627,6 @@ async def acquire_obligation( raise LedgerConflictError( "FINALITY_POLICY_UNPROVEN", "official finality requires a selected policy" ) - if isinstance(selected_policy, ReportingStabilizedPolicy): - self._observation_history_store() # Everything below needs the frozen code: the slice request carries it, # the manifest is checked against it, and the revision is written under # it. Gate here rather than earlier so a settled legacy obligation stays @@ -1919,8 +2259,11 @@ async def _reserve_observation( account_id=obligation.account_id, reporting_obligation_id=obligation.reporting_obligation_id, ) - if latest is not None and latest.is_scheduled: - frozen_policy = latest.acquisition.policy + scheduled = await self._latest_scheduled_snapshot(obligation, latest) + selected = obligation.definition.finality_policy if obligation.definition else None + if scheduled is not None: + frozen_policy = scheduled.acquisition.policy + previous_boundary = scheduled.provisional_until else: if latest is not None: previous_boundary = None @@ -1935,6 +2278,12 @@ async def _reserve_observation( timedelta(0), iso_duration_to_timedelta(configuration.schedule.period_duration) ) ) + if isinstance(selected, ReportingStabilizedPolicy) and ( + obligation.required_finality == "official" + or frozen_policy.official_close_lag is not None + ): + stabilization_history_limit(selected, frozen_policy.cadence) + frozen_policy = replace(frozen_policy, until_stabilized=True) if self._requires_source_authority(obligation): self._source_authority(obligation, request) self._check_source_authorization( @@ -2236,6 +2585,17 @@ async def _commit_revision_from_manifest( control_totals = tuple((total.name, total.value) for total in manifest.control_totals) revision_id = f"rpr_{manifest.publication_id[4:44]}" + selected_policy = obligation.definition.finality_policy if obligation.definition else None + stabilized = finality == "official" and isinstance( + selected_policy, ReportingStabilizedPolicy + ) + if stabilized and selected_policy is not None: + digest = hashlib.sha256( + canonical_json_utf8_v1( + [manifest.publication_id, "official", selected_policy.finality_policy_id] + ) + ).hexdigest() + revision_id = f"rpr_{digest[:40]}" prior = next((item for item in existing if item.reporting_revision_id == revision_id), None) # New revisions bind protocol-shape totals. A replay of a retained # pair-hashed revision must reproduce its original binding, because @@ -2246,7 +2606,7 @@ async def _commit_revision_from_manifest( else None ) retained_official = prior is not None and prior.finality == finality == "official" - proved_candidate = finality == "official" and finality_proof is not None + candidate_proof = finality_proof if finality == "official" else None created_at = ( now if prior is None or (retained_official and finality_proof is not None) @@ -2331,8 +2691,18 @@ async def _commit_revision_from_manifest( raise LedgerConflictError( "PUBLICATION_TIME_INVALID", "proof is later than the publication clock" ) - finality_proof.validate(obligation, manifest, proof_anchor) - if retained_official and not verified_source_rows: + if stabilized: + # A single source publication cannot prove stabilization. The account + # publication lock protects the retained-history check at commit. + finality_proof.validate_time(obligation, proof_anchor) + if acquisition is not None: + raise LedgerConflictError( + "OBSERVATION_CONFLICT", + "commit the snapshot observation before stabilization", + ) + else: + finality_proof.validate(obligation, manifest, proof_anchor) + if stabilized or (retained_official and not verified_source_rows): acquisition = None elif not verified_source_rows: acquisition, rows = await self._verify_direct_official_publication( @@ -2356,7 +2726,7 @@ async def _commit_revision_from_manifest( "producer clock regressed during staged verification", ) created_at = now = completed_at - if finality == "official": + if finality == "official" and not stabilized: finality_proof = manifest_finality_proof(obligation, manifest, created_at) revision = ReportingRevisionRecord( reporting_revision_id=revision_id, @@ -2434,7 +2804,7 @@ async def _commit_revision_from_manifest( finality, now, acquisition, - proved_candidate=proved_candidate, + candidate_proof=candidate_proof, retained_revision_id=( retained.reporting_revision_id if retained is not None else None ), @@ -2516,7 +2886,9 @@ async def _retained_snapshot_replay( ) ): raise LedgerConflictError("REVISION_IMMUTABLE", "retained source totals differ") - validate_retained_provisional_observation(revision, observation, rows) + validate_retained_provisional_observation( + revision, latest if is_latest else observation, rows + ) if is_latest: return revision # A later observation closes this ordinal to new writes. The store @@ -2637,7 +3009,7 @@ async def _commit_publication( now: datetime, acquisition: ProvisionalAcquisition | None, *, - proved_candidate: bool, + candidate_proof: FinalityProof | None, retained_revision_id: str | None = None, ) -> tuple[ReportingRevisionRecord, bool]: async with self._revision_publication( @@ -2645,13 +3017,36 @@ async def _commit_publication( ): replay = None if finality == "official" and revision is not None: + selected = obligation.definition.finality_policy if obligation.definition else None + if candidate_proof is not None and isinstance(selected, ReportingStabilizedPolicy): + scheduled = await self._scheduled_snapshot_history(obligation, 1) + evidence = ( + await self._stabilized_publication(obligation, selected, scheduled[-1]) + if scheduled + else None + ) + if ( + evidence is None + or evidence.manifest != manifest + or evidence.proof != candidate_proof + or rfc8785.dumps(list(rows)) != rfc8785.dumps(list(evidence.rows)) + ): + raise LedgerConflictError( + "FINALITY_POLICY_UNPROVEN", + "retained snapshot history does not prove this official publication", + ) replay = await self._store.get_revision( account_id=obligation.account_id, reporting_revision_id=revision.reporting_revision_id, ) if replay is not None and replay.finality == "official": revision = await self._official_replay_revision( - revision, replay, manifest, acquisition, proved_candidate=proved_candidate + obligation, + revision, + replay, + manifest, + acquisition, + proved_candidate=candidate_proof is not None, ) acquisition = None if acquisition is None: @@ -2676,6 +3071,7 @@ async def _commit_publication( async def _official_replay_revision( self, + obligation: ReportingObligationRecord, candidate: ReportingRevisionRecord, retained: ReportingRevisionRecord, manifest: SourceBatchManifestV1, @@ -2684,16 +3080,23 @@ async def _official_replay_revision( proved_candidate: bool, ) -> ReportingRevisionRecord: """Compare official source evidence inside the account publication lock.""" - store = self._optional_observation_store() - observation = ( - await store.get_provisional_observation( - account_id=retained.account_id, - reporting_obligation_id=retained.reporting_obligation_id, - ) - if store is not None - else None - ) - if observation is None or observation.revision_id != retained.reporting_revision_id: + observation = None + if retained.finality_basis == "stabilized": + scheduled = await self._scheduled_snapshot_history(obligation, 1) + observation = scheduled[-1] if scheduled else None + else: + store = self._optional_observation_store() + if store is not None: + observation = await store.get_provisional_observation( + account_id=retained.account_id, + reporting_obligation_id=retained.reporting_obligation_id, + ) + if ( + observation is not None + and observation.revision_id != retained.reporting_revision_id + ): + observation = None + if observation is None: if proved_candidate: raise LedgerConflictError( "REVISION_IMMUTABLE", "the winning official source evidence is unavailable" diff --git a/src/adcp/reporting/ledger/provisional.py b/src/adcp/reporting/ledger/provisional.py index 90f073ca3..e443ea09b 100644 --- a/src/adcp/reporting/ledger/provisional.py +++ b/src/adcp/reporting/ledger/provisional.py @@ -8,7 +8,7 @@ import json from collections.abc import Sequence -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any, Literal, Protocol, runtime_checkable @@ -30,6 +30,7 @@ class ProvisionalPolicy: window: timedelta cadence: timedelta official_close_lag: timedelta | None = None + until_stabilized: bool = False def __post_init__(self) -> None: if self.window < timedelta(0) or self.cadence <= timedelta(0): @@ -39,6 +40,7 @@ def __post_init__(self) -> None: def to_wire(self) -> dict[str, int | None]: return { + **({"until_stabilized": True} if self.until_stabilized else {}), "window_us": self.window // timedelta(microseconds=1), "cadence_us": self.cadence // timedelta(microseconds=1), "official_close_lag_us": ( @@ -55,6 +57,7 @@ def from_wire(cls, value: dict[str, Any]) -> ProvisionalPolicy: timedelta(microseconds=value["window_us"]), timedelta(microseconds=value["cadence_us"]), timedelta(microseconds=lag) if lag is not None else None, + value.get("until_stabilized", False), ) @@ -101,6 +104,22 @@ def finality(self) -> Literal["snapshot", "official"]: return "official" return "snapshot" + def observation_deadlines( + self, checked_at: datetime, manifest: SourceBatchManifestV1 + ) -> tuple[datetime, datetime | None]: + boundary = utc( + manifest.finality_evidence.provisional_until + or (utc(manifest.period.end) + self.policy.window) + ) + if self.finality != "snapshot" or ( + not self.policy.until_stabilized and checked_at >= boundary + ): + return boundary, None + next_poll = utc(checked_at) + self.policy.cadence + if self.policy.until_stabilized: + return boundary, next_poll + return boundary, min(next_poll, boundary) + def binds(self, obligation: ReportingObligationRecord) -> bool: request = self.request() identity = request.identity @@ -156,11 +175,35 @@ class ProvisionalObservation: provisional_until: datetime next_due_at: datetime | None manifest_json: str + revision_publication_id: str | None = None @property def is_scheduled(self) -> bool: return self.acquisition.request().trigger == "scheduled_poll" + @property + def is_scheduled_snapshot(self) -> bool: + request = self.acquisition.request() + return ( + request.trigger == "scheduled_poll" + and request.publication_class == "PROVISIONAL_SNAPSHOT" + and self.acquisition.output_finality is None + ) + + @property + def next_stabilization_due_at(self) -> datetime: + return self.next_due_at or self.checked_at + self.acquisition.policy.cadence + + def bind_revision(self, revision: ReportingRevisionRecord) -> ProvisionalObservation: + """Keep the full originating publication ID when an unchanged read reuses a snapshot.""" + if self.acquisition.policy.until_stabilized and revision.finality == "snapshot": + return replace( + self, + revision_id=revision.reporting_revision_id, + revision_publication_id=revision.source_publication_id, + ) + return self + @classmethod def from_manifest( cls, @@ -170,15 +213,7 @@ def from_manifest( manifest: SourceBatchManifestV1, ) -> ProvisionalObservation: checked_at = utc(checked_at) - boundary = utc( - manifest.finality_evidence.provisional_until - or (utc(manifest.period.end) + acquisition.policy.window) - ) - next_due = ( - min(checked_at + acquisition.policy.cadence, boundary) - if acquisition.finality == "snapshot" and checked_at < boundary - else None - ) + boundary, next_due = acquisition.observation_deadlines(checked_at, manifest) return cls( acquisition, revision_id, checked_at, boundary, next_due, manifest.model_dump_json() ) @@ -198,6 +233,11 @@ def to_wire(self) -> dict[str, Any]: return { "acquisition": self.acquisition.to_wire(), "revision_id": self.revision_id, + **( + {"revision_publication_id": self.revision_publication_id} + if self.revision_publication_id is not None + else {} + ), "checked_at": utc(self.checked_at).isoformat(), "provisional_until": utc(self.provisional_until).isoformat(), "next_due_at": ( @@ -219,6 +259,7 @@ def from_wire(cls, value: dict[str, Any]) -> ProvisionalObservation: else None ), json.dumps(value["manifest"], sort_keys=True, separators=(",", ":")), + value.get("revision_publication_id"), ) @@ -258,11 +299,17 @@ async def commit_provisional_observation( @runtime_checkable class ProvisionalObservationStore(ProvisionalObservationPublicationStore, Protocol): async def list_provisional_observations( - self, *, account_id: str, reporting_obligation_id: str, limit: int + self, + *, + account_id: str, + reporting_obligation_id: str, + limit: int, + scheduled_snapshots_only: bool = False, ) -> tuple[ProvisionalObservation, ...]: """Return up to ``limit`` latest observations in ascending ordinal order. The read is scoped to the requested account and reporting obligation. + A scheduled snapshot filter applies before the limit. ``limit`` must be a positive integer. """ pass diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index 6840afa78..560b03ba0 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -711,6 +711,21 @@ def validate_retained_provisional_observation( require_readable: bool = True, ) -> None: """Validate an unchanged observation against immutable predecessor content.""" + validate_snapshot_observation_binding( + revision, observation, require_publication=observation.acquisition.policy.until_stabilized + ) + validate_snapshot_rows(revision, rows, require_readable=require_readable) + + +def validate_snapshot_observation_binding( + revision: ReportingRevisionRecord, + observation: ProvisionalObservation, + *, + require_publication: bool = True, +) -> None: + """An unchanged observation has its own publication and the retained snapshot's content.""" + from adcp.reporting.source import deterministic_source_publication_id_v1 + manifest = json.loads(observation.manifest_json) if ( revision.source_manifest_sha256 is None @@ -719,6 +734,49 @@ def validate_retained_provisional_observation( raise LedgerConflictError( "REVISION_IMMUTABLE", "observation content differs from the snapshot predecessor" ) + if not require_publication: + return + source_id = revision.source_publication_id + observation_source = manifest.get("publication_id") + request = observation.acquisition.request() + expected_publication = deterministic_source_publication_id_v1( + source_execution_key=request.identity.source_execution_key, + logical_slice_fingerprint=request.identity.logical_slice_fingerprint, + publication_class=request.publication_class, + revision_kind=request.revision_kind, + supersedes_publication_id=request.supersedes_publication_id, + ) + if ( + revision.reporting_revision_id != observation.revision_id + or source_id is None + or ( + observation.revision_publication_id is not None + and source_id != observation.revision_publication_id + ) + or revision.reporting_revision_id != f"rpr_{source_id[4:44]}" + or observation_source != expected_publication + or ( + observation_source != source_id + and ( + observation.acquisition.predecessor_revision_id != observation.revision_id + or f"rpr_{observation_source[4:44]}" == observation.revision_id + ) + ) + or manifest.get("row_count") != revision.row_count + or tuple((t["name"], t["value"]) for t in manifest.get("control_totals", ())) + != revision.control_totals + ): + raise LedgerConflictError( + "REVISION_IMMUTABLE", "retained snapshot source publication differs" + ) + + +def validate_snapshot_rows( + revision: ReportingRevisionRecord, + rows: Sequence[dict[str, Any]], + *, + require_readable: bool = True, +) -> None: from adcp.reporting.ledger.producer import recompute_revision_content_sha256 if len(rows) != revision.row_count: @@ -1357,12 +1415,23 @@ async def get_provisional_observation( return history[0] if history else None async def list_provisional_observations( - self, *, account_id: str, reporting_obligation_id: str, limit: int + self, + *, + account_id: str, + reporting_obligation_id: str, + limit: int, + scheduled_snapshots_only: bool = False, ) -> tuple[ProvisionalObservation, ...]: if type(limit) is not int or limit <= 0: raise ValueError("observation history limit must be a positive integer") observations = self._provisional_observations.get((account_id, reporting_obligation_id), {}) - newest_first = list(islice(reversed(observations.values()), min(limit, len(observations)))) + candidates = reversed(observations.values()) + newest_first = list( + islice( + (o for o in candidates if not scheduled_snapshots_only or o.is_scheduled_snapshot), + min(limit, len(observations)), + ) + ) return tuple(reversed(newest_first)) async def commit_provisional_observation( @@ -1407,7 +1476,7 @@ async def commit_provisional_observation( reuse_candidate and existing.revision_id == observation.revision_id ): validate_retained_provisional_observation( - self._revisions[existing.revision_id], observation, rows + self._revisions[existing.revision_id], existing, rows ) return self._revisions[existing.revision_id] return await self.commit_revision(revision, rows) @@ -1466,6 +1535,7 @@ async def commit_provisional_observation( else: committed = await self.commit_revision(revision, rows) # The checkpoint enforces ascending ordinals for distinct observations. + observation = observation.bind_revision(committed) self._provisional_observations.setdefault(owner, {})[acquisition.ordinal] = observation return committed diff --git a/src/adcp/reporting/production/offerings.py b/src/adcp/reporting/production/offerings.py index d012f5229..85b67dc0a 100644 --- a/src/adcp/reporting/production/offerings.py +++ b/src/adcp/reporting/production/offerings.py @@ -13,13 +13,21 @@ from adcp.reporting.canonical_json import canonical_json_utf8_v1 from adcp.reporting.inline_source import InlineReportingSource from adcp.reporting.ledger.delivery_models import ReportingDestinationBinding -from adcp.reporting.ledger.finality import ReportingStabilizedPolicy, finality_policy_source_class +from adcp.reporting.ledger.finality import ( + ReportingStabilizedPolicy, + finality_policy_source_class, + stabilization_history_limit, +) from adcp.reporting.ledger.models import ( ReportingConfiguration, _schedule_clock, iso_duration_to_timedelta, ) -from adcp.reporting.ledger.producer import ReportingProducer, _supports_official_source_formats +from adcp.reporting.ledger.producer import ( + ReportingProducer, + _supports_official_source_formats, + snapshot_restatement_cadence, +) from adcp.reporting.ledger.store import LedgerConflictError from adcp.reporting.materializer.contracts import ReportingVerificationKey, failure from adcp.reporting.materializer.verification import _same_definition @@ -197,6 +205,13 @@ def configured_source( if isinstance(policy, ReportingStabilizedPolicy): try: self.producer._observation_history_store() + assert isinstance(proof_source, ProvisionalSnapshotOfferingV1) + stabilization_history_limit( + policy, + snapshot_restatement_cadence( + proof_source, self.wire()["schedule"]["period_duration"] + ), + ) except LedgerConflictError: raise failure("UNSUPPORTED_VERIFICATION") from None if snapshot is None or snapshot.offering_id == proof_source.offering_id: diff --git a/src/adcp/reporting/service.py b/src/adcp/reporting/service.py index 30afed3f1..98bf1e4a6 100644 --- a/src/adcp/reporting/service.py +++ b/src/adcp/reporting/service.py @@ -53,10 +53,12 @@ ReportingFinalityPolicy, ReportingStabilizedPolicy, finality_policy_source_class, + stabilization_history_limit, ) from adcp.reporting.ledger.producer import ( _advertised_reporting_delivery, _supports_official_source_formats, + snapshot_restatement_cadence, ) from adcp.reporting.ledger.store import LedgerConflictError, decode_cursor from adcp.reporting.service_lifecycle import ( @@ -889,6 +891,18 @@ def _validate_offerings( raise ReliableReportingConfigurationError( f"selected finality policy requires a matching {source_class} source offering" ) + if isinstance(policy, ReportingStabilizedPolicy) and isinstance( + source, ProvisionalSnapshotOfferingV1 + ): + try: + stabilization_history_limit( + policy, + snapshot_restatement_cadence( + source, configuration.schedule.period_duration + ), + ) + except LedgerConflictError as error: + raise ReliableReportingConfigurationError(str(error)) from error used = [source] if ( configuration.required_finality == "snapshot" diff --git a/tests/conformance/reporting/test_reporting_production_configuration.py b/tests/conformance/reporting/test_reporting_production_configuration.py index 6cddb3052..b6a694768 100644 --- a/tests/conformance/reporting/test_reporting_production_configuration.py +++ b/tests/conformance/reporting/test_reporting_production_configuration.py @@ -48,6 +48,15 @@ (CUTOFF, "snapshot", ["official"], False, False, True, True), (CUTOFF, "snapshot", ["snapshot", "official"], False, False, True, True), (STABILIZED, "snapshot", ["official"], False, False, True, True), + ( + ReportingStabilizedPolicy("too-long", "PT0S", "P10000D"), + "snapshot", + ["official"], + False, + False, + True, + False, + ), (STABILIZED, "snapshot", ["official"], False, False, False, False), (None, "official", ["official"], False, True, True, False), (SOURCE_FINAL, "snapshot", ["official"], False, False, True, False), diff --git a/tests/test_reliable_reporting_service.py b/tests/test_reliable_reporting_service.py index 46134b41c..fca4bc5e1 100644 --- a/tests/test_reliable_reporting_service.py +++ b/tests/test_reliable_reporting_service.py @@ -401,6 +401,24 @@ class SnapshotStore(InMemoryReportingLedgerStore): await service.configure(config) +async def test_stabilized_configuration_rejects_excessive_history_before_admission() -> None: + config = replace( + _configuration(), + required_finality="official", + definition=replace( + DEFINITION, finality_policy=ReportingStabilizedPolicy("too-long", "PT0S", "P10000D") + ), + ) + service = _finality_service(config, snapshot=True, official=False) + await service.initialize() + with pytest.raises(ReliableReportingConfigurationError, match="history"): + await service.configure(config) + state = await service.store.read_status_snapshot( + caller=ReportingStatusCaller(config.account_id, config.consumer_id) + ) + assert not state.configurations and not state.obligations + + @pytest.mark.parametrize("media_type", ["application/json", "application/vnd.apache.parquet"]) async def test_service_refuses_an_unreadable_official_source_format_before_admission( media_type: str, diff --git a/tests/test_reporting_official_finality.py b/tests/test_reporting_official_finality.py index d09a9b090..763619b2a 100644 --- a/tests/test_reporting_official_finality.py +++ b/tests/test_reporting_official_finality.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from collections.abc import AsyncIterator from dataclasses import replace from datetime import timedelta @@ -30,6 +31,7 @@ ReportingProducer, ReportingRevisionRecord, ReportingSourceFinalPolicy, + ReportingStabilizedPolicy, revision_content_sha256, ) from adcp.reporting.ledger.finality import ( @@ -50,6 +52,8 @@ from tests.test_reporting_source_availability import ( END, PROVIDER_ROW, + ProgressStore, + _capabilities, _closing_capabilities, _harness, ) @@ -57,6 +61,387 @@ ROWS = [{"campaign_id": "campaign-redacted-1", "impressions": 10}] +async def _stabilized_harness( + store_factory: Any, *, minimum_age: str = "PT0S", official: bool = True +) -> tuple[Any, ...]: + result = await _harness( + store_factory, + official=official, + capabilities=_capabilities( + restatement_window="PT6H", restatement_cadence="PT6H", official_close_lag="PT6H" + ), + finality_policy=ReportingStabilizedPolicy("stable-12h", minimum_age, "PT12H"), + ) + result[-1][0] = END + timedelta(hours=1) + return result + + +@pytest.mark.parametrize("official", [False, True]) +async def test_stabilized_close_restarts_from_the_qualifying_observation( + store_factory: Any, monkeypatch: pytest.MonkeyPatch, official: bool +) -> None: + producer, store, config, obligation, calls, clock = await _stabilized_harness( + store_factory, official=official + ) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + await producer.run_configuration(config) + clock[0] += timedelta(hours=6) + await producer.run_configuration(config) + clock[0] += timedelta(hours=6) + qualified_at = clock[0] + original = store.commit_revision + + async def interrupted(revision: Any, rows: Any) -> Any: + if revision.finality == "official": + raise OSError("interrupted official commit") + return await original(revision, rows) + + with monkeypatch.context() as patch: + patch.setattr(store, "commit_revision", interrupted) + with pytest.raises(OSError, match="interrupted official commit"): + await producer.run_configuration(config) + history = await store.list_provisional_observations(**owner, limit=4) + assert [o.checked_at for o in history] == [END + timedelta(hours=h) for h in (1, 7, 13)] + (snapshot,) = await store.list_revisions(**owner) + assert all(o.revision_id == snapshot.reporting_revision_id for o in history) + clock[0] += timedelta(minutes=10) + fresh = ReportingProducer( + source=producer._source, + offerings=producer._offerings, + store=store, + object_reader=producer._object_reader, + clock=lambda: clock[0], + ) + turn = await fresh.run_configuration(config) + closed = next(r for r in await store.list_revisions(**owner) if r.finality == "official") + assert turn.revisions_committed == [closed.reporting_revision_id] + assert closed.reporting_revision_id != snapshot.reporting_revision_id + assert closed.supersedes_reporting_revision_id is None + assert closed.finality_policy_id == "stable-12h" and closed.finality_basis == "stabilized" + assert closed.finalized_at == qualified_at and closed.created_at == clock[0] + manifest = SourceBatchManifestV1.model_validate_json(history[-1].manifest_json) + assert closed.source_publication_id == manifest.publication_id + assert ( + await store.read_revision_rows( + account_id=config.account_id, reporting_revision_id=closed.reporting_revision_id + ) + ).rows == tuple(ROWS) + assert ( + await fresh.commit_revision_from_manifest( + obligation, manifest, rows=ROWS, finality="official" + ) + == closed + ) + assert not (await fresh.run_configuration(config)).revisions_committed + assert len(calls) == 3 + + +@pytest.mark.parametrize( + ("reset", "hours"), + [("changed", (1, 7, 13, 19)), ("gap", (1, 8, 14, 20)), ("age", (1, 7, 13, 19))], +) +async def test_stabilized_requires_a_complete_unchanged_run( + reset: str, hours: tuple[int, ...], monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, obligation, calls, clock = await _stabilized_harness( + ProgressStore, minimum_age="PT19H" if reset == "age" else "PT0S" + ) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + for idx, hour in enumerate(hours): + clock[0] = END + timedelta(hours=hour) + if reset == "changed" and idx == 1: + monkeypatch.setattr( + "tests.test_reporting_source_availability.PROVIDER_ROW", + PROVIDER_ROW.replace("10", "11"), + ) + await producer.run_configuration(config) + revisions = await store.list_revisions(**owner) + assert any(r.finality == "official" for r in revisions) == (idx == 3) + assert (obligation.reporting_obligation_id in store.pending) == (idx != 3) + assert len(calls) == 4 + assert sum(r.finality == "snapshot" for r in revisions) == (2 if reset == "changed" else 1) + + +@pytest.mark.parametrize("corruption", ["unreadable", "publication", "owner", "unchanged-manifest"]) +async def test_stabilized_requires_readable_history_and_retained_provenance( + store_factory: Any, monkeypatch: pytest.MonkeyPatch, corruption: str +) -> None: + producer, store, config, obligation, _, clock = await _stabilized_harness(store_factory) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + for hour in (1, 7): + clock[0] = END + timedelta(hours=hour) + await producer.acquire_obligation( + config, obligation, target_finality="snapshot", restate=True, track_settling=True + ) + (snapshot,) = await store.list_revisions(**owner) + if corruption == "unreadable": + await store.set_revision_readable( + account_id=config.account_id, + reporting_revision_id=snapshot.reporting_revision_id, + readable=False, + ) + elif corruption in {"publication", "owner"}: + get = store.get_revision + + async def corrupt_revision(**kwargs: Any) -> Any: + revision = await get(**kwargs) + if revision == snapshot: + if corruption == "owner": + return replace(revision, account_id="different-account") + return replace(revision, source_publication_id="src-" + "0" * 64) + return revision + + monkeypatch.setattr(store, "get_revision", corrupt_revision) + else: + read = store.list_provisional_observations + + async def corrupt_history(**kwargs: Any) -> Any: + observations = await read(**kwargs) + return tuple( + ( + replace( + o, + manifest_json=json.dumps( + {**json.loads(o.manifest_json), "publication_id": "src-" + "0" * 64} + ), + ) + if o.acquisition.ordinal == 1 + else o + ) + for o in observations + ) + + monkeypatch.setattr(store, "list_provisional_observations", corrupt_history) + clock[0] += timedelta(hours=6) + with pytest.raises(LedgerConflictError, match="history"): + await producer.run_configuration(config) + assert all(r.finality == "snapshot" for r in await store.list_revisions(**owner)) + + +async def test_stabilized_manual_history_cannot_move_the_due_time_or_displace_scheduled_reads( + store_factory: Any, +) -> None: + producer, store, config, obligation, calls, clock = await _stabilized_harness(store_factory) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + await producer.acquire_obligation( + config, obligation, target_finality="snapshot", track_settling=True + ) + first = await store.get_provisional_observation(**owner) + for ordinal in range(1, 7): + clock[0] += timedelta(minutes=5) + request = producer._build_slice( + config, + obligation, + SNAPSHOT_OFFERING_ID, + finality="snapshot", + now=clock[0], + observation=ordinal, + trigger="manual_replay", + ) + await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + request.model_dump_json(), ordinal, first.acquisition.policy, first.revision_id + ) + ) + await producer.acquire_obligation( + config, obligation, target_finality="snapshot", restate=True, manual_replay=True + ) + await producer.run_configuration(config) + assert len(calls) == 7 + for hour in (7, 13): + clock[0] = END + timedelta(hours=hour) + await producer.run_configuration(config) + history = await store.list_provisional_observations(**owner, limit=10) + assert [o.acquisition.ordinal for o in history if o.is_scheduled] == [0, 7, 8] + assert [r.finality for r in await store.list_revisions(**owner)].count("official") == 1 + + +async def test_stabilized_progress_survives_a_temporarily_unreadable_history() -> None: + producer, store, config, obligation, _, clock = await _stabilized_harness(ProgressStore) + await producer.run_configuration(config) + clock[0] += timedelta(hours=6) + await producer.run_configuration(config) + (snapshot,) = await store.list_revisions( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + key = dict(account_id=config.account_id, reporting_revision_id=snapshot.reporting_revision_id) + await store.set_revision_readable(**key, readable=False) + clock[0] += timedelta(hours=6) + turn = await producer.run_configuration(config) + assert obligation.reporting_obligation_id in turn.slices_failed + assert obligation.reporting_obligation_id in store.pending + await store.set_revision_readable(**key, readable=True) + assert (await producer.run_configuration(config)).revisions_committed + assert obligation.reporting_obligation_id not in store.pending + + +async def test_stabilized_history_rejects_an_extended_frozen_deadline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + producer, store, config, _, _, clock = await _stabilized_harness(InMemoryReportingLedgerStore) + await producer.run_configuration(config) + read = store.list_provisional_observations + + async def extended(**kwargs: Any) -> Any: + return tuple( + replace(o, next_due_at=o.next_due_at + timedelta(hours=1)) for o in await read(**kwargs) + ) + + monkeypatch.setattr(store, "list_provisional_observations", extended) + clock[0] += timedelta(hours=7) + with pytest.raises(LedgerConflictError, match="history"): + await producer.run_configuration(config) + + +async def test_stabilized_unchanged_suffix_binds_the_complete_origin_publication( + store_factory: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, obligation, _, clock = await _stabilized_harness(store_factory) + for hour in (1, 7, 13, 19, 25): + clock[0] = END + timedelta(hours=hour) + snapshot = await producer.acquire_obligation( + config, obligation, target_finality="snapshot", restate=True, track_settling=True + ) + assert snapshot is not None + original = snapshot.source_publication_id + forged = original[:-1] + ("1" if original[-1] == "0" else "0") + get = store.get_revision + + async def replace_origin(**kwargs: Any) -> Any: + revision = await get(**kwargs) + return replace(revision, source_publication_id=forged) if revision == snapshot else revision + + monkeypatch.setattr(store, "get_revision", replace_origin) + with pytest.raises(LedgerConflictError, match="history"): + await producer.acquire_obligation(config, obligation) + + +async def test_stabilized_replay_compares_the_detached_qualifying_manifest() -> None: + producer, store, config, obligation, _, clock = await _stabilized_harness( + InMemoryReportingLedgerStore + ) + for hour in (1, 7, 13): + clock[0] = END + timedelta(hours=hour) + await producer.run_configuration(config) + observed = await store.get_provisional_observation( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + manifest = SourceBatchManifestV1.model_validate_json(observed.manifest_json) + other = manifest.model_copy(update={"warnings": ["other source evidence"]}) + with pytest.raises(LedgerConflictError) as conflict: + await producer.commit_revision_from_manifest( + obligation, other, rows=ROWS, finality="official" + ) + assert conflict.value.code == "REVISION_IMMUTABLE" + + +async def test_settled_stabilized_obligation_needs_no_acquisition_history() -> None: + from tests.test_reporting_official_acquisition import CoreStore + + producer, store, config, obligation, calls, clock = await _stabilized_harness( + InMemoryReportingLedgerStore + ) + for hour in (1, 7, 13): + clock[0] = END + timedelta(hours=hour) + await producer.run_configuration(config) + reader = ReportingProducer( + source=producer._source, + offerings=producer._offerings, + store=CoreStore(store), + retry_store=store, + clock=lambda: clock[0], + ) + assert await reader.acquire_obligation(config, obligation) is None + assert not (await reader.run_configuration(config)).did_work + assert len(calls) == 3 + + +async def test_stabilized_unchanged_replay_binds_the_observations_own_publication( + monkeypatch: pytest.MonkeyPatch, +) -> None: + producer, store, config, obligation, _, clock = await _stabilized_harness( + InMemoryReportingLedgerStore + ) + for hour in (1, 7): + clock[0] = END + timedelta(hours=hour) + await producer.run_configuration(config) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + observed = await store.get_provisional_observation(**owner) + other = SourceBatchManifestV1.model_validate_json(observed.manifest_json).model_copy( + update={"publication_id": "src-" + "0" * 64} + ) + + async def corrupted(**kwargs: Any) -> Any: + return replace(observed, manifest_json=other.model_dump_json()) + + monkeypatch.setattr(store, "get_provisional_observation", corrupted) + with pytest.raises(LedgerConflictError) as conflict: + await producer.commit_revision_from_manifest( + obligation, other, rows=ROWS, finality="snapshot", acquisition=observed.acquisition + ) + assert conflict.value.code == "REVISION_IMMUTABLE" + + +async def test_stabilized_workers_cannot_consume_a_future_observation( + store_factory: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, obligation, calls, clock = await _stabilized_harness(store_factory) + await producer.run_configuration(config) + clock[0] += timedelta(hours=6) + other = ReportingProducer( + source=producer._source, + offerings=producer._offerings, + store=store, + object_reader=producer._object_reader, + clock=lambda: clock[0], + ) + acquire = producer.acquire_obligation + + async def after_other_worker(*args: Any, **kwargs: Any) -> Any: + await other.run_configuration(config) + return await acquire(*args, **kwargs) + + monkeypatch.setattr(producer, "acquire_obligation", after_other_worker) + await producer.run_configuration(config) + history = await store.list_provisional_observations( + account_id=config.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + limit=4, + ) + assert [o.acquisition.ordinal for o in history] == [0, 1] + assert len(calls) == 2 + + +async def test_stabilized_restart_reads_at_the_admitted_cadence_after_capabilities_change() -> None: + producer, store, config, obligation, calls, clock = await _harness( + official=True, + capabilities=_capabilities(restatement_window="P1D", restatement_cadence="PT6H"), + finality_policy=ReportingStabilizedPolicy("stable", "PT0S", "P30D"), + ) + clock[0] = END + timedelta(hours=1) + await producer.run_configuration(config) + producer._source.capabilities = _capabilities( + restatement_window="P1D", restatement_cadence="PT1H" + ) + clock[0] += timedelta(hours=6) + await producer.run_configuration(config) + assert len(calls) == 2 + observed = await store.get_provisional_observation( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + assert observed.next_due_at == clock[0] + timedelta(hours=6) + + @pytest.fixture(params=["memory", "postgres"]) async def store_factory(request: pytest.FixtureRequest) -> AsyncIterator[Any]: if request.param == "memory": diff --git a/tests/test_reporting_provisional_observations.py b/tests/test_reporting_provisional_observations.py index 9933793a7..be6f6e676 100644 --- a/tests/test_reporting_provisional_observations.py +++ b/tests/test_reporting_provisional_observations.py @@ -71,6 +71,74 @@ async def latest(store): return observation +async def test_stabilization_history_selects_only_scheduled_snapshot_outputs(make_harness): + producer, store, _, clock = await make_harness(_capabilities(restatement_window="P3D")) + await producer.run_worker() + scheduled = await latest(store) + owner = dict( + account_id=scheduled.acquisition.account_id, + reporting_obligation_id=scheduled.acquisition.obligation_id, + ) + # Retained history can interleave durable manual retries and official acquisitions. + payloads = [] + for ordinal, (trigger, source_class, output) in enumerate( + [ + ("scheduled_poll", "PROVISIONAL_SNAPSHOT", None), + ("manual_replay", "PROVISIONAL_SNAPSHOT", None), + ("scheduled_poll", "AUTHORITATIVE", None), + ("scheduled_poll", "PROVISIONAL_SNAPSHOT", "official"), + ] + ): + wire = scheduled.to_wire() + acquisition = wire["acquisition"] + acquisition["ordinal"] = ordinal + acquisition["request"]["trigger"] = trigger + acquisition["request"]["publication_class"] = source_class + if source_class == "AUTHORITATIVE": + acquisition["request"]["revision_kind"] = "authoritative" + if output: + acquisition["output_finality"] = output + payloads.append(wire) + if isinstance(store, PgReportingLedgerStore): + from psycopg.types.json import Jsonb + + async with store.transaction(), store._connection() as connection: + for ordinal, payload in enumerate(payloads[1:], start=1): + await connection.execute( + "INSERT INTO reporting_provisional_acquisitions" + " (account_id,reporting_obligation_id,ordinal,source_execution_key,payload)" + " VALUES (%s,%s,%s,%s,%s)", + ( + owner["account_id"], + owner["reporting_obligation_id"], + ordinal, + f"execution-{ordinal}", + Jsonb(payload["acquisition"]), + ), + ) + await connection.execute( + "INSERT INTO reporting_provisional_observations" + " (account_id,reporting_obligation_id,ordinal,reporting_revision_id,payload)" + " VALUES (%s,%s,%s,%s,%s)", + ( + owner["account_id"], + owner["reporting_obligation_id"], + ordinal, + scheduled.revision_id, + Jsonb(payload), + ), + ) + else: + history = store._provisional_observations[ + (owner["account_id"], owner["reporting_obligation_id"]) + ] + for ordinal, payload in enumerate(payloads[1:], start=1): + history[ordinal] = type(scheduled).from_wire(payload) + assert await store.list_provisional_observations( + **owner, limit=1, scheduled_snapshots_only=True + ) == (scheduled,) + + async def test_jitter_phase_is_persisted_by_memory_and_postgres_observations(make_harness): producer, store, _, clock = await make_harness(_capabilities(restatement_window="P3D")) configurations = await store.list_configurations( @@ -206,6 +274,19 @@ async def _publication_effect_counts(store): ) +async def test_snapshot_only_unchanged_read_reuses_retained_content_without_source_id(): + producer, store, _, clock = await _harness(_capabilities(restatement_window="P3D")) + await producer.run_worker() + (original,) = await _revisions(store) + store._revisions[original.reporting_revision_id] = replace(original, source_publication_id=None) + clock[0] += timedelta(hours=1) + turn = await producer.run_worker() + assert not turn.revisions_committed + observation = await latest(store) + assert observation.acquisition.ordinal == 1 + assert observation.revision_id == original.reporting_revision_id + + async def test_adapter_without_window_rereads_with_sdk_default(make_harness): producer, store, fetch, clock = await make_harness(_capabilities(restatement_window=None)) first = await producer.run_worker() From c62a274aff1b4d51cec87a5cb21111e6430530b1 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 16:20:25 +0530 Subject: [PATCH 16/39] fix(reporting): recover and verify stabilized snapshot evidence --- .../src/adcp_reporting_storage/ledger/pg.py | 38 ++- src/adcp/reporting/ledger/finality.py | 8 +- src/adcp/reporting/ledger/producer.py | 246 +++++++++++++++--- src/adcp/reporting/ledger/provisional.py | 24 +- src/adcp/reporting/ledger/store.py | 36 ++- src/adcp/reporting/production/offerings.py | 2 +- src/adcp/reporting/service.py | 42 ++- ..._reporting_prepared_observation_adapter.py | 6 +- ...test_reporting_production_configuration.py | 15 ++ tests/test_reliable_reporting_service.py | 81 ++++++ tests/test_reporting_official_finality.py | 230 ++++++++++++++++ 11 files changed, 658 insertions(+), 70 deletions(-) diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py index 98bbf1059..7283b5d27 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py @@ -136,7 +136,11 @@ issue_evidence, validate_scope_refinement, ) -from adcp.reporting.ledger.provisional import ProvisionalAcquisition, ProvisionalObservation +from adcp.reporting.ledger.provisional import ( + ProvisionalAcquisition, + ProvisionalObservation, + ProvisionalPolicy, +) from adcp.reporting.ledger.store import ( LeasedConfiguration, LedgerConflictError, @@ -1812,6 +1816,36 @@ async def get_provisional_observation( ) return history[0] if history else None + async def get_stabilization_schedule( + self, generation: ReportingConfigurationGenerationKey + ) -> ProvisionalPolicy | None: + async with self._connection() as connection: + await self._require_provisional_schema(connection) + row = await ( + await connection.execute( + "SELECT h.payload #> '{acquisition,policy}' FROM reporting_obligations o" + " JOIN reporting_provisional_observations h" + " USING(account_id, reporting_obligation_id)" + " WHERE o.account_id=%s AND o.consumer_id=%s AND o.delivery_config_id=%s" + " AND o.delivery_config_version=%s" + " AND h.payload #>> '{acquisition,request,trigger}' = 'scheduled_poll'" + " AND h.payload #>> '{acquisition,request,publication_class}'" + " = 'PROVISIONAL_SNAPSHOT'" + " AND h.payload #>> '{acquisition,output_finality}' IS NULL" + " AND (o.required_finality='official'" + " OR h.payload #>> '{acquisition,policy,until_stabilized}' = 'true'" + " OR h.payload #>> '{acquisition,policy,official_close_lag_us}' IS NOT NULL)" + " ORDER BY o.period_start, h.ordinal DESC LIMIT 1", + ( + generation.account_id, + generation.consumer_id, + generation.delivery_config_id, + generation.delivery_config_version, + ), + ) + ).fetchone() + return ProvisionalPolicy.from_wire(row[0]) if row is not None else None + async def list_provisional_observations( self, *, @@ -1959,7 +1993,7 @@ async def commit_provisional_observation( "OBSERVATION_CONFLICT", "snapshot predecessor is no longer current" ) validate_retained_provisional_observation( - committed, observation, rows, require_readable=False + committed, observation.bind_revision(committed), rows, require_readable=False ) if not committed.readable: if revision is None: diff --git a/src/adcp/reporting/ledger/finality.py b/src/adcp/reporting/ledger/finality.py index b71edd221..b88fc86e2 100644 --- a/src/adcp/reporting/ledger/finality.py +++ b/src/adcp/reporting/ledger/finality.py @@ -162,21 +162,21 @@ def stabilization_history_limit(policy: ReportingStabilizedPolicy, cadence: time def prove_stabilized_finality( obligation: ReportingObligationRecord, policy: ReportingStabilizedPolicy, - history: Sequence[tuple[ProvisionalObservation, str]], + history: Sequence[tuple[ProvisionalObservation, str, bytes]], ) -> FinalityProof | None: """Evaluate a bounded, verified sequence of scheduled snapshot observations.""" if not history: return None - latest, _ = history[-1] + latest, _, _ = history[-1] run_start = latest.checked_at - for (prior, prior_fingerprint), (current, fingerprint) in zip( + for (prior, prior_fingerprint, prior_rows), (current, fingerprint, current_rows) in zip( reversed(history[:-1]), reversed(history[1:]) ): dispatch_at = current.acquisition.request().period.source_read_cutoff_at if ( prior.next_due_at is None or dispatch_at > prior.next_due_at - or prior.revision_id != current.revision_id + or prior_rows != current_rows or prior_fingerprint != fingerprint ): break diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 512973947..43170d91e 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -73,6 +73,7 @@ ) from adcp.reporting.ledger.models import ( ReportingConfiguration, + ReportingConfigurationGenerationKey, ReportingDeliveryEscalation, ReportingObligationRecord, ReportingPeriodBoundary, @@ -86,6 +87,7 @@ ProvisionalObservationPublicationStore, ProvisionalObservationStore, ProvisionalPolicy, + StabilizationScheduleStore, ) from adcp.reporting.ledger.store import ( LeasedConfiguration, @@ -479,6 +481,9 @@ def __init__( self._post_deadline_retry_interval = post_deadline_retry_interval self._retry_store = retry_store self._read_jitter_window = read_jitter_window + self._recovered_stabilization: dict[ + ReportingConfigurationGenerationKey, ProvisionalPolicy + ] = {} @property def store(self) -> ReportingLedgerStore: @@ -700,19 +705,32 @@ async def _close_elapsed_periods( selected = ( configuration.definition.finality_policy if configuration.definition else None ) - if ( - isinstance(selected, ReportingStabilizedPolicy) - and self._offerings.snapshot_offering_id - ): - offering = self._source.capabilities.offering(self._offerings.snapshot_offering_id) - if isinstance(offering, ProvisionalSnapshotOfferingV1) and ( - configuration.required_finality == "official" - or offering.official_close_lag is not None + if isinstance(selected, ReportingStabilizedPolicy): + offering_id = self._offerings.snapshot_offering_id + offering = self._source.capabilities.offering(offering_id) if offering_id else None + frozen = self._recovered_stabilization.get(configuration.generation_key) + if ( + frozen is not None + or configuration.required_finality == "official" + or ( + isinstance(offering, ProvisionalSnapshotOfferingV1) + and offering.official_close_lag is not None + ) ): + self._stabilization_store() + if not isinstance(offering, ProvisionalSnapshotOfferingV1): + raise LedgerConflictError( + "SNAPSHOT_OFFERING_REQUIRED", + "stabilized finality requires a snapshot source offering", + ) stabilization_history_limit( selected, - snapshot_restatement_cadence( - offering, configuration.schedule.period_duration + ( + frozen.cadence + if frozen is not None + else snapshot_restatement_cadence( + offering, configuration.schedule.period_duration + ) ), ) obligation = ReportingObligationRecord( @@ -843,7 +861,10 @@ async def _acquire_pending( now=now, ) except (ReportingCurrencyError, LedgerConflictError) as error: - if isinstance(error, LedgerConflictError) and error.code != "OBSERVATION_CONFLICT": + if isinstance(error, LedgerConflictError) and error.code not in { + "OBSERVATION_CONFLICT", + "STABILIZATION_ORIGIN_UNAVAILABLE", + }: raise # Currency interpretation and competing observations are local # slice failures. Let later periods make progress; a later turn @@ -893,6 +914,7 @@ async def _acquire_progress( except (ReportingCurrencyError, LedgerConflictError) as error: if isinstance(error, LedgerConflictError) and error.code not in { "HISTORY_UNAVAILABLE", + "STABILIZATION_ORIGIN_UNAVAILABLE", "EMPTY_DENOMINATOR", "OBSERVATION_CONFLICT", }: @@ -1044,6 +1066,9 @@ def _settling_policy( offering = self._source.capabilities.offering(offering_id) if not isinstance(offering, ProvisionalSnapshotOfferingV1): return None + frozen = self._recovered_stabilization.get(configuration.generation_key) + if frozen is not None: + return _SettlingPolicy(frozen.window, frozen.cadence, frozen.official_close_lag) cadence = snapshot_restatement_cadence(offering, configuration.schedule.period_duration) close_lag = ( timedelta(milliseconds=iso_duration_milliseconds_v1(offering.official_close_lag)) @@ -1079,15 +1104,6 @@ async def _acquire_with_settling_policy( if any(item.finality == "official" for item in revisions): return True selected = obligation.definition.finality_policy if obligation.definition else None - if isinstance(selected, ReportingStabilizedPolicy) and ( - obligation.required_finality == "official" or policy.official_close_lag is not None - ): - await self._acquire_stabilized(configuration, obligation, selected, turn, now) - revisions = await self._store.list_revisions( - account_id=obligation.account_id, - reporting_obligation_id=obligation.reporting_obligation_id, - ) - return any(r.finality == "official" for r in revisions) checkpoint_store = self._restatement_store() observation_store = self._observation_store() latest = await observation_store.get_provisional_observation( @@ -1098,6 +1114,13 @@ async def _acquire_with_settling_policy( if scheduled is not None: frozen = scheduled.acquisition.policy policy = _SettlingPolicy(frozen.window, frozen.cadence, frozen.official_close_lag) + if isinstance(selected, ReportingStabilizedPolicy) and ( + obligation.required_finality == "official" + or policy.official_close_lag is not None + or (scheduled is not None and scheduled.acquisition.policy.until_stabilized) + ): + result = await self._acquire_stabilized(configuration, obligation, selected, turn, now) + return result is not None and result.finality == "official" explicit_close = ( policy.official_close_lag is not None and self._offerings.offering_for(_source_finality(obligation, "official")) is not None @@ -1274,6 +1297,66 @@ async def _latest_scheduled_snapshot( raise return history[-1] if history else None + async def _recover_stabilization_schedule( + self, + configuration: ReportingConfiguration, + ledger: ReportingLedgerStore, + ) -> ProvisionalPolicy | None: + """Restore a bound generation's schedule from a scheduled snapshot it retained.""" + if isinstance(ledger, StabilizationScheduleStore): + policy = await ledger.get_stabilization_schedule(configuration.generation_key) + else: + policy = await self._find_stabilization_schedule(configuration, ledger) + if policy is not None: + self._recovered_stabilization[configuration.generation_key] = policy + return policy + + async def _find_stabilization_schedule( + self, + configuration: ReportingConfiguration, + ledger: ReportingLedgerStore, + ) -> ProvisionalPolicy | None: + store = self._optional_observation_store() + if store is None: + return None + caller = ReportingDeliveryPrincipal( + account_id=configuration.account_id, consumer_id=configuration.consumer_id + ) + snapshot = await ledger.open_snapshot( + caller=caller, filters_fingerprint="stabilization-schedule" + ) + offset = 0 + while True: + page = await ledger.read_page( + snapshot=snapshot, + consumer_id=configuration.consumer_id, + delivery_config_ids=[configuration.delivery_config_id], + media_buy_ids=None, + offset=offset, + limit=128, + changes_after_sequence=None, + ) + for obligation in page.obligations: + if obligation.generation_key != configuration.generation_key: + continue + latest = await store.get_provisional_observation( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + observation = await self._latest_scheduled_snapshot(obligation, latest) + if observation is None: + continue + policy = observation.acquisition.policy + if ( + configuration.required_finality == "official" + or policy.until_stabilized + or policy.official_close_lag is not None + ): + return policy + if not page.has_more: + return None + offset += 128 + async def _acquire_stabilized( self, configuration: ReportingConfiguration, @@ -1292,10 +1375,35 @@ async def _acquire_stabilized( revisions = await self._store.list_revisions(**owner) if any(r.finality == "official" for r in revisions): return None - store = self._observation_history_store() - checkpoint = await self._restatement_store().get_restatement_checkpoint(**owner) + store = self._stabilization_store() + checkpoint_store = self._restatement_store() + checkpoint = await checkpoint_store.get_restatement_checkpoint(**owner) ordinal = checkpoint.next_observation if checkpoint else len(revisions) pending = await self._pending_acquisition(obligation, ordinal=ordinal) + if pending is not None and ( + pending.output_finality == "official" + and pending.request().publication_class == "PROVISIONAL_SNAPSHOT" + and not pending.policy.until_stabilized + ): + # The reserved output can never supply a scheduled snapshot observation. + # Advance only this cursor, preserving the immutable reservation for replay. + request = pending.request() + async with self._stabilization_lock().source_publication(obligation.account_id): + self._check_source_authorization( + configuration, request.offering_id, tuple(request.coverage.constituents) + ) + checkpoint = await checkpoint_store.get_restatement_checkpoint(**owner) + if checkpoint is not None and checkpoint.next_observation != ordinal: + return None + await self._retire_acquisition( + obligation, + ordinal=ordinal, + checked_at=now, + checkpoint_store=checkpoint_store, + checkpoint=checkpoint, + ) + ordinal += 1 + pending = await self._pending_acquisition(obligation, ordinal=ordinal) if pending is not None and pending.finality != "snapshot": raise LedgerConflictError( "OBSERVATION_CONFLICT", "stabilized finality requires a reserved snapshot output" @@ -1304,7 +1412,16 @@ async def _acquire_stabilized( obligation, await store.get_provisional_observation(**owner) ) if latest is not None and pending is None: - official = await self._promote_stabilized(obligation, policy, latest, turn) + try: + official = await self._promote_stabilized(obligation, policy, latest, turn) + except LedgerConflictError as error: + if ( + error.code != "STABILIZATION_ORIGIN_UNAVAILABLE" + or manual + or (_utc(now) < _utc(latest.next_stabilization_due_at)) + ): + raise + official = None if official is not None: return official if not manual and _utc(now) < _utc(latest.next_stabilization_due_at): @@ -1368,7 +1485,8 @@ async def _stabilized_publication( raise LedgerConflictError( "OBSERVATION_CONFLICT", "stabilization history changed during proof" ) - verified: list[tuple[ProvisionalObservation, str]] = [] + verified: list[tuple[ProvisionalObservation, str, bytes]] = [] + previous: ProvisionalObservation | None = None revision = None rows: list[dict[str, Any]] = [] for observation in history: @@ -1377,17 +1495,17 @@ async def _stabilized_publication( not observation.is_scheduled_snapshot or not acquisition.binds(obligation) or ( - verified + previous is not None and ( - acquisition.ordinal <= verified[-1][0].acquisition.ordinal - or acquisition.request().period.source_read_cutoff_at - < verified[-1][0].checked_at + acquisition.ordinal <= previous.acquisition.ordinal + or acquisition.request().period.source_read_cutoff_at < previous.checked_at ) ) ): raise LedgerConflictError( "HISTORY_UNAVAILABLE", "stabilization history acquisition differs" ) + previous = observation if revision is None or revision.reporting_revision_id != observation.revision_id: revision = await self._store.get_revision( account_id=obligation.account_id, reporting_revision_id=observation.revision_id @@ -1403,6 +1521,7 @@ async def _stabilized_publication( "HISTORY_UNAVAILABLE", "stabilization history requires a readable snapshot" ) rows = await self._retained_snapshot_rows(revision) + row_binding = hashlib.sha256(rfc8785.dumps(rows)).digest() try: manifest = SourceBatchManifestV1.model_validate_json(observation.manifest_json) self._validate_source_manifest( @@ -1434,20 +1553,22 @@ async def _stabilized_publication( ): raise ValueError("snapshot control totals differ") except (ValueError, LedgerConflictError) as error: + if ( + isinstance(error, LedgerConflictError) + and error.code == "STABILIZATION_ORIGIN_UNAVAILABLE" + ): + if observation == latest: + raise + verified.clear() + continue raise LedgerConflictError( "HISTORY_UNAVAILABLE", "stabilization history differs from its snapshot publication", ) from error - verified.append((observation, manifest.content_fingerprint)) + verified.append((observation, manifest.content_fingerprint, row_binding)) proof = prove_stabilized_finality(obligation, policy, verified) if proof is None: return None - revisions = await self._store.list_revisions( - account_id=obligation.account_id, - reporting_obligation_id=obligation.reporting_obligation_id, - ) - if self._current_snapshot_leaf(revisions) != latest.revision_id: - return None return _StabilizedPublication(proof, manifest, rows) async def _retained_snapshot_rows( @@ -1812,7 +1933,7 @@ async def acquire_obligation( await self._schedule_retry( obligation, turn, now=published_at, replayed=manual_replay ) - await self._retire_cutoff_attempt( + await self._retire_acquisition( obligation, ordinal=observation, checked_at=published_at, @@ -1849,7 +1970,7 @@ async def acquire_obligation( def _cutoff_ordinal_key(obligation: ReportingObligationRecord) -> str: return f"cutoff:{obligation.account_id}:{obligation.reporting_obligation_id}" - async def _retire_cutoff_attempt( + async def _retire_acquisition( self, obligation: ReportingObligationRecord, *, @@ -2131,6 +2252,12 @@ async def _revision_publication( ) -> AsyncIterator[None]: from adcp.reporting.production.contracts import ReportingProductionSource + if ( + official + and obligation.definition + and isinstance(obligation.definition.finality_policy, ReportingStabilizedPolicy) + ): + self._stabilization_lock() if not isinstance(self._source, ReportingProductionSource): if official and isinstance(self._store, ReportingSourcePublicationStore): async with self._store.source_publication(obligation.account_id): @@ -2243,6 +2370,30 @@ def _observation_history_store(self) -> ProvisionalObservationStore: ) return store + def _stabilization_lock(self) -> ReportingSourcePublicationStore: + store = self._store + stub = ReportingSourcePublicationStore.source_publication + declared = inspect.getattr_static(store, "source_publication", None) + implementation = ( + getattr(store, "source_publication", None) if declared is not None else None + ) + if ( + not callable(implementation) + or getattr(implementation, "__func__", implementation) is stub + or not isinstance(store, ReportingSourcePublicationStore) + ): + raise LedgerConflictError( + "STABILIZATION_PUBLICATION_NOT_SUPPORTED", + "stabilized finality requires an account publication lock", + ) + return store + + def _stabilization_store(self) -> ProvisionalObservationStore: + history = self._observation_history_store() + self._restatement_store() + self._stabilization_lock() + return history + async def _reserve_observation( self, configuration: ReportingConfiguration, @@ -2887,7 +3038,7 @@ async def _retained_snapshot_replay( ): raise LedgerConflictError("REVISION_IMMUTABLE", "retained source totals differ") validate_retained_provisional_observation( - revision, latest if is_latest else observation, rows + revision, latest if is_latest else observation.bind_revision(revision), rows ) if is_latest: return revision @@ -3081,7 +3232,8 @@ async def _official_replay_revision( ) -> ReportingRevisionRecord: """Compare official source evidence inside the account publication lock.""" observation = None - if retained.finality_basis == "stabilized": + policy = obligation.definition.finality_policy if obligation.definition else None + if isinstance(policy, ReportingStabilizedPolicy): scheduled = await self._scheduled_snapshot_history(obligation, 1) observation = scheduled[-1] if scheduled else None else: @@ -3097,7 +3249,7 @@ async def _official_replay_revision( ): observation = None if observation is None: - if proved_candidate: + if proved_candidate or isinstance(policy, ReportingStabilizedPolicy): raise LedgerConflictError( "REVISION_IMMUTABLE", "the winning official source evidence is unavailable" ) @@ -3105,6 +3257,20 @@ async def _official_replay_revision( acquisition is not None and observation.acquisition != acquisition ): raise LedgerConflictError("REVISION_IMMUTABLE", "official source observation differs") + if isinstance(policy, ReportingStabilizedPolicy) and not proved_candidate: + assert observation is not None + evidence = await self._stabilized_publication(obligation, policy, observation) + if ( + evidence is None + or evidence.proof.basis != retained.finality_basis + or evidence.proof.finality_policy_id != retained.finality_policy_id + or evidence.proof.finalized_at != retained.finalized_at + or evidence.manifest.publication_id != retained.source_publication_id + ): + raise LedgerConflictError( + "REVISION_IMMUTABLE", + "the retained history does not prove the official evidence", + ) # Creation and contractual finality are sampled by each producer. A # source-final timestamp belongs to the provider's immutable evidence. replay = replace( diff --git a/src/adcp/reporting/ledger/provisional.py b/src/adcp/reporting/ledger/provisional.py index e443ea09b..018ce8e0e 100644 --- a/src/adcp/reporting/ledger/provisional.py +++ b/src/adcp/reporting/ledger/provisional.py @@ -15,7 +15,11 @@ from adcp.reporting.source import ReportingSourceSliceRequestV1 if TYPE_CHECKING: - from adcp.reporting.ledger.models import ReportingObligationRecord, ReportingRevisionRecord + from adcp.reporting.ledger.models import ( + ReportingConfigurationGenerationKey, + ReportingObligationRecord, + ReportingRevisionRecord, + ) from adcp.reporting.source import SourceBatchManifestV1 @@ -197,6 +201,15 @@ def next_stabilization_due_at(self) -> datetime: def bind_revision(self, revision: ReportingRevisionRecord) -> ProvisionalObservation: """Keep the full originating publication ID when an unchanged read reuses a snapshot.""" if self.acquisition.policy.until_stabilized and revision.finality == "snapshot": + if ( + self.revision_publication_id is not None + and self.revision_publication_id != revision.source_publication_id + ): + from adcp.reporting.ledger.store import LedgerConflictError + + raise LedgerConflictError( + "REVISION_IMMUTABLE", "observation snapshot origin differs" + ) return replace( self, revision_id=revision.reporting_revision_id, @@ -263,6 +276,15 @@ def from_wire(cls, value: dict[str, Any]) -> ProvisionalObservation: ) +@runtime_checkable +class StabilizationScheduleStore(Protocol): + """Optional generation-scoped lookup of an admitted scheduled snapshot's policy.""" + + async def get_stabilization_schedule( + self, generation: ReportingConfigurationGenerationKey + ) -> ProvisionalPolicy | None: ... + + @runtime_checkable class PendingProvisionalAcquisitionStore(Protocol): """Optional lookup of a frozen acquisition before selecting its readiness window.""" diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index 560b03ba0..74a216790 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -76,7 +76,11 @@ issue_evidence, validate_scope_refinement, ) -from adcp.reporting.ledger.provisional import ProvisionalAcquisition, ProvisionalObservation +from adcp.reporting.ledger.provisional import ( + ProvisionalAcquisition, + ProvisionalObservation, + ProvisionalPolicy, +) if TYPE_CHECKING: from adcp.reporting.inline_source import ReportingSealStore @@ -749,10 +753,6 @@ def validate_snapshot_observation_binding( if ( revision.reporting_revision_id != observation.revision_id or source_id is None - or ( - observation.revision_publication_id is not None - and source_id != observation.revision_publication_id - ) or revision.reporting_revision_id != f"rpr_{source_id[4:44]}" or observation_source != expected_publication or ( @@ -769,6 +769,13 @@ def validate_snapshot_observation_binding( raise LedgerConflictError( "REVISION_IMMUTABLE", "retained snapshot source publication differs" ) + if ( + observation_source != source_id or observation.revision_publication_id is not None + ) and observation.revision_publication_id != source_id: + raise LedgerConflictError( + "STABILIZATION_ORIGIN_UNAVAILABLE", + "stabilization history lacks the exact source publication of its referenced snapshot", + ) def validate_snapshot_rows( @@ -1414,6 +1421,23 @@ async def get_provisional_observation( ) return history[0] if history else None + async def get_stabilization_schedule( + self, generation: ReportingConfigurationGenerationKey + ) -> ProvisionalPolicy | None: + for obligation in self._obligations.values(): + if obligation.generation_key != generation: + continue + owner = (obligation.account_id, obligation.reporting_obligation_id) + for observation in reversed(self._provisional_observations.get(owner, {}).values()): + policy = observation.acquisition.policy + if observation.is_scheduled_snapshot and ( + obligation.required_finality == "official" + or policy.until_stabilized + or policy.official_close_lag is not None + ): + return policy + return None + async def list_provisional_observations( self, *, @@ -1523,7 +1547,7 @@ async def commit_provisional_observation( "OBSERVATION_CONFLICT", "snapshot predecessor is no longer current" ) validate_retained_provisional_observation( - committed, observation, rows, require_readable=False + committed, observation.bind_revision(committed), rows, require_readable=False ) if not committed.readable: if revision is None: diff --git a/src/adcp/reporting/production/offerings.py b/src/adcp/reporting/production/offerings.py index 85b67dc0a..53e795e38 100644 --- a/src/adcp/reporting/production/offerings.py +++ b/src/adcp/reporting/production/offerings.py @@ -204,7 +204,7 @@ def configured_source( raise failure("UNSUPPORTED_VERIFICATION") if isinstance(policy, ReportingStabilizedPolicy): try: - self.producer._observation_history_store() + self.producer._stabilization_store() assert isinstance(proof_source, ProvisionalSnapshotOfferingV1) stabilization_history_limit( policy, diff --git a/src/adcp/reporting/service.py b/src/adcp/reporting/service.py index 98bf1e4a6..c759de9aa 100644 --- a/src/adcp/reporting/service.py +++ b/src/adcp/reporting/service.py @@ -60,6 +60,7 @@ _supports_official_source_formats, snapshot_restatement_cadence, ) +from adcp.reporting.ledger.provisional import ProvisionalPolicy from adcp.reporting.ledger.store import LedgerConflictError, decode_cursor from adcp.reporting.service_lifecycle import ( ReliableReportingServiceError, @@ -743,7 +744,6 @@ async def _configure( "registered source capabilities must be scoped to an effective account" ) try: - policy = self._validate_offerings(configuration, context, registration) offerings = context.producer_offerings() if not isinstance(offerings, ProducerOfferings): raise TypeError("producer_offerings must return ProducerOfferings") @@ -765,13 +765,23 @@ async def _configure( else {} ), ) - if isinstance(policy, ReportingStabilizedPolicy): - try: - producer._observation_history_store() - except LedgerConflictError as error: - raise ReliableReportingConfigurationError( - "stabilized finality requires durable provisional observation history" - ) from error + try: + frozen = None + if ( + recovering + and configuration.definition + and isinstance(configuration.definition.finality_policy, ReportingStabilizedPolicy) + ): + frozen = await producer._recover_stabilization_schedule(configuration, self.store) + policy = self._validate_offerings(configuration, context, registration, frozen=frozen) + if isinstance(policy, ReportingStabilizedPolicy): + producer._stabilization_store() + except LedgerConflictError as error: + raise ReliableReportingConfigurationError(str(error)) from error + except Exception as error: + if retrying or (recovering and isinstance(error, _MissingFinalityPolicyError)): + raise _AccountContextResolutionError(error, permanent=True) from error + raise binding = _Binding(configuration=configuration, context=context, producer=producer) if persist: if self._initialized: @@ -825,6 +835,8 @@ def _validate_offerings( configuration: ReportingConfiguration, context: ReportingAccountContext, registration: AdapterRegistration, + *, + frozen: ProvisionalPolicy | None = None, ) -> ReportingFinalityPolicy | None: capabilities = registration.executor.capabilities if capabilities.source_scope != _thaw(context.source_scope): @@ -878,8 +890,10 @@ def _validate_offerings( "snapshot configuration requires a matching snapshot source offering" ) policy = None - if configuration.required_finality == "official" or ( - snapshot is not None and snapshot.official_close_lag is not None + if ( + frozen is not None + or configuration.required_finality == "official" + or (snapshot is not None and snapshot.official_close_lag is not None) ): definition = configuration.definition if definition is None or definition.finality_policy is None: @@ -897,8 +911,12 @@ def _validate_offerings( try: stabilization_history_limit( policy, - snapshot_restatement_cadence( - source, configuration.schedule.period_duration + ( + frozen.cadence + if frozen is not None + else snapshot_restatement_cadence( + source, configuration.schedule.period_duration + ) ), ) except LedgerConflictError as error: diff --git a/tests/conformance/reporting/test_reporting_prepared_observation_adapter.py b/tests/conformance/reporting/test_reporting_prepared_observation_adapter.py index 590a12261..edd54e54f 100644 --- a/tests/conformance/reporting/test_reporting_prepared_observation_adapter.py +++ b/tests/conformance/reporting/test_reporting_prepared_observation_adapter.py @@ -41,7 +41,7 @@ async def list_provisional_observations( "publisher", [_PreparedRevisionStore, _PublisherWithProtocolStub, _PublisherWithNoncallableHistory], ) -async def test_stabilized_acquisition_requires_publisher_to_expose_history( +async def test_stabilized_admission_requires_publisher_to_expose_history( backend, publisher, monkeypatch ) -> None: async with reliable_factory(backend) as h: @@ -58,10 +58,8 @@ async def test_stabilized_acquisition_requires_publisher_to_expose_history( script = ScriptedSource(eur=[]) producer = h.producer(h.source(script.async_fetch)) monkeypatch.setattr(producer, "_store", publisher(h)) - (obligation,) = await producer.close_elapsed_periods(config) - with pytest.raises(LedgerConflictError) as error: - await producer.acquire_obligation(config, obligation) + await producer.close_elapsed_periods(config) assert error.value.code == "PROVISIONAL_OBSERVATION_HISTORY_NOT_SUPPORTED" assert script.requests == [] diff --git a/tests/conformance/reporting/test_reporting_production_configuration.py b/tests/conformance/reporting/test_reporting_production_configuration.py index b6a694768..e17484974 100644 --- a/tests/conformance/reporting/test_reporting_production_configuration.py +++ b/tests/conformance/reporting/test_reporting_production_configuration.py @@ -58,6 +58,7 @@ False, ), (STABILIZED, "snapshot", ["official"], False, False, False, False), + (STABILIZED, "snapshot", ["official"], False, False, "unlocked", False), (None, "official", ["official"], False, True, True, False), (SOURCE_FINAL, "snapshot", ["official"], False, False, True, False), (CUTOFF, "official", ["official"], False, True, True, False), @@ -89,6 +90,20 @@ async def test_production_offering_validates_selected_finality_policy( get_provisional_observation=h.store.get_provisional_observation, commit_provisional_observation=h.store.commit_provisional_observation, ) + if history == "unlocked": + store = SimpleNamespace( + **{ + name: getattr(h.store, name) + for name in ( + "reserve_provisional_acquisition", + "get_provisional_observation", + "commit_provisional_observation", + "list_provisional_observations", + "get_restatement_checkpoint", + "record_restatement_checkpoint", + ) + } + ) producer = ReportingProducer( source=source, store=store, diff --git a/tests/test_reliable_reporting_service.py b/tests/test_reliable_reporting_service.py index fca4bc5e1..7bf46546d 100644 --- a/tests/test_reliable_reporting_service.py +++ b/tests/test_reliable_reporting_service.py @@ -160,6 +160,7 @@ def _finality_service( results: tuple[InlineFetchResult, ...] = (), clock: DeterministicReportingClock | None = None, source_format: str | None = None, + cadence: str | None = None, products: tuple[str, ...] = (), adapter_type: type[ScriptedReportingAdapter] = ScriptedReportingAdapter, ) -> ReliableReportingService: @@ -179,6 +180,8 @@ def _finality_service( offering["windowing"]["minimum_window"] = "PT1H" if close: raw["offerings"][0].update(restatement_window="P1D", official_close_lag="P2D") + if cadence is not None: + raw["offerings"][0].update(restatement_window="P1D", restatement_cadence=cadence) raw["offerings"] = [ offering for offering, enabled in zip(raw["offerings"], (snapshot, official)) if enabled ] @@ -401,6 +404,84 @@ class SnapshotStore(InMemoryReportingLedgerStore): await service.configure(config) +async def test_stabilized_admission_requires_publication_lock_before_persisting() -> None: + class UnlockedHistory(InMemoryReportingLedgerStore): + source_publication = None + + config = replace( + _configuration(), + required_finality="official", + definition=replace( + DEFINITION, finality_policy=ReportingStabilizedPolicy("stable", "PT0S", "PT1H") + ), + ) + store = UnlockedHistory() + service = _finality_service(config, snapshot=True, official=False, store=store) + await service.initialize() + with pytest.raises(ReliableReportingConfigurationError, match="publication"): + await service.configure(config) + assert not await store.list_all_configurations() + + +async def test_initialize_recovers_frozen_stabilization_cadence_and_healthy_binding() -> None: + clock = DeterministicReportingClock(NOW + timedelta(days=1)) + store = InMemoryReportingLedgerStore(clock=clock) + config = replace( + _configuration(), + required_finality="official", + activated_at=NOW.replace(hour=1, minute=0), + deactivated_at=NOW.replace(hour=2, minute=0), + definition=replace( + DEFINITION, finality_policy=ReportingStabilizedPolicy("stable", "PT0S", "P30D") + ), + ) + original = _finality_service( + config, + snapshot=True, + official=False, + cadence="PT6H", + store=store, + clock=clock, + results=(InlineFetchResult(rows=_rows(10)),), + ) + await original.configure(config) + turn = await original.run_worker() + assert len(turn.configurations[config.generation_key].revisions_committed) == 1 + healthy = replace( + config, + delivery_config_id="healthy", + definition=replace( + DEFINITION, finality_policy=ReportingStabilizedPolicy("short", "PT0S", "PT6H") + ), + ) + await store.put_configuration(healthy) + recovered = _finality_service( + config, + snapshot=True, + official=False, + cadence="PT1H", + store=store, + clock=clock, + results=(InlineFetchResult(rows=_rows(10)),), + ) + await recovered.initialize() + assert set(recovered._bindings) == {config.generation_key, healthy.generation_key} + assert not recovered.initialization_errors + restarted = recovered._bindings[config.generation_key].producer + (obligation_id,) = turn.configurations[config.generation_key].obligations_committed + owner = dict(account_id=config.account_id, reporting_obligation_id=obligation_id) + first = await store.get_provisional_observation(**owner) + assert first is not None + clock.advance(timedelta(hours=1)) + await restarted.run_configuration(config) + assert await store.get_provisional_observation(**owner) == first + clock.advance(timedelta(hours=5)) + await restarted.run_configuration(config) + second = await store.get_provisional_observation(**owner) + assert second is not None and second.checked_at == clock() + assert second.acquisition.policy.cadence == timedelta(hours=6) + + async def test_stabilized_configuration_rejects_excessive_history_before_admission() -> None: config = replace( _configuration(), diff --git a/tests/test_reporting_official_finality.py b/tests/test_reporting_official_finality.py index 763619b2a..9e23beffc 100644 --- a/tests/test_reporting_official_finality.py +++ b/tests/test_reporting_official_finality.py @@ -61,6 +61,236 @@ ROWS = [{"campaign_id": "campaign-redacted-1", "impressions": 10}] +async def test_standalone_stabilization_requires_account_lock_before_obligations() -> None: + class UnlockedHistory(InMemoryReportingLedgerStore): + source_publication = None + + store = UnlockedHistory() + with pytest.raises(LedgerConflictError, match="publication"): + await _stabilized_harness(lambda **kwargs: store) + assert not store._obligations + + +async def test_standalone_stabilization_rejects_an_unbound_snapshot_source() -> None: + producer, _, config, _, _, _ = await _stabilized_harness(InMemoryReportingLedgerStore) + store = InMemoryReportingLedgerStore() + await store.put_configuration(config) + producer._store = store + producer._offerings = replace(producer._offerings, snapshot_offering_id=None) + with pytest.raises(LedgerConflictError, match="snapshot"): + await producer.close_elapsed_periods(config) + assert not store._obligations + + +async def test_native_recovery_reads_the_schedule_for_the_generation( + store_factory: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, _, _, _ = await _stabilized_harness(store_factory) + await producer.run_configuration(config) + + async def forbid_full_ledger(**kwargs: Any) -> Any: + pytest.fail("recovery should look up the generation's scheduled observation") + + monkeypatch.setattr(store, "read_page", forbid_full_ledger) + frozen = await producer._recover_stabilization_schedule(config, store) + assert frozen and frozen.cadence == timedelta(hours=6) + other = replace(config, delivery_config_version=2) + assert await producer._recover_stabilization_schedule(other, store) is None + + +@pytest.mark.parametrize("tracked", [False, True]) +async def test_manual_change_between_equal_scheduled_reads_preserves_stabilization( + store_factory: Any, monkeypatch: pytest.MonkeyPatch, tracked: bool +) -> None: + producer, store, config, obligation, calls, clock = await _stabilized_harness(store_factory) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + for hour in (1, 7): + clock[0] = END + timedelta(hours=hour) + await producer.run_configuration(config) + clock[0] = END + timedelta(hours=8) + prior = await store.get_provisional_observation(**owner) + if tracked: + request = producer._build_slice( + config, + obligation, + SNAPSHOT_OFFERING_ID, + finality="snapshot", + now=clock[0], + observation=2, + trigger="manual_replay", + ) + await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + request.model_dump_json(), 2, prior.acquisition.policy, prior.revision_id + ) + ) + with monkeypatch.context() as patch: + patch.setattr( + "tests.test_reporting_source_availability.PROVIDER_ROW", + PROVIDER_ROW.replace("10", "11"), + ) + manual = await producer.acquire_obligation( + config, obligation, restate=True, target_finality="snapshot", manual_replay=True + ) + assert manual is not None and manual.reporting_revision_id != prior.revision_id + clock[0] = END + timedelta(hours=13) + await producer.run_configuration(config) + revisions = await store.list_revisions(**owner) + assert [r.finalized_at for r in revisions if r.finality == "official"] == [clock[0]] + assert len(calls) == 4 + + +@pytest.mark.parametrize("origin", [None, "different"]) +async def test_unbound_reused_stabilization_history_recovers_after_fresh_evidence( + store_factory: Any, monkeypatch: pytest.MonkeyPatch, origin: str | None +) -> None: + producer, store, config, obligation, _, clock = await _stabilized_harness(store_factory) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + for hour in (1, 7, 13, 19, 25): + clock[0] = END + timedelta(hours=hour) + await producer.acquire_obligation( + config, obligation, target_finality="snapshot", restate=True, track_settling=True + ) + read = store.list_provisional_observations + + async def base_history(**kwargs: Any) -> Any: + return tuple( + replace(o, revision_publication_id=origin) if 0 < o.acquisition.ordinal <= 4 else o + for o in await read(**kwargs) + ) + + monkeypatch.setattr(store, "list_provisional_observations", base_history) + with pytest.raises(LedgerConflictError, match="exact source publication"): + await producer.acquire_obligation(config, obligation) + assert all(r.finality == "snapshot" for r in await store.list_revisions(**owner)) + for hour in (31, 37, 43): + clock[0] = END + timedelta(hours=hour) + await producer.acquire_obligation(config, obligation) + official = [r for r in await store.list_revisions(**owner) if r.finality == "official"] + assert [r.finalized_at for r in official] == ([clock[0]] if hour == 43 else []) + + +async def test_stabilized_close_keeps_frozen_route_when_offering_drops_close_lag( + store_factory: Any, +) -> None: + producer, store, config, obligation, _, clock = await _harness( + store_factory, + capabilities=_capabilities( + restatement_window="P3D", restatement_cadence="PT6H", official_close_lag="P4D" + ), + finality_policy=ReportingStabilizedPolicy("stable-12h", "PT0S", "PT12H"), + ) + clock[0] = END + timedelta(hours=1) + await producer.run_configuration(config) + producer._source.capabilities = _capabilities( + restatement_window="PT6H", restatement_cadence="PT6H", official_close_lag=None + ) + for hour in (7, 13): + clock[0] = END + timedelta(hours=hour) + await producer.run_configuration(config) + revisions = await store.list_revisions( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + assert [r.finalized_at for r in revisions if r.finality == "official"] == [clock[0]] + + +async def test_stabilized_retires_base_shaped_derived_official_reservation( + store_factory: Any, +) -> None: + producer, store, config, obligation, calls, clock = await _stabilized_harness(store_factory) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + request = producer._build_slice( + config, obligation, SNAPSHOT_OFFERING_ID, finality="official", now=clock[0] + ) + reservation = ProvisionalAcquisition( + request.model_dump_json(), + 0, + ProvisionalPolicy(timedelta(hours=6), timedelta(hours=6), timedelta(hours=6)), + output_finality="official", + ) + await store.reserve_provisional_acquisition(reservation) + assert "until_stabilized" not in reservation.to_wire()["policy"] + fresh = ReportingProducer( + source=producer._source, + store=store, + offerings=producer._offerings, + object_reader=producer._object_reader, + clock=lambda: clock[0], + read_jitter_window=timedelta(0), + ) + for hour in (1, 1, 7, 13): + clock[0] = END + timedelta(hours=hour) + await fresh.run_configuration(config) + history = await store.list_provisional_observations(**owner, limit=8) + assert [o.acquisition.ordinal for o in history] == [1, 2, 3] + assert all(o.is_scheduled_snapshot for o in history) + assert len(calls) == 3 and all( + r.identity.source_execution_key != reservation.execution_key + for r in producer._source.requests + ) + assert sorted(r.finality for r in await store.list_revisions(**owner)) == [ + "official", + "snapshot", + ] + + +@pytest.mark.parametrize("change", ["none", "warnings", "basis"]) +async def test_proofless_stabilized_replay_requires_its_qualifying_observation( + monkeypatch: pytest.MonkeyPatch, change: str +) -> None: + producer, store, config, obligation, _, clock = await _stabilized_harness( + InMemoryReportingLedgerStore + ) + for hour in (1, 7, 13): + clock[0] = END + timedelta(hours=hour) + await producer.run_configuration(config) + manifest = producer._source.manifest + if change == "warnings": + manifest = manifest.model_copy(update={"warnings": ["different evidence"]}) + if change == "basis": + (official,) = (r for r in store._revisions.values() if r.finality == "official") + store._revisions[official.reporting_revision_id] = replace( + official, finality_basis="source_final" + ) + + async def missing(**kwargs: Any) -> tuple[Any, ...]: + return () + + monkeypatch.setattr(store, "list_provisional_observations", missing) + assert await producer.acquire_obligation(config, obligation) is None + with pytest.raises(LedgerConflictError, match="evidence|history"): + await producer.commit_revision_from_manifest( + obligation, manifest, rows=ROWS, finality="official" + ) + + +async def test_proofless_stabilized_replay_requires_the_qualifying_interval( + monkeypatch: pytest.MonkeyPatch, +) -> None: + producer, store, config, obligation, _, clock = await _stabilized_harness( + InMemoryReportingLedgerStore + ) + for hour in (1, 7, 13): + clock[0] = END + timedelta(hours=hour) + await producer.run_configuration(config) + read = store.list_provisional_observations + + async def truncated(**kwargs: Any) -> Any: + return (await read(**kwargs))[-1:] + + monkeypatch.setattr(store, "list_provisional_observations", truncated) + with pytest.raises(LedgerConflictError, match="history|evidence"): + await producer.commit_revision_from_manifest( + obligation, producer._source.manifest, rows=ROWS, finality="official" + ) + + async def _stabilized_harness( store_factory: Any, *, minimum_age: str = "PT0S", official: bool = True ) -> tuple[Any, ...]: From b0f3379502cb556443c677fb4157c5d6d01ce685 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 16:44:46 +0530 Subject: [PATCH 17/39] fix(reporting): fence stabilized reservations and verify snapshot origins --- .../src/adcp_reporting_storage/ledger/pg.py | 61 ++++- src/adcp/reporting/ledger/producer.py | 221 +++++++++++++----- src/adcp/reporting/ledger/provisional.py | 55 ++++- src/adcp/reporting/ledger/store.py | 63 +++-- src/adcp/reporting/service.py | 6 +- tests/test_reliable_reporting_service.py | 48 +++- tests/test_reporting_official_finality.py | 166 ++++++++++++- 7 files changed, 522 insertions(+), 98 deletions(-) diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py index 7283b5d27..4d1684ab8 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py @@ -158,6 +158,7 @@ validate_provisional_revision, validate_retained_provisional_observation, validate_revision_currency, + validate_snapshot_origin, ) from adcp_reporting_storage.ledger.row_storage import ( REPORTING_ROW_STORAGE_DDL_PATH, @@ -1816,26 +1817,68 @@ async def get_provisional_observation( ) return history[0] if history else None + async def verify_snapshot_origin(self, revision: ReportingRevisionRecord) -> None: + async with self._connection() as connection: + witness = await ( + await connection.execute( + "SELECT origin FROM (SELECT COALESCE(" + " CASE WHEN payload #>> '{acquisition,predecessor_revision_id}'" + " IS DISTINCT FROM reporting_revision_id" + " THEN payload #>> '{manifest,publication_id}' END," + " payload->>'revision_publication_id') AS origin, ordinal" + " FROM reporting_provisional_observations WHERE account_id=%s" + " AND reporting_obligation_id=%s AND reporting_revision_id=%s) refs" + " WHERE origin IS NOT NULL ORDER BY ordinal LIMIT 1", + ( + revision.account_id, + revision.reporting_obligation_id, + revision.reporting_revision_id, + ), + ) + ).fetchone() + if witness is not None: + validate_snapshot_origin(revision, witness[0]) + async def get_stabilization_schedule( self, generation: ReportingConfigurationGenerationKey ) -> ProvisionalPolicy | None: async with self._connection() as connection: + # configure() can precede schema creation on a brand-new installation. + installed = await ( + await connection.execute( + "SELECT to_regclass('reporting_provisional_acquisitions') IS NOT NULL" + " OR to_regclass('reporting_provisional_observations') IS NOT NULL" + ) + ).fetchone() + if installed is not None and not installed[0]: + return None await self._require_provisional_schema(connection) row = await ( await connection.execute( - "SELECT h.payload #> '{acquisition,policy}' FROM reporting_obligations o" - " JOIN reporting_provisional_observations h" + "SELECT a.payload->'policy' FROM reporting_obligations o" + " JOIN reporting_provisional_acquisitions a" + " USING(account_id, reporting_obligation_id)" + " LEFT JOIN reporting_provisional_observations h" + " USING(account_id, reporting_obligation_id, ordinal)" + " LEFT JOIN reporting_restatement_checkpoints c" " USING(account_id, reporting_obligation_id)" " WHERE o.account_id=%s AND o.consumer_id=%s AND o.delivery_config_id=%s" " AND o.delivery_config_version=%s" - " AND h.payload #>> '{acquisition,request,trigger}' = 'scheduled_poll'" - " AND h.payload #>> '{acquisition,request,publication_class}'" + " AND a.payload #>> '{request,trigger}' = 'scheduled_poll'" + " AND a.payload #>> '{request,publication_class}'" " = 'PROVISIONAL_SNAPSHOT'" - " AND h.payload #>> '{acquisition,output_finality}' IS NULL" + " AND (a.payload->>'output_finality' IS NULL OR (h.ordinal IS NULL" + " AND a.payload->>'output_finality'='official'" + " AND a.payload #>> '{policy,until_stabilized}' IS DISTINCT FROM 'true'))" + " AND (h.ordinal IS NOT NULL OR c.next_observation IS NULL" + " OR c.next_observation=a.ordinal)" " AND (o.required_finality='official'" - " OR h.payload #>> '{acquisition,policy,until_stabilized}' = 'true'" - " OR h.payload #>> '{acquisition,policy,official_close_lag_us}' IS NOT NULL)" - " ORDER BY o.period_start, h.ordinal DESC LIMIT 1", + " OR a.payload #>> '{policy,until_stabilized}' = 'true'" + " OR a.payload #>> '{policy,official_close_lag_us}' IS NOT NULL)" + " ORDER BY (h.ordinal IS NOT NULL) DESC," + " COALESCE(h.payload->>'checked_at'," + " a.payload #>> '{request,period,source_read_cutoff_at}')::timestamptz DESC," + " a.ordinal DESC LIMIT 1", ( generation.account_id, generation.consumer_id, @@ -1992,6 +2035,8 @@ async def commit_provisional_observation( raise LedgerConflictError( "OBSERVATION_CONFLICT", "snapshot predecessor is no longer current" ) + if acquisition.policy.until_stabilized: + await self.verify_snapshot_origin(committed) validate_retained_provisional_observation( committed, observation.bind_revision(committed), rows, require_readable=False ) diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 43170d91e..bdb80e48b 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -87,6 +87,7 @@ ProvisionalObservationPublicationStore, ProvisionalObservationStore, ProvisionalPolicy, + SnapshotOriginStore, StabilizationScheduleStore, ) from adcp.reporting.ledger.store import ( @@ -103,6 +104,7 @@ managed_revision_metadata, validate_retained_provisional_observation, validate_snapshot_observation_binding, + validate_snapshot_origin, validate_snapshot_rows, ) from adcp.reporting.revision_selection import select_reporting_revision @@ -864,6 +866,7 @@ async def _acquire_pending( if isinstance(error, LedgerConflictError) and error.code not in { "OBSERVATION_CONFLICT", "STABILIZATION_ORIGIN_UNAVAILABLE", + "STABILIZATION_ORIGIN_CONFLICT", }: raise # Currency interpretation and competing observations are local @@ -915,6 +918,7 @@ async def _acquire_progress( if isinstance(error, LedgerConflictError) and error.code not in { "HISTORY_UNAVAILABLE", "STABILIZATION_ORIGIN_UNAVAILABLE", + "STABILIZATION_ORIGIN_CONFLICT", "EMPTY_DENOMINATOR", "OBSERVATION_CONFLICT", }: @@ -1325,6 +1329,8 @@ async def _find_stabilization_schedule( snapshot = await ledger.open_snapshot( caller=caller, filters_fingerprint="stabilization-schedule" ) + best: tuple[bool, datetime, int] | None = None + result = None offset = 0 while True: page = await ledger.read_page( @@ -1344,17 +1350,42 @@ async def _find_stabilization_schedule( reporting_obligation_id=obligation.reporting_obligation_id, ) observation = await self._latest_scheduled_snapshot(obligation, latest) - if observation is None: - continue - policy = observation.acquisition.policy - if ( - configuration.required_finality == "official" - or policy.until_stabilized - or policy.official_close_lag is not None - ): - return policy + if observation is not None: + acquisition = observation.acquisition + else: + owner = dict( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + checkpoints = self._optional_restatement_store() + checkpoint = ( + await checkpoints.get_restatement_checkpoint(**owner) + if checkpoints is not None + else None + ) + ordinal = ( + checkpoint.next_observation + if checkpoint + else len(await ledger.list_revisions(**owner)) + ) + pending = await self._pending_acquisition(obligation, ordinal=ordinal) + if pending is None: + continue + acquisition = pending + policy = acquisition.stabilization_schedule( + obligation.required_finality, pending=observation is None + ) + if policy is not None: + at = ( + observation.checked_at + if observation + else acquisition.request().period.source_read_cutoff_at + ) + order = (observation is not None, at, acquisition.ordinal) + if best is None or order > best: + best, result = order, policy if not page.has_more: - return None + return result offset += 128 async def _acquire_stabilized( @@ -1380,28 +1411,11 @@ async def _acquire_stabilized( checkpoint = await checkpoint_store.get_restatement_checkpoint(**owner) ordinal = checkpoint.next_observation if checkpoint else len(revisions) pending = await self._pending_acquisition(obligation, ordinal=ordinal) - if pending is not None and ( - pending.output_finality == "official" - and pending.request().publication_class == "PROVISIONAL_SNAPSHOT" - and not pending.policy.until_stabilized - ): - # The reserved output can never supply a scheduled snapshot observation. - # Advance only this cursor, preserving the immutable reservation for replay. - request = pending.request() - async with self._stabilization_lock().source_publication(obligation.account_id): - self._check_source_authorization( - configuration, request.offering_id, tuple(request.coverage.constituents) - ) - checkpoint = await checkpoint_store.get_restatement_checkpoint(**owner) - if checkpoint is not None and checkpoint.next_observation != ordinal: - return None - await self._retire_acquisition( - obligation, - ordinal=ordinal, - checked_at=now, - checkpoint_store=checkpoint_store, - checkpoint=checkpoint, - ) + if pending is not None and pending.legacy_stabilized_close: + if not await self._retire_stabilized_acquisition( + configuration, obligation, pending, now + ): + return None ordinal += 1 pending = await self._pending_acquisition(obligation, ordinal=ordinal) if pending is not None and pending.finality != "snapshot": @@ -1416,7 +1430,8 @@ async def _acquire_stabilized( official = await self._promote_stabilized(obligation, policy, latest, turn) except LedgerConflictError as error: if ( - error.code != "STABILIZATION_ORIGIN_UNAVAILABLE" + error.code + not in {"STABILIZATION_ORIGIN_UNAVAILABLE", "STABILIZATION_ORIGIN_CONFLICT"} or manual or (_utc(now) < _utc(latest.next_stabilization_due_at)) ): @@ -1546,6 +1561,10 @@ async def _stabilized_publication( ): raise ValueError("originating snapshot differs from its publication") validate_snapshot_observation_binding(revision, observation) + if isinstance(self._store, SnapshotOriginStore) and ( + not verified or verified[-1][0].revision_id != observation.revision_id + ): + await self._store.verify_snapshot_origin(revision) if ( revision.managed_control_totals is not None and revision.managed_control_totals @@ -1553,11 +1572,14 @@ async def _stabilized_publication( ): raise ValueError("snapshot control totals differ") except (ValueError, LedgerConflictError) as error: - if ( - isinstance(error, LedgerConflictError) - and error.code == "STABILIZATION_ORIGIN_UNAVAILABLE" - ): - if observation == latest: + if isinstance(error, LedgerConflictError) and error.code in { + "STABILIZATION_ORIGIN_UNAVAILABLE", + "STABILIZATION_ORIGIN_CONFLICT", + }: + if observation == latest or ( + error.code == "STABILIZATION_ORIGIN_CONFLICT" + and observation.revision_id == latest.revision_id + ): raise verified.clear() continue @@ -1801,7 +1823,17 @@ async def acquire_obligation( previous_boundary=checkpoint.provisional_until if checkpoint else None, derived_close=derived_close, ) + if acquisition is None: + return None if acquisition is not None: + if ( + isinstance(selected_policy, ReportingStabilizedPolicy) + and acquisition.legacy_stabilized_close + ): + await self._retire_stabilized_acquisition( + configuration, obligation, acquisition, now + ) + return None request = acquisition.request(deadline_at=_utc(now) + self._offerings.slice_timeout) finality = acquisition.finality if acquisition.output_finality and _utc(now) < _utc( @@ -1951,18 +1983,36 @@ async def acquire_obligation( untracked_restatement = ( finality == "snapshot" and manual_observation and acquisition is None ) - committed = await self._commit_revision_from_manifest( - obligation, - manifest, - rows=rows, - finality=finality, - finality_proof=proof, - now=published_at, - turn=turn, - acquisition=acquisition, - verified_source_rows=True, - suppress_untracked_unchanged=untracked_restatement, - ) + try: + committed = await self._commit_revision_from_manifest( + obligation, + manifest, + rows=rows, + finality=finality, + finality_proof=proof, + now=published_at, + turn=turn, + acquisition=acquisition, + verified_source_rows=True, + suppress_untracked_unchanged=untracked_restatement, + ) + except LedgerConflictError as error: + if ( + error.code == "STABILIZATION_ORIGIN_CONFLICT" + and acquisition is not None + and acquisition.is_scheduled_snapshot + ): + # A sealed unchanged read cannot repair its predecessor's conflicting origin. + if await self._retire_stabilized_acquisition( + configuration, obligation, acquisition, published_at + ): + await self._schedule_retry( + obligation, + turn, + now=published_at, + retry_after_seconds=acquisition.policy.cadence.total_seconds(), + ) + raise await self._clear_retry(obligation, turn=turn, now=published_at) return committed @@ -1970,6 +2020,38 @@ async def acquire_obligation( def _cutoff_ordinal_key(obligation: ReportingObligationRecord) -> str: return f"cutoff:{obligation.account_id}:{obligation.reporting_obligation_id}" + async def _retire_stabilized_acquisition( + self, + configuration: ReportingConfiguration, + obligation: ReportingObligationRecord, + acquisition: ProvisionalAcquisition, + now: datetime, + ) -> bool: + """Advance an unusable reservation only while it still owns the observation cursor.""" + store = self._restatement_store() + request = acquisition.request() + owner = dict( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + async with self._stabilization_lock().source_publication(obligation.account_id): + self._check_source_authorization( + configuration, request.offering_id, tuple(request.coverage.constituents) + ) + checkpoint = await store.get_restatement_checkpoint(**owner) + if checkpoint is not None and checkpoint.next_observation != acquisition.ordinal: + return False + if any(r.finality == "official" for r in await self._store.list_revisions(**owner)): + return False + await self._retire_acquisition( + obligation, + ordinal=acquisition.ordinal, + checked_at=now, + checkpoint_store=store, + checkpoint=checkpoint, + ) + return True + async def _retire_acquisition( self, obligation: ReportingObligationRecord, @@ -2404,7 +2486,7 @@ async def _reserve_observation( revisions: Sequence[ReportingRevisionRecord], previous_boundary: datetime | None, derived_close: bool, - ) -> ProvisionalAcquisition: + ) -> ProvisionalAcquisition | None: store = self._observation_store() latest = await store.get_provisional_observation( account_id=obligation.account_id, @@ -2440,16 +2522,25 @@ async def _reserve_observation( self._check_source_authorization( configuration, request.offering_id, tuple(request.coverage.constituents) ) - acquisition = await store.reserve_provisional_acquisition( - ProvisionalAcquisition( - request.model_dump_json(), - ordinal, - frozen_policy, - self._current_snapshot_leaf(revisions), - previous_boundary, - output_finality="official" if derived_close else None, - ) + candidate = ProvisionalAcquisition( + request.model_dump_json(), + ordinal, + frozen_policy, + self._current_snapshot_leaf(revisions), + previous_boundary, + output_finality="official" if derived_close else None, ) + if frozen_policy.until_stabilized: + async with self._stabilization_lock().source_publication(obligation.account_id): + current = await self._store.list_revisions( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + if any(revision.finality == "official" for revision in current): + return None + acquisition = await store.reserve_provisional_acquisition(candidate) + else: + acquisition = await store.reserve_provisional_acquisition(candidate) if not acquisition.binds(obligation): raise LedgerConflictError( "OBSERVATION_CONFLICT", @@ -2945,6 +3036,13 @@ async def _commit_revision_from_manifest( retained = predecessor if acquisition is None: return retained + if acquisition.policy.until_stabilized and isinstance( + selected_policy, ReportingStabilizedPolicy + ): + limit = stabilization_history_limit(selected_policy, acquisition.policy.cadence) + for observed in await self._scheduled_snapshot_history(obligation, limit): + if observed.revision_id == predecessor.reporting_revision_id: + validate_snapshot_origin(predecessor, observed.snapshot_origin) staged = staging is not None and bool(await staging.stage_revision_rows(revision, rows)) try: committed, replayed = await self._commit_publication( @@ -3124,7 +3222,8 @@ async def _verify_direct_official_publication( derived_close=request.publication_class == "PROVISIONAL_SNAPSHOT", ) if ( - (acquisition is not None and acquisition != winner) + winner is None + or (acquisition is not None and acquisition != winner) or winner.finality != "official" or not self._matches_reservation(request, winner) ): diff --git a/src/adcp/reporting/ledger/provisional.py b/src/adcp/reporting/ledger/provisional.py index 018ce8e0e..dcaebf03a 100644 --- a/src/adcp/reporting/ledger/provisional.py +++ b/src/adcp/reporting/ledger/provisional.py @@ -108,6 +108,39 @@ def finality(self) -> Literal["snapshot", "official"]: return "official" return "snapshot" + @property + def legacy_stabilized_close(self) -> bool: + return ( + self.output_finality == "official" + and not self.policy.until_stabilized + and self.request().publication_class == "PROVISIONAL_SNAPSHOT" + ) + + @property + def is_scheduled_snapshot(self) -> bool: + request = self.request() + return ( + request.trigger == "scheduled_poll" + and request.publication_class == "PROVISIONAL_SNAPSHOT" + and self.output_finality is None + ) + + def stabilization_schedule( + self, required_finality: str, *, pending: bool + ) -> ProvisionalPolicy | None: + if not self.is_scheduled_snapshot and not ( + pending and self.legacy_stabilized_close and self.request().trigger == "scheduled_poll" + ): + return None + policy = self.policy + if ( + required_finality == "official" + or policy.until_stabilized + or policy.official_close_lag is not None + ): + return policy + return None + def observation_deadlines( self, checked_at: datetime, manifest: SourceBatchManifestV1 ) -> tuple[datetime, datetime | None]: @@ -187,17 +220,20 @@ def is_scheduled(self) -> bool: @property def is_scheduled_snapshot(self) -> bool: - request = self.acquisition.request() - return ( - request.trigger == "scheduled_poll" - and request.publication_class == "PROVISIONAL_SNAPSHOT" - and self.acquisition.output_finality is None - ) + return self.acquisition.is_scheduled_snapshot @property def next_stabilization_due_at(self) -> datetime: return self.next_due_at or self.checked_at + self.acquisition.policy.cadence + @property + def snapshot_origin(self) -> str | None: + if self.acquisition.predecessor_revision_id != self.revision_id: + origin = json.loads(self.manifest_json).get("publication_id") + if isinstance(origin, str): + return origin + return self.revision_publication_id + def bind_revision(self, revision: ReportingRevisionRecord) -> ProvisionalObservation: """Keep the full originating publication ID when an unchanged read reuses a snapshot.""" if self.acquisition.policy.until_stabilized and revision.finality == "snapshot": @@ -276,6 +312,13 @@ def from_wire(cls, value: dict[str, Any]) -> ProvisionalObservation: ) +@runtime_checkable +class SnapshotOriginStore(Protocol): + """Check a revision against the first retained witness of its full publication ID.""" + + async def verify_snapshot_origin(self, revision: ReportingRevisionRecord) -> None: ... + + @runtime_checkable class StabilizationScheduleStore(Protocol): """Optional generation-scoped lookup of an admitted scheduled snapshot's policy.""" diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index 74a216790..07cbf82d3 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -721,6 +721,14 @@ def validate_retained_provisional_observation( validate_snapshot_rows(revision, rows, require_readable=require_readable) +def validate_snapshot_origin(revision: ReportingRevisionRecord, origin: str | None) -> None: + if origin is not None and revision.source_publication_id != origin: + raise LedgerConflictError( + "STABILIZATION_ORIGIN_CONFLICT", + "stabilization history contradicts the snapshot's exact source publication", + ) + + def validate_snapshot_observation_binding( revision: ReportingRevisionRecord, observation: ProvisionalObservation, @@ -741,6 +749,8 @@ def validate_snapshot_observation_binding( if not require_publication: return source_id = revision.source_publication_id + if source_id is not None and revision.reporting_revision_id == f"rpr_{source_id[4:44]}": + validate_snapshot_origin(revision, observation.snapshot_origin) observation_source = manifest.get("publication_id") request = observation.acquisition.request() expected_publication = deterministic_source_publication_id_v1( @@ -769,9 +779,8 @@ def validate_snapshot_observation_binding( raise LedgerConflictError( "REVISION_IMMUTABLE", "retained snapshot source publication differs" ) - if ( - observation_source != source_id or observation.revision_publication_id is not None - ) and observation.revision_publication_id != source_id: + validate_snapshot_origin(revision, observation.revision_publication_id) + if observation_source != source_id and observation.revision_publication_id is None: raise LedgerConflictError( "STABILIZATION_ORIGIN_UNAVAILABLE", "stabilization history lacks the exact source publication of its referenced snapshot", @@ -1421,22 +1430,46 @@ async def get_provisional_observation( ) return history[0] if history else None + async def verify_snapshot_origin(self, revision: ReportingRevisionRecord) -> None: + owner = (revision.account_id, revision.reporting_obligation_id) + for prior in self._provisional_observations.get(owner, {}).values(): + if prior.revision_id == revision.reporting_revision_id: + origin = prior.snapshot_origin + if origin is not None: + validate_snapshot_origin(revision, origin) + return + async def get_stabilization_schedule( self, generation: ReportingConfigurationGenerationKey ) -> ProvisionalPolicy | None: - for obligation in self._obligations.values(): + latest: tuple[bool, datetime, int] | None = None + result = None + for (account, identifier, ordinal), acquisition in self._provisional_acquisitions.items(): + obligation = self._obligations[identifier] if obligation.generation_key != generation: continue - owner = (obligation.account_id, obligation.reporting_obligation_id) - for observation in reversed(self._provisional_observations.get(owner, {}).values()): - policy = observation.acquisition.policy - if observation.is_scheduled_snapshot and ( - obligation.required_finality == "official" - or policy.until_stabilized - or policy.official_close_lag is not None - ): - return policy - return None + observed = self._provisional_observations.get((account, identifier), {}).get(ordinal) + policy = acquisition.stabilization_schedule( + obligation.required_finality, pending=observed is None + ) + if policy is None: + continue + checkpoint = self._restatement_checkpoints.get(identifier) + if ( + observed is None + and checkpoint is not None + and ordinal != checkpoint.next_observation + ): + continue + at = ( + observed.checked_at + if observed + else acquisition.request().period.source_read_cutoff_at + ) + order = (observed is not None, at, ordinal) + if latest is None or order > latest: + latest, result = order, policy + return result async def list_provisional_observations( self, @@ -1546,6 +1579,8 @@ async def commit_provisional_observation( raise LedgerConflictError( "OBSERVATION_CONFLICT", "snapshot predecessor is no longer current" ) + if acquisition.policy.until_stabilized: + await self.verify_snapshot_origin(committed) validate_retained_provisional_observation( committed, observation.bind_revision(committed), rows, require_readable=False ) diff --git a/src/adcp/reporting/service.py b/src/adcp/reporting/service.py index c759de9aa..a962af7e0 100644 --- a/src/adcp/reporting/service.py +++ b/src/adcp/reporting/service.py @@ -767,10 +767,8 @@ async def _configure( ) try: frozen = None - if ( - recovering - and configuration.definition - and isinstance(configuration.definition.finality_policy, ReportingStabilizedPolicy) + if configuration.definition and isinstance( + configuration.definition.finality_policy, ReportingStabilizedPolicy ): frozen = await producer._recover_stabilization_schedule(configuration, self.store) policy = self._validate_offerings(configuration, context, registration, frozen=frozen) diff --git a/tests/test_reliable_reporting_service.py b/tests/test_reliable_reporting_service.py index 7bf46546d..dbf0bdea7 100644 --- a/tests/test_reliable_reporting_service.py +++ b/tests/test_reliable_reporting_service.py @@ -157,7 +157,7 @@ def _finality_service( close: bool = False, store: InMemoryReportingLedgerStore | None = None, producer_factory: ProducerFactory = ReportingProducer, - results: tuple[InlineFetchResult, ...] = (), + results: tuple[InlineFetchResult | None, ...] = (), clock: DeterministicReportingClock | None = None, source_format: str | None = None, cadence: str | None = None, @@ -482,6 +482,52 @@ async def test_initialize_recovers_frozen_stabilization_cadence_and_healthy_bind assert second.acquisition.policy.cadence == timedelta(hours=6) +@pytest.mark.parametrize( + "reserved_only,configure_before_init", [(True, False), (True, True), (False, True)] +) +async def test_stabilization_startup_recovers_admitted_schedule( + reserved_only: bool, configure_before_init: bool +) -> None: + clock = DeterministicReportingClock(NOW + timedelta(days=1)) + config = replace( + _configuration(), + required_finality="official", + activated_at=NOW.replace(hour=1, minute=0), + deactivated_at=NOW.replace(hour=2, minute=0), + definition=replace( + DEFINITION, finality_policy=ReportingStabilizedPolicy("stable", "PT0S", "P30D") + ), + ) + store = InMemoryReportingLedgerStore(clock=clock) + original = _finality_service( + config, + snapshot=True, + official=False, + cadence="PT6H", + clock=clock, + store=store, + results=(None,) if reserved_only else (InlineFetchResult(rows=_rows(10)),), + ) + await original.configure(config) + await original.run_worker() + (obligation,) = store._obligations.values() + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + first = await store.get_provisional_acquisition(**owner, ordinal=0) + assert first is not None and first.policy.cadence == timedelta(hours=6) + assert (await store.get_provisional_observation(**owner) is None) == reserved_only + recovered = _finality_service( + config, snapshot=True, official=False, cadence="PT1H", clock=clock, store=store + ) + if configure_before_init: + await recovered.configure(config) + await recovered.initialize() + assert config.generation_key in recovered._bindings + with pytest.raises(ReliableReportingConfigurationError, match="history"): + await recovered.configure(replace(config, delivery_config_version=2)) + + async def test_stabilized_configuration_rejects_excessive_history_before_admission() -> None: config = replace( _configuration(), diff --git a/tests/test_reporting_official_finality.py b/tests/test_reporting_official_finality.py index 9e23beffc..11665d2a0 100644 --- a/tests/test_reporting_official_finality.py +++ b/tests/test_reporting_official_finality.py @@ -2,8 +2,10 @@ from __future__ import annotations +import asyncio import json from collections.abc import AsyncIterator +from contextlib import asynccontextmanager, suppress from dataclasses import replace from datetime import timedelta from functools import partial @@ -27,6 +29,7 @@ from adcp.reporting.ledger import ( InMemoryReportingLedgerStore, LedgerConflictError, + ReportingConfigurationGenerationKey, ReportingContractualCutoffPolicy, ReportingProducer, ReportingRevisionRecord, @@ -98,6 +101,40 @@ async def forbid_full_ledger(**kwargs: Any) -> Any: assert await producer._recover_stabilization_schedule(other, store) is None +@pytest.mark.parametrize("legacy", [False, True]) +async def test_recovery_finds_the_pending_snapshot_before_the_first_observation( + store_factory: Any, legacy: bool +) -> None: + store = store_factory() + assert ( + await store.get_stabilization_schedule( + ReportingConfigurationGenerationKey("account-redacted", "buyer", "availability", 1) + ) + is None + ) + producer, _, config, obligation, _, clock = await _stabilized_harness(lambda **kwargs: store) + request = producer._build_slice( + config, + obligation, + SNAPSHOT_OFFERING_ID, + finality="official" if legacy else "snapshot", + now=clock[0], + ) + policy = ProvisionalPolicy(timedelta(days=1), timedelta(hours=6), until_stabilized=not legacy) + await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + request.model_dump_json(), 0, policy, output_finality="official" if legacy else None + ) + ) + assert await producer._recover_stabilization_schedule(config, store) == policy + assert ( + await producer._recover_stabilization_schedule( + replace(config, delivery_config_version=2), store + ) + is None + ) + + @pytest.mark.parametrize("tracked", [False, True]) async def test_manual_change_between_equal_scheduled_reads_preserves_stabilization( store_factory: Any, monkeypatch: pytest.MonkeyPatch, tracked: bool @@ -142,9 +179,8 @@ async def test_manual_change_between_equal_scheduled_reads_preserves_stabilizati assert len(calls) == 4 -@pytest.mark.parametrize("origin", [None, "different"]) async def test_unbound_reused_stabilization_history_recovers_after_fresh_evidence( - store_factory: Any, monkeypatch: pytest.MonkeyPatch, origin: str | None + store_factory: Any, monkeypatch: pytest.MonkeyPatch ) -> None: producer, store, config, obligation, _, clock = await _stabilized_harness(store_factory) owner = dict( @@ -159,7 +195,7 @@ async def test_unbound_reused_stabilization_history_recovers_after_fresh_evidenc async def base_history(**kwargs: Any) -> Any: return tuple( - replace(o, revision_publication_id=origin) if 0 < o.acquisition.ordinal <= 4 else o + replace(o, revision_publication_id=None) if 0 < o.acquisition.ordinal <= 4 else o for o in await read(**kwargs) ) @@ -198,8 +234,9 @@ async def test_stabilized_close_keeps_frozen_route_when_offering_drops_close_lag assert [r.finalized_at for r in revisions if r.finality == "official"] == [clock[0]] +@pytest.mark.parametrize("pending_lookup", [True, False]) async def test_stabilized_retires_base_shaped_derived_official_reservation( - store_factory: Any, + store_factory: Any, monkeypatch: pytest.MonkeyPatch, pending_lookup: bool ) -> None: producer, store, config, obligation, calls, clock = await _stabilized_harness(store_factory) owner = dict( @@ -215,6 +252,8 @@ async def test_stabilized_retires_base_shaped_derived_official_reservation( output_finality="official", ) await store.reserve_provisional_acquisition(reservation) + if not pending_lookup: + monkeypatch.setattr(store, "get_provisional_acquisition", None) assert "until_stabilized" not in reservation.to_wire()["policy"] fresh = ReportingProducer( source=producer._source, @@ -240,6 +279,125 @@ async def test_stabilized_retires_base_shaped_derived_official_reservation( ] +async def test_stabilization_cannot_rebind_a_contradictory_snapshot_origin( + store_factory: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, obligation, calls, clock = await _stabilized_harness(store_factory) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + for hour in (1, 7, 13, 19, 25): + clock[0] = END + timedelta(hours=hour) + await producer.acquire_obligation( + config, obligation, target_finality="snapshot", restate=True, track_settling=True + ) + (snapshot,) = await store.list_revisions(**owner) + origin = snapshot.source_publication_id + forged = origin[:-1] + ("1" if origin[-1] == "0" else "0") + if isinstance(store, InMemoryReportingLedgerStore): + store._revisions[snapshot.reporting_revision_id] = replace( + snapshot, source_publication_id=forged + ) + else: + async with store._pool.connection() as connection: + await connection.execute( + "UPDATE reporting_revisions SET source_publication_id=%s" + " WHERE reporting_revision_id=%s", + (forged, snapshot.reporting_revision_id), + ) + # A previous publisher may already have copied the bad ID into an entire suffix. + read = store.list_provisional_observations + + async def rebound_suffix(**kwargs: Any) -> Any: + return tuple( + replace(o, revision_publication_id=forged) if o.acquisition.ordinal > 0 else o + for o in await read(**kwargs) + ) + + with monkeypatch.context() as patch: + patch.setattr(store, "list_provisional_observations", rebound_suffix) + with pytest.raises(LedgerConflictError, match="history"): + await producer.acquire_obligation(config, obligation) + for hour in (31, 31, 37, 43, 49): + clock[0] = END + timedelta(hours=hour) + with suppress(LedgerConflictError): + await producer.acquire_obligation(config, obligation) + history = await store.list_provisional_observations(**owner, limit=16) + assert all(o.revision_publication_id != forged for o in history) + assert all(r.finality == "snapshot" for r in await store.list_revisions(**owner)) + assert len(calls) == 9 + assert len({r.identity.source_execution_key for r in producer._source.requests}) == 9 + + monkeypatch.setattr( + "tests.test_reporting_source_availability.PROVIDER_ROW", PROVIDER_ROW.replace("10", "11") + ) + for hour in (55, 61, 67): + clock[0] = END + timedelta(hours=hour) + await producer.acquire_obligation(config, obligation) + official = [r for r in await store.list_revisions(**owner) if r.finality == "official"] + assert [r.finalized_at for r in official] == ([clock[0]] if hour == 67 else []) + + +async def test_scheduled_reservation_waiting_behind_stabilized_official_is_terminal( + store_factory: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, obligation, calls, clock = await _stabilized_harness(store_factory) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + for hour in (1, 7, 13): + clock[0] = END + timedelta(hours=hour) + await producer.acquire_obligation( + config, obligation, target_finality="snapshot", restate=True, track_settling=True + ) + clock[0] = END + timedelta(hours=19) + inside_official = asyncio.Event() + reaching_reservation = asyncio.Event() + release = asyncio.Event() + reserve = store.reserve_provisional_acquisition + publication_lock = store.source_publication + commit = store.commit_revision + reservations = [] + + async def hold_official(*args: Any, **kwargs: Any) -> Any: + inside_official.set() + await release.wait() + return await commit(*args, **kwargs) + + @asynccontextmanager + async def wait_for_lock(*args: Any, **kwargs: Any) -> AsyncIterator[Any]: + if inside_official.is_set(): + reaching_reservation.set() + async with publication_lock(*args, **kwargs) as publisher: + yield publisher + + async def at_reservation(*args: Any, **kwargs: Any) -> Any: + reaching_reservation.set() + reservations.append(args[0]) + return await reserve(*args, **kwargs) + + monkeypatch.setattr(store, "source_publication", wait_for_lock) + monkeypatch.setattr(store, "reserve_provisional_acquisition", at_reservation) + monkeypatch.setattr(store, "commit_revision", hold_official) + close = asyncio.create_task(producer.acquire_obligation(config, obligation)) + await asyncio.wait_for(inside_official.wait(), timeout=5) + attempt = asyncio.create_task( + producer.acquire_obligation( + config, obligation, target_finality="snapshot", restate=True, track_settling=True + ) + ) + try: + await asyncio.wait_for(reaching_reservation.wait(), timeout=5) + finally: + release.set() + official = await asyncio.wait_for(close, timeout=5) + assert official is not None and official.finality == "official" + assert await asyncio.wait_for(attempt, timeout=5) is None + assert reservations == [] + assert await store.get_provisional_acquisition(**owner, ordinal=3) is None + assert len(calls) == 3 + + @pytest.mark.parametrize("change", ["none", "warnings", "basis"]) async def test_proofless_stabilized_replay_requires_its_qualifying_observation( monkeypatch: pytest.MonkeyPatch, change: str From 88835ed23222d37714bc9276c07ab13ae067b9c0 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 17:06:16 +0530 Subject: [PATCH 18/39] fix(reporting): preserve stabilized recovery and terminal publication --- .../src/adcp_reporting_storage/ledger/pg.py | 50 +++- .../ledger/reporting_ledger.sql | 4 + src/adcp/reporting/ledger/producer.py | 205 +++++++------- src/adcp/reporting/ledger/provisional.py | 11 + src/adcp/reporting/ledger/store.py | 57 +++- src/adcp/reporting/service.py | 4 +- ...test_reporting_production_configuration.py | 8 + .../test_reporting_reconciliation_changes.py | 2 +- tests/test_reliable_reporting_service.py | 69 +++++ tests/test_reporting_official_finality.py | 266 ++++++++++++++++-- 10 files changed, 539 insertions(+), 137 deletions(-) diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py index 4d1684ab8..081bab92f 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py @@ -1518,6 +1518,20 @@ async def commit_revision( raise LedgerConflictError( "OBSERVATION_CONFLICT", "a source acquisition is reserved" ) + if ( + revision.finality == "snapshot" + and await ( + await connection.execute( + "SELECT 1 FROM reporting_revisions WHERE account_id=%s" + " AND reporting_obligation_id=%s AND finality='official' LIMIT 1", + (revision.account_id, revision.reporting_obligation_id), + ) + ).fetchone() + ): + raise LedgerConflictError( + "OFFICIAL_REVISION_TERMINAL", + "a new snapshot cannot follow the official revision", + ) if revision.supersedes_reporting_revision_id: await self._require_current_leaf(connection, revision) write_error = None @@ -1853,6 +1867,7 @@ async def get_stabilization_schedule( if installed is not None and not installed[0]: return None await self._require_provisional_schema(connection) + # The checkpoint's frozen policy can also be absent on pre-upgrade tables. row = await ( await connection.execute( "SELECT a.payload->'policy' FROM reporting_obligations o" @@ -1871,7 +1886,8 @@ async def get_stabilization_schedule( " AND a.payload->>'output_finality'='official'" " AND a.payload #>> '{policy,until_stabilized}' IS DISTINCT FROM 'true'))" " AND (h.ordinal IS NOT NULL OR c.next_observation IS NULL" - " OR c.next_observation=a.ordinal)" + " OR c.next_observation=a.ordinal OR (c.next_observation>a.ordinal" + " AND to_jsonb(c)->'stabilization_policy'=a.payload->'policy'))" " AND (o.required_finality='official'" " OR a.payload #>> '{policy,until_stabilized}' = 'true'" " OR a.payload #>> '{policy,official_close_lag_us}' IS NOT NULL)" @@ -1997,12 +2013,8 @@ async def commit_provisional_observation( # Consume the reservation under the account lock. Failure of the # ordinary revision write also rolls this checkpoint back. await self.record_restatement_checkpoint( - RestatementCheckpoint( - acquisition.account_id, - acquisition.obligation_id, - observation.checked_at, - acquisition.ordinal + 1, - observation.provisional_until, + RestatementCheckpoint.after_acquisition( + acquisition, observation.checked_at, observation.provisional_until, checkpoint ) ) if revision is None or reuse_candidate: @@ -2119,7 +2131,8 @@ async def get_restatement_checkpoint( row = await ( await connection.execute( "SELECT account_id, reporting_obligation_id, checked_at," - " next_observation, provisional_until" + " next_observation, provisional_until, stabilization_policy," + " next_stabilization_due_at" " FROM reporting_restatement_checkpoints" " WHERE account_id = %s AND reporting_obligation_id = %s", (account_id, reporting_obligation_id), @@ -2133,6 +2146,8 @@ async def get_restatement_checkpoint( checked_at=_utc(row[2]), next_observation=int(row[3]), provisional_until=_utc(row[4]) if row[4] else None, + stabilization_policy=ProvisionalPolicy.from_wire(row[5]) if row[5] else None, + next_stabilization_due_at=_utc(row[6]) if row[6] else None, ) async def record_restatement_checkpoint( @@ -2155,12 +2170,14 @@ async def record_restatement_checkpoint( await connection.execute( "INSERT INTO reporting_restatement_checkpoints" " (account_id, reporting_obligation_id, checked_at, next_observation," - " provisional_until)" - " VALUES (%s, %s, %s, %s, %s)" + " provisional_until, stabilization_policy, next_stabilization_due_at)" + " VALUES (%s, %s, %s, %s, %s, %s::jsonb, %s)" " ON CONFLICT (reporting_obligation_id) DO UPDATE SET" " checked_at = EXCLUDED.checked_at," " next_observation = EXCLUDED.next_observation," - " provisional_until = EXCLUDED.provisional_until" + " provisional_until = EXCLUDED.provisional_until," + " stabilization_policy = EXCLUDED.stabilization_policy," + " next_stabilization_due_at = EXCLUDED.next_stabilization_due_at" " WHERE reporting_restatement_checkpoints.account_id = EXCLUDED.account_id" " AND (reporting_restatement_checkpoints.next_observation" " < EXCLUDED.next_observation" @@ -2169,13 +2186,20 @@ async def record_restatement_checkpoint( " AND reporting_restatement_checkpoints.checked_at" " < EXCLUDED.checked_at))" " RETURNING account_id, reporting_obligation_id, checked_at," - " next_observation, provisional_until", + " next_observation, provisional_until, stabilization_policy," + " next_stabilization_due_at", ( checkpoint.account_id, checkpoint.reporting_obligation_id, checkpoint.checked_at, checkpoint.next_observation, checkpoint.provisional_until, + ( + json.dumps(checkpoint.stabilization_policy.to_wire()) + if checkpoint.stabilization_policy + else None + ), + checkpoint.next_stabilization_due_at, ), ) ).fetchone() @@ -2196,6 +2220,8 @@ async def record_restatement_checkpoint( checked_at=_utc(row[2]), next_observation=int(row[3]), provisional_until=_utc(row[4]) if row[4] else None, + stabilization_policy=ProvisionalPolicy.from_wire(row[5]) if row[5] else None, + next_stabilization_due_at=_utc(row[6]) if row[6] else None, ) async def get_revision( diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger.sql b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger.sql index cb57362cc..224a727e1 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger.sql +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger.sql @@ -199,6 +199,10 @@ CREATE TABLE IF NOT EXISTS reporting_restatement_checkpoints ( CREATE INDEX IF NOT EXISTS reporting_restatement_checkpoints_account_idx ON reporting_restatement_checkpoints (account_id, checked_at); +ALTER TABLE reporting_restatement_checkpoints + ADD COLUMN IF NOT EXISTS stabilization_policy JSONB, + ADD COLUMN IF NOT EXISTS next_stabilization_due_at TIMESTAMPTZ; + -- Mutable scheduling state; it does not enter the immutable ledger change feed. CREATE TABLE IF NOT EXISTS adcp_reporting_producer_retry_schedules ( scope_key TEXT COLLATE "C" PRIMARY KEY, diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index bdb80e48b..febda9da7 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -89,6 +89,7 @@ ProvisionalPolicy, SnapshotOriginStore, StabilizationScheduleStore, + StabilizationStore, ) from adcp.reporting.ledger.store import ( LeasedConfiguration, @@ -674,8 +675,27 @@ async def close_elapsed_periods( "REPORTING_GENERATION_QUARANTINED", "legacy generations cannot run" ) turn = WorkerTurn() + if self._stabilization_requested(configuration): + self._stabilization_store() return await self._close_elapsed_periods(configuration, turn, now=now or self._clock()) + def _stabilization_requested(self, configuration: ReportingConfiguration) -> bool: + definition = configuration.definition + if definition is None or not isinstance( + definition.finality_policy, ReportingStabilizedPolicy + ): + return False + return ( + configuration.required_finality == "official" + or configuration.generation_key in self._recovered_stabilization + or any( + isinstance(offering, ProvisionalSnapshotOfferingV1) + and offering.offering_id == self._offerings.snapshot_offering_id + and offering.official_close_lag is not None + for offering in self._source.capabilities.offerings + ) + ) + async def _close_elapsed_periods( self, configuration: ReportingConfiguration, @@ -720,6 +740,8 @@ async def _close_elapsed_periods( ) ): self._stabilization_store() + if frozen is None: + frozen = await self._recover_stabilization_schedule(configuration) if not isinstance(offering, ProvisionalSnapshotOfferingV1): raise LedgerConflictError( "SNAPSHOT_OFFERING_REQUIRED", @@ -1304,90 +1326,21 @@ async def _latest_scheduled_snapshot( async def _recover_stabilization_schedule( self, configuration: ReportingConfiguration, - ledger: ReportingLedgerStore, + *, + required: bool = True, ) -> ProvisionalPolicy | None: - """Restore a bound generation's schedule from a scheduled snapshot it retained.""" - if isinstance(ledger, StabilizationScheduleStore): - policy = await ledger.get_stabilization_schedule(configuration.generation_key) - else: - policy = await self._find_stabilization_schedule(configuration, ledger) + """Restore the exact generation's frozen schedule before validating its history budget.""" + try: + store = self._stabilization_store() + except LedgerConflictError: + if required: + raise + return None + policy = await store.get_stabilization_schedule(configuration.generation_key) if policy is not None: self._recovered_stabilization[configuration.generation_key] = policy return policy - async def _find_stabilization_schedule( - self, - configuration: ReportingConfiguration, - ledger: ReportingLedgerStore, - ) -> ProvisionalPolicy | None: - store = self._optional_observation_store() - if store is None: - return None - caller = ReportingDeliveryPrincipal( - account_id=configuration.account_id, consumer_id=configuration.consumer_id - ) - snapshot = await ledger.open_snapshot( - caller=caller, filters_fingerprint="stabilization-schedule" - ) - best: tuple[bool, datetime, int] | None = None - result = None - offset = 0 - while True: - page = await ledger.read_page( - snapshot=snapshot, - consumer_id=configuration.consumer_id, - delivery_config_ids=[configuration.delivery_config_id], - media_buy_ids=None, - offset=offset, - limit=128, - changes_after_sequence=None, - ) - for obligation in page.obligations: - if obligation.generation_key != configuration.generation_key: - continue - latest = await store.get_provisional_observation( - account_id=obligation.account_id, - reporting_obligation_id=obligation.reporting_obligation_id, - ) - observation = await self._latest_scheduled_snapshot(obligation, latest) - if observation is not None: - acquisition = observation.acquisition - else: - owner = dict( - account_id=obligation.account_id, - reporting_obligation_id=obligation.reporting_obligation_id, - ) - checkpoints = self._optional_restatement_store() - checkpoint = ( - await checkpoints.get_restatement_checkpoint(**owner) - if checkpoints is not None - else None - ) - ordinal = ( - checkpoint.next_observation - if checkpoint - else len(await ledger.list_revisions(**owner)) - ) - pending = await self._pending_acquisition(obligation, ordinal=ordinal) - if pending is None: - continue - acquisition = pending - policy = acquisition.stabilization_schedule( - obligation.required_finality, pending=observation is None - ) - if policy is not None: - at = ( - observation.checked_at - if observation - else acquisition.request().period.source_read_cutoff_at - ) - order = (observation is not None, at, acquisition.ordinal) - if best is None or order > best: - best, result = order, policy - if not page.has_more: - return result - offset += 128 - async def _acquire_stabilized( self, configuration: ReportingConfiguration, @@ -1409,9 +1362,16 @@ async def _acquire_stabilized( store = self._stabilization_store() checkpoint_store = self._restatement_store() checkpoint = await checkpoint_store.get_restatement_checkpoint(**owner) + if checkpoint is not None and checkpoint.stabilization_policy is not None: + self._recovered_stabilization[configuration.generation_key] = ( + checkpoint.stabilization_policy + ) + if not manual and checkpoint is not None and checkpoint.waiting_for_stabilization(now): + return None ordinal = checkpoint.next_observation if checkpoint else len(revisions) pending = await self._pending_acquisition(obligation, ordinal=ordinal) if pending is not None and pending.legacy_stabilized_close: + self._recovered_stabilization[configuration.generation_key] = pending.policy if not await self._retire_stabilized_acquisition( configuration, obligation, pending, now ): @@ -1561,10 +1521,8 @@ async def _stabilized_publication( ): raise ValueError("originating snapshot differs from its publication") validate_snapshot_observation_binding(revision, observation) - if isinstance(self._store, SnapshotOriginStore) and ( - not verified or verified[-1][0].revision_id != observation.revision_id - ): - await self._store.verify_snapshot_origin(revision) + if not verified or verified[-1][0].revision_id != observation.revision_id: + await self._stabilization_store().verify_snapshot_origin(revision) if ( revision.managed_control_totals is not None and revision.managed_control_totals @@ -1741,6 +1699,12 @@ async def acquire_obligation( else None ) observation = checkpoint.next_observation if checkpoint else len(revisions) + if ( + not manual_replay + and checkpoint is not None + and checkpoint.waiting_for_stabilization(now) + ): + return None pending = ( await self._pending_acquisition(obligation, ordinal=observation) if checkpoint_store is not None @@ -1997,15 +1961,18 @@ async def acquire_obligation( suppress_untracked_unchanged=untracked_restatement, ) except LedgerConflictError as error: + if error.code == "OFFICIAL_REVISION_TERMINAL" and untracked_restatement: + return None if ( error.code == "STABILIZATION_ORIGIN_CONFLICT" and acquisition is not None - and acquisition.is_scheduled_snapshot + and acquisition.policy.until_stabilized ): # A sealed unchanged read cannot repair its predecessor's conflicting origin. - if await self._retire_stabilized_acquisition( + retired = await self._retire_stabilized_acquisition( configuration, obligation, acquisition, published_at - ): + ) + if retired and acquisition.is_scheduled_snapshot: await self._schedule_retry( obligation, turn, @@ -2043,12 +2010,14 @@ async def _retire_stabilized_acquisition( return False if any(r.finality == "official" for r in await self._store.list_revisions(**owner)): return False - await self._retire_acquisition( - obligation, - ordinal=acquisition.ordinal, - checked_at=now, - checkpoint_store=store, - checkpoint=checkpoint, + await store.record_restatement_checkpoint( + RestatementCheckpoint.after_acquisition( + acquisition, + now, + checkpoint.provisional_until if checkpoint else None, + checkpoint, + retired_stabilized=True, + ) ) return True @@ -2341,7 +2310,7 @@ async def _revision_publication( ): self._stabilization_lock() if not isinstance(self._source, ReportingProductionSource): - if official and isinstance(self._store, ReportingSourcePublicationStore): + if isinstance(self._store, ReportingSourcePublicationStore): async with self._store.source_publication(obligation.account_id): yield else: @@ -2470,10 +2439,32 @@ def _stabilization_lock(self) -> ReportingSourcePublicationStore: ) return store - def _stabilization_store(self) -> ProvisionalObservationStore: + def _stabilization_store(self) -> StabilizationStore: history = self._observation_history_store() self._restatement_store() self._stabilization_lock() + for stub in ( + ProvisionalObservationPublicationStore.reserve_provisional_acquisition, + ProvisionalObservationPublicationStore.get_provisional_observation, + ProvisionalObservationPublicationStore.commit_provisional_observation, + RestatementCheckpointStore.get_restatement_checkpoint, + RestatementCheckpointStore.record_restatement_checkpoint, + SnapshotOriginStore.verify_snapshot_origin, + PendingProvisionalAcquisitionStore.get_provisional_acquisition, + StabilizationScheduleStore.get_stabilization_schedule, + ): + name = stub.__name__ + method = ( + getattr(history, name, None) + if inspect.getattr_static(history, name, None) is not None + else None + ) + if not callable(method) or getattr(method, "__func__", method) is stub: + raise LedgerConflictError( + "STABILIZATION_STORAGE_NOT_SUPPORTED", + f"stabilized finality requires explicitly implemented {name}", + ) + assert isinstance(history, StabilizationStore) return history async def _reserve_observation( @@ -2511,6 +2502,20 @@ async def _reserve_observation( timedelta(0), iso_duration_to_timedelta(configuration.schedule.period_duration) ) ) + if ( + isinstance(selected, ReportingStabilizedPolicy) + and (checkpoint_store := self._optional_restatement_store()) is not None + ): + checkpoint = await checkpoint_store.get_restatement_checkpoint( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + if ( + checkpoint is not None + and checkpoint.next_observation == ordinal + and checkpoint.stabilization_policy is not None + ): + frozen_policy = checkpoint.stabilization_policy if isinstance(selected, ReportingStabilizedPolicy) and ( obligation.required_finality == "official" or frozen_policy.official_close_lag is not None @@ -3299,6 +3304,18 @@ async def _commit_publication( proved_candidate=candidate_proof is not None, ) acquisition = None + if finality == "snapshot" and acquisition is None and revision is not None: + siblings = await self._store.list_revisions( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + if any(r.finality == "official" for r in siblings) and not any( + r.reporting_revision_id == revision.reporting_revision_id for r in siblings + ): + raise LedgerConflictError( + "OFFICIAL_REVISION_TERMINAL", + "a new snapshot cannot follow the official revision", + ) if acquisition is None: assert revision is not None committed = await self._store.commit_revision(revision, rows) diff --git a/src/adcp/reporting/ledger/provisional.py b/src/adcp/reporting/ledger/provisional.py index dcaebf03a..d53924aa2 100644 --- a/src/adcp/reporting/ledger/provisional.py +++ b/src/adcp/reporting/ledger/provisional.py @@ -378,3 +378,14 @@ async def list_provisional_observations( ``limit`` must be a positive integer. """ pass + + +@runtime_checkable +class StabilizationStore( + ProvisionalObservationStore, + SnapshotOriginStore, + PendingProvisionalAcquisitionStore, + StabilizationScheduleStore, + Protocol, +): + """Durable evidence and recovery methods required for stabilized publication.""" diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index 07cbf82d3..53870df63 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -149,7 +149,7 @@ def generation_key(self) -> ReportingConfigurationGenerationKey: @dataclass(frozen=True) class RestatementCheckpoint: - """Durable scheduling state for successful source observations. + """Durable scheduling state for consumed source acquisition ordinals. ``next_observation`` advances even when the source content is unchanged. Without that durable ordinal the next scheduled refresh would replay the @@ -161,6 +161,8 @@ class RestatementCheckpoint: checked_at: datetime next_observation: int provisional_until: datetime | None = None + stabilization_policy: ProvisionalPolicy | None = None + next_stabilization_due_at: datetime | None = None def __post_init__(self) -> None: if self.next_observation < 1: @@ -168,6 +170,43 @@ def __post_init__(self) -> None: _utc(self.checked_at) if self.provisional_until is not None: _utc(self.provisional_until) + if self.next_stabilization_due_at is not None: + _utc(self.next_stabilization_due_at) + + def waiting_for_stabilization(self, now: datetime) -> bool: + return self.next_stabilization_due_at is not None and _utc(now) < _utc( + self.next_stabilization_due_at + ) + + @classmethod + def after_acquisition( + cls, + acquisition: ProvisionalAcquisition, + checked_at: datetime, + provisional_until: datetime | None, + previous: RestatementCheckpoint | None, + *, + retired_stabilized: bool = False, + ) -> RestatementCheckpoint: + """Consume the ordinal and carry any outstanding scheduled deadline across manual reads.""" + policy = None + due = None + if acquisition.request().trigger == "manual_replay": + if previous is not None: + policy, due = previous.stabilization_policy, previous.next_stabilization_due_at + elif retired_stabilized: + policy = acquisition.policy + if not acquisition.legacy_stabilized_close: + due = checked_at + policy.cadence + return cls( + acquisition.account_id, + acquisition.obligation_id, + checked_at, + acquisition.ordinal + 1, + provisional_until, + policy, + due, + ) @runtime_checkable @@ -1303,9 +1342,7 @@ async def commit_revision( raise LedgerConflictError( "OBSERVATION_CONFLICT", "a source acquisition is reserved" ) - if revision.finality == "official" and any( - item.finality == "official" for item in siblings - ): + if any(item.finality == "official" for item in siblings): raise LedgerConflictError( "OFFICIAL_REVISION_TERMINAL", "an official revision already exists for this obligation; publish a later " @@ -1459,6 +1496,10 @@ async def get_stabilization_schedule( observed is None and checkpoint is not None and ordinal != checkpoint.next_observation + and not ( + ordinal < checkpoint.next_observation + and checkpoint.stabilization_policy == policy + ) ): continue at = ( @@ -1546,12 +1587,8 @@ async def commit_provisional_observation( # Consume the reservation inside this transaction before its revision # uses the ordinary commit path. Any publication failure rolls back both. await self.record_restatement_checkpoint( - RestatementCheckpoint( - acquisition.account_id, - acquisition.obligation_id, - observation.checked_at, - acquisition.ordinal + 1, - observation.provisional_until, + RestatementCheckpoint.after_acquisition( + acquisition, observation.checked_at, observation.provisional_until, checkpoint ) ) if revision is None or reuse_candidate: diff --git a/src/adcp/reporting/service.py b/src/adcp/reporting/service.py index a962af7e0..3f536d1ee 100644 --- a/src/adcp/reporting/service.py +++ b/src/adcp/reporting/service.py @@ -770,7 +770,9 @@ async def _configure( if configuration.definition and isinstance( configuration.definition.finality_policy, ReportingStabilizedPolicy ): - frozen = await producer._recover_stabilization_schedule(configuration, self.store) + frozen = await producer._recover_stabilization_schedule( + configuration, required=producer._stabilization_requested(configuration) + ) policy = self._validate_offerings(configuration, context, registration, frozen=frozen) if isinstance(policy, ReportingStabilizedPolicy): producer._stabilization_store() diff --git a/tests/conformance/reporting/test_reporting_production_configuration.py b/tests/conformance/reporting/test_reporting_production_configuration.py index e17484974..5016ff128 100644 --- a/tests/conformance/reporting/test_reporting_production_configuration.py +++ b/tests/conformance/reporting/test_reporting_production_configuration.py @@ -59,6 +59,12 @@ ), (STABILIZED, "snapshot", ["official"], False, False, False, False), (STABILIZED, "snapshot", ["official"], False, False, "unlocked", False), + (STABILIZED, "snapshot", ["official"], False, False, "verify_snapshot_origin", False), + (STABILIZED, "snapshot", ["official"], False, False, "get_provisional_acquisition", False), + (STABILIZED, "snapshot", ["official"], False, False, "get_stabilization_schedule", False), + (CUTOFF, "snapshot", ["official"], False, False, "verify_snapshot_origin", True), + (SOURCE_FINAL, "official", ["official"], False, True, "get_stabilization_schedule", True), + (STABILIZED, "snapshot", ["snapshot"], False, False, "get_stabilization_schedule", True), (None, "official", ["official"], False, True, True, False), (SOURCE_FINAL, "snapshot", ["official"], False, False, True, False), (CUTOFF, "official", ["official"], False, True, True, False), @@ -84,6 +90,8 @@ async def test_production_offering_validates_selected_finality_policy( config = replace(h.item.config, definition=definition, required_finality=finalities[-1]) source.bind_generation(config) store = h.store + if isinstance(history, str) and history != "unlocked": + setattr(store, history, None) if not history: store = SimpleNamespace( reserve_provisional_acquisition=h.store.reserve_provisional_acquisition, diff --git a/tests/conformance/reporting/test_reporting_reconciliation_changes.py b/tests/conformance/reporting/test_reporting_reconciliation_changes.py index cb1a2ba64..693950323 100644 --- a/tests/conformance/reporting/test_reporting_reconciliation_changes.py +++ b/tests/conformance/reporting/test_reporting_reconciliation_changes.py @@ -499,7 +499,7 @@ async def test_distinct_concurrent_writes_take_dense_unique_feed_sequences( ) -> None: """Order-independent records for one principal must not collide or lose a sequence.""" store, clock = reconciliation_store - s = await scenario(store) + s = await scenario(store, billing=False, finality="snapshot") clock.now = END + timedelta(seconds=60) await store.commit_materialization(s.outcome) rows = ( diff --git a/tests/test_reliable_reporting_service.py b/tests/test_reliable_reporting_service.py index dbf0bdea7..a21a88c07 100644 --- a/tests/test_reliable_reporting_service.py +++ b/tests/test_reliable_reporting_service.py @@ -423,6 +423,75 @@ class UnlockedHistory(InMemoryReportingLedgerStore): assert not await store.list_all_configurations() +@pytest.mark.parametrize( + "missing", + ["verify_snapshot_origin", "get_provisional_acquisition", "get_stabilization_schedule"], +) +async def test_stabilized_admission_rejects_missing_evidence_api_before_reading_storage( + missing: str, +) -> None: + class IncompletePublisher(InMemoryReportingLedgerStore): + async def get_stabilization_schedule(self, generation: Any) -> Any: + pytest.fail("admission opened an incomplete publisher") + + async def open_snapshot(self, **kwargs: Any) -> Any: + pytest.fail("admission attempted an uninitialized ledger read") + + async def create_schema(self) -> None: + pytest.fail("stabilized admission attempted schema initialization") + + store = IncompletePublisher() + setattr(store, missing, None) + config = replace( + _configuration(), + required_finality="official", + definition=replace( + DEFINITION, finality_policy=ReportingStabilizedPolicy("stable", "PT0S", "PT1H") + ), + ) + service = _finality_service(config, snapshot=True, official=False, store=store) + with pytest.raises(ReliableReportingConfigurationError, match=missing): + await service.configure(config) + assert not await store.list_all_configurations() + + +async def test_plain_snapshot_custom_publisher_needs_no_stabilization_apis() -> None: + class SnapshotPublisher(InMemoryReportingLedgerStore): + get_stabilization_schedule = None + verify_snapshot_origin = None + + config = replace( + _configuration(), + definition=replace( + DEFINITION, finality_policy=ReportingStabilizedPolicy("optional", "PT0S", "PT6H") + ), + ) + service = _finality_service(config, snapshot=True, official=False, store=SnapshotPublisher()) + await service.configure(config) + await service.initialize() + assert config.generation_key in service._bindings + + +@pytest.mark.parametrize("required", ["snapshot", "official"]) +async def test_stabilized_admission_reports_an_unregistered_snapshot_offering(required) -> None: + config = replace( + _configuration(), + required_finality=required, + definition=replace( + DEFINITION, finality_policy=ReportingStabilizedPolicy("stable", "PT0S", "PT1H") + ), + ) + service = ReliableReportingService( + store=InMemoryReportingLedgerStore(), + account_context=lambda configuration: replace( + _account_context(configuration), snapshot_offering_id="retired" + ), + ) + service.sources.register("gam", ScriptedReportingAdapter(redacted_capabilities(), [])) + with pytest.raises(ReliableReportingConfigurationError, match="snapshot offering.*registered"): + await service.configure(config) + + async def test_initialize_recovers_frozen_stabilization_cadence_and_healthy_binding() -> None: clock = DeterministicReportingClock(NOW + timedelta(days=1)) store = InMemoryReportingLedgerStore(clock=clock) diff --git a/tests/test_reporting_official_finality.py b/tests/test_reporting_official_finality.py index 11665d2a0..e10d1942a 100644 --- a/tests/test_reporting_official_finality.py +++ b/tests/test_reporting_official_finality.py @@ -85,6 +85,22 @@ async def test_standalone_stabilization_rejects_an_unbound_snapshot_source() -> assert not store._obligations +@pytest.mark.parametrize( + "missing", + ["verify_snapshot_origin", "get_provisional_acquisition", "get_stabilization_schedule"], +) +async def test_standalone_stabilization_checks_durable_apis_before_storage(missing: str) -> None: + class IncompletePublisher(InMemoryReportingLedgerStore): + async def find_obligation(self, **kwargs: Any) -> Any: + pytest.fail("admission read obligations from an incomplete publisher") + + store = IncompletePublisher() + setattr(store, missing, None) + with pytest.raises(LedgerConflictError, match=missing): + await _stabilized_harness(lambda **kwargs: store) + assert not store._obligations + + async def test_native_recovery_reads_the_schedule_for_the_generation( store_factory: Any, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -95,10 +111,10 @@ async def forbid_full_ledger(**kwargs: Any) -> Any: pytest.fail("recovery should look up the generation's scheduled observation") monkeypatch.setattr(store, "read_page", forbid_full_ledger) - frozen = await producer._recover_stabilization_schedule(config, store) + frozen = await producer._recover_stabilization_schedule(config) assert frozen and frozen.cadence == timedelta(hours=6) other = replace(config, delivery_config_version=2) - assert await producer._recover_stabilization_schedule(other, store) is None + assert await producer._recover_stabilization_schedule(other) is None @pytest.mark.parametrize("legacy", [False, True]) @@ -126,15 +142,30 @@ async def test_recovery_finds_the_pending_snapshot_before_the_first_observation( request.model_dump_json(), 0, policy, output_finality="official" if legacy else None ) ) - assert await producer._recover_stabilization_schedule(config, store) == policy + assert await producer._recover_stabilization_schedule(config) == policy assert ( - await producer._recover_stabilization_schedule( - replace(config, delivery_config_version=2), store - ) + await producer._recover_stabilization_schedule(replace(config, delivery_config_version=2)) is None ) +async def test_previous_checkpoint_schema_recovers_schedule_before_initialize() -> None: + async with isolated_reporting_pool() as pool: + producer, store, config, _, _, _ = await _stabilized_harness( + partial(PgReportingLedgerStore, pool=pool) + ) + await producer.run_configuration(config) + async with pool.connection() as connection: + await connection.execute( + "ALTER TABLE reporting_restatement_checkpoints DROP COLUMN stabilization_policy," + " DROP COLUMN next_stabilization_due_at" + ) + policy = await producer._recover_stabilization_schedule(config) + assert policy is not None and policy.cadence == timedelta(hours=6) + await store.create_schema() + assert await producer._recover_stabilization_schedule(config) == policy + + @pytest.mark.parametrize("tracked", [False, True]) async def test_manual_change_between_equal_scheduled_reads_preserves_stabilization( store_factory: Any, monkeypatch: pytest.MonkeyPatch, tracked: bool @@ -234,11 +265,18 @@ async def test_stabilized_close_keeps_frozen_route_when_offering_drops_close_lag assert [r.finalized_at for r in revisions if r.finality == "official"] == [clock[0]] -@pytest.mark.parametrize("pending_lookup", [True, False]) +@pytest.mark.parametrize("unchanged", ["PT12H", "P30D"]) +@pytest.mark.parametrize("wrapped", [False, True]) async def test_stabilized_retires_base_shaped_derived_official_reservation( - store_factory: Any, monkeypatch: pytest.MonkeyPatch, pending_lookup: bool + store_factory: Any, unchanged: str, wrapped: bool ) -> None: - producer, store, config, obligation, calls, clock = await _stabilized_harness(store_factory) + producer, store, config, obligation, calls, clock = await _harness( + store_factory, + official=True, + capabilities=_capabilities(restatement_window="P1D", restatement_cadence="PT6H"), + finality_policy=ReportingStabilizedPolicy("stable", "PT0S", unchanged), + ) + clock[0] = END + timedelta(hours=1) owner = dict( account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id ) @@ -252,31 +290,72 @@ async def test_stabilized_retires_base_shaped_derived_official_reservation( output_finality="official", ) await store.reserve_provisional_acquisition(reservation) - if not pending_lookup: - monkeypatch.setattr(store, "get_provisional_acquisition", None) + producer._source.capabilities = _capabilities( + restatement_window="P1D", restatement_cadence="PT1H" + ) assert "until_stabilized" not in reservation.to_wire()["policy"] + + class Publisher: + def __getattr__(self, name: str) -> Any: + return getattr(store, name) + + publisher = Publisher() if wrapped else store + if wrapped: + for name in ( + "verify_snapshot_origin", + "get_provisional_acquisition", + "get_stabilization_schedule", + "list_provisional_observations", + "source_publication", + "get_restatement_checkpoint", + "record_restatement_checkpoint", + "reserve_provisional_acquisition", + "get_provisional_observation", + "commit_provisional_observation", + ): + setattr(publisher, name, getattr(store, name)) fresh = ReportingProducer( source=producer._source, - store=store, + store=publisher, + retry_store=store, offerings=producer._offerings, object_reader=producer._object_reader, clock=lambda: clock[0], read_jitter_window=timedelta(0), ) - for hour in (1, 1, 7, 13): + + async def interrupt_replacement(*args: Any, **kwargs: Any) -> Any: + raise OSError("interrupted before successor reservation") + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(fresh, "_reserve_observation", interrupt_replacement) + with pytest.raises(OSError, match="before successor reservation"): + await fresh.acquire_obligation(config, obligation) + assert await store.get_stabilization_schedule(config.generation_key) == reservation.policy + fresh = ReportingProducer( + source=producer._source, + store=publisher, + retry_store=store, + offerings=producer._offerings, + object_reader=producer._object_reader, + clock=lambda: clock[0], + read_jitter_window=timedelta(0), + ) + for hour in (1, 1, 2, 7, 13): clock[0] = END + timedelta(hours=hour) - await fresh.run_configuration(config) + await fresh.acquire_obligation(config, obligation) history = await store.list_provisional_observations(**owner, limit=8) assert [o.acquisition.ordinal for o in history] == [1, 2, 3] + assert [o.checked_at for o in history] == [END + timedelta(hours=h) for h in (1, 7, 13)] + assert all(o.acquisition.policy.cadence == timedelta(hours=6) for o in history) assert all(o.is_scheduled_snapshot for o in history) assert len(calls) == 3 and all( r.identity.source_execution_key != reservation.execution_key for r in producer._source.requests ) - assert sorted(r.finality for r in await store.list_revisions(**owner)) == [ - "official", - "snapshot", - ] + assert sorted(r.finality for r in await store.list_revisions(**owner)) == ( + ["official", "snapshot"] if unchanged == "PT12H" else ["snapshot"] + ) async def test_stabilization_cannot_rebind_a_contradictory_snapshot_origin( @@ -318,7 +397,23 @@ async def rebound_suffix(**kwargs: Any) -> Any: patch.setattr(store, "list_provisional_observations", rebound_suffix) with pytest.raises(LedgerConflictError, match="history"): await producer.acquire_obligation(config, obligation) - for hour in (31, 31, 37, 43, 49): + + async def crash_before_retry(*args: Any, **kwargs: Any) -> Any: + raise OSError("interrupted retry write") + + clock[0] = END + timedelta(hours=31) + with monkeypatch.context() as patch: + patch.setattr(producer, "_schedule_retry", crash_before_retry) + with pytest.raises(OSError, match="interrupted retry write"): + await producer.acquire_obligation(config, obligation) + producer = ReportingProducer( + source=producer._source, + offerings=producer._offerings, + store=store, + object_reader=producer._object_reader, + clock=lambda: clock[0], + ) + for hour in (31, 36, 37, 43, 49): clock[0] = END + timedelta(hours=hour) with suppress(LedgerConflictError): await producer.acquire_obligation(config, obligation) @@ -398,6 +493,139 @@ async def at_reservation(*args: Any, **kwargs: Any) -> Any: assert len(calls) == 3 +async def test_untracked_manual_publication_cannot_follow_stabilized_official( + store_factory: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, obligation, _, clock = await _stabilized_harness(store_factory) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + for hour in (1, 7, 13): + clock[0] = END + timedelta(hours=hour) + await producer.acquire_obligation( + config, obligation, target_finality="snapshot", restate=True, track_settling=True + ) + clock[0] = END + timedelta(hours=14) + paused = asyncio.Event() + resumed = asyncio.Event() + commit = producer._commit_publication + candidates = [] + + async def hold_manual(*args: Any, **kwargs: Any) -> Any: + if args[4] == "snapshot": + candidates.append((args[2], args[3])) + paused.set() + await resumed.wait() + return await commit(*args, **kwargs) + + monkeypatch.setattr(producer, "_commit_publication", hold_manual) + monkeypatch.setattr( + "tests.test_reporting_source_availability.PROVIDER_ROW", PROVIDER_ROW.replace("10", "11") + ) + manual = asyncio.create_task( + producer.acquire_obligation( + config, obligation, target_finality="snapshot", restate=True, manual_replay=True + ) + ) + try: + await asyncio.wait_for(paused.wait(), timeout=5) + official = await producer.acquire_obligation(config, obligation) + assert official is not None and official.finality == "official" + finally: + resumed.set() + assert await asyncio.wait_for(manual, timeout=5) is None + (snapshot,) = [r for r in await store.list_revisions(**owner) if r.finality == "snapshot"] + assert await store.commit_revision(snapshot, ROWS) == snapshot + with pytest.raises(LedgerConflictError, match="official"): + await store.commit_revision(*candidates[0]) + + +@pytest.mark.parametrize("publish_manual", [True, False]) +async def test_tracked_manual_snapshot_preserves_retired_scheduled_deadline( + store_factory: Any, publish_manual: bool +) -> None: + producer, store, config, obligation, calls, clock = await _stabilized_harness(store_factory) + await producer.run_configuration(config) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + first = await store.get_provisional_observation(**owner) + for ordinal, hour, trigger in [(1, 7, "scheduled_poll"), (2, 8, "manual_replay")]: + clock[0] = END + timedelta(hours=hour) + request = producer._build_slice( + config, + obligation, + SNAPSHOT_OFFERING_ID, + finality="snapshot", + now=clock[0], + observation=ordinal, + trigger=trigger, + ) + acquisition = await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + request.model_dump_json(), ordinal, first.acquisition.policy, first.revision_id + ) + ) + if trigger == "scheduled_poll": + assert await producer._retire_stabilized_acquisition( + config, obligation, acquisition, clock[0] + ) + elif not publish_manual: + + async def reject_manual(*args: Any, **kwargs: Any) -> Any: + raise LedgerConflictError("STABILIZATION_ORIGIN_CONFLICT", "conflicting origin") + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(store, "commit_provisional_observation", reject_manual) + with pytest.raises(LedgerConflictError, match="conflicting origin"): + await producer.acquire_obligation( + config, + obligation, + target_finality="snapshot", + restate=True, + manual_replay=True, + ) + else: + await producer.acquire_obligation( + config, obligation, target_finality="snapshot", restate=True, manual_replay=True + ) + checkpoint = await store.get_restatement_checkpoint(**owner) + assert checkpoint.next_observation == 3 + assert checkpoint.next_stabilization_due_at == END + timedelta(hours=13) + await producer.acquire_obligation(config, obligation) + assert len(calls) == 2 + clock[0] = END + timedelta(hours=13) + await producer.acquire_obligation(config, obligation) + assert len(calls) == 3 + + +async def test_native_ledger_rejects_a_new_bare_snapshot_after_official(store_factory: Any) -> None: + producer, store, config, obligation, _, clock = await _stabilized_harness(store_factory) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + for hour in (1, 7, 13): + clock[0] = END + timedelta(hours=hour) + await producer.run_configuration(config) + (snapshot,) = [r for r in await store.list_revisions(**owner) if r.finality == "snapshot"] + later_id = "rpr_" + "f" * 40 + later = replace( + snapshot, + reporting_revision_id=later_id, + supersedes_reporting_revision_id=snapshot.reporting_revision_id, + revision_content_sha256=revision_content_sha256( + reporting_revision_id=later_id, + row_count=snapshot.row_count, + control_totals=snapshot.control_totals, + reporting_rows=ROWS, + control_total_evidence=snapshot.managed_control_totals, + ), + ) + with pytest.raises(LedgerConflictError, match="official"): + await store.commit_revision(later, ROWS) + assert await store.commit_revision(snapshot, ROWS) == snapshot + + @pytest.mark.parametrize("change", ["none", "warnings", "basis"]) async def test_proofless_stabilized_replay_requires_its_qualifying_observation( monkeypatch: pytest.MonkeyPatch, change: str From 16c4bf133fbbfcfeaf4ecce82aa0c59b38173b14 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 17:38:14 +0530 Subject: [PATCH 19/39] fix(reporting): retain scheduled cadence and verify snapshot reuse --- src/adcp/reporting/ledger/producer.py | 253 ++++++++++++--------- tests/test_reliable_reporting_service.py | 48 +++- tests/test_reporting_official_finality.py | 255 ++++++++++++++++++---- 3 files changed, 416 insertions(+), 140 deletions(-) diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index febda9da7..087797dac 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -427,6 +427,14 @@ class _StabilizedPublication: rows: Sequence[dict[str, Any]] +def _implements_store_method(store: object, stub: Callable[..., Any]) -> bool: + name = stub.__name__ + if inspect.getattr_static(store, name, None) is None: + return False + method = getattr(store, name, None) + return callable(method) and getattr(method, "__func__", method) is not stub + + class ReportingProducer: """Closes periods, drives the source, and commits immutable revisions.""" @@ -705,6 +713,10 @@ async def _close_elapsed_periods( ) -> list[ReportingObligationRecord]: from adcp.reporting.ledger.producer_progress import ReportingProducerProgress + selected = configuration.definition.finality_policy if configuration.definition else None + frozen = None + if isinstance(selected, ReportingStabilizedPolicy): + frozen = await self._recover_stabilization_schedule(configuration, required=False) progress = self._store if isinstance(self._store, ReportingProducerProgress) else None after = None if progress is None else await progress.producer_closed_through(configuration) committed: list[ReportingObligationRecord] = [] @@ -724,39 +736,27 @@ async def _close_elapsed_periods( ) after = boundary.end continue - selected = ( - configuration.definition.finality_policy if configuration.definition else None - ) - if isinstance(selected, ReportingStabilizedPolicy): + if isinstance(selected, ReportingStabilizedPolicy) and self._stabilization_requested( + configuration + ): + self._stabilization_store() offering_id = self._offerings.snapshot_offering_id offering = self._source.capabilities.offering(offering_id) if offering_id else None - frozen = self._recovered_stabilization.get(configuration.generation_key) - if ( - frozen is not None - or configuration.required_finality == "official" - or ( - isinstance(offering, ProvisionalSnapshotOfferingV1) - and offering.official_close_lag is not None + if not isinstance(offering, ProvisionalSnapshotOfferingV1): + raise LedgerConflictError( + "SNAPSHOT_OFFERING_REQUIRED", + "stabilized finality requires a snapshot source offering", ) - ): - self._stabilization_store() - if frozen is None: - frozen = await self._recover_stabilization_schedule(configuration) - if not isinstance(offering, ProvisionalSnapshotOfferingV1): - raise LedgerConflictError( - "SNAPSHOT_OFFERING_REQUIRED", - "stabilized finality requires a snapshot source offering", + stabilization_history_limit( + selected, + ( + frozen.cadence + if frozen is not None + else snapshot_restatement_cadence( + offering, configuration.schedule.period_duration ) - stabilization_history_limit( - selected, - ( - frozen.cadence - if frozen is not None - else snapshot_restatement_cadence( - offering, configuration.schedule.period_duration - ) - ), - ) + ), + ) obligation = ReportingObligationRecord( reporting_obligation_id=self._obligation_id(configuration, boundary), account_id=configuration.account_id, @@ -1142,11 +1142,19 @@ async def _acquire_with_settling_policy( policy = _SettlingPolicy(frozen.window, frozen.cadence, frozen.official_close_lag) if isinstance(selected, ReportingStabilizedPolicy) and ( obligation.required_finality == "official" + or configuration.generation_key in self._recovered_stabilization or policy.official_close_lag is not None or (scheduled is not None and scheduled.acquisition.policy.until_stabilized) ): result = await self._acquire_stabilized(configuration, obligation, selected, turn, now) return result is not None and result.finality == "official" + # A snapshot-only schedule can carry an unused definition policy. Its + # automatic reads need only the ordinary observation publisher. + acquire = ( + self._acquire_obligation + if isinstance(selected, ReportingStabilizedPolicy) + else self.acquire_obligation + ) explicit_close = ( policy.official_close_lag is not None and self._offerings.offering_for(_source_finality(obligation, "official")) is not None @@ -1164,7 +1172,7 @@ async def _acquire_with_settling_policy( request = pending.request() if _utc(now) < self._source_available_at(obligation, offering_id=request.offering_id): return False - await self.acquire_obligation( + await acquire( configuration, obligation, restate=bool(revisions), @@ -1192,7 +1200,7 @@ async def _acquire_with_settling_policy( ) return False if not revisions: - await self.acquire_obligation( + await acquire( configuration, obligation, turn=turn, @@ -1225,7 +1233,7 @@ async def _acquire_with_settling_policy( if next_due is None: return not explicit_close if _utc(now) >= _utc(next_due): - await self.acquire_obligation( + await acquire( configuration, obligation, restate=True, @@ -1254,7 +1262,7 @@ async def _acquire_with_settling_policy( self._source_available_at(obligation, target_finality="official"), ) if _utc(now) >= closes_at: - await self.acquire_obligation( + await acquire( configuration, obligation, restate=True, @@ -1330,15 +1338,22 @@ async def _recover_stabilization_schedule( required: bool = True, ) -> ProvisionalPolicy | None: """Restore the exact generation's frozen schedule before validating its history budget.""" - try: - store = self._stabilization_store() - except LedgerConflictError: - if required: - raise + key = configuration.generation_key + frozen = self._recovered_stabilization.get(key) + if required or frozen is not None: + self._stabilization_store() + if frozen is not None: + return frozen + if not _implements_store_method( + self._store, StabilizationScheduleStore.get_stabilization_schedule + ): return None - policy = await store.get_stabilization_schedule(configuration.generation_key) + assert isinstance(self._store, StabilizationScheduleStore) + policy = await self._store.get_stabilization_schedule(key) if policy is not None: - self._recovered_stabilization[configuration.generation_key] = policy + if not required: + self._stabilization_store() + self._recovered_stabilization[key] = policy return policy async def _acquire_stabilized( @@ -1359,19 +1374,16 @@ async def _acquire_stabilized( revisions = await self._store.list_revisions(**owner) if any(r.finality == "official" for r in revisions): return None - store = self._stabilization_store() + await self._recover_stabilization_schedule(configuration) + store = self._store + assert isinstance(store, StabilizationStore) checkpoint_store = self._restatement_store() checkpoint = await checkpoint_store.get_restatement_checkpoint(**owner) - if checkpoint is not None and checkpoint.stabilization_policy is not None: - self._recovered_stabilization[configuration.generation_key] = ( - checkpoint.stabilization_policy - ) if not manual and checkpoint is not None and checkpoint.waiting_for_stabilization(now): return None ordinal = checkpoint.next_observation if checkpoint else len(revisions) pending = await self._pending_acquisition(obligation, ordinal=ordinal) if pending is not None and pending.legacy_stabilized_close: - self._recovered_stabilization[configuration.generation_key] = pending.policy if not await self._retire_stabilized_acquisition( configuration, obligation, pending, now ): @@ -1617,6 +1629,31 @@ async def acquire_obligation( ``now`` freezes dispatch and the source read cutoff. Revision creation uses a fresh producer clock sample after the staged objects are read. """ + return await self._acquire_obligation( + configuration, + obligation, + restate=restate, + turn=turn, + now=now, + target_finality=target_finality, + track_settling=track_settling, + manual_replay=manual_replay, + admit_tracked_stabilization=True, + ) + + async def _acquire_obligation( + self, + configuration: ReportingConfiguration, + obligation: ReportingObligationRecord, + *, + restate: bool = False, + turn: WorkerTurn | None = None, + now: datetime | None = None, + target_finality: str | None = None, + track_settling: bool = False, + manual_replay: bool = False, + admit_tracked_stabilization: bool = False, + ) -> ReportingRevisionRecord | None: if configuration.generation_key != obligation.generation_key: raise LedgerConflictError( "CONFIGURATION_GENERATION_MISMATCH", @@ -1710,6 +1747,16 @@ async def acquire_obligation( if checkpoint_store is not None else None ) + if ( + admit_tracked_stabilization + and isinstance(selected_policy, ReportingStabilizedPolicy) + and ( + track_settling + or pending is not None + or (checkpoint_store is not None and not restate and not manual_replay) + ) + ): + await self._recover_stabilization_schedule(configuration) if pending is not None: finality = pending.finality elif ( @@ -1966,7 +2013,12 @@ async def acquire_obligation( if ( error.code == "STABILIZATION_ORIGIN_CONFLICT" and acquisition is not None - and acquisition.policy.until_stabilized + and isinstance(selected_policy, ReportingStabilizedPolicy) + and (acquisition.policy.until_stabilized or acquisition.is_scheduled_snapshot) + and ( + admit_tracked_stabilization + or self._uses_stabilization_schedule(obligation, acquisition.policy) + ) ): # A sealed unchanged read cannot repair its predecessor's conflicting origin. retired = await self._retire_stabilized_acquisition( @@ -2404,17 +2456,9 @@ def _observation_store(self) -> ProvisionalObservationPublicationStore: def _observation_history_store(self) -> ProvisionalObservationStore: store = self._observation_store() - name = "list_provisional_observations" - stub = ProvisionalObservationStore.list_provisional_observations - declared = inspect.getattr_static(store, name, None) - implementation = ( - getattr(store, name, None) if declared is not None and declared is not stub else None - ) - if ( - not callable(implementation) - or getattr(implementation, "__func__", implementation) is stub - or not isinstance(store, ProvisionalObservationStore) - ): + if not _implements_store_method( + store, ProvisionalObservationStore.list_provisional_observations + ) or not isinstance(store, ProvisionalObservationStore): raise LedgerConflictError( "PROVISIONAL_OBSERVATION_HISTORY_NOT_SUPPORTED", "stabilized finality requires explicitly implemented bounded observation history", @@ -2423,16 +2467,9 @@ def _observation_history_store(self) -> ProvisionalObservationStore: def _stabilization_lock(self) -> ReportingSourcePublicationStore: store = self._store - stub = ReportingSourcePublicationStore.source_publication - declared = inspect.getattr_static(store, "source_publication", None) - implementation = ( - getattr(store, "source_publication", None) if declared is not None else None - ) - if ( - not callable(implementation) - or getattr(implementation, "__func__", implementation) is stub - or not isinstance(store, ReportingSourcePublicationStore) - ): + if not _implements_store_method( + store, ReportingSourcePublicationStore.source_publication + ) or not isinstance(store, ReportingSourcePublicationStore): raise LedgerConflictError( "STABILIZATION_PUBLICATION_NOT_SUPPORTED", "stabilized finality requires an account publication lock", @@ -2453,16 +2490,10 @@ def _stabilization_store(self) -> StabilizationStore: PendingProvisionalAcquisitionStore.get_provisional_acquisition, StabilizationScheduleStore.get_stabilization_schedule, ): - name = stub.__name__ - method = ( - getattr(history, name, None) - if inspect.getattr_static(history, name, None) is not None - else None - ) - if not callable(method) or getattr(method, "__func__", method) is stub: + if not _implements_store_method(history, stub): raise LedgerConflictError( "STABILIZATION_STORAGE_NOT_SUPPORTED", - f"stabilized finality requires explicitly implemented {name}", + f"stabilized finality requires explicitly implemented {stub.__name__}", ) assert isinstance(history, StabilizationStore) return history @@ -2502,23 +2533,8 @@ async def _reserve_observation( timedelta(0), iso_duration_to_timedelta(configuration.schedule.period_duration) ) ) - if ( - isinstance(selected, ReportingStabilizedPolicy) - and (checkpoint_store := self._optional_restatement_store()) is not None - ): - checkpoint = await checkpoint_store.get_restatement_checkpoint( - account_id=obligation.account_id, - reporting_obligation_id=obligation.reporting_obligation_id, - ) - if ( - checkpoint is not None - and checkpoint.next_observation == ordinal - and checkpoint.stabilization_policy is not None - ): - frozen_policy = checkpoint.stabilization_policy - if isinstance(selected, ReportingStabilizedPolicy) and ( - obligation.required_finality == "official" - or frozen_policy.official_close_lag is not None + if isinstance(selected, ReportingStabilizedPolicy) and self._uses_stabilization_schedule( + obligation, frozen_policy ): stabilization_history_limit(selected, frozen_policy.cadence) frozen_policy = replace(frozen_policy, until_stabilized=True) @@ -3041,13 +3057,6 @@ async def _commit_revision_from_manifest( retained = predecessor if acquisition is None: return retained - if acquisition.policy.until_stabilized and isinstance( - selected_policy, ReportingStabilizedPolicy - ): - limit = stabilization_history_limit(selected_policy, acquisition.policy.cadence) - for observed in await self._scheduled_snapshot_history(obligation, limit): - if observed.revision_id == predecessor.reporting_revision_id: - validate_snapshot_origin(predecessor, observed.snapshot_origin) staged = staging is not None and bool(await staging.stage_revision_rows(revision, rows)) try: committed, replayed = await self._commit_publication( @@ -3254,6 +3263,47 @@ async def _verify_supplied_rows( ) return staged_rows + def _uses_stabilization_schedule( + self, obligation: ReportingObligationRecord, policy: ProvisionalPolicy + ) -> bool: + return ( + obligation.required_finality == "official" + or policy.until_stabilized + or policy.official_close_lag is not None + or obligation.generation_key in self._recovered_stabilization + ) + + async def _verify_reused_snapshot_origin( + self, + obligation: ReportingObligationRecord, + acquisition: ProvisionalAcquisition, + revision_id: str, + ) -> None: + selected = obligation.definition.finality_policy if obligation.definition else None + if not isinstance(selected, ReportingStabilizedPolicy) or not ( + acquisition.policy.until_stabilized or acquisition.is_scheduled_snapshot + ): + return + active = self._uses_stabilization_schedule(obligation, acquisition.policy) + if active: + witness: SnapshotOriginStore = self._stabilization_store() + elif _implements_store_method(self._store, SnapshotOriginStore.verify_snapshot_origin): + assert isinstance(self._store, SnapshotOriginStore) + witness = self._store + else: + return + predecessor = await self._store.get_revision( + account_id=obligation.account_id, reporting_revision_id=revision_id + ) + if predecessor is None: + raise LedgerConflictError("HISTORY_UNAVAILABLE", "snapshot predecessor is missing") + await witness.verify_snapshot_origin(predecessor) + if active: + limit = stabilization_history_limit(selected, acquisition.policy.cadence) + for observed in await self._scheduled_snapshot_history(obligation, limit): + if observed.revision_id == revision_id: + validate_snapshot_origin(predecessor, observed.snapshot_origin) + async def _commit_publication( self, obligation: ReportingObligationRecord, @@ -3324,6 +3374,9 @@ async def _commit_publication( # The bounds check above already rejects acquired_at after now. if retained_revision_id is not None: observation_revision_id = retained_revision_id + await self._verify_reused_snapshot_origin( + obligation, acquisition, retained_revision_id + ) else: assert revision is not None observation_revision_id = revision.reporting_revision_id diff --git a/tests/test_reliable_reporting_service.py b/tests/test_reliable_reporting_service.py index a21a88c07..a0b4572df 100644 --- a/tests/test_reliable_reporting_service.py +++ b/tests/test_reliable_reporting_service.py @@ -462,14 +462,60 @@ class SnapshotPublisher(InMemoryReportingLedgerStore): config = replace( _configuration(), + activated_at=NOW.replace(hour=1, minute=0), + deactivated_at=NOW.replace(hour=2, minute=0), definition=replace( DEFINITION, finality_policy=ReportingStabilizedPolicy("optional", "PT0S", "PT6H") ), ) - service = _finality_service(config, snapshot=True, official=False, store=SnapshotPublisher()) + service = _finality_service( + config, + snapshot=True, + official=False, + store=SnapshotPublisher(), + clock=DeterministicReportingClock(NOW + timedelta(days=1)), + results=(InlineFetchResult(rows=_rows(10)),), + ) await service.configure(config) await service.initialize() assert config.generation_key in service._bindings + turn = await service.run_worker() + assert len(turn.configurations[config.generation_key].revisions_committed) == 1 + + +async def test_recovered_snapshot_close_requires_origin_witness_after_live_lag_removal( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = DeterministicReportingClock(NOW + timedelta(days=1)) + store = InMemoryReportingLedgerStore(clock=clock) + config = replace( + _configuration(), + activated_at=NOW.replace(hour=1, minute=0), + deactivated_at=NOW.replace(hour=2, minute=0), + definition=replace( + DEFINITION, finality_policy=ReportingStabilizedPolicy("stable", "PT0S", "PT12H") + ), + ) + original = _finality_service( + config, + snapshot=True, + official=False, + close=True, + cadence="PT6H", + store=store, + clock=clock, + results=(InlineFetchResult(rows=_rows(10)),), + ) + await original.configure(config) + turn = await original.run_worker() + assert len(turn.configurations[config.generation_key].revisions_committed) == 1 + monkeypatch.setattr(store, "verify_snapshot_origin", None) + recovered = _finality_service( + config, snapshot=True, official=False, cadence="PT1H", store=store, clock=clock + ) + with pytest.raises(ReliableReportingConfigurationError, match="verify_snapshot_origin"): + await recovered.initialize() + assert config.generation_key not in recovered._bindings @pytest.mark.parametrize("required", ["snapshot", "official"]) diff --git a/tests/test_reporting_official_finality.py b/tests/test_reporting_official_finality.py index e10d1942a..7fb67f208 100644 --- a/tests/test_reporting_official_finality.py +++ b/tests/test_reporting_official_finality.py @@ -7,7 +7,7 @@ from collections.abc import AsyncIterator from contextlib import asynccontextmanager, suppress from dataclasses import replace -from datetime import timedelta +from datetime import datetime, timedelta from functools import partial from typing import Any @@ -163,7 +163,7 @@ async def test_previous_checkpoint_schema_recovers_schedule_before_initialize() policy = await producer._recover_stabilization_schedule(config) assert policy is not None and policy.cadence == timedelta(hours=6) await store.create_schema() - assert await producer._recover_stabilization_schedule(config) == policy + assert await store.get_stabilization_schedule(config.generation_key) == policy @pytest.mark.parametrize("tracked", [False, True]) @@ -265,6 +265,217 @@ async def test_stabilized_close_keeps_frozen_route_when_offering_drops_close_lag assert [r.finalized_at for r in revisions if r.finality == "official"] == [clock[0]] +async def test_manual_official_reservation_cannot_replace_the_scheduled_generation_cadence( + store_factory: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, _, _, clock = await _stabilized_harness(store_factory) + for hour in (1, 7, 13): + clock[0] = END + timedelta(hours=hour) + await producer.run_configuration(config) + period_end = END + timedelta(days=1) + clock[0] = period_end + timedelta(hours=1) + (obligation,) = await producer.close_elapsed_periods(config) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + request = producer._build_slice( + config, + obligation, + SNAPSHOT_OFFERING_ID, + finality="official", + trigger="manual_replay", + now=clock[0], + ) + await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + request.model_dump_json(), + 0, + ProvisionalPolicy(timedelta(days=1), timedelta(hours=12)), + output_finality="official", + ) + ) + producer._source.capabilities = _capabilities( + restatement_window="P1D", restatement_cadence="PT1H", official_close_lag="P2D" + ) + restarted = _restart_producer(producer, store, clock) + for hour, impressions in ((1, 10), (7, 11), (13, 10)): + clock[0] = period_end + timedelta(hours=hour) + monkeypatch.setattr( + "tests.test_reporting_source_availability.PROVIDER_ROW", + PROVIDER_ROW.replace("10", str(impressions)), + ) + await restarted.run_configuration(config) + revisions = await store.list_revisions(**owner) + assert all(revision.finality == "snapshot" for revision in revisions) + observations = await store.list_provisional_observations(**owner, limit=4) + assert [item.checked_at for item in observations] == [ + period_end + timedelta(hours=h) for h in (1, 7, 13) + ] + assert [item.acquisition.ordinal for item in observations] == [1, 2, 3] + assert all(item.acquisition.policy.cadence == timedelta(hours=6) for item in observations) + for item, impressions in zip(observations, (10, 11, 10)): + page = await store.read_revision_rows( + account_id=config.account_id, reporting_revision_id=item.revision_id + ) + assert page.rows == ({"campaign_id": "campaign-redacted-1", "impressions": impressions},) + + +@pytest.mark.parametrize("period_already_created", [False, True]) +async def test_standalone_period_recovers_generation_schedule_after_close_lag_removal( + store_factory: Any, period_already_created: bool +) -> None: + producer, store, config, _, _, clock = await _stabilized_harness(store_factory, official=False) + for hour in (1, 7, 13): + clock[0] = END + timedelta(hours=hour) + await producer.run_configuration(config) + period_end = END + timedelta(days=1) + clock[0] = period_end + timedelta(hours=1) + if period_already_created: + await producer.close_elapsed_periods(config) + producer._source.capabilities = _capabilities( + restatement_window="PT1H", restatement_cadence="PT1H", official_close_lag=None + ) + restarted = _restart_producer(producer, store, clock) + for hour in (1, 2, 7, 13): + clock[0] = period_end + timedelta(hours=hour) + await restarted.run_configuration(config) + obligation = await store.find_obligation( + account_id=config.account_id, + consumer_id=config.consumer_id, + delivery_config_id=config.delivery_config_id, + delivery_config_version=config.delivery_config_version, + period_start=END, + period_end=period_end, + ) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + observations = await store.list_provisional_observations(**owner, limit=5) + assert [item.checked_at for item in observations] == [ + period_end + timedelta(hours=h) for h in (1, 7, 13) + ] + assert all(item.acquisition.policy.cadence == timedelta(hours=6) for item in observations) + assert all(item.acquisition.policy.until_stabilized for item in observations) + assert [ + r.finalized_at for r in await store.list_revisions(**owner) if r.finality == "official" + ] == [clock[0]] + + +@pytest.mark.parametrize( + "missing", + ["verify_snapshot_origin", "get_provisional_acquisition", "get_stabilization_schedule"], +) +@pytest.mark.parametrize("official,track", [(False, True), (True, True), (True, False)]) +async def test_direct_tracked_snapshot_requires_stabilization_capabilities_before_dispatch( + store_factory: Any, missing: str, official: bool, track: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, obligation, calls, _ = await _stabilized_harness( + store_factory, official=official + ) + producer._source.capabilities = _capabilities(official_close_lag=None) + monkeypatch.setattr(store, missing, None) + with pytest.raises(LedgerConflictError, match=missing): + await producer.acquire_obligation( + config, obligation, target_finality="snapshot", track_settling=track + ) + assert not calls + assert ( + await store.get_provisional_observation( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + is None + ) + + +def _stabilization_publisher(store: Any) -> Any: + class Publisher: + def __getattr__(self, name: str) -> Any: + return getattr(store, name) + + publisher = Publisher() + for name in ( + "verify_snapshot_origin", + "get_provisional_acquisition", + "get_stabilization_schedule", + "list_provisional_observations", + "source_publication", + "get_restatement_checkpoint", + "record_restatement_checkpoint", + "reserve_provisional_acquisition", + "get_provisional_observation", + "commit_provisional_observation", + ): + setattr(publisher, name, getattr(store, name)) + return publisher + + +def _restart_producer( + producer: ReportingProducer, publisher: Any, clock: list[datetime] +) -> ReportingProducer: + return ReportingProducer( + source=producer._source, + store=publisher, + retry_store=producer._retry_store, + offerings=producer._offerings, + object_reader=producer._object_reader, + clock=lambda: clock[0], + read_jitter_window=timedelta(0), + ) + + +async def test_unchanged_scheduled_observation_respects_the_publishers_origin_witness( + store_factory: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, obligation, calls, clock = await _stabilized_harness(store_factory) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + await producer.run_configuration(config) + first = await store.get_provisional_observation(**owner) + publisher = _stabilization_publisher(store) + + async def conflicting_origin(revision: ReportingRevisionRecord) -> None: + assert revision.reporting_revision_id == first.revision_id + raise LedgerConflictError("STABILIZATION_ORIGIN_CONFLICT", "publisher origin differs") + + monkeypatch.setattr(publisher, "verify_snapshot_origin", conflicting_origin) + producer._store = publisher + clock[0] += timedelta(hours=6) + await producer.run_configuration(config) + assert await store.list_provisional_observations(**owner, limit=3) == (first,) + assert all(r.finality == "snapshot" for r in await store.list_revisions(**owner)) + clock[0] += timedelta(hours=1) + await producer.run_configuration(config) + assert len(calls) == 2 + + +async def test_direct_tracked_unchanged_snapshot_checks_the_admitted_witness_without_live_close( + store_factory: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, obligation, _, clock = await _stabilized_harness( + store_factory, official=False + ) + producer._source.capabilities = _capabilities( + restatement_window="P3D", restatement_cadence="PT6H", official_close_lag=None + ) + publisher = _stabilization_publisher(store) + producer._store = publisher + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + await producer.acquire_obligation(config, obligation, track_settling=True) + first = await store.get_provisional_observation(**owner) + + async def conflicting_origin(revision: ReportingRevisionRecord) -> None: + raise LedgerConflictError("STABILIZATION_ORIGIN_CONFLICT", "publisher origin differs") + + monkeypatch.setattr(publisher, "verify_snapshot_origin", conflicting_origin) + clock[0] += timedelta(hours=6) + with pytest.raises(LedgerConflictError, match="publisher origin differs"): + await producer.acquire_obligation(config, obligation, restate=True, track_settling=True) + assert await store.list_provisional_observations(**owner, limit=3) == (first,) + + @pytest.mark.parametrize("unchanged", ["PT12H", "P30D"]) @pytest.mark.parametrize("wrapped", [False, True]) async def test_stabilized_retires_base_shaped_derived_official_reservation( @@ -295,34 +506,8 @@ async def test_stabilized_retires_base_shaped_derived_official_reservation( ) assert "until_stabilized" not in reservation.to_wire()["policy"] - class Publisher: - def __getattr__(self, name: str) -> Any: - return getattr(store, name) - - publisher = Publisher() if wrapped else store - if wrapped: - for name in ( - "verify_snapshot_origin", - "get_provisional_acquisition", - "get_stabilization_schedule", - "list_provisional_observations", - "source_publication", - "get_restatement_checkpoint", - "record_restatement_checkpoint", - "reserve_provisional_acquisition", - "get_provisional_observation", - "commit_provisional_observation", - ): - setattr(publisher, name, getattr(store, name)) - fresh = ReportingProducer( - source=producer._source, - store=publisher, - retry_store=store, - offerings=producer._offerings, - object_reader=producer._object_reader, - clock=lambda: clock[0], - read_jitter_window=timedelta(0), - ) + publisher = _stabilization_publisher(store) if wrapped else store + fresh = _restart_producer(producer, publisher, clock) async def interrupt_replacement(*args: Any, **kwargs: Any) -> Any: raise OSError("interrupted before successor reservation") @@ -332,15 +517,7 @@ async def interrupt_replacement(*args: Any, **kwargs: Any) -> Any: with pytest.raises(OSError, match="before successor reservation"): await fresh.acquire_obligation(config, obligation) assert await store.get_stabilization_schedule(config.generation_key) == reservation.policy - fresh = ReportingProducer( - source=producer._source, - store=publisher, - retry_store=store, - offerings=producer._offerings, - object_reader=producer._object_reader, - clock=lambda: clock[0], - read_jitter_window=timedelta(0), - ) + fresh = _restart_producer(producer, publisher, clock) for hour in (1, 1, 2, 7, 13): clock[0] = END + timedelta(hours=hour) await fresh.acquire_obligation(config, obligation) From 5354345be87614dc470daeeef80b37272520ad35 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 18:03:33 +0530 Subject: [PATCH 20/39] fix(reporting): verify recovered schedules and tracked snapshot origins --- .../src/adcp_reporting_storage/ledger/pg.py | 14 +- src/adcp/reporting/ledger/producer.py | 115 +++++++---- src/adcp/reporting/ledger/provisional.py | 24 ++- src/adcp/reporting/ledger/store.py | 30 ++- src/adcp/reporting/service.py | 55 +++-- tests/test_reliable_reporting_service.py | 41 +++- tests/test_reporting_official_finality.py | 192 +++++++++++++++++- 7 files changed, 397 insertions(+), 74 deletions(-) diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py index 081bab92f..3397ac90f 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py @@ -1958,6 +1958,7 @@ async def commit_provisional_observation( "OBSERVATION_CONFLICT", "observation replay differs: acquisition does not bind this obligation", ) + require_origin = acquisition.requires_snapshot_origin(obligation) existing = await ( await connection.execute( "SELECT reporting_revision_id,payload FROM reporting_provisional_observations" @@ -1987,6 +1988,9 @@ async def commit_provisional_observation( raise LedgerConflictError( "HISTORY_UNAVAILABLE", "observation revision is missing" ) + if require_origin and retained_revision.finality == "snapshot": + await self.verify_snapshot_origin(retained_revision) + validate_snapshot_origin(retained_revision, observation.revision_publication_id) # Reuse the immutable publication replay checks while holding # the observation's account lock and transaction. if revision is None or (reuse_candidate and existing[0] == observation.revision_id): @@ -2047,10 +2051,14 @@ async def commit_provisional_observation( raise LedgerConflictError( "OBSERVATION_CONFLICT", "snapshot predecessor is no longer current" ) - if acquisition.policy.until_stabilized: + if require_origin: await self.verify_snapshot_origin(committed) validate_retained_provisional_observation( - committed, observation.bind_revision(committed), rows, require_readable=False + committed, + observation.bind_revision(committed, require_origin=require_origin), + rows, + require_readable=False, + require_origin=require_origin, ) if not committed.readable: if revision is None: @@ -2061,7 +2069,7 @@ async def commit_provisional_observation( observation = replace(observation, revision_id=committed.reporting_revision_id) else: committed = await self.commit_revision(revision, rows) - observation = observation.bind_revision(committed) + observation = observation.bind_revision(committed, require_origin=require_origin) await connection.execute( "INSERT INTO reporting_provisional_observations" " (account_id,reporting_obligation_id,ordinal,reporting_revision_id,payload)" diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 087797dac..3835d80af 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -1344,18 +1344,24 @@ async def _recover_stabilization_schedule( self._stabilization_store() if frozen is not None: return frozen - if not _implements_store_method( - self._store, StabilizationScheduleStore.get_stabilization_schedule - ): + store = self._stabilization_schedule_store() + if store is None: return None - assert isinstance(self._store, StabilizationScheduleStore) - policy = await self._store.get_stabilization_schedule(key) + policy = await store.get_stabilization_schedule(key) if policy is not None: if not required: self._stabilization_store() self._recovered_stabilization[key] = policy return policy + def _stabilization_schedule_store(self) -> StabilizationScheduleStore | None: + if _implements_store_method( + self._store, StabilizationScheduleStore.get_stabilization_schedule + ): + assert isinstance(self._store, StabilizationScheduleStore) + return self._store + return None + async def _acquire_stabilized( self, configuration: ReportingConfiguration, @@ -2006,6 +2012,7 @@ async def _acquire_obligation( acquisition=acquisition, verified_source_rows=True, suppress_untracked_unchanged=untracked_restatement, + admit_tracked_stabilization=admit_tracked_stabilization, ) except LedgerConflictError as error: if error.code == "OFFICIAL_REVISION_TERMINAL" and untracked_restatement: @@ -2799,8 +2806,25 @@ async def _commit_revision_from_manifest( acquisition: ProvisionalAcquisition | None, verified_source_rows: bool, suppress_untracked_unchanged: bool = False, + admit_tracked_stabilization: bool = True, ) -> ReportingRevisionRecord: obligation = await self._stored_obligation(obligation) + selected_policy = obligation.definition.finality_policy if obligation.definition else None + origin_store: SnapshotOriginStore | None = None + if finality == "snapshot" and acquisition is not None: + if acquisition.requires_snapshot_origin( + obligation, + explicitly_tracked=( + admit_tracked_stabilization + or obligation.generation_key in self._recovered_stabilization + ), + ): + origin_store = self._stabilization_store() + elif isinstance( + selected_policy, ReportingStabilizedPolicy + ) and _implements_store_method(self._store, SnapshotOriginStore.verify_snapshot_origin): + assert isinstance(self._store, SnapshotOriginStore) + origin_store = self._store self._validate_manifest_currency(obligation, manifest) if any(cell.status == "unsupported" for cell in manifest.metric_availability): try: @@ -2848,7 +2872,6 @@ async def _commit_revision_from_manifest( control_totals = tuple((total.name, total.value) for total in manifest.control_totals) revision_id = f"rpr_{manifest.publication_id[4:44]}" - selected_policy = obligation.definition.finality_policy if obligation.definition else None stabilized = finality == "official" and isinstance( selected_policy, ReportingStabilizedPolicy ) @@ -2912,7 +2935,7 @@ async def _commit_revision_from_manifest( ) if not verified_source_rows: replay = await self._retained_snapshot_replay( - obligation, manifest, rows, acquisition, prior, now + obligation, manifest, rows, acquisition, prior, now, origin_store ) if replay is not None: return replay @@ -3068,6 +3091,7 @@ async def _commit_revision_from_manifest( now, acquisition, candidate_proof=candidate_proof, + origin_store=origin_store, retained_revision_id=( retained.reporting_revision_id if retained is not None else None ), @@ -3094,6 +3118,7 @@ async def _retained_snapshot_replay( acquisition: ProvisionalAcquisition, prior: ReportingRevisionRecord | None, now: datetime, + origin_store: SnapshotOriginStore | None, ) -> ReportingRevisionRecord | None: store = self._observation_store() latest = await store.get_provisional_observation( @@ -3152,6 +3177,10 @@ async def _retained_snapshot_replay( validate_retained_provisional_observation( revision, latest if is_latest else observation.bind_revision(revision), rows ) + if origin_store is not None: + await self._verify_tracked_snapshot_origin( + obligation, acquisition, origin_store, revision + ) if is_latest: return revision # A later observation closes this ordinal to new writes. The store @@ -3273,55 +3302,57 @@ def _uses_stabilization_schedule( or obligation.generation_key in self._recovered_stabilization ) - async def _verify_reused_snapshot_origin( + async def _verify_tracked_snapshot_origin( self, obligation: ReportingObligationRecord, acquisition: ProvisionalAcquisition, - revision_id: str, + witness: SnapshotOriginStore, + revision: ReportingRevisionRecord, + *, + retained_revision_id: str | None = None, ) -> None: + if retained_revision_id is not None: + predecessor = await self._store.get_revision( + account_id=obligation.account_id, reporting_revision_id=retained_revision_id + ) + if predecessor is None: + raise LedgerConflictError("HISTORY_UNAVAILABLE", "snapshot predecessor is missing") + if not predecessor.readable: + # The atomic publisher can use the fresh snapshot when the + # predecessor loses readability during staging. + await witness.verify_snapshot_origin(revision) + revision = predecessor + await witness.verify_snapshot_origin(revision) + if revision.reporting_revision_id != acquisition.predecessor_revision_id: + return selected = obligation.definition.finality_policy if obligation.definition else None - if not isinstance(selected, ReportingStabilizedPolicy) or not ( - acquisition.policy.until_stabilized or acquisition.is_scheduled_snapshot + if isinstance(selected, ReportingStabilizedPolicy) and self._uses_stabilization_schedule( + obligation, acquisition.policy ): - return - active = self._uses_stabilization_schedule(obligation, acquisition.policy) - if active: - witness: SnapshotOriginStore = self._stabilization_store() - elif _implements_store_method(self._store, SnapshotOriginStore.verify_snapshot_origin): - assert isinstance(self._store, SnapshotOriginStore) - witness = self._store - else: - return - predecessor = await self._store.get_revision( - account_id=obligation.account_id, reporting_revision_id=revision_id - ) - if predecessor is None: - raise LedgerConflictError("HISTORY_UNAVAILABLE", "snapshot predecessor is missing") - await witness.verify_snapshot_origin(predecessor) - if active: limit = stabilization_history_limit(selected, acquisition.policy.cadence) for observed in await self._scheduled_snapshot_history(obligation, limit): - if observed.revision_id == revision_id: - validate_snapshot_origin(predecessor, observed.snapshot_origin) + if observed.revision_id == revision.reporting_revision_id: + validate_snapshot_origin(revision, observed.snapshot_origin) async def _commit_publication( self, obligation: ReportingObligationRecord, manifest: SourceBatchManifestV1, - revision: ReportingRevisionRecord | None, + revision: ReportingRevisionRecord, rows: Sequence[dict[str, Any]], finality: str, now: datetime, acquisition: ProvisionalAcquisition | None, *, candidate_proof: FinalityProof | None, + origin_store: SnapshotOriginStore | None, retained_revision_id: str | None = None, ) -> tuple[ReportingRevisionRecord, bool]: async with self._revision_publication( obligation, manifest, official=finality == "official" ): replay = None - if finality == "official" and revision is not None: + if finality == "official": selected = obligation.definition.finality_policy if obligation.definition else None if candidate_proof is not None and isinstance(selected, ReportingStabilizedPolicy): scheduled = await self._scheduled_snapshot_history(obligation, 1) @@ -3354,7 +3385,7 @@ async def _commit_publication( proved_candidate=candidate_proof is not None, ) acquisition = None - if finality == "snapshot" and acquisition is None and revision is not None: + if finality == "snapshot" and acquisition is None: siblings = await self._store.list_revisions( account_id=obligation.account_id, reporting_obligation_id=obligation.reporting_obligation_id, @@ -3367,22 +3398,24 @@ async def _commit_publication( "a new snapshot cannot follow the official revision", ) if acquisition is None: - assert revision is not None committed = await self._store.commit_revision(revision, rows) else: # ``now`` is the publication anchor, sampled after row acquisition. # The bounds check above already rejects acquired_at after now. - if retained_revision_id is not None: - observation_revision_id = retained_revision_id - await self._verify_reused_snapshot_origin( - obligation, acquisition, retained_revision_id + if origin_store is not None: + await self._verify_tracked_snapshot_origin( + obligation, + acquisition, + origin_store, + revision, + retained_revision_id=retained_revision_id, ) - else: - assert revision is not None - observation_revision_id = revision.reporting_revision_id committed = await self._observation_store().commit_provisional_observation( ProvisionalObservation.from_manifest( - acquisition, observation_revision_id, now, manifest + acquisition, + retained_revision_id or revision.reporting_revision_id, + now, + manifest, ), revision, rows, diff --git a/src/adcp/reporting/ledger/provisional.py b/src/adcp/reporting/ledger/provisional.py index d53924aa2..6c9f67b7b 100644 --- a/src/adcp/reporting/ledger/provisional.py +++ b/src/adcp/reporting/ledger/provisional.py @@ -12,6 +12,7 @@ from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any, Literal, Protocol, runtime_checkable +from adcp.reporting.ledger.finality import ReportingStabilizedPolicy from adcp.reporting.source import ReportingSourceSliceRequestV1 if TYPE_CHECKING: @@ -125,6 +126,21 @@ def is_scheduled_snapshot(self) -> bool: and self.output_finality is None ) + def requires_snapshot_origin( + self, obligation: ReportingObligationRecord, *, explicitly_tracked: bool = False + ) -> bool: + definition = obligation.definition + return self.policy.until_stabilized or ( + definition is not None + and isinstance(definition.finality_policy, ReportingStabilizedPolicy) + and ( + explicitly_tracked + or obligation.required_finality == "official" + or self.policy.official_close_lag is not None + or not self.is_scheduled_snapshot + ) + ) + def stabilization_schedule( self, required_finality: str, *, pending: bool ) -> ProvisionalPolicy | None: @@ -234,9 +250,13 @@ def snapshot_origin(self) -> str | None: return origin return self.revision_publication_id - def bind_revision(self, revision: ReportingRevisionRecord) -> ProvisionalObservation: + def bind_revision( + self, revision: ReportingRevisionRecord, *, require_origin: bool = False + ) -> ProvisionalObservation: """Keep the full originating publication ID when an unchanged read reuses a snapshot.""" - if self.acquisition.policy.until_stabilized and revision.finality == "snapshot": + if revision.finality == "snapshot" and ( + require_origin or self.acquisition.policy.until_stabilized + ): if ( self.revision_publication_id is not None and self.revision_publication_id != revision.source_publication_id diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index 53870df63..f27bd29e7 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -752,10 +752,17 @@ def validate_retained_provisional_observation( rows: Sequence[dict[str, Any]], *, require_readable: bool = True, + require_origin: bool = False, ) -> None: """Validate an unchanged observation against immutable predecessor content.""" validate_snapshot_observation_binding( - revision, observation, require_publication=observation.acquisition.policy.until_stabilized + revision, + observation, + require_publication=( + require_origin + or observation.acquisition.policy.until_stabilized + or observation.revision_publication_id is not None + ), ) validate_snapshot_rows(revision, rows, require_readable=require_readable) @@ -1550,6 +1557,7 @@ async def commit_provisional_observation( "OBSERVATION_CONFLICT", "observation replay differs: acquisition does not bind this obligation", ) + require_origin = acquisition.requires_snapshot_origin(obligation) if existing is not None: if ( existing.acquisition != acquisition @@ -1568,15 +1576,17 @@ async def commit_provisional_observation( raise LedgerConflictError( "HISTORY_UNAVAILABLE", "observation revision is missing" ) + retained_revision = self._revisions[existing.revision_id] + if require_origin and retained_revision.finality == "snapshot": + await self.verify_snapshot_origin(retained_revision) + validate_snapshot_origin(retained_revision, observation.revision_publication_id) # A matching observation identity does not make conflicting # publication content an idempotent replay. if revision is None or ( reuse_candidate and existing.revision_id == observation.revision_id ): - validate_retained_provisional_observation( - self._revisions[existing.revision_id], existing, rows - ) - return self._revisions[existing.revision_id] + validate_retained_provisional_observation(retained_revision, existing, rows) + return retained_revision return await self.commit_revision(revision, rows) retained = self._provisional_acquisitions.get(key) if retained != acquisition: @@ -1616,10 +1626,14 @@ async def commit_provisional_observation( raise LedgerConflictError( "OBSERVATION_CONFLICT", "snapshot predecessor is no longer current" ) - if acquisition.policy.until_stabilized: + if require_origin: await self.verify_snapshot_origin(committed) validate_retained_provisional_observation( - committed, observation.bind_revision(committed), rows, require_readable=False + committed, + observation.bind_revision(committed, require_origin=require_origin), + rows, + require_readable=False, + require_origin=require_origin, ) if not committed.readable: if revision is None: @@ -1631,7 +1645,7 @@ async def commit_provisional_observation( else: committed = await self.commit_revision(revision, rows) # The checkpoint enforces ascending ordinals for distinct observations. - observation = observation.bind_revision(committed) + observation = observation.bind_revision(committed, require_origin=require_origin) self._provisional_observations.setdefault(owner, {})[acquisition.ordinal] = observation return committed diff --git a/src/adcp/reporting/service.py b/src/adcp/reporting/service.py index 3f536d1ee..c8d8ca54d 100644 --- a/src/adcp/reporting/service.py +++ b/src/adcp/reporting/service.py @@ -437,6 +437,15 @@ class _Binding: context: ReportingAccountContext producer: ReportingProducer + @property + def missing_stabilization_lookup(self) -> bool: + definition = self.configuration.definition + return ( + definition is not None + and isinstance(definition.finality_policy, ReportingStabilizedPolicy) + and self.producer._stabilization_schedule_store() is None + ) + class _MissingFinalityPolicyError(ReliableReportingConfigurationError): """A retained generation may still serve reads while its producer is unbound.""" @@ -564,9 +573,7 @@ def __init__( self._reconciled_billing = reconciled_billing self._worker_error_handler = worker_error_handler self._bindings: dict[ReportingConfigurationGenerationKey, _Binding] = {} - self._pending_configurations: dict[ - ReportingConfigurationGenerationKey, ReportingConfiguration - ] = {} + self._pending_bindings: dict[ReportingConfigurationGenerationKey, _Binding] = {} self._initialized = False self._configuration_lock = asyncio.Lock() self._initialization_errors: dict[ReportingConfigurationGenerationKey, Exception] = {} @@ -702,7 +709,7 @@ async def _configure( retrying: bool = False, ) -> None: key = configuration.generation_key - existing = self._bindings.get(key) + existing = self._bindings.get(key) or self._pending_bindings.get(key) if existing is not None: if existing.configuration != configuration: raise ReliableReportingConfigurationError( @@ -783,15 +790,37 @@ async def _configure( raise _AccountContextResolutionError(error, permanent=True) from error raise binding = _Binding(configuration=configuration, context=context, producer=producer) - if persist: - if self._initialized: - await self.store.put_configuration(configuration) - else: - self._pending_configurations[key] = configuration - self._bindings[key] = binding + if persist and not self._initialized: + self._pending_bindings[key] = binding + # Schema initialization makes it possible to distinguish a new snapshot + # generation from an existing one whose frozen close needs recovery. + if not binding.missing_stabilization_lookup: + self._bindings[key] = binding + else: + await self._install_binding(binding, persist=persist) self._initialization_errors.pop(key, None) self._pending_recovery.pop(key, None) + async def _install_binding(self, binding: _Binding, *, persist: bool) -> None: + configuration = binding.configuration + key = configuration.generation_key + if binding.missing_stabilization_lookup and ( + not persist + or any( + stored.generation_key == key + for stored in await self.store.list_configurations( + caller=ReportingStatusCaller(key.account_id, key.consumer_id), + delivery_config_ids=[key.delivery_config_id], + ) + ) + ): + raise ReliableReportingConfigurationError( + "recovering a selected stabilized generation requires get_stabilization_schedule" + ) + if persist: + await self.store.put_configuration(configuration) + self._bindings[key] = binding + @property def initialization_errors(self) -> Mapping[ReportingConfigurationGenerationKey, Exception]: """Read-only view of unresolved stored configuration bindings. @@ -1023,9 +1052,9 @@ async def _initialize(self) -> None: await self.store.create_schema() if self._initialize_source_storage is not None: await self._initialize_source_storage() - for configuration in self._pending_configurations.values(): - await self.store.put_configuration(configuration) - self._pending_configurations.clear() + for binding in self._pending_bindings.values(): + await self._install_binding(binding, persist=True) + self._pending_bindings.clear() if self._production is None and self.sources.names: # Older custom stores can retain explicit configure() startup. # Built-in stores recover all generations in one scan, without diff --git a/tests/test_reliable_reporting_service.py b/tests/test_reliable_reporting_service.py index a0b4572df..dbe5d1799 100644 --- a/tests/test_reliable_reporting_service.py +++ b/tests/test_reliable_reporting_service.py @@ -455,7 +455,10 @@ async def create_schema(self) -> None: assert not await store.list_all_configurations() -async def test_plain_snapshot_custom_publisher_needs_no_stabilization_apis() -> None: +@pytest.mark.parametrize("configure_before_init", [True, False]) +async def test_plain_snapshot_custom_publisher_needs_no_stabilization_apis( + configure_before_init: bool, +) -> None: class SnapshotPublisher(InMemoryReportingLedgerStore): get_stabilization_schedule = None verify_snapshot_origin = None @@ -468,23 +471,46 @@ class SnapshotPublisher(InMemoryReportingLedgerStore): DEFINITION, finality_policy=ReportingStabilizedPolicy("optional", "PT0S", "PT6H") ), ) + clock = DeterministicReportingClock(NOW) + result = InlineFetchResult(rows=_rows(10), data_through=config.deactivated_at) service = _finality_service( config, snapshot=True, official=False, store=SnapshotPublisher(), - clock=DeterministicReportingClock(NOW + timedelta(days=1)), - results=(InlineFetchResult(rows=_rows(10)),), + clock=clock, + cadence="PT1H", + results=(result, result), ) + if not configure_before_init: + await service.initialize() await service.configure(config) await service.initialize() assert config.generation_key in service._bindings turn = await service.run_worker() assert len(turn.configurations[config.generation_key].revisions_committed) == 1 + (identifier,) = turn.configurations[config.generation_key].obligations_committed + clock.advance(timedelta(hours=1)) + turn = await service.run_worker() + assert not turn.configuration_errors + history = await service.store.list_provisional_observations( + account_id=config.account_id, reporting_obligation_id=identifier, limit=2 + ) + assert len(history) == 2 and history[0].revision_id == history[1].revision_id -async def test_recovered_snapshot_close_requires_origin_witness_after_live_lag_removal( +@pytest.mark.parametrize( + "missing,configure_before_init", + [ + ("verify_snapshot_origin", False), + ("get_stabilization_schedule", False), + ("get_stabilization_schedule", True), + ], +) +async def test_recovered_snapshot_close_requires_evidence_after_live_lag_removal( monkeypatch: pytest.MonkeyPatch, + missing: str, + configure_before_init: bool, ) -> None: clock = DeterministicReportingClock(NOW + timedelta(days=1)) store = InMemoryReportingLedgerStore(clock=clock) @@ -509,11 +535,14 @@ async def test_recovered_snapshot_close_requires_origin_witness_after_live_lag_r await original.configure(config) turn = await original.run_worker() assert len(turn.configurations[config.generation_key].revisions_committed) == 1 - monkeypatch.setattr(store, "verify_snapshot_origin", None) + monkeypatch.setattr(store, missing, None) recovered = _finality_service( config, snapshot=True, official=False, cadence="PT1H", store=store, clock=clock ) - with pytest.raises(ReliableReportingConfigurationError, match="verify_snapshot_origin"): + if configure_before_init: + await recovered.configure(config) + assert config.generation_key not in recovered._bindings + with pytest.raises(ReliableReportingConfigurationError, match=missing): await recovered.initialize() assert config.generation_key not in recovered._bindings diff --git a/tests/test_reporting_official_finality.py b/tests/test_reporting_official_finality.py index 7fb67f208..b1d3d4fb5 100644 --- a/tests/test_reporting_official_finality.py +++ b/tests/test_reporting_official_finality.py @@ -42,7 +42,11 @@ manifest_finality_proof, source_finality_proof, ) -from adcp.reporting.ledger.provisional import ProvisionalAcquisition, ProvisionalPolicy +from adcp.reporting.ledger.provisional import ( + ProvisionalAcquisition, + ProvisionalObservation, + ProvisionalPolicy, +) from adcp.reporting.source import ( ReportingFinalityEvidenceV1, ReportingSourceExecutorResult, @@ -423,6 +427,192 @@ def _restart_producer( ) +@pytest.mark.parametrize( + "failure", + [ + "get_stabilization_schedule", + "get_provisional_acquisition", + "verify_snapshot_origin", + "origin", + ], +) +@pytest.mark.parametrize("official", [True, False]) +async def test_reserved_manifest_requires_stabilization_evidence_before_first_snapshot( + store_factory: Any, failure: str, official: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, obligation, _, clock = await _stabilized_harness( + store_factory, official=official + ) + producer._source.capabilities = _capabilities(official_close_lag=None) + request = producer._build_slice( + config, obligation, SNAPSHOT_OFFERING_ID, finality="snapshot", now=clock[0] + ) + reservation = await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + request.model_dump_json(), 0, ProvisionalPolicy(timedelta(days=1), timedelta(hours=6)) + ) + ) + await producer._source.execute(request) + manifest = producer._source.manifest + publisher = _stabilization_publisher(store) + if failure == "origin": + + async def contradict(candidate: ReportingRevisionRecord) -> None: + assert candidate.source_publication_id == manifest.publication_id + assert not await store.list_revisions(**owner) + assert await store.get_provisional_observation(**owner) is None + assert await store.get_restatement_checkpoint(**owner) is None + raise LedgerConflictError("STABILIZATION_ORIGIN_CONFLICT", "publisher origin differs") + + monkeypatch.setattr(publisher, "verify_snapshot_origin", contradict) + else: + monkeypatch.setattr(publisher, failure, None) + fresh = _restart_producer(producer, publisher, clock) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + with pytest.raises(LedgerConflictError, match=failure): + await fresh.commit_revision_from_manifest( + replace(obligation, definition=None), + manifest, + rows=ROWS, + finality="snapshot", + acquisition=reservation, + ) + assert not await store.list_revisions(**owner) + assert await store.get_provisional_observation(**owner) is None + assert await store.get_restatement_checkpoint(**owner) is None + assert await store.get_provisional_acquisition(**owner, ordinal=0) == reservation + fresh._store = store + published = await fresh.commit_revision_from_manifest( + obligation, manifest, rows=ROWS, finality="snapshot", acquisition=reservation + ) + assert published.source_publication_id == manifest.publication_id + assert (await store.get_restatement_checkpoint(**owner)).next_observation == 1 + + +@pytest.mark.parametrize("entry", ["manifest", "acquire", "ledger"]) +@pytest.mark.parametrize("official", [True, False]) +async def test_manual_snapshot_reservation_without_stabilization_flag_checks_predecessor_origin( + store_factory: Any, entry: str, official: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, obligation, _, clock = await _stabilized_harness( + store_factory, official=official + ) + producer._source.capabilities = _capabilities( + restatement_window="P3D", restatement_cadence="PT6H", official_close_lag=None + ) + first = await producer.acquire_obligation( + config, obligation, target_finality="snapshot", track_settling=True + ) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + observations = await store.list_provisional_observations(**owner, limit=2) + checkpoint = await store.get_restatement_checkpoint(**owner) + clock[0] += timedelta(hours=1) + request = producer._build_slice( + config, + obligation, + SNAPSHOT_OFFERING_ID, + finality="snapshot", + trigger="manual_replay", + observation=1, + now=clock[0], + ) + reservation = await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + request.model_dump_json(), + 1, + ProvisionalPolicy(timedelta(days=3), timedelta(hours=6)), + first.reporting_revision_id, + ) + ) + assert "until_stabilized" not in reservation.to_wire()["policy"] + publisher = _stabilization_publisher(store) + + async def contradict(predecessor: ReportingRevisionRecord) -> None: + assert predecessor == first + raise LedgerConflictError("STABILIZATION_ORIGIN_CONFLICT", "publisher origin differs") + + monkeypatch.setattr( + store if entry == "ledger" else publisher, "verify_snapshot_origin", contradict + ) + fresh = _restart_producer(producer, publisher, clock) + with pytest.raises(LedgerConflictError, match="publisher origin differs"): + if entry == "ledger": + await producer._source.execute(request) + observation = ProvisionalObservation.from_manifest( + reservation, first.reporting_revision_id, clock[0], producer._source.manifest + ) + await store.commit_provisional_observation(observation, None, ROWS) + elif entry == "manifest": + await producer._source.execute(request) + await fresh.commit_revision_from_manifest( + obligation, + producer._source.manifest, + rows=ROWS, + finality="snapshot", + acquisition=reservation, + ) + else: + await fresh.acquire_obligation( + config, obligation, target_finality="snapshot", manual_replay=True, restate=True + ) + assert await store.list_provisional_observations(**owner, limit=2) == observations + assert await store.get_restatement_checkpoint(**owner) == checkpoint + assert await store.get_provisional_acquisition(**owner, ordinal=1) == reservation + monkeypatch.undo() + await producer._source.execute(request) + manifest = producer._source.manifest + if entry == "ledger": + observation = ProvisionalObservation.from_manifest( + reservation, first.reporting_revision_id, clock[0], manifest + ) + origin = first.source_publication_id + different_origin = origin[:-1] + ("0" if origin[-1] != "0" else "1") + with pytest.raises(LedgerConflictError, match="origin|publication"): + await store.commit_provisional_observation( + replace(observation, revision_publication_id=different_origin), None, ROWS + ) + assert await store.get_restatement_checkpoint(**owner) == checkpoint + retried = await fresh.commit_revision_from_manifest( + obligation, manifest, rows=ROWS, finality="snapshot", acquisition=reservation + ) + assert retried == first + latest = await store.get_provisional_observation(**owner) + assert latest.revision_publication_id == first.source_publication_id + if entry == "ledger": + with pytest.raises(LedgerConflictError, match="origin|publication"): + await store.commit_provisional_observation( + replace(latest, revision_publication_id=different_origin), None, ROWS + ) + + +async def test_reserved_snapshot_replay_checks_the_publishers_witness(store_factory: Any) -> None: + producer, store, config, obligation, _, _ = await _stabilized_harness(store_factory) + await producer.run_configuration(config) + first = await store.get_provisional_observation( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + publisher = _stabilization_publisher(store) + + async def contradict(candidate: ReportingRevisionRecord) -> None: + assert candidate.reporting_revision_id == first.revision_id + raise LedgerConflictError("STABILIZATION_ORIGIN_CONFLICT", "publisher origin differs") + + publisher.verify_snapshot_origin = contradict + producer._store = publisher + with pytest.raises(LedgerConflictError, match="publisher origin differs"): + await producer.commit_revision_from_manifest( + obligation, + producer._source.manifest, + rows=ROWS, + finality="snapshot", + acquisition=first.acquisition, + ) + + async def test_unchanged_scheduled_observation_respects_the_publishers_origin_witness( store_factory: Any, monkeypatch: pytest.MonkeyPatch ) -> None: From ee49fecdb8833224e48de9cca5b4de020817ddc5 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 18:33:59 +0530 Subject: [PATCH 21/39] fix(reporting): enforce explicit official finality on admission and replay --- docs/reporting-provisional-observations.md | 39 +++- docs/reporting-source-adapters.md | 39 ++++ src/adcp/reporting/ledger/producer.py | 172 +++++++++++++++--- src/adcp/reporting/ledger/provisional.py | 7 +- src/adcp/reporting/production/offerings.py | 4 + src/adcp/reporting/service.py | 20 +- src/adcp/reporting/source.py | 2 +- .../reporting/_production_support.py | 35 +++- .../reporting/_reliable_support.py | 15 ++ .../reporting/_scope_onboarding_server.py | 3 +- .../reporting/test_reporting_currency.py | 24 ++- ...reporting_evidence_currency_integration.py | 33 +++- .../test_reporting_materializer_contracts.py | 33 +++- ...test_reporting_production_configuration.py | 4 + .../test_reporting_production_lifecycle.py | 3 +- .../test_reporting_production_readiness.py | 5 +- .../test_reporting_production_settling.py | 5 +- .../test_reporting_projection_rc6.py | 3 +- .../test_reporting_publication_time.py | 28 ++- .../test_reporting_revision_selection.py | 53 ++++-- .../test_reporting_source_authorization.py | 3 +- tests/test_reliable_reporting_service.py | 26 ++- tests/test_reporting_ledger.py | 4 +- tests/test_reporting_official_acquisition.py | 159 ++++++++++++++++ tests/test_reporting_official_finality.py | 76 +++++++- ...test_reporting_provisional_observations.py | 19 +- tests/test_reporting_scheduling_jitter.py | 12 +- tests/test_reporting_settling.py | 48 ++++- 28 files changed, 774 insertions(+), 100 deletions(-) diff --git a/docs/reporting-provisional-observations.md b/docs/reporting-provisional-observations.md index 36dd596a5..697c14801 100644 --- a/docs/reporting-provisional-observations.md +++ b/docs/reporting-provisional-observations.md @@ -34,10 +34,10 @@ reuse, installed-package checks and independent integration review. Follow the - Snapshot-only reporting retains one final inclusive read at the boundary, including after downtime. Valid new source evidence may extend that boundary. Expiry does not create an official revision. -- Existing explicitly configured official-close behavior remains in force. - `official_close_lag` still requires an explicitly declared `restatement_window` - and an official offering. The SDK fallback does not opt an adapter into - automatic official publication. +- An explicit snapshot-to-official route requires a selected definition policy. + `official_close_lag` requires an explicitly declared `restatement_window` + and schedules a close attempt. The selected policy determines the source + offering and the proof needed to publish. The durable acquisition freezes the source request, predecessor revision and observation ordinal. A retry renews only `deadline_at`, the execution budget for @@ -45,6 +45,37 @@ that attempt. It retains the execution key, run ID, cutoff, scope and remaining request fields even after the original deadline expires. Once reserved, a successful retry completes that observation before a later acquisition starts. +## Selecting official finality + +For a new generation with `required_finality: official`, the adopter copies +one policy from the pinned report definition into +`ReportingDefinitionBinding.finality_policy`. Use the same selection when +configuring an explicit snapshot-to-official close. To migrate unfinished +official work with no policy, configure a new generation with the selected +policy. Retained revisions and exact replays keep their original evidence. + +- A `source_final` policy uses the authoritative offering. Its + `provider_declared` or `provider_job_terminal` evidence must carry the exact + selected signal and the upstream finalization time. +- A `contractual_cutoff` policy uses the snapshot offering. Its full, verified + observation must reach the declared period-end cutoff. +- A `stabilized` policy uses the snapshot offering and durable scheduled + observation history. Changed content or a missed cadence restarts the + unchanged interval. Manual reads do not establish a run. + +`stable_since` is a source observation and `official_close_lag` is scheduling +metadata. Snapshot-only generations need no selection and remain snapshots +regardless of elapsed time. After an official close the producer accepts exact +replay. Later corrections use adjustments. + +Custom Core ledgers that publish official revisions must implement +`ReportingSourcePublicationStore.source_publication(account_id)` with a lock +shared across all publishers for that account. Stabilization additionally +requires durable reservations, bounded scheduled history, checkpoints, exact +snapshot origins and generation schedule recovery. An ordinary snapshot route +can keep using its snapshot publisher when the definition also lists an unused +stabilization policy. + ## Persistence and compatibility `PgReportingLedgerStore.create_schema()` installs two additive private tables: diff --git a/docs/reporting-source-adapters.md b/docs/reporting-source-adapters.md index 907a31552..5b8c21073 100644 --- a/docs/reporting-source-adapters.md +++ b/docs/reporting-source-adapters.md @@ -266,3 +266,42 @@ and the [conformance tests](../tests/conformance/reporting/test_inline_cell_avai Run `run_reporting_source_replay_conformance` against your adapter and staging store to verify that the same execution key returns the original sealed evidence even if source measurements or availability later change. + +## Official source evidence + +For a new generation with `required_finality: official`, copy one policy from +the pinned report definition into `ReportingDefinitionBinding.finality_policy`. +The adopter selects the policy for the generation. An explicit snapshot-to-official +offering also needs a selection. Snapshot-only offerings need no policy. Their +final scheduled read still produces a snapshot, even after prolonged downtime. + +| Selected basis | Source offering | Required evidence | +| --- | --- | --- | +| `source_final` | Authoritative | Return `provider_declared` or `provider_job_terminal` evidence with the exact selected `source_signal` in `evidence_ref`. The provider's timestamp becomes `finalized_at`. | +| `contractual_cutoff` | Snapshot | Read and verify complete staged content at or after period end plus `duration_after_period_end`. The source observation and read cutoff must reach that boundary and the available data must cover the period end. | +| `stabilized` | Snapshot | The SDK checks verified content from consecutive scheduled observations at the frozen cadence against `minimum_age` and `unchanged_for`. | + +For example, if the definition declares the signal `warehouse.close.completed`, +an adapter can return the timestamp of the completed upstream job. + +```python +from adcp.reporting.inline_source import InlineFetchResult +from adcp.reporting.source import ReportingFinalityEvidenceV1 + +return InlineFetchResult( + rows=rows, + data_through=period_end, + finality_evidence=ReportingFinalityEvidenceV1( + basis="provider_job_terminal", + evidence_ref="warehouse.close.completed", + observed_at=job.completed_at, + ), +) +``` + +`stable_since` records source evidence and `official_close_lag` schedules an +explicit close attempt. Proof comes from the selected policy. A custom Core +ledger that admits official work must implement +`ReportingSourcePublicationStore.source_publication(account_id)` and serialize +publication for that account across workers. Exact reads and replays retain +the original policy fields of previously published official revisions. diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 3835d80af..595230189 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -64,6 +64,7 @@ MAX_STABILIZATION_HISTORY_OBSERVATIONS, FinalityProof, ReportingContractualCutoffPolicy, + ReportingSourceFinalPolicy, ReportingStabilizedPolicy, contractual_cutoff_at, finality_policy_source_class, @@ -689,10 +690,15 @@ async def close_elapsed_periods( def _stabilization_requested(self, configuration: ReportingConfiguration) -> bool: definition = configuration.definition - if definition is None or not isinstance( - definition.finality_policy, ReportingStabilizedPolicy - ): - return False + return ( + definition is not None + and isinstance(definition.finality_policy, ReportingStabilizedPolicy) + and self._official_close_requested(configuration) + ) + + def _official_close_requested( + self, configuration: ReportingConfiguration | ReportingObligationRecord + ) -> bool: return ( configuration.required_finality == "official" or configuration.generation_key in self._recovered_stabilization @@ -757,6 +763,8 @@ async def _close_elapsed_periods( ) ), ) + elif selected is not None and self._official_close_requested(configuration): + self._official_publication_lock() obligation = ReportingObligationRecord( reporting_obligation_id=self._obligation_id(configuration, boundary), account_id=configuration.account_id, @@ -1362,6 +1370,56 @@ def _stabilization_schedule_store(self) -> StabilizationScheduleStore | None: return self._store return None + async def _has_retained_stabilization(self, configuration: ReportingConfiguration) -> bool: + """Inspect retained routes when a snapshot publisher lacks the generation lookup.""" + key = configuration.generation_key + snapshot = await self._store.open_snapshot( + caller=ReportingDeliveryPrincipal(key.account_id, key.consumer_id), + filters_fingerprint=f"stabilization:{key.delivery_config_id}:{key.delivery_config_version}", + ) + offset = 0 + page_size = 100 + while True: + page = await self._store.read_page( + snapshot=snapshot, + consumer_id=key.consumer_id, + delivery_config_ids=[key.delivery_config_id], + media_buy_ids=None, + changes_after_sequence=None, + offset=offset, + limit=page_size, + ) + for obligation in page.obligations: + if obligation.generation_key != key: + continue + checkpoints = self._optional_restatement_store() + if checkpoints is None or not _implements_store_method( + self._store, PendingProvisionalAcquisitionStore.get_provisional_acquisition + ): + # Without the retained cursor and its reservations a prior + # stabilized close cannot be distinguished from an ordinary poll. + return True + checkpoint = await checkpoints.get_restatement_checkpoint( + account_id=key.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + if checkpoint is not None and checkpoint.stabilization_policy is not None: + return True + next_ordinal = checkpoint.next_observation if checkpoint is not None else 0 + for ordinal in range(next_ordinal + 1): + acquisition = await self._pending_acquisition(obligation, ordinal=ordinal) + if ( + acquisition is not None + and acquisition.stabilization_schedule( + obligation.required_finality, pending=ordinal == next_ordinal + ) + is not None + ): + return True + if not page.has_more: + return False + offset += page_size + async def _acquire_stabilized( self, configuration: ReportingConfiguration, @@ -1787,6 +1845,12 @@ async def _acquire_obligation( raise LedgerConflictError( "FINALITY_POLICY_UNPROVEN", "official finality requires a selected policy" ) + if selected_policy is not None and ( + finality == "official" or self._official_close_requested(configuration) + ): + self._official_publication_lock( + stabilized=isinstance(selected_policy, ReportingStabilizedPolicy) + ) # Everything below needs the frozen code: the slice request carries it, # the manifest is checked against it, and the revision is written under # it. Gate here rather than earlier so a settled legacy obligation stays @@ -2060,7 +2124,9 @@ async def _retire_stabilized_acquisition( account_id=obligation.account_id, reporting_obligation_id=obligation.reporting_obligation_id, ) - async with self._stabilization_lock().source_publication(obligation.account_id): + async with self._official_publication_lock(stabilized=True).source_publication( + obligation.account_id + ): self._check_source_authorization( configuration, request.offering_id, tuple(request.coverage.constituents) ) @@ -2315,11 +2381,10 @@ async def _source_publication( if not isinstance(self._source, ReportingProductionSource): yield None return - if not isinstance(self._store, ReportingSourcePublicationStore): + store = self._optional_publication_store() + if store is None: raise TypeError("production source publication requires an SDK account lock") - async with self._store.source_publication( - configuration.account_id, seals=seals - ) as publish_seal: + async with store.source_publication(configuration.account_id, seals=seals) as publish_seal: self._check_source_authorization(configuration, offering_id, admitted_constituents) yield publish_seal @@ -2362,15 +2427,18 @@ async def _revision_publication( ) -> AsyncIterator[None]: from adcp.reporting.production.contracts import ReportingProductionSource - if ( - official - and obligation.definition - and isinstance(obligation.definition.finality_policy, ReportingStabilizedPolicy) - ): - self._stabilization_lock() + lock: ReportingSourcePublicationStore | None + if official and obligation.definition and obligation.definition.finality_policy: + lock = self._official_publication_lock( + stabilized=isinstance( + obligation.definition.finality_policy, ReportingStabilizedPolicy + ) + ) + else: + lock = self._optional_publication_store() if not isinstance(self._source, ReportingProductionSource): - if isinstance(self._store, ReportingSourcePublicationStore): - async with self._store.source_publication(obligation.account_id): + if lock is not None: + async with lock.source_publication(obligation.account_id): yield else: yield @@ -2472,21 +2540,33 @@ def _observation_history_store(self) -> ProvisionalObservationStore: ) return store - def _stabilization_lock(self) -> ReportingSourcePublicationStore: + def _optional_publication_store(self) -> ReportingSourcePublicationStore | None: store = self._store - if not _implements_store_method( + if _implements_store_method( store, ReportingSourcePublicationStore.source_publication - ) or not isinstance(store, ReportingSourcePublicationStore): + ) and isinstance(store, ReportingSourcePublicationStore): + return store + return None + + def _official_publication_lock( + self, *, stabilized: bool = False + ) -> ReportingSourcePublicationStore: + store = self._optional_publication_store() + if store is None: raise LedgerConflictError( - "STABILIZATION_PUBLICATION_NOT_SUPPORTED", - "stabilized finality requires an account publication lock", + ( + "STABILIZATION_PUBLICATION_NOT_SUPPORTED" + if stabilized + else "OFFICIAL_PUBLICATION_NOT_SUPPORTED" + ), + "official finality requires an account publication lock", ) return store def _stabilization_store(self) -> StabilizationStore: history = self._observation_history_store() self._restatement_store() - self._stabilization_lock() + self._official_publication_lock(stabilized=True) for stub in ( ProvisionalObservationPublicationStore.reserve_provisional_acquisition, ProvisionalObservationPublicationStore.get_provisional_observation, @@ -2559,7 +2639,9 @@ async def _reserve_observation( output_finality="official" if derived_close else None, ) if frozen_policy.until_stabilized: - async with self._stabilization_lock().source_publication(obligation.account_id): + async with self._official_publication_lock(stabilized=True).source_publication( + obligation.account_id + ): current = await self._store.list_revisions( account_id=obligation.account_id, reporting_obligation_id=obligation.reporting_obligation_id, @@ -2810,6 +2892,12 @@ async def _commit_revision_from_manifest( ) -> ReportingRevisionRecord: obligation = await self._stored_obligation(obligation) selected_policy = obligation.definition.finality_policy if obligation.definition else None + if selected_policy is not None and ( + finality == "official" or self._official_close_requested(obligation) + ): + self._official_publication_lock( + stabilized=isinstance(selected_policy, ReportingStabilizedPolicy) + ) origin_store: SnapshotOriginStore | None = None if finality == "snapshot" and acquisition is not None: if acquisition.requires_snapshot_origin( @@ -3154,6 +3242,9 @@ async def _retained_snapshot_replay( or revision.reporting_obligation_id != obligation.reporting_obligation_id ): raise LedgerConflictError("HISTORY_UNAVAILABLE", "retained snapshot is missing") + source_id = revision.source_publication_id + if source_id is None or revision_id != f"rpr_{source_id[4:44]}": + raise LedgerConflictError("REVISION_IMMUTABLE", "retained snapshot origin differs") if prior is not None and ( revision.source_publication_id != manifest.publication_id or revision.observed_at != manifest.observed_at @@ -3181,16 +3272,24 @@ async def _retained_snapshot_replay( await self._verify_tracked_snapshot_origin( obligation, acquisition, origin_store, revision ) - if is_latest: + elif prior is None and _implements_store_method( + self._store, SnapshotOriginStore.verify_snapshot_origin + ): + assert isinstance(self._store, SnapshotOriginStore) + await self._store.verify_snapshot_origin(revision) + if is_latest and (prior is not None or latest.revision_publication_id is not None): return revision - # A later observation closes this ordinal to new writes. The store - # atomically checks the historical acquisition, manifest and revision. + # The atomic replay also checks older observations without a full + # predecessor publication witness against the original immutable revision. try: - return await store.commit_provisional_observation(observation, revision, rows) + committed = await store.commit_provisional_observation(observation, revision, rows) except LedgerConflictError as error: if error.code != "OBSERVATION_CONFLICT": raise return None + if committed.source_publication_id != source_id: + raise LedgerConflictError("REVISION_IMMUTABLE", "retained snapshot origin differs") + return committed async def _verify_direct_official_publication( self, @@ -3455,6 +3554,23 @@ async def _official_replay_revision( raise LedgerConflictError( "REVISION_IMMUTABLE", "the winning official source evidence is unavailable" ) + if isinstance(policy, ReportingSourceFinalPolicy): + try: + proof = manifest_finality_proof(obligation, manifest, retained.created_at) + except LedgerConflictError as error: + raise LedgerConflictError( + "REVISION_IMMUTABLE", + "the manifest contradicts the retained source finality", + ) from error + if ( + manifest.offering_id != self._offerings.official_offering_id + or proof.finality_policy_id != retained.finality_policy_id + or proof.basis != retained.finality_basis + or proof.finalized_at != retained.finalized_at + ): + raise LedgerConflictError( + "REVISION_IMMUTABLE", "official source evidence differs" + ) elif json.loads(observation.manifest_json) != manifest.model_dump(mode="json") or ( acquisition is not None and observation.acquisition != acquisition ): diff --git a/src/adcp/reporting/ledger/provisional.py b/src/adcp/reporting/ledger/provisional.py index 6c9f67b7b..893f3e46f 100644 --- a/src/adcp/reporting/ledger/provisional.py +++ b/src/adcp/reporting/ledger/provisional.py @@ -255,7 +255,12 @@ def bind_revision( ) -> ProvisionalObservation: """Keep the full originating publication ID when an unchanged read reuses a snapshot.""" if revision.finality == "snapshot" and ( - require_origin or self.acquisition.policy.until_stabilized + require_origin + or self.acquisition.policy.until_stabilized + or ( + self.acquisition.predecessor_revision_id == revision.reporting_revision_id + and revision.source_publication_id is not None + ) ): if ( self.revision_publication_id is not None diff --git a/src/adcp/reporting/production/offerings.py b/src/adcp/reporting/production/offerings.py index 53e795e38..cb8b3b855 100644 --- a/src/adcp/reporting/production/offerings.py +++ b/src/adcp/reporting/production/offerings.py @@ -185,6 +185,10 @@ def configured_source( policy = self.verification_key.definition.finality_policy if policy is None: raise failure("UNSUPPORTED_VERIFICATION") + try: + self.producer._official_publication_lock() + except LedgerConflictError: + raise failure("UNSUPPORTED_VERIFICATION") from None source_class = finality_policy_source_class(policy) proof_source = configured_source(source_class) if not _supports_official_source_formats(proof_source) or ( diff --git a/src/adcp/reporting/service.py b/src/adcp/reporting/service.py index c8d8ca54d..e2bd96b93 100644 --- a/src/adcp/reporting/service.py +++ b/src/adcp/reporting/service.py @@ -783,6 +783,8 @@ async def _configure( policy = self._validate_offerings(configuration, context, registration, frozen=frozen) if isinstance(policy, ReportingStabilizedPolicy): producer._stabilization_store() + elif policy is not None: + producer._official_publication_lock() except LedgerConflictError as error: raise ReliableReportingConfigurationError(str(error)) from error except Exception as error: @@ -804,15 +806,19 @@ async def _configure( async def _install_binding(self, binding: _Binding, *, persist: bool) -> None: configuration = binding.configuration key = configuration.generation_key - if binding.missing_stabilization_lookup and ( - not persist - or any( - stored.generation_key == key - for stored in await self.store.list_configurations( - caller=ReportingStatusCaller(key.account_id, key.consumer_id), - delivery_config_ids=[key.delivery_config_id], + if ( + binding.missing_stabilization_lookup + and ( + not persist + or any( + stored.generation_key == key + for stored in await self.store.list_configurations( + caller=ReportingStatusCaller(key.account_id, key.consumer_id), + delivery_config_ids=[key.delivery_config_id], + ) ) ) + and await binding.producer._has_retained_stabilization(configuration) ): raise ReliableReportingConfigurationError( "recovering a selected stabilized generation requires get_stabilization_schedule" diff --git a/src/adcp/reporting/source.py b/src/adcp/reporting/source.py index 4e280bd32..6a6d0e562 100644 --- a/src/adcp/reporting/source.py +++ b/src/adcp/reporting/source.py @@ -285,7 +285,7 @@ def iso_duration_milliseconds_v1(value: str) -> int: months and years are deliberately absent because their length depends on a calendar the source and the producer may not share. """ - match = _ISO_DURATION_RE.match(value) + match = _ISO_DURATION_RE.fullmatch(value) if match is None: raise ValueError(f"{value!r} is not a supported reporting duration") days, hours, minutes, seconds, fraction = ( diff --git a/tests/conformance/reporting/_production_support.py b/tests/conformance/reporting/_production_support.py index a759a8e61..77da6b6f5 100644 --- a/tests/conformance/reporting/_production_support.py +++ b/tests/conformance/reporting/_production_support.py @@ -333,13 +333,15 @@ def __init__( self.capabilities = ReportingSourceCapabilitiesV1.model_validate(raw) self.bindings = {} self.rows, self.requests = rows, [] + self.clock = clock or (lambda: END) + self.finality_policy = getattr(key.definition, "finality_policy", None) self.inline = InlineReportingSource( capabilities=self.capabilities, fetch=self.fetch, staging=FileSystemStagingStore(path.with_suffix(".staging")), seals=SQLiteSeals(path.with_suffix(".seals")), constituent_of=lambda row, req: req.coverage.constituents[0].constituent_id, - clock=clock or (lambda: END), + clock=self.clock, ) self.reader = self.inline.staging @@ -360,7 +362,19 @@ def configuration_binding(self, configuration): async def fetch(self, request): assert self.rows is not None, "this seeded publication needs no new acquisition" self.requests.append(request) - return InlineFetchResult(self.rows, data_through=request.period.end, currency="USD") + result = InlineFetchResult(self.rows, data_through=request.period.end, currency="USD") + if request.publication_class == "AUTHORITATIVE" and self.finality_policy is not None: + from adcp.reporting.source import ReportingFinalityEvidenceV1 + + return replace( + result, + finality_evidence=ReportingFinalityEvidenceV1( + basis="provider_declared", + observed_at=self.clock(), + evidence_ref="fixture_closed", + ), + ) + return result async def execute(self, request, *, cancel, heartbeat=None): return await self.inline.execute(request, cancel=cancel, heartbeat=heartbeat) @@ -370,6 +384,13 @@ async def account_task(request, context, admit): return {"accounts": []} +def reference_source_final(): + """The provider signal declared by the pinned reference report definition.""" + from adcp.reporting.ledger import ReportingSourceFinalPolicy + + return ReportingSourceFinalPolicy("reference-final", "fixture_closed") + + @asynccontextmanager async def production_harness( backend, @@ -386,6 +407,7 @@ async def production_harness( source_bindings=(), account_handler=None, reconciled=False, + finality_policy=None, feedback=False, identity_prefix="", poll_seconds=60, @@ -427,6 +449,9 @@ async def production_harness( ) verifier = reference_verifier(cap) key = verifier.key + if finality_policy is not None: + key = replace(key, definition=replace(key.definition, finality_policy=finality_policy)) + verifier = replace(verifier, key=key) config = replace( configuration(consumer_id="https://buyer.example.test/agent"), delivery_config_id=identity_prefix + configuration().delivery_config_id, @@ -483,7 +508,11 @@ async def production_harness( END, END, finality_basis="source_final" if reconciled else None, - finality_policy_id="fixture-official-v1" if reconciled else None, + finality_policy_id=( + (finality_policy.finality_policy_id if finality_policy else "fixture-official-v1") + if reconciled + else None + ), finalized_at=END if reconciled else None, canonical_content_digest=reference_digest(verifier, rows), managed_control_totals=totals, diff --git a/tests/conformance/reporting/_reliable_support.py b/tests/conformance/reporting/_reliable_support.py index 81caed671..96aa266ac 100644 --- a/tests/conformance/reporting/_reliable_support.py +++ b/tests/conformance/reporting/_reliable_support.py @@ -35,6 +35,9 @@ def _storage_split(): import pytest +if TYPE_CHECKING: + from adcp.reporting.ledger.finality import FinalityProof + if _storage_split(): # wheels before the storage split (#1510) from adcp.reporting.ledger import PgReportingReconciliationStore from adcp.reporting.outbox import PgReportingOutbox @@ -425,6 +428,16 @@ def capabilities() -> ReportingSourceCapabilitiesV1: update={ "source_timezone": "UTC", "product_ids": [redacted_contract_identity.report_definition_id], + **( + { + "grain": "fixed_window", + "windowing": offering.windowing.model_copy( + update={"minimum_window": "PT1H"} + ), + } + if offering.offering_id == OFFICIAL_OFFERING_ID + else {} + ), "metrics": [ MetricOfferingV1( name=metric, @@ -649,6 +662,7 @@ async def commit_slice( result: ReportingSourceExecutorResult, *, finality: str = "snapshot", + finality_proof: FinalityProof | None = None, ) -> ReportingRevisionRecord: manifest = verified(result) self.manifests[(obligation.account_id, manifest.content_fingerprint)] = manifest @@ -657,6 +671,7 @@ async def commit_slice( manifest, rows=await producer._read_rows(request, manifest), finality=finality, + **({"finality_proof": finality_proof} if finality_proof is not None else {}), ) async def restart(self) -> None: diff --git a/tests/conformance/reporting/_scope_onboarding_server.py b/tests/conformance/reporting/_scope_onboarding_server.py index f4568a614..018ce575d 100644 --- a/tests/conformance/reporting/_scope_onboarding_server.py +++ b/tests/conformance/reporting/_scope_onboarding_server.py @@ -20,7 +20,7 @@ from adcp.server.serve import _build_mcp_and_a2a_app from ._feed_support import MountedFeed, feed_harness, feed_request, mixed_case -from ._production_support import production_harness +from ._production_support import production_harness, reference_source_final from .test_reporting_production_configuration import state_for, wire_configuration @@ -130,6 +130,7 @@ async def exact_fixture(backend, root, count): count=count, source_publication=True, reconciled=True, + finality_policy=reference_source_final(), poll_seconds=0.02, ) as h: await h.production.activate(account_id=h.item.config.account_id) diff --git a/tests/conformance/reporting/test_reporting_currency.py b/tests/conformance/reporting/test_reporting_currency.py index 6591b8c8e..7d04f4336 100644 --- a/tests/conformance/reporting/test_reporting_currency.py +++ b/tests/conformance/reporting/test_reporting_currency.py @@ -50,6 +50,7 @@ ReportingObligationRecord, ReportingProducer, ReportingScheduleSpec, + ReportingSourceFinalPolicy, ReportingStatusCaller, ReportingStatusHandler, require_single_currency, @@ -58,6 +59,7 @@ from adcp.reporting.ledger import ReportingStatusCaller as OwnershipCaller from adcp.reporting.ledger.producer import recompute_revision_content_sha256 from adcp.reporting.source import ( + ReportingFinalityEvidenceV1, ReportingSourceCapabilitiesV1, ReportingSourceExecutorResult, ReportingSourceSliceRequestV1, @@ -570,8 +572,26 @@ def must_not_aggregate(*args: Any) -> None: async def test_official_and_adjustments_keep_frozen_units(store: ReportingLedgerStore) -> None: - config = replace(configuration(pinned_currency="EUR"), required_finality="official") - source, staging = source_for(money_rows) + config = configuration(pinned_currency="EUR") + assert config.definition is not None + config = replace( + config, + required_finality="official", + definition=replace( + config.definition, + finality_policy=ReportingSourceFinalPolicy("billed", "billing.closed"), + ), + ) + + def closed_amounts(request: ReportingSourceSliceRequestV1) -> InlineFetchResult: + return InlineFetchResult( + rows=money_rows(request), + finality_evidence=ReportingFinalityEvidenceV1( + basis="provider_declared", observed_at=NOW, evidence_ref="billing.closed" + ), + ) + + source, staging = source_for(closed_amounts) first = producer_for(store, source, staging, currency="EUR") (obligation,) = await close_admitted_periods(first, config) restarted = producer_for(store, source, staging, currency="USD") diff --git a/tests/conformance/reporting/test_reporting_evidence_currency_integration.py b/tests/conformance/reporting/test_reporting_evidence_currency_integration.py index ecac503b8..6200b05cf 100644 --- a/tests/conformance/reporting/test_reporting_evidence_currency_integration.py +++ b/tests/conformance/reporting/test_reporting_evidence_currency_integration.py @@ -35,9 +35,12 @@ ReportingObligationDeliveryRecord, ReportingObligationRecord, ReportingProducer, + ReportingSourceFinalPolicy, ) +from adcp.reporting.ledger.finality import manifest_finality_proof from adcp.reporting.source import ( MediaBuyConstituentV1, + ReportingFinalityEvidenceV1, ReportingSourceCoverageRequestV1, ReportingSourceSliceRequestV1, coverage_denominator_fingerprint_v1, @@ -841,10 +844,27 @@ def zero(req: ReportingSourceSliceRequestV1) -> InlineFetchResult: }, ) - script = ScriptedSource(eur=[zero, complete_fetch]) + def closed(req: ReportingSourceSliceRequestV1) -> InlineFetchResult: + return replace( + complete_fetch(req), + finality_evidence=ReportingFinalityEvidenceV1( + basis="provider_declared", evidence_ref="delivery.closed", observed_at=h.clock() + ), + ) + + script = ScriptedSource(eur=[zero, closed]) source = h.source(script.async_fetch) producer = h.producer(source) + producer._store.source_publication = h.store.source_publication config = configuration("eur") + assert config.definition is not None + config = replace( + config, + definition=replace( + config.definition, + finality_policy=ReportingSourceFinalPolicy("closed-delivery", "delivery.closed"), + ), + ) await h.store.put_configuration(config) await drain_until_idle(producer, h.clock) request = script.requests[0] @@ -867,13 +887,20 @@ def zero(req: ReportingSourceSliceRequestV1) -> InlineFetchResult: h.clock.advance(timedelta(hours=1)) official_request = producer._build_slice( config, - replace(obligation, required_finality="official"), + obligation, OFFICIAL_OFFERING_ID, + finality="official", + observation=1, now=h.clock(), ) result = await source.execute(official_request, cancel=asyncio.Event()) official = await h.commit_slice( - producer, obligation, official_request, result, finality="official" + producer, + obligation, + official_request, + result, + finality="official", + finality_proof=manifest_finality_proof(obligation, verified(result), h.clock()), ) assert official.finality == "official" and official.supersedes_reporting_revision_id is None official_records = await publication_records(h, obligation, official, suffix="official") diff --git a/tests/conformance/reporting/test_reporting_materializer_contracts.py b/tests/conformance/reporting/test_reporting_materializer_contracts.py index ae7440599..a4d076070 100644 --- a/tests/conformance/reporting/test_reporting_materializer_contracts.py +++ b/tests/conformance/reporting/test_reporting_materializer_contracts.py @@ -14,6 +14,7 @@ InMemoryReportingLedgerStore, ProducerOfferings, ReportingContractualCutoffPolicy, + ReportingDefinitionBinding, ReportingDeliveryPrincipal, ReportingProducer, ReportingSourceFinalPolicy, @@ -21,8 +22,11 @@ from adcp.reporting.ledger.delivery_models import MaterializationFailure from adcp.reporting.materializer import ( ReferenceReportingDestinationWriter, + ReportingCanonicalization, ReportingDestinationPage, ReportingIOContext, + ReportingVerificationKey, + ReportingWriterCapability, ReportingWriterError, ReportingWriterFailure, ReportingWriterFailureCode, @@ -162,7 +166,34 @@ def test_readback_page_repr_never_includes_unverified_provider_body(): def test_verification_key_id_without_finality_policy_preserves_stored_identity() -> None: - key = reference_verifier().key + key = ReportingVerificationKey( + "reference-report-v1", + "paid_media_delivery", + ReportingDefinitionBinding( + "https://contracts.example.test/reference-definition.json", + "2c988a2c008b84097be962248c433dc3acd9663372f89a4b1c7c1235ee635785", + "1.0.0", + "https://contracts.example.test/reference-row-schema.json", + "aa4927d97b5e5889c4d79bd5461da81657cafea75a61d266ab452bfa0fc5d799", + monetary_metric_units=(("spend", "USD"),), + monetary_control_total_units=(("spend", "USD"),), + ), + ReportingCanonicalization( + "reference-jcs-rows-v1", + "https://contracts.example.test/reference-canonicalization.json", + "5566f62a29487d54d755e4e9c35c97661bb243d158d4bf7a712fb1c7eb3bcb80", + ), + ReportingWriterCapability( + "file_transfer", + "reference-memory", + "jsonl", + "canonical_digest", + "producer", + "immutable_location", + "sha256", + "conditional_create", + ), + ) assert ( verification_key_id(key) == "d53ff0456ffc9b62c98c8c1d198818956ce11ae4749e81f5118ac1f653ce8b5d" diff --git a/tests/conformance/reporting/test_reporting_production_configuration.py b/tests/conformance/reporting/test_reporting_production_configuration.py index 5016ff128..7db332f2a 100644 --- a/tests/conformance/reporting/test_reporting_production_configuration.py +++ b/tests/conformance/reporting/test_reporting_production_configuration.py @@ -59,6 +59,10 @@ ), (STABILIZED, "snapshot", ["official"], False, False, False, False), (STABILIZED, "snapshot", ["official"], False, False, "unlocked", False), + (CUTOFF, "snapshot", ["official"], False, False, "unlocked", False), + (SOURCE_FINAL, "official", ["official"], False, True, "unlocked", False), + (SOURCE_FINAL, "snapshot", ["snapshot"], True, True, "unlocked", False), + (CUTOFF, "snapshot", ["snapshot"], False, False, "unlocked", True), (STABILIZED, "snapshot", ["official"], False, False, "verify_snapshot_origin", False), (STABILIZED, "snapshot", ["official"], False, False, "get_provisional_acquisition", False), (STABILIZED, "snapshot", ["official"], False, False, "get_stabilization_schedule", False), diff --git a/tests/conformance/reporting/test_reporting_production_lifecycle.py b/tests/conformance/reporting/test_reporting_production_lifecycle.py index 837332c43..73d3e06ef 100644 --- a/tests/conformance/reporting/test_reporting_production_lifecycle.py +++ b/tests/conformance/reporting/test_reporting_production_lifecycle.py @@ -18,7 +18,7 @@ from adcp.validation.schema_loader import get_validator from ._generation_support import END -from ._production_support import production_harness +from ._production_support import production_harness, reference_source_final from ._production_transport import MountedProduction from ._projection_support import drain from ._receipt_transport import error_code @@ -80,6 +80,7 @@ async def test_production_official_receipts_adjustment_and_frozen_wire_cycle( count=3, source_publication=True, reconciled=True, + finality_policy=reference_source_final(), notifications=notifications, feedback=feedback, ) as h: diff --git a/tests/conformance/reporting/test_reporting_production_readiness.py b/tests/conformance/reporting/test_reporting_production_readiness.py index b71721451..6f6717294 100644 --- a/tests/conformance/reporting/test_reporting_production_readiness.py +++ b/tests/conformance/reporting/test_reporting_production_readiness.py @@ -7,7 +7,7 @@ from adcp.reporting.ledger.notification_models import ReportingNotificationError from adcp.server.a2a_server import create_a2a_server -from ._production_support import production_harness +from ._production_support import production_harness, reference_source_final from ._production_transport import MountedProduction @@ -92,6 +92,7 @@ async def test_discovery_preserves_declaration_while_admission_rechecks_pool( notifications=notifications, notification_delivery=notifications, reconciled=reconciled, + finality_policy=reference_source_final() if reconciled else None, ) as h: support = h.production mounted = MountedProduction(h) @@ -171,6 +172,7 @@ async def test_discovery_preserves_declaration_while_admission_rechecks_queue_po notifications=True, notification_delivery=delivery, reconciled=True, + finality_policy=reference_source_final(), ) as h: mount = MountedProduction(h) mount.authorize(h.item) @@ -231,6 +233,7 @@ async def test_warm_mounted_discovery_does_not_checkout_a_saturated_valid_pool( notifications=notifications, notification_delivery=notifications, reconciled=True, + finality_policy=reference_source_final(), ) as h: mounted = MountedProduction(h) mounted.authorize(h.item) diff --git a/tests/conformance/reporting/test_reporting_production_settling.py b/tests/conformance/reporting/test_reporting_production_settling.py index 563658696..e800cba94 100644 --- a/tests/conformance/reporting/test_reporting_production_settling.py +++ b/tests/conformance/reporting/test_reporting_production_settling.py @@ -13,7 +13,7 @@ from ._generation_support import END from ._materializer_support import reference_rows -from ._production_support import Source, SQLiteSeals, production_harness +from ._production_support import Source, SQLiteSeals, production_harness, reference_source_final from .test_reporting_production_lock_order import source_turn @@ -82,6 +82,7 @@ async def test_progress_retains_policy_until_terminal_publication( source_publication=True, periods=1, source_factory=partial(SettlingSource, close_officially=close_officially), + finality_policy=reference_source_final() if close_officially else None, ) as h: await h.production.activate(account_id=h.item.config.account_id) source = h.production.offerings[0].producer._source @@ -155,6 +156,7 @@ async def test_settling_checkpoint_and_pending_work_survive_fresh_service(backen source_publication=True, periods=1, source_factory=SettlingSource, + finality_policy=reference_source_final(), ) as h: await h.production.activate(account_id=h.item.config.account_id) initial = await source_turn(h.production) @@ -174,6 +176,7 @@ async def test_settling_checkpoint_and_pending_work_survive_fresh_service(backen source_publication=True, periods=1, source_factory=SettlingSource, + finality_policy=reference_source_final(), **existing, ) as fresh: source = fresh.production.offerings[0].producer._source diff --git a/tests/conformance/reporting/test_reporting_projection_rc6.py b/tests/conformance/reporting/test_reporting_projection_rc6.py index 26ad999d7..7e0a4baea 100644 --- a/tests/conformance/reporting/test_reporting_projection_rc6.py +++ b/tests/conformance/reporting/test_reporting_projection_rc6.py @@ -493,7 +493,7 @@ async def test_admitted_producer_verified_artifact_and_receipt_keep_the_future_s from adcp.reporting.ledger import ReportingRevisionReceiptRecord from adcp.reporting.ledger.delivery import receipt_to_wire - from ._production_support import production_harness + from ._production_support import production_harness, reference_source_final from ._production_transport import MountedProduction from ._projection_support import drain from .test_reporting_production_lifecycle import observed_now @@ -505,6 +505,7 @@ async def test_admitted_producer_verified_artifact_and_receipt_keep_the_future_s count=3, source_publication=True, reconciled=reconciled, + finality_policy=reference_source_final() if reconciled else None, adcp_version=version, ) as h: support, item = h.production, h.item diff --git a/tests/conformance/reporting/test_reporting_publication_time.py b/tests/conformance/reporting/test_reporting_publication_time.py index 74fbc1da8..9f79a503a 100644 --- a/tests/conformance/reporting/test_reporting_publication_time.py +++ b/tests/conformance/reporting/test_reporting_publication_time.py @@ -3,7 +3,7 @@ from __future__ import annotations import asyncio -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import datetime, timedelta, timezone import pytest @@ -26,6 +26,7 @@ ReportingStatusCaller, ReportingStatusHandler, ) +from adcp.reporting.ledger.finality import manifest_finality_proof from adcp.reporting.materializer import ReportingWriterCapability, reference_verifier from adcp.reporting.source import SourceBatchManifestV1, parse_verified_source_batch_manifest_v1 from adcp.types import GetReportingStatusRequest, GetReportingStatusResponse @@ -33,7 +34,7 @@ from adcp.validation.schema_loader import get_named_validator from ._generation_support import END, START, isolated_reporting_pool -from ._production_support import Source +from ._production_support import Source, reference_source_final TURN = END + timedelta(hours=1) PUBLISHED = TURN + timedelta(seconds=2) @@ -96,6 +97,11 @@ async def setup( ) ) key = verifier.key + if finality == "official": + key = replace( + key, definition=replace(key.definition, finality_policy=reference_source_final()) + ) + verifier = replace(verifier, key=key) clock = Clock() source = RecordedSource( key, @@ -234,7 +240,9 @@ async def test_real_clock_observation_after_dispatch_is_definitive(store, tmp_pa revision = outcome.ledger.revisions[0] assert outcome.definitive, [o.reasons for o in outcome.obligations] assert request.period.source_read_cutoff_at < manifest.observed_at <= revision.created_at - assert revision.observed_at == revision.finalized_at == manifest.observed_at + assert revision.observed_at == manifest.observed_at + assert revision.finalized_at == manifest.finality_evidence.observed_at + assert request.period.source_read_cutoff_at < revision.finalized_at <= revision.observed_at @pytest.mark.parametrize( @@ -401,7 +409,12 @@ async def record_write(*args, **kwargs): monkeypatch.setattr(store, "commit_revision", record_write) with pytest.raises(LedgerConflictError) as raised: await producer.commit_revision_from_manifest( - obligations[0], manifest, rows=ROWS, finality="official", now=PUBLISHED + obligations[0], + manifest, + rows=ROWS, + finality="official", + now=PUBLISHED, + finality_proof=manifest_finality_proof(obligations[0], manifest, PUBLISHED), ) assert raised.value.code == "PUBLICATION_TIME_INVALID" assert not writes @@ -415,7 +428,12 @@ async def record_write(*args, **kwargs): assert correction_condition_3 else: committed = await producer.commit_revision_from_manifest( - obligations[0], manifest, rows=ROWS, finality="official", now=PUBLISHED + obligations[0], + manifest, + rows=ROWS, + finality="official", + now=PUBLISHED, + finality_proof=manifest_finality_proof(obligations[0], manifest, PUBLISHED), ) assert committed.created_at == PUBLISHED assert committed.observed_at == committed.finalized_at == TURN diff --git a/tests/conformance/reporting/test_reporting_revision_selection.py b/tests/conformance/reporting/test_reporting_revision_selection.py index 028908981..e7fffc173 100644 --- a/tests/conformance/reporting/test_reporting_revision_selection.py +++ b/tests/conformance/reporting/test_reporting_revision_selection.py @@ -11,11 +11,13 @@ InMemoryReportingLedgerStore, LedgerConflictError, ProducerOfferings, + ReportingContractualCutoffPolicy, ReportingProducer, current_required_revision, project_obligation_health, select_reporting_revision, ) +from adcp.reporting.ledger.finality import manifest_finality_proof from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler from adcp.reporting.ledger.status_projection import ( ReportingStatusSnapshot, @@ -28,7 +30,7 @@ from adcp.reporting.revision_selection import RevisionHistoryEntry from ._generation_support import NOW, UncalledSource, configuration, obligation_for, revision_for -from ._reliable_support import complete_fetch, reliable_factory +from ._reliable_support import complete_fetch, reliable_factory, verified from ._reliable_support import configuration as reliable_configuration from .test_reporting_notification_outbox import statement @@ -241,28 +243,42 @@ def test_unreadable_official_never_falls_back_to_materialized_snapshot(): @pytest.mark.parametrize("backend", ["memory", "postgres"]) -async def test_snapshot_restatement_after_an_official_close_keeps_one_chain(backend): - """A retained official close must never be mistaken for the snapshot leaf. - - Whole-history selection answers "what does this obligation currently - publish", and a unique official wins that outright. Reusing that answer to - choose ``supersedes`` roots the next restatement at ``None``, so the - obligation ends up with two snapshot roots -- ``disconnected_snapshot_history`` - over immutable rows, which every selector caller then parks for a repair - nothing in the SDK can perform. - """ +async def test_official_close_preserves_the_snapshot_chain_and_rejects_restatement(backend): async with reliable_factory(backend) as h: config = reliable_configuration("eur") + config = replace( + config, + definition=replace( + config.definition, + finality_policy=ReportingContractualCutoffPolicy("period-close", "PT0S"), + ), + ) await h.store.put_configuration(config) producer = h.producer(h.source(complete_fetch), managed=False) (obligation,) = await producer.close_elapsed_periods(config) async def commit(observation, finality): request = producer._build_slice( # noqa: SLF001 - the suite's slice fixture - config, obligation, SNAPSHOT_OFFERING_ID, now=h.clock(), observation=observation + config, + obligation, + SNAPSHOT_OFFERING_ID, + finality=finality, + now=h.clock(), + observation=observation, ) result = await producer._source.execute(request, cancel=asyncio.Event()) # noqa: SLF001 - return await h.commit_slice(producer, obligation, request, result, finality=finality) + return await h.commit_slice( + producer, + obligation, + request, + result, + finality=finality, + finality_proof=( + manifest_finality_proof(obligation, verified(result), h.clock()) + if finality == "official" + else None + ), + ) first = await commit(0, "snapshot") second = await commit(1, "snapshot") @@ -271,13 +287,20 @@ async def commit(observation, finality): assert second.supersedes_reporting_revision_id == first.reporting_revision_id assert official.supersedes_reporting_revision_id is None - restated = await commit(3, "snapshot") - assert restated.supersedes_reporting_revision_id == second.reporting_revision_id + with pytest.raises(LedgerConflictError) as error: + await commit(3, "snapshot") + assert error.value.code == "OFFICIAL_REVISION_TERMINAL" + assert await commit(1, "snapshot") == second history = await h.store.list_revisions( account_id=obligation.account_id, reporting_obligation_id=obligation.reporting_obligation_id, ) + assert {revision.reporting_revision_id for revision in history} == { + first.reporting_revision_id, + second.reporting_revision_id, + official.reporting_revision_id, + } selection = select_reporting_revision( history, account_id=obligation.account_id, diff --git a/tests/conformance/reporting/test_reporting_source_authorization.py b/tests/conformance/reporting/test_reporting_source_authorization.py index b457a9b7d..9701bfb00 100644 --- a/tests/conformance/reporting/test_reporting_source_authorization.py +++ b/tests/conformance/reporting/test_reporting_source_authorization.py @@ -14,7 +14,7 @@ from ._generation_support import END, configuration, isolated_reporting_pool from ._materializer_support import reference_rows -from ._production_support import production_harness +from ._production_support import production_harness, reference_source_final from .test_reporting_production_lock_order import source_turn from .test_reporting_production_settling import SettlingSource, revisions @@ -67,6 +67,7 @@ def harness(backend, path, *, source_factory=RevocableSource, **kwargs): source_publication=True, periods=1, source_factory=source_factory, + finality_policy=reference_source_final(), **kwargs, ) diff --git a/tests/test_reliable_reporting_service.py b/tests/test_reliable_reporting_service.py index dbe5d1799..ec44336fc 100644 --- a/tests/test_reliable_reporting_service.py +++ b/tests/test_reliable_reporting_service.py @@ -404,19 +404,28 @@ class SnapshotStore(InMemoryReportingLedgerStore): await service.configure(config) -async def test_stabilized_admission_requires_publication_lock_before_persisting() -> None: +@pytest.mark.parametrize( + ("policy", "required", "close"), + [ + (ReportingStabilizedPolicy("stable", "PT0S", "PT1H"), "official", False), + (ReportingSourceFinalPolicy("source", "delivery.final"), "official", False), + (ReportingContractualCutoffPolicy("cutoff", "PT1H"), "official", False), + (ReportingSourceFinalPolicy("source", "delivery.final"), "snapshot", True), + ], +) +async def test_official_admission_requires_publication_lock_before_persisting( + policy, required, close +) -> None: class UnlockedHistory(InMemoryReportingLedgerStore): source_publication = None config = replace( _configuration(), - required_finality="official", - definition=replace( - DEFINITION, finality_policy=ReportingStabilizedPolicy("stable", "PT0S", "PT1H") - ), + required_finality=required, + definition=replace(DEFINITION, finality_policy=policy), ) store = UnlockedHistory() - service = _finality_service(config, snapshot=True, official=False, store=store) + service = _finality_service(config, snapshot=True, official=True, store=store, close=close) await service.initialize() with pytest.raises(ReliableReportingConfigurationError, match="publication"): await service.configure(config) @@ -458,6 +467,7 @@ async def create_schema(self) -> None: @pytest.mark.parametrize("configure_before_init", [True, False]) async def test_plain_snapshot_custom_publisher_needs_no_stabilization_apis( configure_before_init: bool, + monkeypatch: pytest.MonkeyPatch, ) -> None: class SnapshotPublisher(InMemoryReportingLedgerStore): get_stabilization_schedule = None @@ -473,11 +483,13 @@ class SnapshotPublisher(InMemoryReportingLedgerStore): ) clock = DeterministicReportingClock(NOW) result = InlineFetchResult(rows=_rows(10), data_through=config.deactivated_at) + store = SnapshotPublisher() + monkeypatch.setattr(store, "source_publication", None) service = _finality_service( config, snapshot=True, official=False, - store=SnapshotPublisher(), + store=store, clock=clock, cadence="PT1H", results=(result, result), diff --git a/tests/test_reporting_ledger.py b/tests/test_reporting_ledger.py index e071c1e3b..5daccc635 100644 --- a/tests/test_reporting_ledger.py +++ b/tests/test_reporting_ledger.py @@ -194,7 +194,7 @@ def test_definition_binding_retains_selected_finality_policy_off_wire() -> None: assert _definition_from_payload(binding.to_storage()) == binding -@pytest.mark.parametrize("value", ["P1M", "P1Y", "not-a-duration"]) +@pytest.mark.parametrize("value", ["P1M", "P1Y", "not-a-duration", "PT1H\n"]) def test_finality_policy_refuses_unsupported_duration(value: str) -> None: with pytest.raises(ValueError, match="supported reporting duration"): ReportingContractualCutoffPolicy("cutoff-v1", value) @@ -261,7 +261,7 @@ class CustomPolicy(ReportingSourceFinalPolicy): replace(DEFINITION, finality_policy=CustomPolicy("daily-v1", "close")) -@pytest.mark.parametrize("policy_id", ["", "a" * 256, "contains spaces"]) +@pytest.mark.parametrize("policy_id", ["", "a" * 256, "contains spaces", "close\n"]) def test_finality_policy_refuses_invalid_identifier(policy_id: str) -> None: with pytest.raises(ValueError, match="finality_policy_id"): ReportingSourceFinalPolicy(policy_id, "close") diff --git a/tests/test_reporting_official_acquisition.py b/tests/test_reporting_official_acquisition.py index c9a4e2a37..a29b98f3f 100644 --- a/tests/test_reporting_official_acquisition.py +++ b/tests/test_reporting_official_acquisition.py @@ -77,6 +77,165 @@ def __getattr__(self, name: str) -> Any: raise AttributeError(name) return getattr(self.store, name) + def source_publication(self, account_id: str, **kwargs: Any) -> Any: + return self.store.source_publication(account_id, **kwargs) + + +@pytest.mark.parametrize( + "policy", + [ + ReportingSourceFinalPolicy("provider", "delivery.is_final"), + ReportingContractualCutoffPolicy("cutoff", "PT0S"), + ], +) +async def test_standalone_official_admission_requires_account_lock(policy: Any) -> None: + producer, store, config, obligation, calls, _ = await _harness( + official=True, finality_policy=policy + ) + + class UnlockedCore(CoreStore): + source_publication = None + + producer._store = UnlockedCore(store) + changed = replace(config, delivery_config_version=2) + await store.put_configuration(changed) + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.close_elapsed_periods(changed) + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.acquire_obligation(config, obligation) + assert not calls + + +@pytest.mark.parametrize("finality", ["official", "snapshot"]) +async def test_direct_official_work_requires_lock_before_reserving( + monkeypatch: pytest.MonkeyPatch, finality: str +) -> None: + policy = ReportingSourceFinalPolicy("provider", "source-evidence-redacted") + producer, store, config, obligation, _, clock = await _harness( + official=True, finality_policy=policy + ) + offering = SNAPSHOT_OFFERING_ID if finality == "snapshot" else OFFICIAL_OFFERING_ID + request = producer._build_slice(config, obligation, offering, finality=finality, now=clock[0]) + _, manifest, producer._object_reader = redacted_completed_result( + request, page_bodies=[PROVIDER_ROW] + ) + monkeypatch.setattr(store, "source_publication", None) + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.commit_revision_from_manifest( + obligation, + manifest, + rows=ROWS, + finality=finality, + finality_proof=( + manifest_finality_proof(obligation, manifest, clock[0]) + if finality == "official" + else None + ), + ) + assert ( + await store.get_provisional_acquisition( + account_id=config.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ordinal=0, + ) + is None + ) + + +@pytest.mark.parametrize("changed", ["signal", "provider_time"]) +async def test_core_source_final_replay_requires_retained_signal_and_timestamp( + changed: str, +) -> None: + producer, store, config, obligation, _, _ = await _harness( + official=True, finality_policy=ReportingSourceFinalPolicy("provider", "delivery.is_final") + ) + producer._store = CoreStore(store) + first = await producer.acquire_obligation(config, obligation) + manifest = producer._source.manifest + updates = ( + {"evidence_ref": "another-signal"} + if changed == "signal" + else {"observed_at": first.finalized_at - timedelta(seconds=1)} + ) + changed_manifest = manifest.model_copy( + update={"finality_evidence": manifest.finality_evidence.model_copy(update=updates)} + ) + with pytest.raises(LedgerConflictError) as error: + await producer.commit_revision_from_manifest( + obligation, changed_manifest, rows=ROWS, finality="official" + ) + assert error.value.code == "REVISION_IMMUTABLE" + assert ( + await producer.commit_revision_from_manifest( + obligation, manifest, rows=ROWS, finality="official" + ) + == first + ) + + +@pytest.mark.parametrize("change", ["prefix", "suffix"]) +@pytest.mark.parametrize("origin_lookup", [True, False]) +@pytest.mark.parametrize( + ("archived", "change_stored_revision"), [(False, False), (True, False), (True, True)] +) +async def test_unchanged_snapshot_replay_checks_the_referenced_publication( + change: str, + origin_lookup: bool, + archived: bool, + change_stored_revision: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + producer, store, config, obligation, _, clock = await _harness( + finality_policy=ReportingContractualCutoffPolicy("later", "P2D") + ) + first = await producer.acquire_obligation(config, obligation, track_settling=True) + clock[0] += timedelta(hours=1) + assert first == await producer.acquire_obligation( + config, obligation, restate=True, track_settling=True + ) + observation = await store.get_provisional_observation( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + if archived: + observation = replace(observation, revision_publication_id=None) + store._provisional_observations[(config.account_id, obligation.reporting_obligation_id)][ + observation.acquisition.ordinal + ] = observation + if not origin_lookup: + monkeypatch.setattr(store, "verify_snapshot_origin", None) + manifest = SourceBatchManifestV1.model_validate_json(observation.manifest_json) + assert first == await producer.commit_revision_from_manifest( + obligation, manifest, rows=ROWS, finality="snapshot", acquisition=observation.acquisition + ) + get_revision = store.get_revision + + async def different_origin(**kwargs: Any) -> Any: + revision = await get_revision(**kwargs) + offset = 4 if change == "prefix" else 44 + identifier = revision.source_publication_id + return replace( + revision, + source_publication_id=identifier[:offset] + + ("a" if identifier[offset] != "a" else "b") + + identifier[offset + 1 :], + ) + + if change_stored_revision: + store._revisions[first.reporting_revision_id] = await different_origin( + account_id=config.account_id, reporting_revision_id=first.reporting_revision_id + ) + else: + monkeypatch.setattr(store, "get_revision", different_origin) + with pytest.raises(LedgerConflictError) as error: + await producer.commit_revision_from_manifest( + obligation, + manifest, + rows=ROWS, + finality="snapshot", + acquisition=observation.acquisition, + ) + assert error.value.code in {"REVISION_IMMUTABLE", "STABILIZATION_ORIGIN_CONFLICT"} + @pytest.fixture(params=["memory", "postgres", "core", "progress"]) async def cutoff_backend(request: Any) -> AsyncIterator[Any]: diff --git a/tests/test_reporting_official_finality.py b/tests/test_reporting_official_finality.py index b1d3d4fb5..46d02b751 100644 --- a/tests/test_reporting_official_finality.py +++ b/tests/test_reporting_official_finality.py @@ -22,6 +22,7 @@ redacted_completed_result, ) from adcp.reporting.inline_source import ( + InlineFetchResult, InlineReportingSource, InMemorySealStore, InMemoryStagingStore, @@ -35,6 +36,8 @@ ReportingRevisionRecord, ReportingSourceFinalPolicy, ReportingStabilizedPolicy, + ReportingStatusCaller, + ReportingStatusHandler, revision_content_sha256, ) from adcp.reporting.ledger.finality import ( @@ -47,6 +50,7 @@ ProvisionalObservation, ProvisionalPolicy, ) +from adcp.reporting.service import ReliableReportingService from adcp.reporting.source import ( ReportingFinalityEvidenceV1, ReportingSourceExecutorResult, @@ -55,6 +59,7 @@ publication_content_fingerprint_v1, source_batch_object_set_sha256_v1, ) +from adcp.reporting.testing import DeterministicReportingClock from tests.conformance.reporting._generation_support import isolated_reporting_pool from tests.test_reporting_source_availability import ( END, @@ -73,8 +78,9 @@ class UnlockedHistory(InMemoryReportingLedgerStore): source_publication = None store = UnlockedHistory() - with pytest.raises(LedgerConflictError, match="publication"): + with pytest.raises(LedgerConflictError, match="publication") as error: await _stabilized_harness(lambda **kwargs: store) + assert error.value.code == "STABILIZATION_PUBLICATION_NOT_SUPPORTED" assert not store._obligations @@ -1434,6 +1440,68 @@ async def store_factory(request: pytest.FixtureRequest) -> AsyncIterator[Any]: yield partial(PgReportingLedgerStore, pool=pool) +@pytest.mark.parametrize("configure_before_init", [True, False]) +async def test_restart_resumes_ordinary_snapshot_with_unused_stabilized_policy( + store_factory: Any, configure_before_init: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + from tests.test_reliable_reporting_service import ( + DEFINITION, + NOW, + _configuration, + _finality_service, + _rows, + ) + + config = replace( + _configuration(), + activated_at=NOW.replace(hour=1, minute=0), + deactivated_at=NOW.replace(hour=2, minute=0), + definition=replace( + DEFINITION, finality_policy=ReportingStabilizedPolicy("unused", "PT0S", "PT1H") + ), + ) + clock = DeterministicReportingClock(NOW) + store = store_factory(clock=clock) + monkeypatch.setattr(store, "get_stabilization_schedule", None) + monkeypatch.setattr(store, "verify_snapshot_origin", None) + + def service() -> ReliableReportingService: + return _finality_service( + config, + snapshot=True, + official=False, + store=store, + clock=clock, + cadence="PT1H", + results=(InlineFetchResult(rows=_rows(10), data_through=config.deactivated_at),), + ) + + original = service() + await original.configure(config) + created = await original.run_worker() + (identifier,) = created.configurations[config.generation_key].revisions_committed + await original.close() + clock.advance(timedelta(hours=1)) + restarted = service() + if configure_before_init: + await restarted.configure(config) + await restarted.initialize() + if not configure_before_init: + await restarted.configure(config) + turn = await restarted.run_worker() + assert not turn.configuration_errors + revision = await store.get_revision( + account_id=config.account_id, reporting_revision_id=identifier + ) + history = await store.list_provisional_observations( + account_id=config.account_id, + reporting_obligation_id=revision.reporting_obligation_id, + limit=3, + ) + assert [o.revision_id for o in history] == [identifier, identifier] + assert revision.finality == "snapshot" + + def test_source_final_evidence_ref_accepts_512_characters() -> None: evidence = ReportingFinalityEvidenceV1( basis="provider_declared", observed_at=END, evidence_ref="s" * 512 @@ -2096,6 +2164,12 @@ async def test_retained_official_replay_preserves_original_policy_fields() -> No source_manifest_sha256=manifest.content_fingerprint.split(":", 1)[1], ) historical = await store.commit_revision(historical, ROWS) + result = await ReportingStatusHandler(store).handle( + {"view": "revision", "reporting_revision_id": revision_id}, + caller=ReportingStatusCaller(config.account_id, config.consumer_id), + ) + assert result["revision"]["finality"] == "official" + assert result["revision"]["finality_policy_id"] == "archived-policy" clock[0] += timedelta(hours=1) assert await producer.acquire_obligation(config, obligation) is None assert ( diff --git a/tests/test_reporting_provisional_observations.py b/tests/test_reporting_provisional_observations.py index be6f6e676..b853a7814 100644 --- a/tests/test_reporting_provisional_observations.py +++ b/tests/test_reporting_provisional_observations.py @@ -15,6 +15,7 @@ from adcp.reporting.ledger import ( InMemoryReportingLedgerStore, ReportingProducer, + ReportingSourceFinalPolicy, revision_content_sha256, ) from adcp.reporting.ledger import ReportingStatusCaller as OwnershipCaller @@ -39,9 +40,10 @@ async def make_harness(request): if request.param == "memory": - async def build(capabilities): + async def build(capabilities, *, finality_policy=None): return await _harness( capabilities, + finality_policy=finality_policy, store_factory=lambda clock: InMemoryReportingLedgerStore( clock=clock, notifications=True ), @@ -51,9 +53,10 @@ async def build(capabilities): else: async with isolated_reporting_pool() as pool: - async def build(capabilities): + async def build(capabilities, *, finality_policy=None): return await _harness( capabilities, + finality_policy=finality_policy, store_factory=lambda clock: PgReportingLedgerStore( pool=pool, clock=clock, notifications=True ), @@ -1128,7 +1131,10 @@ def fail(event): async def test_reserved_snapshot_retry_survives_explicit_close_boundary(make_harness): producer, store, fetch, clock = await make_harness( - _capabilities(restatement_window="PT3H", official_close_lag="PT4H") + _capabilities(restatement_window="PT3H", official_close_lag="PT4H"), + finality_policy=ReportingSourceFinalPolicy( + "delivery-settlement", "settlement.job.complete" + ), ) await producer.run_worker() source = producer._source @@ -1168,7 +1174,12 @@ async def test_restart_retries_frozen_snapshot_before_authoritative_readiness( offering.update(expected_availability_lag="PT8H", worst_case_availability_lag="PT12H") payload["capabilities_sha256"] = reporting_source_capabilities_sha256_v1(payload) capabilities = ReportingSourceCapabilitiesV1.model_validate(payload) - producer, store, _, clock = await make_harness(capabilities) + producer, store, _, clock = await make_harness( + capabilities, + finality_policy=ReportingSourceFinalPolicy( + "delivery-settlement", "settlement.job.complete" + ), + ) await producer.run_worker() source = producer._source original_fetch = source._fetch diff --git a/tests/test_reporting_scheduling_jitter.py b/tests/test_reporting_scheduling_jitter.py index e6fb14a67..a001b48fd 100644 --- a/tests/test_reporting_scheduling_jitter.py +++ b/tests/test_reporting_scheduling_jitter.py @@ -5,7 +5,12 @@ import pytest -from adcp.reporting.ledger import InMemoryReportingLedgerStore, ReportingProducer +from adcp.reporting.ledger import ( + InMemoryReportingLedgerStore, + ReportingContractualCutoffPolicy, + ReportingProducer, + ReportingSourceFinalPolicy, +) from tests.test_reporting_settling import _capabilities as settling_capabilities from tests.test_reporting_source_availability import ( ProgressStore, @@ -152,7 +157,9 @@ async def test_jitter_is_stable_and_bounded_by_sla_short_period_and_cadence(): async def test_jitter_keeps_explicit_official_boundary_and_manual_acquisition(): producer, store, configuration, obligation, calls, clock = await _harness( - read_jitter_window=WINDOW, capabilities=_closing_capabilities() + read_jitter_window=WINDOW, + capabilities=_closing_capabilities(), + finality_policy=ReportingSourceFinalPolicy("provider-final", "delivery.is_final"), ) ready = producer._source_available_at(obligation) clock[0] = ready @@ -234,6 +241,7 @@ async def test_successful_manual_snapshot_keeps_immediate_automatic_official_clo producer, store, configuration, obligation, calls, clock = await _harness( read_jitter_window=WINDOW, capabilities=settling_capabilities(restatement_window="PT0S", official_close_lag="PT0S"), + finality_policy=ReportingContractualCutoffPolicy("immediate", "PT0S"), ) ready = producer._source_available_at(obligation) assert ready < producer._scheduled_source_ready_at(configuration, obligation) diff --git a/tests/test_reporting_settling.py b/tests/test_reporting_settling.py index d7289f35e..7a8f3d118 100644 --- a/tests/test_reporting_settling.py +++ b/tests/test_reporting_settling.py @@ -12,6 +12,7 @@ OFFICIAL_OFFERING_ID, SNAPSHOT_OFFERING_ID, redacted_capabilities, + redacted_contract_identity, ) from adcp.reporting.inline_source import ( InlineFetchResult, @@ -23,14 +24,19 @@ InMemoryReportingLedgerStore, ProducerOfferings, ReportingConfiguration, + ReportingDefinitionBinding, + ReportingFinalityPolicy, ReportingProducer, ReportingScheduleSpec, + ReportingSourceFinalPolicy, + ReportingStabilizedPolicy, ReportingStatusCaller, ReportingStatusHandler, ) from adcp.reporting.ledger import ReportingStatusCaller as OwnershipCaller from adcp.reporting.source import ( ProvisionalSnapshotOfferingV1, + ReportingFinalityEvidenceV1, ReportingSourceCapabilitiesV1, reporting_source_capabilities_sha256_v1, ) @@ -72,6 +78,8 @@ def _capabilities( for item in base.offerings ] for offering in payload["offerings"]: + offering.update(source_timezone="UTC", product_ids=[DEFINITION_ID]) + offering["windowing"]["minimum_window"] = "PT1H" if offering["offering_id"] == OFFICIAL_OFFERING_ID: # Policy fixtures make authoritative data ready independently of # the configured close cadence, just like the snapshot above. @@ -105,6 +113,15 @@ def __call__(self, request: Any) -> InlineFetchResult: } ], data_through=request.period.end, + finality_evidence=( + ReportingFinalityEvidenceV1( + basis="provider_job_terminal", + evidence_ref="settlement.job.complete", + observed_at=request.period.source_read_cutoff_at, + ) + if request.publication_class == "AUTHORITATIVE" + else None + ), provisional_until=( self.provisional_until if request.publication_class == "PROVISIONAL_SNAPSHOT" @@ -117,6 +134,7 @@ async def _harness( capabilities: ReportingSourceCapabilitiesV1, *, store_factory: Any = None, + finality_policy: ReportingFinalityPolicy | None = None, read_jitter_window: timedelta = timedelta(0), ) -> tuple[ ReportingProducer, @@ -150,6 +168,16 @@ async def _harness( feed_purpose="analytics", schedule=SCHEDULE, required_finality="snapshot", + definition=( + ReportingDefinitionBinding( + **redacted_contract_identity.model_dump( + exclude={"report_definition_id", "reporting_profile"} + ), + finality_policy=finality_policy, + ) + if finality_policy is not None + else None + ), activated_at=ACTIVATED_AT, media_buy_ids=("media-buy-redacted",), ) @@ -216,7 +244,6 @@ async def test_unchanged_refresh_honors_cadence_and_commits_an_observation() -> _capabilities( restatement_window="P3D", restatement_cadence="PT1H", - official_close_lag="P3D", ) ) first = await producer.run_worker() @@ -230,7 +257,7 @@ async def test_unchanged_refresh_honors_cadence_and_commits_an_observation() -> unchanged = await producer.run_worker() assert unchanged.revisions_committed == [] assert fetch.calls == ["PROVISIONAL_SNAPSHOT", "PROVISIONAL_SNAPSHOT"] - assert len(await _revisions(store)) == 1 + assert [r.finality for r in await _revisions(store)] == ["snapshot"] # The successful observation advances both the cadence clock and execution # ordinal, so an immediate worker turn neither polls nor replays it. @@ -250,7 +277,8 @@ async def test_changed_refresh_supersedes_snapshot_and_window_expiry_closes_offi restatement_window="P3D", restatement_cadence="PT1H", official_close_lag="P3D", - ) + ), + finality_policy=ReportingStabilizedPolicy("settled-delivery", "P3D", "PT1H"), ) await producer.run_worker() first = (await _revisions(store))[0] @@ -264,6 +292,8 @@ async def test_changed_refresh_supersedes_snapshot_and_window_expiry_closes_offi assert snapshots[1].supersedes_reporting_revision_id == first.reporting_revision_id obligation = await _only_obligation(store) + clock[0] = obligation.period.end + timedelta(days=3, hours=-1) + assert not (await producer.run_worker()).revisions_committed clock[0] = obligation.period.end + timedelta(days=3) closed = await producer.run_worker() assert len(closed.revisions_committed) == 1 @@ -286,8 +316,7 @@ async def test_changed_refresh_supersedes_snapshot_and_window_expiry_closes_offi clock[0] += timedelta(days=1) await producer.run_worker() - assert fetch.calls[-1] == "AUTHORITATIVE" - assert len(fetch.calls) == 3 + assert fetch.calls == ["PROVISIONAL_SNAPSHOT"] * 4 async def test_slice_provisional_until_overrides_declared_window() -> None: @@ -296,7 +325,10 @@ async def test_slice_provisional_until_overrides_declared_window() -> None: restatement_window="P3D", restatement_cadence="P1D", official_close_lag="P3D", - ) + ), + finality_policy=ReportingSourceFinalPolicy( + "delivery-settlement", "settlement.job.complete" + ), ) await producer.run_worker() obligation = await _only_obligation(store) @@ -322,10 +354,10 @@ async def test_source_without_a_window_retains_its_final_default_read() -> None: producer, store, fetch, clock = await _harness(_capabilities(restatement_window=None)) await producer.run_worker() obligation = await _only_obligation(store) - clock[0] = obligation.period.end + timedelta(days=10) + clock[0] = obligation.period.end + timedelta(days=120) await producer.run_worker() assert fetch.calls == ["PROVISIONAL_SNAPSHOT", "PROVISIONAL_SNAPSHOT"] - assert len(await _revisions(store)) == 1 + assert [revision.finality for revision in await _revisions(store)] == ["snapshot"] checkpoint = await store.get_restatement_checkpoint( account_id=ACCOUNT, reporting_obligation_id=obligation.reporting_obligation_id, From 05763b9228d87f5656907801df796f6afeea0627 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 18:53:20 +0530 Subject: [PATCH 22/39] fix(reporting): recover frozen close routes and verify replay origins --- docs/reporting-provisional-observations.md | 10 +- docs/reporting-source-adapters.md | 7 +- .../src/adcp_reporting_storage/ledger/pg.py | 3 + src/adcp/reporting/ledger/producer.py | 90 ++++++++--- src/adcp/reporting/ledger/store.py | 1 + src/adcp/reporting/service.py | 21 ++- tests/test_reporting_official_acquisition.py | 62 ++++++++ tests/test_reporting_official_finality.py | 148 +++++++++++++++++- 8 files changed, 306 insertions(+), 36 deletions(-) diff --git a/docs/reporting-provisional-observations.md b/docs/reporting-provisional-observations.md index 697c14801..a27a4d212 100644 --- a/docs/reporting-provisional-observations.md +++ b/docs/reporting-provisional-observations.md @@ -34,10 +34,12 @@ reuse, installed-package checks and independent integration review. Follow the - Snapshot-only reporting retains one final inclusive read at the boundary, including after downtime. Valid new source evidence may extend that boundary. Expiry does not create an official revision. -- An explicit snapshot-to-official route requires a selected definition policy. - `official_close_lag` requires an explicitly declared `restatement_window` - and schedules a close attempt. The selected policy determines the source - offering and the proof needed to publish. +- Setting `official_close_lag` opts a snapshot offering into an explicit + snapshot-to-official route. This requires a declared `restatement_window` + and a selected definition policy. For `source_final` and + `contractual_cutoff`, the lag can delay the source attempt. For + `stabilized`, the SDK promotes the snapshot when its scheduled observations + satisfy the selected policy, independently of the lag value. The durable acquisition freezes the source request, predecessor revision and observation ordinal. A retry renews only `deadline_at`, the execution budget for diff --git a/docs/reporting-source-adapters.md b/docs/reporting-source-adapters.md index 5b8c21073..8a61a1c36 100644 --- a/docs/reporting-source-adapters.md +++ b/docs/reporting-source-adapters.md @@ -299,8 +299,11 @@ return InlineFetchResult( ) ``` -`stable_since` records source evidence and `official_close_lag` schedules an -explicit close attempt. Proof comes from the selected policy. A custom Core +`stable_since` records source evidence. Setting `official_close_lag` opts a +snapshot offering into an explicit close. For `source_final` and +`contractual_cutoff`, the lag can delay the source attempt. For `stabilized`, +the SDK promotes the snapshot when the scheduled observations satisfy +`minimum_age` and `unchanged_for`, independently of the lag value. A custom Core ledger that admits official work must implement `ReportingSourcePublicationStore.source_publication(account_id)` and serialize publication for that account across workers. Exact reads and replays retain diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py index 3397ac90f..9407b247c 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py @@ -1988,6 +1988,9 @@ async def commit_provisional_observation( raise LedgerConflictError( "HISTORY_UNAVAILABLE", "observation revision is missing" ) + validate_snapshot_origin( + retained_revision, retained_observation.revision_publication_id + ) if require_origin and retained_revision.finality == "snapshot": await self.verify_snapshot_origin(retained_revision) validate_snapshot_origin(retained_revision, observation.revision_publication_id) diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 595230189..f0e66d7aa 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -763,8 +763,8 @@ async def _close_elapsed_periods( ) ), ) - elif selected is not None and self._official_close_requested(configuration): - self._official_publication_lock() + else: + await self._check_official_publication_lock(configuration) obligation = ReportingObligationRecord( reporting_obligation_id=self._obligation_id(configuration, boundary), account_id=configuration.account_id, @@ -1370,12 +1370,14 @@ def _stabilization_schedule_store(self) -> StabilizationScheduleStore | None: return self._store return None - async def _has_retained_stabilization(self, configuration: ReportingConfiguration) -> bool: - """Inspect retained routes when a snapshot publisher lacks the generation lookup.""" - key = configuration.generation_key + async def _has_retained_official_close(self, key: ReportingConfigurationGenerationKey) -> bool: + """Find a frozen scheduled close, including an unfinished first reservation.""" + schedule_store = self._stabilization_schedule_store() + if schedule_store is not None: + return await schedule_store.get_stabilization_schedule(key) is not None snapshot = await self._store.open_snapshot( caller=ReportingDeliveryPrincipal(key.account_id, key.consumer_id), - filters_fingerprint=f"stabilization:{key.delivery_config_id}:{key.delivery_config_version}", + filters_fingerprint=f"official-close:{key.delivery_config_id}:{key.delivery_config_version}", ) offset = 0 page_size = 100 @@ -1397,7 +1399,7 @@ async def _has_retained_stabilization(self, configuration: ReportingConfiguratio self._store, PendingProvisionalAcquisitionStore.get_provisional_acquisition ): # Without the retained cursor and its reservations a prior - # stabilized close cannot be distinguished from an ordinary poll. + # scheduled close cannot be distinguished from an ordinary poll. return True checkpoint = await checkpoints.get_restatement_checkpoint( account_id=key.account_id, @@ -1405,7 +1407,16 @@ async def _has_retained_stabilization(self, configuration: ReportingConfiguratio ) if checkpoint is not None and checkpoint.stabilization_policy is not None: return True - next_ordinal = checkpoint.next_observation if checkpoint is not None else 0 + # The first reservation follows any untracked manual revisions. + if checkpoint is None: + next_ordinal = len( + await self._store.list_revisions( + account_id=key.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + ) + else: + next_ordinal = checkpoint.next_observation for ordinal in range(next_ordinal + 1): acquisition = await self._pending_acquisition(obligation, ordinal=ordinal) if ( @@ -1845,12 +1856,9 @@ async def _acquire_obligation( raise LedgerConflictError( "FINALITY_POLICY_UNPROVEN", "official finality requires a selected policy" ) - if selected_policy is not None and ( - finality == "official" or self._official_close_requested(configuration) - ): - self._official_publication_lock( - stabilized=isinstance(selected_policy, ReportingStabilizedPolicy) - ) + await self._check_official_publication_lock( + obligation, official=finality == "official", frozen=pending.policy if pending else None + ) # Everything below needs the frozen code: the slice request carries it, # the manifest is checked against it, and the revision is written under # it. Gate here rather than earlier so a settled legacy obligation stays @@ -2563,6 +2571,30 @@ def _official_publication_lock( ) return store + async def _check_official_publication_lock( + self, + route: ReportingConfiguration | ReportingObligationRecord, + *, + official: bool = False, + frozen: ProvisionalPolicy | None = None, + check_generation: bool = True, + ) -> None: + policy = route.definition.finality_policy if route.definition else None + if policy is None or self._optional_publication_store() is not None: + return + if ( + official + or self._official_close_requested(route) + or ( + frozen is not None + and (frozen.official_close_lag is not None or frozen.until_stabilized) + ) + or (check_generation and await self._has_retained_official_close(route.generation_key)) + ): + self._official_publication_lock( + stabilized=isinstance(policy, ReportingStabilizedPolicy) + ) + def _stabilization_store(self) -> StabilizationStore: history = self._observation_history_store() self._restatement_store() @@ -2625,6 +2657,11 @@ async def _reserve_observation( ): stabilization_history_limit(selected, frozen_policy.cadence) frozen_policy = replace(frozen_policy, until_stabilized=True) + await self._check_official_publication_lock( + obligation, + official=derived_close or request.publication_class == "AUTHORITATIVE", + frozen=frozen_policy, + ) if self._requires_source_authority(obligation): self._source_authority(obligation, request) self._check_source_authorization( @@ -2892,12 +2929,14 @@ async def _commit_revision_from_manifest( ) -> ReportingRevisionRecord: obligation = await self._stored_obligation(obligation) selected_policy = obligation.definition.finality_policy if obligation.definition else None - if selected_policy is not None and ( - finality == "official" or self._official_close_requested(obligation) - ): - self._official_publication_lock( - stabilized=isinstance(selected_policy, ReportingStabilizedPolicy) - ) + # A matching historical observation already retained its own route. + # Check generation-wide evidence before admitting any fresh snapshot. + await self._check_official_publication_lock( + obligation, + official=finality == "official", + frozen=acquisition.policy if acquisition else None, + check_generation=acquisition is None or verified_source_rows, + ) origin_store: SnapshotOriginStore | None = None if finality == "snapshot" and acquisition is not None: if acquisition.requires_snapshot_origin( @@ -3027,6 +3066,9 @@ async def _commit_revision_from_manifest( ) if replay is not None: return replay + await self._check_official_publication_lock( + obligation, frozen=acquisition.policy + ) self._validate_source_manifest( obligation, request, @@ -3265,9 +3307,8 @@ async def _retained_snapshot_replay( ) ): raise LedgerConflictError("REVISION_IMMUTABLE", "retained source totals differ") - validate_retained_provisional_observation( - revision, latest if is_latest else observation.bind_revision(revision), rows - ) + if is_latest: + validate_retained_provisional_observation(revision, latest, rows) if origin_store is not None: await self._verify_tracked_snapshot_origin( obligation, acquisition, origin_store, revision @@ -3279,8 +3320,7 @@ async def _retained_snapshot_replay( await self._store.verify_snapshot_origin(revision) if is_latest and (prior is not None or latest.revision_publication_id is not None): return revision - # The atomic replay also checks older observations without a full - # predecessor publication witness against the original immutable revision. + # The atomic publisher reads this exact ordinal, its stored witness and revision. try: committed = await store.commit_provisional_observation(observation, revision, rows) except LedgerConflictError as error: diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index f27bd29e7..e167a2643 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -1577,6 +1577,7 @@ async def commit_provisional_observation( "HISTORY_UNAVAILABLE", "observation revision is missing" ) retained_revision = self._revisions[existing.revision_id] + validate_snapshot_origin(retained_revision, existing.revision_publication_id) if require_origin and retained_revision.finality == "snapshot": await self.verify_snapshot_origin(retained_revision) validate_snapshot_origin(retained_revision, observation.revision_publication_id) diff --git a/src/adcp/reporting/service.py b/src/adcp/reporting/service.py index e2bd96b93..025831a72 100644 --- a/src/adcp/reporting/service.py +++ b/src/adcp/reporting/service.py @@ -446,6 +446,14 @@ def missing_stabilization_lookup(self) -> bool: and self.producer._stabilization_schedule_store() is None ) + @property + def needs_close_recovery_check(self) -> bool: + return self.missing_stabilization_lookup or ( + self.configuration.definition is not None + and self.configuration.definition.finality_policy is not None + and self.producer._optional_publication_store() is None + ) + class _MissingFinalityPolicyError(ReliableReportingConfigurationError): """A retained generation may still serve reads while its producer is unbound.""" @@ -796,7 +804,7 @@ async def _configure( self._pending_bindings[key] = binding # Schema initialization makes it possible to distinguish a new snapshot # generation from an existing one whose frozen close needs recovery. - if not binding.missing_stabilization_lookup: + if not binding.needs_close_recovery_check: self._bindings[key] = binding else: await self._install_binding(binding, persist=persist) @@ -807,7 +815,7 @@ async def _install_binding(self, binding: _Binding, *, persist: bool) -> None: configuration = binding.configuration key = configuration.generation_key if ( - binding.missing_stabilization_lookup + binding.needs_close_recovery_check and ( not persist or any( @@ -818,10 +826,15 @@ async def _install_binding(self, binding: _Binding, *, persist: bool) -> None: ) ) ) - and await binding.producer._has_retained_stabilization(configuration) + and await binding.producer._has_retained_official_close(key) ): + if binding.missing_stabilization_lookup: + raise ReliableReportingConfigurationError( + "recovering a selected stabilized generation requires " + "get_stabilization_schedule" + ) raise ReliableReportingConfigurationError( - "recovering a selected stabilized generation requires get_stabilization_schedule" + "official finality requires an account publication lock" ) if persist: await self.store.put_configuration(configuration) diff --git a/tests/test_reporting_official_acquisition.py b/tests/test_reporting_official_acquisition.py index a29b98f3f..28174f186 100644 --- a/tests/test_reporting_official_acquisition.py +++ b/tests/test_reporting_official_acquisition.py @@ -248,6 +248,68 @@ async def cutoff_backend(request: Any) -> AsyncIterator[Any]: ), request.param == "core" +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +@pytest.mark.parametrize("witness", [True, False]) +@pytest.mark.parametrize("atomic", [True, False]) +async def test_historical_unchanged_replay_validates_stored_observation_origin( + cutoff_backend: Any, + witness: bool, + atomic: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + factory, _ = cutoff_backend + producer, store, config, obligation, _, clock = await _harness( + factory, finality_policy=ReportingContractualCutoffPolicy("cutoff", "P2D") + ) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + first = await producer.acquire_obligation(config, obligation, track_settling=True) + for _ in range(2): + clock[0] += timedelta(hours=1) + assert first == await producer.acquire_obligation( + config, obligation, restate=True, track_settling=True + ) + _, middle, latest = await store.list_provisional_observations(**owner, limit=3) + assert middle.acquisition.ordinal == 1 and latest.acquisition.ordinal == 2 + assert middle.revision_publication_id == first.source_publication_id + manifest = SourceBatchManifestV1.model_validate_json(middle.manifest_json) + stored_id = first.source_publication_id + wrong_id = stored_id[:-1] + ("0" if stored_id[-1] != "0" else "1") + if isinstance(store, PgReportingLedgerStore): + async with store.storage_identity.connection() as connection, connection.transaction(): + # Simulate an already inconsistent historical row in this fixture's private schema. + await connection.execute( + "ALTER TABLE reporting_provisional_observations DISABLE TRIGGER " + "reporting_provisional_observation_immutable" + ) + await connection.execute( + "UPDATE reporting_provisional_observations SET payload = " + "jsonb_set(payload, '{revision_publication_id}', to_jsonb(%s::text)) " + "WHERE account_id=%s AND reporting_obligation_id=%s AND ordinal=1", + (wrong_id, config.account_id, obligation.reporting_obligation_id), + ) + await connection.execute( + "ALTER TABLE reporting_provisional_observations ENABLE TRIGGER " + "reporting_provisional_observation_immutable" + ) + else: + store._provisional_observations[(config.account_id, obligation.reporting_obligation_id)][ + 1 + ] = replace(middle, revision_publication_id=wrong_id) + if not witness: + monkeypatch.setattr(store, "verify_snapshot_origin", None) + with pytest.raises(LedgerConflictError) as error: + if atomic: + await store.commit_provisional_observation(middle, first, ROWS) + else: + await producer.commit_revision_from_manifest( + obligation, manifest, rows=ROWS, finality="snapshot", acquisition=middle.acquisition + ) + assert error.value.code in {"REVISION_IMMUTABLE", "STABILIZATION_ORIGIN_CONFLICT"} + assert await store.get_provisional_observation(**owner) == latest + + async def test_stale_cutoff_seal_retries_with_fresh_source_work( cutoff_backend: Any, monkeypatch: Any ) -> None: diff --git a/tests/test_reporting_official_finality.py b/tests/test_reporting_official_finality.py index 46d02b751..3fe98d8cb 100644 --- a/tests/test_reporting_official_finality.py +++ b/tests/test_reporting_official_finality.py @@ -50,7 +50,7 @@ ProvisionalObservation, ProvisionalPolicy, ) -from adcp.reporting.service import ReliableReportingService +from adcp.reporting.service import ReliableReportingConfigurationError, ReliableReportingService from adcp.reporting.source import ( ReportingFinalityEvidenceV1, ReportingSourceExecutorResult, @@ -1502,6 +1502,152 @@ def service() -> ReliableReportingService: assert revision.finality == "snapshot" +@pytest.mark.parametrize("configure_before_init", [True, False]) +@pytest.mark.parametrize( + "policy,missing", + [ + (ReportingStabilizedPolicy("stable", "PT0S", "PT6H"), "get_stabilization_schedule"), + (ReportingSourceFinalPolicy("provider", "closed"), "source_publication"), + (ReportingContractualCutoffPolicy("cutoff", "P2D"), "source_publication"), + ], +) +async def test_recovery_checks_pending_close_after_untracked_manual_revision( + store_factory: Any, + configure_before_init: bool, + policy: Any, + missing: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from tests.test_reliable_reporting_service import ( + DEFINITION, + NOW, + _configuration, + _finality_service, + _rows, + ) + + config = replace( + _configuration(), + activated_at=NOW.replace(hour=1, minute=0), + deactivated_at=NOW.replace(hour=2, minute=0), + definition=replace(DEFINITION, finality_policy=policy), + ) + clock = DeterministicReportingClock(NOW) + store = store_factory(clock=clock) + original = _finality_service( + config, + snapshot=True, + official=isinstance(policy, ReportingSourceFinalPolicy), + close=True, + store=store, + clock=clock, + cadence="PT1H", + results=(InlineFetchResult(rows=_rows(10), data_through=config.deactivated_at), None), + ) + await original.configure(config) + await original.initialize() + producer = original._bindings[config.generation_key].producer + (obligation,) = await producer.close_elapsed_periods(config) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + manual = await producer.acquire_obligation(config, obligation, manual_replay=True) + assert manual is not None + await producer.run_configuration(config) + assert await store.get_restatement_checkpoint(**owner) is None + assert await store.get_provisional_acquisition(**owner, ordinal=0) is None + pending = await store.get_provisional_acquisition(**owner, ordinal=1) + assert pending is not None and pending.policy.official_close_lag is not None + await original.close() + monkeypatch.setattr(store, missing, None) + if missing == "source_publication": + monkeypatch.setattr(store, "get_stabilization_schedule", None) + recovered = _finality_service( + config, + snapshot=True, + official=isinstance(policy, ReportingSourceFinalPolicy), + cadence="PT1H", + store=store, + clock=clock, + ) + expected_error = "publication lock" if missing == "source_publication" else missing + with pytest.raises(ReliableReportingConfigurationError, match=expected_error): + if configure_before_init: + await recovered.configure(config) + await recovered.initialize() + assert config.generation_key not in recovered._bindings + assert await store.get_restatement_checkpoint(**owner) is None + assert await store.get_provisional_acquisition(**owner, ordinal=1) == pending + + +@pytest.mark.parametrize( + "policy", + [ + ReportingSourceFinalPolicy("provider", "delivery.is_final"), + ReportingContractualCutoffPolicy("cutoff", "P2D"), + ], +) +@pytest.mark.parametrize("operation", ["reserve", "acquire", "direct", "next_period"]) +async def test_frozen_explicit_close_requires_lock_before_snapshot_work( + store_factory: Any, + policy: Any, + operation: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + producer, store, config, obligation, calls, clock = await _harness( + store_factory, + capabilities=_capabilities( + restatement_window="P3D", restatement_cadence="PT1H", official_close_lag="P3D" + ), + finality_policy=policy, + ) + clock[0] = END + timedelta(hours=1) + await producer.run_configuration(config) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + first = await store.get_provisional_observation(**owner) + assert first.acquisition.policy.official_close_lag == timedelta(days=3) + revisions = await store.list_revisions(**owner) + producer._source.capabilities = _capabilities( + restatement_window="P3D", restatement_cadence="PT1H" + ) + clock[0] += timedelta(hours=1) + restarted = _restart_producer(producer, store, clock) + monkeypatch.setattr(store, "source_publication", None) + request = restarted._build_slice( + config, obligation, SNAPSHOT_OFFERING_ID, finality="snapshot", now=clock[0], observation=1 + ) + with pytest.raises(LedgerConflictError, match="publication lock"): + if operation == "reserve": + await restarted._reserve_observation( + config, + obligation, + request, + ordinal=1, + revisions=revisions, + previous_boundary=first.provisional_until, + derived_close=False, + ) + elif operation == "acquire": + await restarted.acquire_obligation( + config, obligation, restate=True, target_finality="snapshot" + ) + elif operation == "direct": + _, manifest, restarted._object_reader = redacted_completed_result( + request, page_bodies=[PROVIDER_ROW] + ) + await restarted.commit_revision_from_manifest( + obligation, manifest, rows=ROWS, finality="snapshot" + ) + else: + clock[0] += timedelta(days=1) + await restarted.close_elapsed_periods(config) + assert len(calls) == 1 + assert await store.get_provisional_acquisition(**owner, ordinal=1) is None + assert await store.list_revisions(**owner) == revisions + + def test_source_final_evidence_ref_accepts_512_characters() -> None: evidence = ReportingFinalityEvidenceV1( basis="provider_declared", observed_at=END, evidence_ref="s" * 512 From f2b48f33e0bfda883ba315a6e6cad29be0808119 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 19:15:00 +0530 Subject: [PATCH 23/39] fix(reporting): validate historical snapshots and recover frozen close routes --- .../src/adcp_reporting_storage/ledger/pg.py | 4 +- src/adcp/reporting/ledger/producer.py | 98 +++++--- src/adcp/reporting/ledger/store.py | 6 +- tests/test_reporting_official_acquisition.py | 71 ++++++ tests/test_reporting_official_finality.py | 237 ++++++++++++++++++ 5 files changed, 380 insertions(+), 36 deletions(-) diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py index 9407b247c..17be1ebc1 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py @@ -1996,10 +1996,12 @@ async def commit_provisional_observation( validate_snapshot_origin(retained_revision, observation.revision_publication_id) # Reuse the immutable publication replay checks while holding # the observation's account lock and transaction. - if revision is None or (reuse_candidate and existing[0] == observation.revision_id): + reused = reuse_candidate and existing[0] == observation.revision_id + if retained_revision.finality == "snapshot" or revision is None or reused: validate_retained_provisional_observation( retained_revision, retained_observation, rows ) + if revision is None or reused: return retained_revision return await self.commit_revision(revision, rows) reserved = await ( diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index f0e66d7aa..7b1cf1722 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -496,6 +496,7 @@ def __init__( self._recovered_stabilization: dict[ ReportingConfigurationGenerationKey, ProvisionalPolicy ] = {} + self._new_snapshot_obligations: set[tuple[str, str]] = set() @property def store(self) -> ReportingLedgerStore: @@ -798,6 +799,16 @@ async def _close_elapsed_periods( configuration, obligation, previous_end=after ) ) + if ( + selected is not None + and self._optional_publication_store() is None + and self._stabilization_schedule_store() is None + ): + # Until the first scheduled observation, only this producer knows + # it admitted this particular period on an ordinary snapshot route. + self._new_snapshot_obligations.add( + (stored.account_id, stored.reporting_obligation_id) + ) after = boundary.end committed.append(stored) turn.obligations_committed.append(stored.reporting_obligation_id) @@ -1394,43 +1405,55 @@ async def _has_retained_official_close(self, key: ReportingConfigurationGenerati for obligation in page.obligations: if obligation.generation_key != key: continue - checkpoints = self._optional_restatement_store() - if checkpoints is None or not _implements_store_method( - self._store, PendingProvisionalAcquisitionStore.get_provisional_acquisition - ): - # Without the retained cursor and its reservations a prior - # scheduled close cannot be distinguished from an ordinary poll. + if await self._obligation_has_retained_close(obligation): return True - checkpoint = await checkpoints.get_restatement_checkpoint( - account_id=key.account_id, - reporting_obligation_id=obligation.reporting_obligation_id, - ) - if checkpoint is not None and checkpoint.stabilization_policy is not None: - return True - # The first reservation follows any untracked manual revisions. - if checkpoint is None: - next_ordinal = len( - await self._store.list_revisions( - account_id=key.account_id, - reporting_obligation_id=obligation.reporting_obligation_id, - ) - ) - else: - next_ordinal = checkpoint.next_observation - for ordinal in range(next_ordinal + 1): - acquisition = await self._pending_acquisition(obligation, ordinal=ordinal) - if ( - acquisition is not None - and acquisition.stabilization_schedule( - obligation.required_finality, pending=ordinal == next_ordinal - ) - is not None - ): - return True if not page.has_more: return False offset += page_size + async def _obligation_has_retained_close(self, obligation: ReportingObligationRecord) -> bool: + identity = (obligation.account_id, obligation.reporting_obligation_id) + owner = dict( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + observations = self._optional_observation_store() + latest = await observations.get_provisional_observation(**owner) if observations else None + scheduled = await self._latest_scheduled_snapshot(obligation, latest) + if scheduled is not None: + self._new_snapshot_obligations.discard(identity) + schedule = scheduled.acquisition.stabilization_schedule( + obligation.required_finality, pending=False + ) + if schedule is not None: + return True + checkpoints = self._optional_restatement_store() + checkpoint = await checkpoints.get_restatement_checkpoint(**owner) if checkpoints else None + if checkpoint is not None and checkpoint.stabilization_policy is not None: + return True + lookup = _implements_store_method( + self._store, PendingProvisionalAcquisitionStore.get_provisional_acquisition + ) + if lookup: + # An untracked manual revision can precede the first scheduled reservation. + ordinal = ( + checkpoint.next_observation + if checkpoint + else len(await self._store.list_revisions(**owner)) + ) + pending = await self._pending_acquisition(obligation, ordinal=ordinal) + if ( + pending is not None + and pending.stabilization_schedule(obligation.required_finality, pending=True) + is not None + ): + return True + return ( + scheduled is None + and identity not in self._new_snapshot_obligations + and (checkpoints is None or not lookup or checkpoint is not None) + ) + async def _acquire_stabilized( self, configuration: ReportingConfiguration, @@ -2694,6 +2717,15 @@ async def _reserve_observation( "frozen source acquisition does not bind the obligation's consumer; " "operator reconciliation and a new configuration generation are required", ) + if acquisition != candidate: + # A pending-only lookup can be absent. Reservation then returns the + # winner's frozen route, which may differ from the proposed snapshot. + await self._check_official_publication_lock( + obligation, + official=acquisition.finality == "official", + frozen=acquisition.policy, + check_generation=False, + ) return acquisition async def _pending_acquisition( @@ -3309,6 +3341,8 @@ async def _retained_snapshot_replay( raise LedgerConflictError("REVISION_IMMUTABLE", "retained source totals differ") if is_latest: validate_retained_provisional_observation(revision, latest, rows) + else: + validate_snapshot_rows(revision, rows) if origin_store is not None: await self._verify_tracked_snapshot_origin( obligation, acquisition, origin_store, revision diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index e167a2643..b13b21f81 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -1583,10 +1583,10 @@ async def commit_provisional_observation( validate_snapshot_origin(retained_revision, observation.revision_publication_id) # A matching observation identity does not make conflicting # publication content an idempotent replay. - if revision is None or ( - reuse_candidate and existing.revision_id == observation.revision_id - ): + reused = reuse_candidate and existing.revision_id == observation.revision_id + if retained_revision.finality == "snapshot" or revision is None or reused: validate_retained_provisional_observation(retained_revision, existing, rows) + if revision is None or reused: return retained_revision return await self.commit_revision(revision, rows) retained = self._provisional_acquisitions.get(key) diff --git a/tests/test_reporting_official_acquisition.py b/tests/test_reporting_official_acquisition.py index 28174f186..94b18fe69 100644 --- a/tests/test_reporting_official_acquisition.py +++ b/tests/test_reporting_official_acquisition.py @@ -310,6 +310,77 @@ async def test_historical_unchanged_replay_validates_stored_observation_origin( assert await store.get_provisional_observation(**owner) == latest +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +@pytest.mark.parametrize("ordinal", [0, 1]) +@pytest.mark.parametrize("change", ["rows", "unreadable"]) +@pytest.mark.parametrize("entry", ["producer", "atomic", "archived_core"]) +async def test_historical_snapshot_replay_checks_rows_and_readability_at_each_ordinal( + cutoff_backend: Any, ordinal: int, change: str, entry: str, monkeypatch: pytest.MonkeyPatch +) -> None: + factory, _ = cutoff_backend + producer, store, config, obligation, _, clock = await _harness( + factory, + finality_policy=( + None if entry == "archived_core" else ReportingContractualCutoffPolicy("unused", "P2D") + ), + ) + if entry == "archived_core": + # Older Core revisions hash the control-total pairs without typed evidence. + assert obligation.definition is None + monkeypatch.setattr("adcp.reporting.ledger.producer._typed_control_totals", lambda *_: None) + revision = await producer.acquire_obligation(config, obligation, track_settling=True) + for _ in range(2): + clock[0] += timedelta(hours=1) + assert revision == await producer.acquire_obligation( + config, obligation, restate=True, track_settling=True + ) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + history = await store.list_provisional_observations(**owner, limit=3) + assert [item.acquisition.ordinal for item in history] == [0, 1, 2] + item = history[ordinal] + manifest = SourceBatchManifestV1.model_validate_json(item.manifest_json) + page = await store.read_revision_rows( + account_id=config.account_id, reporting_revision_id=revision.reporting_revision_id + ) + rows = list(page.rows) + if entry == "archived_core": + assert revision.managed_control_totals is None + producer._store = SimpleNamespace( + get_obligation=store.get_obligation, + list_revisions=store.list_revisions, + get_revision=store.get_revision, + reserve_provisional_acquisition=store.reserve_provisional_acquisition, + get_provisional_observation=store.get_provisional_observation, + commit_provisional_observation=store.commit_provisional_observation, + ) + producer._object_reader = None + + async def replay(rows: list[dict[str, Any]]) -> ReportingRevisionRecord: + if entry == "atomic": + return await store.commit_provisional_observation(item, revision, rows) + return await producer.commit_revision_from_manifest( + obligation, manifest, rows=rows, finality="snapshot", acquisition=item.acquisition + ) + + assert await replay(rows) == revision + if change == "unreadable": + await store.set_revision_readable( + account_id=config.account_id, + reporting_revision_id=revision.reporting_revision_id, + readable=False, + ) + else: + rows = [dict(rows[0], campaign_id="changed")] + with pytest.raises(LedgerConflictError) as error: + await replay(rows) + assert error.value.code == ( + "HISTORY_UNAVAILABLE" if change == "unreadable" else "REVISION_IMMUTABLE" + ) + assert await store.list_provisional_observations(**owner, limit=3) == history + + async def test_stale_cutoff_seal_retries_with_fresh_source_work( cutoff_backend: Any, monkeypatch: Any ) -> None: diff --git a/tests/test_reporting_official_finality.py b/tests/test_reporting_official_finality.py index 3fe98d8cb..c2319ad33 100644 --- a/tests/test_reporting_official_finality.py +++ b/tests/test_reporting_official_finality.py @@ -1502,6 +1502,72 @@ def service() -> ReliableReportingService: assert revision.finality == "snapshot" +@pytest.mark.parametrize("interleave", [True, False]) +async def test_standalone_new_snapshot_with_unused_policy_needs_no_optional_lookups( + store_factory: Any, monkeypatch: pytest.MonkeyPatch, interleave: bool +) -> None: + store = store_factory() + for name in ("source_publication", "get_stabilization_schedule", "get_provisional_acquisition"): + monkeypatch.setattr(store, name, None) + producer, _, config, obligation, _, clock = await _harness( + lambda **kwargs: store, finality_policy=ReportingContractualCutoffPolicy("unused", "P2D") + ) + if interleave: + other = replace(config, delivery_config_id="other") + await store.put_configuration(other) + await producer.close_elapsed_periods(other) + clock[0] = END + timedelta(hours=1) + turn = await producer.run_configuration(config) + (identifier,) = turn.revisions_committed + revision = await store.get_revision( + account_id=config.account_id, reporting_revision_id=identifier + ) + assert ( + revision.reporting_obligation_id == obligation.reporting_obligation_id + and revision.finality == "snapshot" + ) + + +@pytest.mark.parametrize("period", ["admitted", "external"]) +async def test_snapshot_admission_checks_another_publishers_pending_close( + store_factory: Any, monkeypatch: pytest.MonkeyPatch, period: str +) -> None: + store = store_factory() + for name in ("source_publication", "get_stabilization_schedule", "get_provisional_acquisition"): + monkeypatch.setattr(store, name, None) + producer, _, config, obligation, calls, clock = await _harness( + lambda **kwargs: store, finality_policy=ReportingContractualCutoffPolicy("cutoff", "P2D") + ) + if period == "external": + clock[0] += timedelta(days=1) + boundary = producer._elapsed_periods(config, now=clock[0])[-1] + obligation = await store.commit_obligation( + replace( + obligation, + reporting_obligation_id=producer._obligation_id(config, boundary), + period=boundary, + scope_resolved_at=boundary.end, + automated_recovery_deadline_at=boundary.expected_at, + ) + ) + request = producer._build_slice( + config, obligation, SNAPSHOT_OFFERING_ID, finality="snapshot", now=clock[0] + ) + await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + request.model_dump_json(), + 0, + ProvisionalPolicy(timedelta(days=3), timedelta(hours=1), timedelta(days=2)), + ) + ) + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.acquire_obligation(config, obligation, target_finality="snapshot") + assert calls == [] + assert not await store.list_revisions( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + + @pytest.mark.parametrize("configure_before_init", [True, False]) @pytest.mark.parametrize( "policy,missing", @@ -1648,6 +1714,177 @@ async def test_frozen_explicit_close_requires_lock_before_snapshot_work( assert await store.list_revisions(**owner) == revisions +@pytest.mark.parametrize("configure_before_init", [True, False]) +@pytest.mark.parametrize( + "policy", + [ + ReportingSourceFinalPolicy("source", "closed"), + ReportingContractualCutoffPolicy("cutoff", "P1D"), + ReportingStabilizedPolicy("stable", "PT0S", "PT6H"), + ], +) +@pytest.mark.parametrize( + "state", ["new", "ordinary", "completed", "manual_after_close", "ambiguous"] +) +async def test_snapshot_admission_distinguishes_new_from_retained_close( + store_factory: Any, + configure_before_init: bool, + policy: Any, + state: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from tests.test_reliable_reporting_service import ( + DEFINITION, + NOW, + _configuration, + _finality_service, + _rows, + ) + + config = replace( + _configuration(), + activated_at=NOW.replace(hour=1, minute=0), + deactivated_at=NOW.replace(hour=2, minute=0), + definition=replace(DEFINITION, finality_policy=policy), + ) + clock = DeterministicReportingClock(NOW) + store = store_factory(clock=clock) + result = InlineFetchResult(rows=_rows(10), data_through=config.deactivated_at) + if state != "new": + original = _finality_service( + config, + snapshot=True, + official=isinstance(policy, ReportingSourceFinalPolicy), + close=state != "ordinary", + store=store, + clock=clock, + cadence="PT1H", + results=(result, result), + ) + await original.configure(config) + await original.initialize() + producer = original._bindings[config.generation_key].producer + (obligation,) = await producer.close_elapsed_periods(config) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + if state != "ambiguous": + await producer.run_configuration(config) + retained = await store.get_provisional_observation(**owner) + assert retained.acquisition.policy.official_close_lag == ( + None if state == "ordinary" else timedelta(days=2) + ) + if state == "manual_after_close": + clock.advance(timedelta(minutes=5)) + request = producer._build_slice( + config, + obligation, + SNAPSHOT_OFFERING_ID, + finality="snapshot", + now=clock(), + observation=1, + trigger="manual_replay", + ) + await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + request.model_dump_json(), + 1, + retained.acquisition.policy, + retained.revision_id, + ) + ) + await producer.acquire_obligation( + config, + obligation, + restate=True, + target_finality="snapshot", + manual_replay=True, + track_settling=True, + ) + latest = await store.get_provisional_observation(**owner) + assert latest.acquisition.ordinal == 1 and not latest.is_scheduled + await original.close() + if state == "ordinary": + clock.advance(timedelta(hours=1)) + + monkeypatch.setattr(store, "source_publication", None) + monkeypatch.setattr(store, "get_stabilization_schedule", None) + monkeypatch.setattr(store, "verify_snapshot_origin", None) + if state in {"completed", "manual_after_close"}: + + async def only_pending(**kwargs: Any) -> None: + # A completed reservation is no longer returned by the pending-only API. + return None + + monkeypatch.setattr(store, "get_provisional_acquisition", only_pending) + checkpoint = store.get_restatement_checkpoint + + async def historical_checkpoint(**kwargs: Any) -> Any: + value = await checkpoint(**kwargs) + return ( + replace(value, stabilization_policy=None, next_stabilization_due_at=None) + if value + else value + ) + + monkeypatch.setattr(store, "get_restatement_checkpoint", historical_checkpoint) + if state == "completed": + # A custom atomic publisher can expose its latest read without a history interface. + read = store.list_provisional_observations + + async def read_latest(**kwargs: Any) -> ProvisionalObservation | None: + history = await read(**kwargs, limit=1) + return history[-1] if history else None + + monkeypatch.setattr(store, "get_provisional_observation", read_latest) + monkeypatch.setattr(store, "list_provisional_observations", None) + else: + monkeypatch.setattr(store, "get_provisional_acquisition", None) + service = _finality_service( + config, + snapshot=True, + official=isinstance(policy, ReportingSourceFinalPolicy), + store=store, + clock=clock, + cadence="PT1H", + results=(result,), + ) + + async def start() -> None: + if configure_before_init: + await service.configure(config) + await service.initialize() + if not configure_before_init: + await service.configure(config) + + if state == "new": + await start() + turn = await service.run_worker() + (identifier,) = turn.configurations[config.generation_key].revisions_committed + revision = await store.get_revision( + account_id=config.account_id, reporting_revision_id=identifier + ) + assert revision.finality == "snapshot" + elif state == "ordinary": + await start() + turn = await service.run_worker() + assert not turn.configuration_errors + history = await store.list_provisional_observations(**owner, limit=3) + assert [item.revision_id for item in history] == [ + retained.revision_id, + retained.revision_id, + ] + else: + expected = ( + "get_stabilization_schedule" + if isinstance(policy, ReportingStabilizedPolicy) + else "publication lock" + ) + with pytest.raises(ReliableReportingConfigurationError, match=expected): + await start() + assert config.generation_key not in service._bindings + + def test_source_final_evidence_ref_accepts_512_characters() -> None: evidence = ReportingFinalityEvidenceV1( basis="provider_declared", observed_at=END, evidence_ref="s" * 512 From c6e07dfc6b9c4ce14bf2fe14a618b3c46146f568 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 19:34:43 +0530 Subject: [PATCH 24/39] fix(reporting): verify pending and completed close routes before dispatch --- docs/reporting-provisional-observations.md | 7 +- src/adcp/reporting/ledger/producer.py | 127 ++++++++---- tests/test_reporting_official_finality.py | 217 ++++++++++++++++++++- 3 files changed, 299 insertions(+), 52 deletions(-) diff --git a/docs/reporting-provisional-observations.md b/docs/reporting-provisional-observations.md index a27a4d212..0c8067614 100644 --- a/docs/reporting-provisional-observations.md +++ b/docs/reporting-provisional-observations.md @@ -76,7 +76,12 @@ shared across all publishers for that account. Stabilization additionally requires durable reservations, bounded scheduled history, checkpoints, exact snapshot origins and generation schedule recovery. An ordinary snapshot route can keep using its snapshot publisher when the definition also lists an unused -stabilization policy. +stabilization policy. Each scheduled read checks the winner of its own atomic +reservation. With a selected policy and no account publication lock, new manual +acquisitions and later periods also need the pending-acquisition lookup to rule out +a competing close. Recovery checks retained scheduled observations for +`source_final` and `contractual_cutoff` when the stabilization lookup returns +no schedule. ## Persistence and compatibility diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 7b1cf1722..1777c0a27 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -496,7 +496,6 @@ def __init__( self._recovered_stabilization: dict[ ReportingConfigurationGenerationKey, ProvisionalPolicy ] = {} - self._new_snapshot_obligations: set[tuple[str, str]] = set() @property def store(self) -> ReportingLedgerStore: @@ -799,19 +798,19 @@ async def _close_elapsed_periods( configuration, obligation, previous_end=after ) ) + after = boundary.end + committed.append(stored) + turn.obligations_committed.append(stored.reporting_obligation_id) if ( selected is not None and self._optional_publication_store() is None - and self._stabilization_schedule_store() is None - ): - # Until the first scheduled observation, only this producer knows - # it admitted this particular period on an ordinary snapshot route. - self._new_snapshot_obligations.add( - (stored.account_id, stored.reporting_obligation_id) + and not _implements_store_method( + self._store, PendingProvisionalAcquisitionStore.get_provisional_acquisition ) - after = boundary.end - committed.append(stored) - turn.obligations_committed.append(stored.reporting_obligation_id) + ): + # Publish this period through its atomic reservation before + # considering a later period with an unobservable pending slot. + break return committed def _elapsed_periods( @@ -1381,11 +1380,20 @@ def _stabilization_schedule_store(self) -> StabilizationScheduleStore | None: return self._store return None - async def _has_retained_official_close(self, key: ReportingConfigurationGenerationKey) -> bool: - """Find a frozen scheduled close, including an unfinished first reservation.""" + async def _has_retained_official_close( + self, + key: ReportingConfigurationGenerationKey, + *, + publishing: bool = False, + scheduled_reservation: tuple[str, int] | None = None, + ) -> bool: + """Check completed close evidence and any reservations that could affect this operation.""" schedule_store = self._stabilization_schedule_store() if schedule_store is not None: - return await schedule_store.get_stabilization_schedule(key) is not None + schedule = await schedule_store.get_stabilization_schedule(key) + if schedule is not None: + return True + # A negative stabilization lookup still requires checking the other close policies. snapshot = await self._store.open_snapshot( caller=ReportingDeliveryPrincipal(key.account_id, key.consumer_id), filters_fingerprint=f"official-close:{key.delivery_config_id}:{key.delivery_config_version}", @@ -1405,23 +1413,33 @@ async def _has_retained_official_close(self, key: ReportingConfigurationGenerati for obligation in page.obligations: if obligation.generation_key != key: continue - if await self._obligation_has_retained_close(obligation): + if await self._obligation_has_retained_close( + obligation, + publishing=publishing, + scheduled_reservation=scheduled_reservation, + ): return True if not page.has_more: return False offset += page_size - async def _obligation_has_retained_close(self, obligation: ReportingObligationRecord) -> bool: - identity = (obligation.account_id, obligation.reporting_obligation_id) + async def _obligation_has_retained_close( + self, + obligation: ReportingObligationRecord, + *, + publishing: bool, + scheduled_reservation: tuple[str, int] | None, + ) -> bool: owner = dict( account_id=obligation.account_id, reporting_obligation_id=obligation.reporting_obligation_id, ) observations = self._optional_observation_store() latest = await observations.get_provisional_observation(**owner) if observations else None + if latest is not None and latest.acquisition.finality == "official": + return True scheduled = await self._latest_scheduled_snapshot(obligation, latest) if scheduled is not None: - self._new_snapshot_obligations.discard(identity) schedule = scheduled.acquisition.stabilization_schedule( obligation.required_finality, pending=False ) @@ -1434,25 +1452,30 @@ async def _obligation_has_retained_close(self, obligation: ReportingObligationRe lookup = _implements_store_method( self._store, PendingProvisionalAcquisitionStore.get_provisional_acquisition ) + if not lookup and ( + scheduled_reservation is None + or scheduled_reservation[0] != obligation.reporting_obligation_id + ): + # A manual read or another period cannot inspect this reservation by claiming it. + return publishing or scheduled is None + # An untracked manual revision can precede the first scheduled reservation. + ordinal = ( + checkpoint.next_observation + if checkpoint + else len(await self._store.list_revisions(**owner)) + ) if lookup: - # An untracked manual revision can precede the first scheduled reservation. - ordinal = ( - checkpoint.next_observation - if checkpoint - else len(await self._store.list_revisions(**owner)) - ) pending = await self._pending_acquisition(obligation, ordinal=ordinal) - if ( - pending is not None - and pending.stabilization_schedule(obligation.required_finality, pending=True) + if pending is not None and ( + pending.finality == "official" + or pending.stabilization_schedule(obligation.required_finality, pending=True) is not None ): return True - return ( - scheduled is None - and identity not in self._new_snapshot_obligations - and (checkpoints is None or not lookup or checkpoint is not None) - ) + elif scheduled_reservation != (obligation.reporting_obligation_id, ordinal): + # A completed ordinal cannot prove which publisher owns the next one. + return True + return scheduled is None and (checkpoints is None or checkpoint is not None) async def _acquire_stabilized( self, @@ -1879,8 +1902,15 @@ async def _acquire_obligation( raise LedgerConflictError( "FINALITY_POLICY_UNPROVEN", "official finality requires a selected policy" ) + manual_observation = manual_replay or (restate and not track_settling) + reserve_snapshot = ( + finality == "snapshot" and not manual_observation and checkpoint_store is not None + ) await self._check_official_publication_lock( - obligation, official=finality == "official", frozen=pending.policy if pending else None + obligation, + official=finality == "official", + frozen=pending.policy if pending else None, + scheduled_ordinal=observation if reserve_snapshot else None, ) # Everything below needs the frozen code: the slice request carries it, # the manifest is checked against it, and the revision is written under @@ -1900,7 +1930,6 @@ async def _acquire_obligation( constituents = self._check_source_authorization(configuration, offering_id, constituents) derived_close = finality == "official" and source_finality == "snapshot" - manual_observation = manual_replay or (restate and not track_settling) manual_snapshot = manual_observation and finality == "snapshot" and pending is None if derived_close and isinstance(selected_policy, ReportingContractualCutoffPolicy): if _utc(now) < contractual_cutoff_at(obligation, selected_policy): @@ -2601,10 +2630,14 @@ async def _check_official_publication_lock( official: bool = False, frozen: ProvisionalPolicy | None = None, check_generation: bool = True, + scheduled_ordinal: int | None = None, ) -> None: policy = route.definition.finality_policy if route.definition else None if policy is None or self._optional_publication_store() is not None: return + reservation = None + if scheduled_ordinal is not None and isinstance(route, ReportingObligationRecord): + reservation = (route.reporting_obligation_id, scheduled_ordinal) if ( official or self._official_close_requested(route) @@ -2612,7 +2645,12 @@ async def _check_official_publication_lock( frozen is not None and (frozen.official_close_lag is not None or frozen.until_stabilized) ) - or (check_generation and await self._has_retained_official_close(route.generation_key)) + or ( + check_generation + and await self._has_retained_official_close( + route.generation_key, publishing=True, scheduled_reservation=reservation + ) + ) ): self._official_publication_lock( stabilized=isinstance(policy, ReportingStabilizedPolicy) @@ -2684,6 +2722,7 @@ async def _reserve_observation( obligation, official=derived_close or request.publication_class == "AUTHORITATIVE", frozen=frozen_policy, + scheduled_ordinal=ordinal if request.trigger == "scheduled_poll" else None, ) if self._requires_source_authority(obligation): self._source_authority(obligation, request) @@ -2717,15 +2756,13 @@ async def _reserve_observation( "frozen source acquisition does not bind the obligation's consumer; " "operator reconciliation and a new configuration generation are required", ) - if acquisition != candidate: - # A pending-only lookup can be absent. Reservation then returns the - # winner's frozen route, which may differ from the proposed snapshot. - await self._check_official_publication_lock( - obligation, - official=acquisition.finality == "official", - frozen=acquisition.policy, - check_generation=False, - ) + # The returned ordinal may already have completed while the reservation was in flight. + await self._check_official_publication_lock( + obligation, + official=acquisition.finality == "official", + frozen=acquisition.policy, + scheduled_ordinal=acquisition.ordinal if acquisition.is_scheduled_snapshot else None, + ) return acquisition async def _pending_acquisition( @@ -2963,11 +3000,15 @@ async def _commit_revision_from_manifest( selected_policy = obligation.definition.finality_policy if obligation.definition else None # A matching historical observation already retained its own route. # Check generation-wide evidence before admitting any fresh snapshot. + scheduled_ordinal = None + if verified_source_rows and acquisition is not None and acquisition.is_scheduled_snapshot: + scheduled_ordinal = acquisition.ordinal await self._check_official_publication_lock( obligation, official=finality == "official", frozen=acquisition.policy if acquisition else None, check_generation=acquisition is None or verified_source_rows, + scheduled_ordinal=scheduled_ordinal, ) origin_store: SnapshotOriginStore | None = None if finality == "snapshot" and acquisition is not None: diff --git a/tests/test_reporting_official_finality.py b/tests/test_reporting_official_finality.py index c2319ad33..b061bbad8 100644 --- a/tests/test_reporting_official_finality.py +++ b/tests/test_reporting_official_finality.py @@ -4,7 +4,7 @@ import asyncio import json -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager, suppress from dataclasses import replace from datetime import datetime, timedelta @@ -1503,12 +1503,15 @@ def service() -> ReliableReportingService: @pytest.mark.parametrize("interleave", [True, False]) +@pytest.mark.parametrize("schedule_lookup", [True, False]) async def test_standalone_new_snapshot_with_unused_policy_needs_no_optional_lookups( - store_factory: Any, monkeypatch: pytest.MonkeyPatch, interleave: bool + store_factory: Any, monkeypatch: pytest.MonkeyPatch, interleave: bool, schedule_lookup: bool ) -> None: store = store_factory() - for name in ("source_publication", "get_stabilization_schedule", "get_provisional_acquisition"): + for name in ("source_publication", "get_provisional_acquisition"): monkeypatch.setattr(store, name, None) + if not schedule_lookup: + monkeypatch.setattr(store, "get_stabilization_schedule", None) producer, _, config, obligation, _, clock = await _harness( lambda **kwargs: store, finality_policy=ReportingContractualCutoffPolicy("unused", "P2D") ) @@ -1568,6 +1571,192 @@ async def test_snapshot_admission_checks_another_publishers_pending_close( ) +@pytest.mark.parametrize("completed_ordinary", [False, True]) +@pytest.mark.parametrize("operation", ["manual", "next_period"]) +async def test_first_snapshot_admission_cannot_hide_a_later_publishers_close( + store_factory: Any, + monkeypatch: pytest.MonkeyPatch, + completed_ordinary: bool, + operation: str, +) -> None: + store = store_factory() + publication = store.source_publication + lookup = store.get_provisional_acquisition + for name in ("source_publication", "get_stabilization_schedule", "get_provisional_acquisition"): + monkeypatch.setattr(store, name, None) + producer, _, config, obligation, calls, clock = await _harness( + lambda **kwargs: store, finality_policy=ReportingContractualCutoffPolicy("cutoff", "P2D") + ) + clock[0] = END + timedelta(hours=1) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + if completed_ordinary: + await producer.run_configuration(config) + ordinary = await store.get_provisional_observation(**owner) + assert ordinary.acquisition.policy.official_close_lag is None + revisions = await store.list_revisions(**owner) + previous_calls = list(calls) + ordinal = int(completed_ordinary) + request = producer._build_slice( + config, + obligation, + SNAPSHOT_OFFERING_ID, + finality="snapshot", + now=clock[0], + observation=ordinal, + ) + async with publication(config.account_id): + winning = await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + request.model_dump_json(), + ordinal, + ProvisionalPolicy(timedelta(days=3), timedelta(hours=1), timedelta(days=2)), + revisions[-1].reporting_revision_id if revisions else None, + ) + ) + assert winning.policy.official_close_lag == timedelta(days=2) + clock[0] += timedelta(days=1) + with pytest.raises(LedgerConflictError, match="publication lock"): + if operation == "manual": + await producer.acquire_obligation( + config, obligation, target_finality="snapshot", restate=True, manual_replay=True + ) + else: + await producer.close_elapsed_periods(config) + assert calls == previous_calls + assert await lookup(**owner, ordinal=ordinal) == winning + assert await store.list_revisions(**owner) == revisions + later = await store.find_obligation( + account_id=config.account_id, + consumer_id=config.consumer_id, + delivery_config_id=config.delivery_config_id, + delivery_config_version=config.delivery_config_version, + period_start=END, + period_end=END + timedelta(days=1), + ) + assert later is None + + +def _stabilized_only_lookup( + store: Any, +) -> Callable[[ReportingConfigurationGenerationKey], Awaitable[ProvisionalPolicy | None]]: + lookup = store.get_stabilization_schedule + + async def read(generation: ReportingConfigurationGenerationKey) -> ProvisionalPolicy | None: + schedule = await lookup(generation) + return schedule if schedule is not None and schedule.until_stabilized else None + + return read + + +async def test_completed_reservation_cannot_authorize_dispatch_past_a_competing_close( + store_factory: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + store = store_factory() + publication = store.source_publication + lookup = store.get_provisional_acquisition + reserve = store.reserve_provisional_acquisition + for name in ("source_publication", "get_stabilization_schedule", "get_provisional_acquisition"): + monkeypatch.setattr(store, name, None) + producer, _, config, obligation, calls, clock = await _harness( + lambda **kwargs: store, finality_policy=ReportingContractualCutoffPolicy("cutoff", "P2D") + ) + clock[0] = END + timedelta(hours=1) + + async def finish_and_reserve_close(candidate: ProvisionalAcquisition) -> ProvisionalAcquisition: + winning = await reserve(candidate) + # The locked publisher finishes this ordinal and reserves the next + # close before the unlocked worker gets its reservation back. + with monkeypatch.context() as locked: + locked.setattr(store, "source_publication", publication) + locked.setattr(store, "get_provisional_acquisition", lookup) + locked.setattr(store, "reserve_provisional_acquisition", reserve) + peer = _restart_producer(producer, store, clock) + revision = await peer.acquire_obligation(config, obligation, target_finality="snapshot") + request = peer._build_slice( + config, obligation, SNAPSHOT_OFFERING_ID, now=clock[0], observation=1 + ) + async with publication(config.account_id): + await reserve( + ProvisionalAcquisition( + request.model_dump_json(), + 1, + replace(candidate.policy, official_close_lag=timedelta(days=2)), + revision.reporting_revision_id, + ) + ) + return winning + + monkeypatch.setattr(store, "reserve_provisional_acquisition", finish_and_reserve_close) + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.acquire_obligation(config, obligation, target_finality="snapshot") + assert calls == [clock[0]] + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + assert (await store.get_restatement_checkpoint(**owner)).next_observation == 1 + assert (await lookup(**owner, ordinal=1)).policy.official_close_lag == timedelta(days=2) + + +async def test_new_ordinary_worker_can_publish_its_first_overdue_period( + store_factory: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, _, calls, clock = await _harness( + store_factory, finality_policy=ReportingContractualCutoffPolicy("unused", "P2D") + ) + for name in ("source_publication", "get_stabilization_schedule", "get_provisional_acquisition"): + monkeypatch.setattr(store, name, None) + fresh = replace(config, delivery_config_id="new-overdue") + await store.put_configuration(fresh) + clock[0] = END + timedelta(days=1, hours=1) + turn = await producer.run_configuration(fresh) + (identifier,) = turn.revisions_committed + revision = await store.get_revision( + account_id=fresh.account_id, reporting_revision_id=identifier + ) + obligation = await store.get_obligation( + account_id=fresh.account_id, reporting_obligation_id=revision.reporting_obligation_id + ) + assert len(calls) == 1 + assert obligation.period.end == END and revision.finality == "snapshot" + + +@pytest.mark.parametrize("completed", [False, True]) +async def test_new_period_checks_an_authoritative_close_with_an_ordinary_predecessor( + store_factory: Any, monkeypatch: pytest.MonkeyPatch, completed: bool +) -> None: + producer, store, config, obligation, calls, clock = await _harness( + store_factory, finality_policy=ReportingSourceFinalPolicy("source", "delivery.is_final") + ) + clock[0] = END + timedelta(hours=1) + await producer.run_configuration(config) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + ordinary = await store.get_provisional_observation(**owner) + assert ordinary.acquisition.policy.official_close_lag is None + request = producer._build_slice( + config, obligation, OFFICIAL_OFFERING_ID, finality="official", now=clock[0], observation=1 + ) + async with store.source_publication(config.account_id): + reserved = await store.reserve_provisional_acquisition( + ProvisionalAcquisition(request.model_dump_json(), 1, ordinary.acquisition.policy) + ) + assert reserved.finality == "official" + if completed: + closed = await producer.acquire_obligation(config, obligation, target_finality="official") + assert closed.finality == "official" + before = list(calls) + monkeypatch.setattr(store, "source_publication", None) + monkeypatch.setattr(store, "get_stabilization_schedule", _stabilized_only_lookup(store)) + assert await store.get_stabilization_schedule(config.generation_key) is None + clock[0] += timedelta(days=1) + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.close_elapsed_periods(config) + assert calls == before + + @pytest.mark.parametrize("configure_before_init", [True, False]) @pytest.mark.parametrize( "policy,missing", @@ -1654,10 +1843,12 @@ async def test_recovery_checks_pending_close_after_untracked_manual_revision( ], ) @pytest.mark.parametrize("operation", ["reserve", "acquire", "direct", "next_period"]) +@pytest.mark.parametrize("stabilized_only_lookup", [False, True]) async def test_frozen_explicit_close_requires_lock_before_snapshot_work( store_factory: Any, policy: Any, operation: str, + stabilized_only_lookup: bool, monkeypatch: pytest.MonkeyPatch, ) -> None: producer, store, config, obligation, calls, clock = await _harness( @@ -1681,6 +1872,9 @@ async def test_frozen_explicit_close_requires_lock_before_snapshot_work( clock[0] += timedelta(hours=1) restarted = _restart_producer(producer, store, clock) monkeypatch.setattr(store, "source_publication", None) + if stabilized_only_lookup: + monkeypatch.setattr(store, "get_stabilization_schedule", _stabilized_only_lookup(store)) + assert await store.get_stabilization_schedule(config.generation_key) is None request = restarted._build_slice( config, obligation, SNAPSHOT_OFFERING_ID, finality="snapshot", now=clock[0], observation=1 ) @@ -1724,7 +1918,8 @@ async def test_frozen_explicit_close_requires_lock_before_snapshot_work( ], ) @pytest.mark.parametrize( - "state", ["new", "ordinary", "completed", "manual_after_close", "ambiguous"] + "state", + ["new", "ordinary", "completed", "stabilized_lookup", "manual_after_close", "ambiguous"], ) async def test_snapshot_admission_distinguishes_new_from_retained_close( store_factory: Any, @@ -1808,9 +2003,13 @@ async def test_snapshot_admission_distinguishes_new_from_retained_close( clock.advance(timedelta(hours=1)) monkeypatch.setattr(store, "source_publication", None) - monkeypatch.setattr(store, "get_stabilization_schedule", None) + monkeypatch.setattr( + store, + "get_stabilization_schedule", + _stabilized_only_lookup(store) if state == "stabilized_lookup" else None, + ) monkeypatch.setattr(store, "verify_snapshot_origin", None) - if state in {"completed", "manual_after_close"}: + if state in {"completed", "stabilized_lookup", "manual_after_close"}: async def only_pending(**kwargs: Any) -> None: # A completed reservation is no longer returned by the pending-only API. @@ -1828,7 +2027,9 @@ async def historical_checkpoint(**kwargs: Any) -> Any: ) monkeypatch.setattr(store, "get_restatement_checkpoint", historical_checkpoint) - if state == "completed": + if state == "completed" or ( + state == "stabilized_lookup" and not isinstance(policy, ReportingStabilizedPolicy) + ): # A custom atomic publisher can expose its latest read without a history interface. read = store.list_provisional_observations @@ -1877,7 +2078,7 @@ async def start() -> None: else: expected = ( "get_stabilization_schedule" - if isinstance(policy, ReportingStabilizedPolicy) + if isinstance(policy, ReportingStabilizedPolicy) and state != "stabilized_lookup" else "publication lock" ) with pytest.raises(ReliableReportingConfigurationError, match=expected): From c83f172c2b499c06528f1c827b9f6a94e545e474 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 20:04:48 +0530 Subject: [PATCH 25/39] fix(reporting): restore first Core snapshots and detect retained official closes --- docs/reporting-provisional-observations.md | 20 +- src/adcp/reporting/ledger/producer.py | 117 +++++-- tests/test_reporting_official_acquisition.py | 310 ++++++++++++++++++- tests/test_reporting_official_finality.py | 56 ++++ 4 files changed, 465 insertions(+), 38 deletions(-) diff --git a/docs/reporting-provisional-observations.md b/docs/reporting-provisional-observations.md index 0c8067614..12d8b7ec0 100644 --- a/docs/reporting-provisional-observations.md +++ b/docs/reporting-provisional-observations.md @@ -75,13 +75,19 @@ Custom Core ledgers that publish official revisions must implement shared across all publishers for that account. Stabilization additionally requires durable reservations, bounded scheduled history, checkpoints, exact snapshot origins and generation schedule recovery. An ordinary snapshot route -can keep using its snapshot publisher when the definition also lists an unused -stabilization policy. Each scheduled read checks the winner of its own atomic -reservation. With a selected policy and no account publication lock, new manual -acquisitions and later periods also need the pending-acquisition lookup to rule out -a competing close. Recovery checks retained scheduled observations for -`source_final` and `contractual_cutoff` when the stabilization lookup returns -no schedule. +can keep using its snapshot publisher when the definition lists an unused +finality policy. Each scheduled read checks the winner of its own atomic reservation. + +A Core producer without observation or checkpoint support can call +`close_elapsed_periods` and then `acquire_obligation` for the first snapshot of +the period it just created. It consumes that allowance before calling the source. +After a retryable fetch failure, the same producer can retry when that slice's +recorded retry becomes due. A manual replay cannot claim the scheduled retry. +If another publisher reserves the period, the ledger's atomic revision commit +rejects the snapshot. With a selected policy and no account publication lock, +additional reads and later periods need durable evidence ruling out a competing +close. Recovery checks immutable official revisions as well as retained +observations and reservations when the stabilization lookup returns no schedule. ## Persistence and compatibility diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 1777c0a27..8c2e5faf8 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -496,6 +496,10 @@ def __init__( self._recovered_stabilization: dict[ ReportingConfigurationGenerationKey, ProvisionalPolicy ] = {} + self._first_core_deliveries: dict[ + tuple[ReportingConfigurationGenerationKey, str], + Literal["ready", "attempted"] | RetryScheduleEntry, + ] = {} @property def store(self) -> ReportingLedgerStore: @@ -686,7 +690,9 @@ async def close_elapsed_periods( turn = WorkerTurn() if self._stabilization_requested(configuration): self._stabilization_store() - return await self._close_elapsed_periods(configuration, turn, now=now or self._clock()) + return await self._close_elapsed_periods( + configuration, turn, now=now or self._clock(), admit_first_core_delivery=True + ) def _stabilization_requested(self, configuration: ReportingConfiguration) -> bool: definition = configuration.definition @@ -716,6 +722,7 @@ async def _close_elapsed_periods( turn: WorkerTurn, *, now: datetime, + admit_first_core_delivery: bool = False, ) -> list[ReportingObligationRecord]: from adcp.reporting.ledger.producer_progress import ReportingProducerProgress @@ -801,16 +808,22 @@ async def _close_elapsed_periods( after = boundary.end committed.append(stored) turn.obligations_committed.append(stored.reporting_obligation_id) - if ( - selected is not None - and self._optional_publication_store() is None - and not _implements_store_method( + if selected is not None and self._optional_publication_store() is None: + if ( + admit_first_core_delivery + and stored == obligation + and self._observation_checkpoint_store() is None + ): + # Overlapping period closes share the same first delivery. + self._first_core_deliveries.setdefault( + (stored.generation_key, stored.reporting_obligation_id), "ready" + ) + if not _implements_store_method( self._store, PendingProvisionalAcquisitionStore.get_provisional_acquisition - ) - ): - # Publish this period through its atomic reservation before - # considering a later period with an unobservable pending slot. - break + ): + # Attempt this period before considering a later period with + # an unobservable pending slot. + break return committed def _elapsed_periods( @@ -1386,6 +1399,7 @@ async def _has_retained_official_close( *, publishing: bool = False, scheduled_reservation: tuple[str, int] | None = None, + first_core_obligation: str | None = None, ) -> bool: """Check completed close evidence and any reservations that could affect this operation.""" schedule_store = self._stabilization_schedule_store() @@ -1417,6 +1431,7 @@ async def _has_retained_official_close( obligation, publishing=publishing, scheduled_reservation=scheduled_reservation, + first_core_delivery=first_core_obligation == obligation.reporting_obligation_id, ): return True if not page.has_more: @@ -1429,11 +1444,15 @@ async def _obligation_has_retained_close( *, publishing: bool, scheduled_reservation: tuple[str, int] | None, + first_core_delivery: bool, ) -> bool: owner = dict( account_id=obligation.account_id, reporting_obligation_id=obligation.reporting_obligation_id, ) + revisions = await self._store.list_revisions(**owner) + if any(revision.finality == "official" for revision in revisions): + return True observations = self._optional_observation_store() latest = await observations.get_provisional_observation(**owner) if observations else None if latest is not None and latest.acquisition.finality == "official": @@ -1452,6 +1471,9 @@ async def _obligation_has_retained_close( lookup = _implements_store_method( self._store, PendingProvisionalAcquisitionStore.get_provisional_acquisition ) + if first_core_delivery and not revisions and latest is None and checkpoint is None: + # An ordinary commit arbitrates reservations hidden behind a Core publisher. + return lookup and await self._pending_acquisition(obligation, ordinal=0) is not None if not lookup and ( scheduled_reservation is None or scheduled_reservation[0] != obligation.reporting_obligation_id @@ -1459,11 +1481,7 @@ async def _obligation_has_retained_close( # A manual read or another period cannot inspect this reservation by claiming it. return publishing or scheduled is None # An untracked manual revision can precede the first scheduled reservation. - ordinal = ( - checkpoint.next_observation - if checkpoint - else len(await self._store.list_revisions(**owner)) - ) + ordinal = checkpoint.next_observation if checkpoint else len(revisions) if lookup: pending = await self._pending_acquisition(obligation, ordinal=ordinal) if pending is not None and ( @@ -1786,6 +1804,7 @@ async def _acquire_obligation( finality = target_finality or obligation.required_finality selected_policy = obligation.definition.finality_policy if obligation.definition else None if finality == "official" and isinstance(selected_policy, ReportingStabilizedPolicy): + await self._claim_first_core_delivery(obligation, now=now, retry=False) return await self._acquire_stabilized( configuration, obligation, @@ -1828,6 +1847,11 @@ async def _acquire_obligation( offering_id=retry_offering_id, ) return None + first_core_obligation = await self._claim_first_core_delivery( + obligation, + now=now, + retry=finality == "snapshot" and not (manual_replay or restate or track_settling), + ) revisions = await self._store.list_revisions( account_id=obligation.account_id, reporting_obligation_id=obligation.reporting_obligation_id, @@ -1911,6 +1935,7 @@ async def _acquire_obligation( official=finality == "official", frozen=pending.policy if pending else None, scheduled_ordinal=observation if reserve_snapshot else None, + first_core_obligation=first_core_obligation, ) # Everything below needs the frozen code: the slice request carries it, # the manifest is checked against it, and the revision is written under @@ -2010,7 +2035,13 @@ async def _acquire_obligation( # the execution explicitly until even synchronous work has settled. await cancel_and_settle(execution) turn.slices_failed.append(obligation.reporting_obligation_id) - await self._schedule_retry(obligation, turn, now=now, replayed=manual_replay) + await self._schedule_retry( + obligation, + turn, + now=now, + replayed=manual_replay, + first_core_delivery=first_core_obligation is not None, + ) self._note_escalation( obligation, turn, @@ -2045,6 +2076,7 @@ async def _acquire_obligation( retry_after_seconds=error.retry_after_seconds, blocked=error.retry == "terminal", replayed=manual_replay, + first_core_delivery=first_core_obligation is not None, ) self._note_escalation( obligation, @@ -2135,6 +2167,7 @@ async def _acquire_obligation( turn=turn, acquisition=acquisition, verified_source_rows=True, + first_core_obligation=first_core_obligation, suppress_untracked_unchanged=untracked_restatement, admit_tracked_stabilization=admit_tracked_stabilization, ) @@ -2256,6 +2289,26 @@ def _retry_keys(self, obligation: ReportingObligationRecord) -> tuple[str, str, f"source:{source}", ) + async def _claim_first_core_delivery( + self, obligation: ReportingObligationRecord, *, now: datetime, retry: bool + ) -> str | None: + """Consume a first attempt or its own due slice retry before source dispatch.""" + key = (obligation.generation_key, obligation.reporting_obligation_id) + state = self._first_core_deliveries.get(key) + if isinstance(state, RetryScheduleEntry): + if not retry or _utc(now) < _utc(state.retry_not_before): + return None + retained = await self._get_retry(state.scope_key) + if self._first_core_deliveries.get(key) is not state: + return None + if retained != state: + self._first_core_deliveries[key] = "attempted" + return None + elif state != "ready": + return None + self._first_core_deliveries[key] = "attempted" + return obligation.reporting_obligation_id + async def _get_retry(self, key: str) -> RetryScheduleEntry | None: return await self._retry_store.get_retry_schedule(scope_key=key) @@ -2305,6 +2358,7 @@ async def _schedule_retry( retry_after_seconds: float | None = None, blocked: bool = False, replayed: bool = False, + first_core_delivery: bool = False, ) -> None: # Dispatch ``now`` can be synthetic in tests and integrations. A slow # source may finish long after it, so anchor the cooldown at completion. @@ -2341,17 +2395,20 @@ async def _schedule_retry( retry_at = completed_at + delay except OverflowError: retry_at = datetime.max.replace(tzinfo=timezone.utc) - stored = await self._record_retry( - RetryScheduleEntry( - key, - completed_at if blocked else retry_at, - attempt, - blocked, - recorded_at=completed_at, - replayed=replayed, - ) - ) + entry = RetryScheduleEntry( + key, + completed_at if blocked else retry_at, + attempt, + blocked, + recorded_at=completed_at, + replayed=replayed, + ) + stored = await self._record_retry(entry) turn._retry_entries[key] = stored + if first_core_delivery and key == keys[0] and not blocked and stored == entry: + self._first_core_deliveries[ + (obligation.generation_key, obligation.reporting_obligation_id) + ] = stored if replayed: # A manual probe supersedes an earlier terminal blast radius. The # new error's declared scope governs subsequent automatic work. @@ -2631,6 +2688,7 @@ async def _check_official_publication_lock( frozen: ProvisionalPolicy | None = None, check_generation: bool = True, scheduled_ordinal: int | None = None, + first_core_obligation: str | None = None, ) -> None: policy = route.definition.finality_policy if route.definition else None if policy is None or self._optional_publication_store() is not None: @@ -2648,7 +2706,10 @@ async def _check_official_publication_lock( or ( check_generation and await self._has_retained_official_close( - route.generation_key, publishing=True, scheduled_reservation=reservation + route.generation_key, + publishing=True, + scheduled_reservation=reservation, + first_core_obligation=first_core_obligation, ) ) ): @@ -2993,6 +3054,7 @@ async def _commit_revision_from_manifest( turn: WorkerTurn | None, acquisition: ProvisionalAcquisition | None, verified_source_rows: bool, + first_core_obligation: str | None = None, suppress_untracked_unchanged: bool = False, admit_tracked_stabilization: bool = True, ) -> ReportingRevisionRecord: @@ -3009,6 +3071,7 @@ async def _commit_revision_from_manifest( frozen=acquisition.policy if acquisition else None, check_generation=acquisition is None or verified_source_rows, scheduled_ordinal=scheduled_ordinal, + first_core_obligation=first_core_obligation, ) origin_store: SnapshotOriginStore | None = None if finality == "snapshot" and acquisition is not None: diff --git a/tests/test_reporting_official_acquisition.py b/tests/test_reporting_official_acquisition.py index 94b18fe69..6bb3665b1 100644 --- a/tests/test_reporting_official_acquisition.py +++ b/tests/test_reporting_official_acquisition.py @@ -34,6 +34,7 @@ ReportingRevisionRecord, ReportingSourceFinalPolicy, ReportingStabilizedPolicy, + RetryScheduleEntry, WorkerTurn, revision_content_sha256, ) @@ -55,7 +56,8 @@ source_batch_object_set_sha256_v1, ) from tests.conformance.reporting._generation_support import isolated_reporting_pool -from tests.test_reporting_official_finality import ROWS +from tests.test_reporting_official_finality import ROWS, _restart_producer +from tests.test_reporting_producer_retry import FailingSource from tests.test_reporting_source_availability import ( END, PROVIDER_ROW, @@ -81,6 +83,11 @@ def source_publication(self, account_id: str, **kwargs: Any) -> Any: return self.store.source_publication(account_id, **kwargs) +class UnlockedCore(CoreStore): + source_publication = None + get_stabilization_schedule = None + + @pytest.mark.parametrize( "policy", [ @@ -93,9 +100,6 @@ async def test_standalone_official_admission_requires_account_lock(policy: Any) official=True, finality_policy=policy ) - class UnlockedCore(CoreStore): - source_publication = None - producer._store = UnlockedCore(store) changed = replace(config, delivery_config_version=2) await store.put_configuration(changed) @@ -248,6 +252,304 @@ async def cutoff_backend(request: Any) -> AsyncIterator[Any]: ), request.param == "core" +async def _core_snapshot_harness(factory: Any, policy: Any) -> tuple[Any, ...]: + template, store, config, _, calls, clock = await _harness(factory, finality_policy=policy) + config = replace(config, delivery_config_id="core-snapshot") + await store.put_configuration(config) + producer = _restart_producer(template, UnlockedCore(store), clock) + (obligation,) = await producer.close_elapsed_periods(config) + return producer, store, config, obligation, calls, clock + + +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +@pytest.mark.parametrize( + "policy", + [ + None, + ReportingSourceFinalPolicy("unused-provider", "delivery.is_final"), + ReportingContractualCutoffPolicy("unused-cutoff", "P2D"), + ReportingStabilizedPolicy("unused-stable", "PT0S", "PT12H"), + ], +) +@pytest.mark.parametrize("manual", [False, True]) +async def test_first_core_snapshot_with_unused_policy( + cutoff_backend: Any, policy: Any, manual: bool +) -> None: + producer, store, config, obligation, calls, clock = await _core_snapshot_harness( + cutoff_backend[0], policy + ) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + if policy is not None: + peer = _restart_producer(producer, UnlockedCore(store), clock) + assert await peer.close_elapsed_periods(config) == [] + with pytest.raises(LedgerConflictError, match="publication lock"): + await peer.acquire_obligation(config, obligation) + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.close_elapsed_periods(config, now=clock[0] + timedelta(days=1)) + assert calls == [] + snapshot = await producer.acquire_obligation(config, obligation, manual_replay=manual) + assert snapshot is not None and snapshot.finality == "snapshot" + assert snapshot.finality_policy_id is None + assert calls == [clock[0]] + assert await store.list_revisions(**owner) == (snapshot,) + assert await store.get_provisional_observation(**owner) is None + assert await store.get_restatement_checkpoint(**owner) is None + clock[0] += timedelta(hours=1) + if policy is None: + await producer.acquire_obligation(config, obligation, restate=True, manual_replay=True) + assert len(calls) == 2 + else: + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.acquire_obligation(config, obligation, restate=True, manual_replay=True) + assert len(calls) == 1 + + +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +@pytest.mark.parametrize("policy", [None, ReportingContractualCutoffPolicy("unused", "P2D")]) +async def test_failed_first_core_delivery_retries_on_its_recorded_schedule( + cutoff_backend: Any, policy: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, obligation, calls, clock = await _core_snapshot_harness( + cutoff_backend[0], policy + ) + failure = FailingSource() + with monkeypatch.context() as patch: + patch.setattr(producer._source, "execute", failure.execute) + assert await producer.acquire_obligation(config, obligation) is None + assert failure.calls == 1 + retry = await store.get_retry_schedule(scope_key=producer._retry_keys(obligation)[0]) + assert retry is not None and retry.attempt == 1 + assert await producer.acquire_obligation(config, obligation) is None + if policy is not None: + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.acquire_obligation(config, obligation, manual_replay=True) + assert calls == [] + clock[0] = retry.retry_not_before + with monkeypatch.context() as patch: + patch.setattr(producer._source, "execute", failure.execute) + assert await producer.acquire_obligation(config, obligation) is None + assert failure.calls == 2 + retry = await store.get_retry_schedule(scope_key=retry.scope_key) + assert retry.attempt == 2 + clock[0] = retry.retry_not_before + recovered = await producer.acquire_obligation(config, obligation) + assert recovered is not None and recovered.finality == "snapshot" + assert calls == [clock[0]] + assert (await store.get_retry_schedule(scope_key=retry.scope_key)).attempt == 0 + if policy is not None: + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.acquire_obligation(config, obligation, restate=True, manual_replay=True) + assert calls == [clock[0]] + + +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +@pytest.mark.parametrize("competing", ["persisting_retry", "retry", "reservation"]) +async def test_core_retry_checks_the_durable_winner( + cutoff_backend: Any, competing: str, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, obligation, calls, clock = await _core_snapshot_harness( + cutoff_backend[0], ReportingSourceFinalPolicy("provider", "delivery.is_final") + ) + with monkeypatch.context() as patch: + if competing == "persisting_retry": + record = store.record_retry_schedule + + async def race(entry: RetryScheduleEntry) -> RetryScheduleEntry: + await record( + replace( + entry, + attempt=entry.attempt + 1, + recorded_at=clock[0] + timedelta(microseconds=1), + ) + ) + return await record(entry) + + patch.setattr(store, "record_retry_schedule", race) + patch.setattr(producer._source, "execute", FailingSource().execute) + assert await producer.acquire_obligation(config, obligation) is None + retry = await store.get_retry_schedule(scope_key=producer._retry_keys(obligation)[0]) + if competing == "retry": + retry = await store.record_retry_schedule( + replace(retry, attempt=2, retry_not_before=retry.retry_not_before + timedelta(hours=1)) + ) + assert retry.attempt == 2 + elif competing == "reservation": + request = producer._build_slice( + config, obligation, OFFICIAL_OFFERING_ID, finality="official", now=clock[0] + ) + async with store.source_publication(config.account_id): + await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + request.model_dump_json(), 0, ProvisionalPolicy(timedelta(0), timedelta(days=1)) + ) + ) + clock[0] = retry.retry_not_before + with pytest.raises(LedgerConflictError) as error: + await producer.acquire_obligation(config, obligation) + assert error.value.code == ( + "OBSERVATION_CONFLICT" + if competing == "reservation" + else "OFFICIAL_PUBLICATION_NOT_SUPPORTED" + ) + assert len(calls) == int(competing == "reservation") + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.acquire_obligation(config, obligation, manual_replay=True) + assert not await store.list_revisions( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + + +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +@pytest.mark.parametrize( + "policy", + [ + ReportingSourceFinalPolicy("provider", "delivery.is_final"), + ReportingContractualCutoffPolicy("cutoff", "PT0S"), + ReportingStabilizedPolicy("stable", "PT0S", "PT12H"), + ], +) +async def test_rejected_official_attempt_consumes_first_core_delivery( + cutoff_backend: Any, policy: Any +) -> None: + producer, _, config, obligation, calls, _ = await _core_snapshot_harness( + cutoff_backend[0], policy + ) + with pytest.raises(LedgerConflictError): + await producer.acquire_obligation(config, obligation, target_finality="official") + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.acquire_obligation(config, obligation, manual_replay=True) + assert calls == [] + + +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +async def test_first_core_snapshot_waits_for_shared_source_retry(cutoff_backend: Any) -> None: + producer, store, config, obligation, calls, clock = await _core_snapshot_harness( + cutoff_backend[0], ReportingStabilizedPolicy("unused", "PT0S", "PT12H") + ) + retry_at = clock[0] + timedelta(minutes=15) + await store.record_retry_schedule( + RetryScheduleEntry(producer._retry_keys(obligation)[2], retry_at, 1, recorded_at=clock[0]) + ) + assert await producer.acquire_obligation(config, obligation) is None + assert calls == [] + clock[0] = retry_at + snapshot = await producer.acquire_obligation(config, obligation) + assert snapshot is not None and snapshot.finality == "snapshot" + assert calls == [retry_at] + + +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +@pytest.mark.parametrize("visible", [False, True]) +async def test_competing_close_blocks_first_core_snapshot( + cutoff_backend: Any, visible: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, obligation, calls, clock = await _core_snapshot_harness( + cutoff_backend[0], ReportingSourceFinalPolicy("provider", "delivery.is_final") + ) + request = producer._build_slice( + config, obligation, OFFICIAL_OFFERING_ID, finality="official", now=clock[0] + ) + async with store.source_publication(config.account_id): + winner = await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + request.model_dump_json(), 0, ProvisionalPolicy(timedelta(0), timedelta(days=1)) + ) + ) + execute = producer._source.execute + dispatches = 0 + + async def during_delivery(*args: Any, **kwargs: Any) -> Any: + nonlocal dispatches + dispatches += 1 + assert dispatches == 1 + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.acquire_obligation(config, obligation, manual_replay=True) + return await execute(*args, **kwargs) + + monkeypatch.setattr(producer._source, "execute", during_delivery) + if visible: + monkeypatch.setattr( + producer.store, + "get_provisional_acquisition", + store.get_provisional_acquisition, + raising=False, + ) + with pytest.raises(LedgerConflictError) as conflict: + await producer.acquire_obligation(config, obligation) + if visible: + assert conflict.value.code == "OFFICIAL_PUBLICATION_NOT_SUPPORTED" + assert calls == [] + monkeypatch.delattr(producer.store, "get_provisional_acquisition") + else: + assert conflict.value.code == "OBSERVATION_CONFLICT" + assert calls == [clock[0]] + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.acquire_obligation(config, obligation) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + assert await store.get_provisional_acquisition(**owner, ordinal=0) == winner + assert await store.list_revisions(**owner) == () + + +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +async def test_overlapping_closes_cannot_rearm_an_attempted_core_delivery( + cutoff_backend: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, _, calls, _ = await _core_snapshot_harness( + cutoff_backend[0], ReportingContractualCutoffPolicy("unused", "P2D") + ) + config = replace(config, delivery_config_version=2) + await store.put_configuration(config) + commit = store.commit_obligation + pending = asyncio.Event() + resume = asyncio.Event() + + async def delayed(candidate: Any) -> Any: + if not pending.is_set(): + pending.set() + await resume.wait() + return await commit(candidate) + + monkeypatch.setattr(store, "commit_obligation", delayed) + overlapping = asyncio.create_task(producer.close_elapsed_periods(config)) + try: + await asyncio.wait_for(pending.wait(), timeout=5) + (obligation,) = await producer.close_elapsed_periods(config) + failure = FailingSource() + with monkeypatch.context() as patch: + patch.setattr(producer._source, "execute", failure.execute) + assert await producer.acquire_obligation(config, obligation) is None + assert failure.calls == 1 + finally: + resume.set() + completed = await overlapping + assert completed == [obligation] + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.acquire_obligation(config, obligation, manual_replay=True) + assert calls == [] + + +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +async def test_first_core_snapshot_checks_other_periods_for_a_close(cutoff_backend: Any) -> None: + producer, store, config, obligation, calls, clock = await _core_snapshot_harness( + cutoff_backend[0], ReportingSourceFinalPolicy("provider", "delivery.is_final") + ) + locked = _restart_producer(producer, CoreStore(store), clock) + clock[0] += timedelta(days=1) + (other,) = await locked.close_elapsed_periods(config) + official = await locked.acquire_obligation(config, other, target_finality="official") + assert official is not None and official.finality == "official" + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.acquire_obligation(config, obligation) + assert calls == [clock[0]] + assert not await store.list_revisions( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + + @pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) @pytest.mark.parametrize("witness", [True, False]) @pytest.mark.parametrize("atomic", [True, False]) diff --git a/tests/test_reporting_official_finality.py b/tests/test_reporting_official_finality.py index b061bbad8..26a69fdf0 100644 --- a/tests/test_reporting_official_finality.py +++ b/tests/test_reporting_official_finality.py @@ -1757,6 +1757,62 @@ async def test_new_period_checks_an_authoritative_close_with_an_ordinary_predece assert calls == before +@pytest.mark.parametrize( + "policy", + [ + ReportingSourceFinalPolicy("provider", "delivery.is_final"), + ReportingContractualCutoffPolicy("cutoff", "PT0S"), + ], +) +@pytest.mark.parametrize("prior_observation", [False, True]) +async def test_core_official_revision_blocks_unlocked_next_period( + store_factory: Any, policy: Any, prior_observation: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + from tests.test_reporting_official_acquisition import CoreStore + + producer, store, config, obligation, calls, clock = await _harness( + store_factory, finality_policy=policy + ) + clock[0] = END + timedelta(hours=1) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + if prior_observation: + await producer.run_configuration(config) + observation = await store.get_provisional_observation(**owner) + checkpoint = await store.get_restatement_checkpoint(**owner) + assert (observation is not None) == prior_observation + core = _restart_producer(producer, CoreStore(store), clock) + official = await core.acquire_obligation( + config, obligation, restate=prior_observation, target_finality="official" + ) + assert official is not None and official.finality_policy_id == policy.finality_policy_id + assert await store.get_provisional_observation(**owner) == observation + assert await store.get_restatement_checkpoint(**owner) == checkpoint + assert await store.get_provisional_acquisition(**owner, ordinal=int(prior_observation)) is None + revisions = await store.list_revisions(**owner) + before = list(calls) + monkeypatch.setattr(store, "source_publication", None) + monkeypatch.setattr(store, "get_stabilization_schedule", _stabilized_only_lookup(store)) + assert await store.get_stabilization_schedule(config.generation_key) is None + rebound = _restart_producer(producer, store, clock) + assert await rebound._has_retained_official_close(config.generation_key) + clock[0] += timedelta(days=1) + with pytest.raises(LedgerConflictError, match="publication lock"): + await rebound.run_configuration(config) + assert calls == before + assert await store.list_revisions(**owner) == revisions + next_period = await store.find_obligation( + account_id=config.account_id, + consumer_id=config.consumer_id, + delivery_config_id=config.delivery_config_id, + delivery_config_version=config.delivery_config_version, + period_start=END, + period_end=END + timedelta(days=1), + ) + assert next_period is None + + @pytest.mark.parametrize("configure_before_init", [True, False]) @pytest.mark.parametrize( "policy,missing", From 9ceabe0984ea5368a49462247d054c6c306f2e79 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 20:33:18 +0530 Subject: [PATCH 26/39] fix(reporting): serialize snapshot roots and recover interrupted Core reads --- docs/reporting-provisional-observations.md | 23 +- .../src/adcp_reporting_storage/ledger/pg.py | 13 ++ src/adcp/reporting/ledger/producer.py | 147 ++++++++++--- src/adcp/reporting/ledger/store.py | 4 + .../test_reporting_notification_security.py | 3 +- ...est_reporting_notification_transactions.py | 1 + .../test_reporting_reconciliation_changes.py | 4 +- .../test_reporting_row_storage_pg.py | 6 +- tests/test_reporting_official_acquisition.py | 207 ++++++++++++++++++ 9 files changed, 363 insertions(+), 45 deletions(-) diff --git a/docs/reporting-provisional-observations.md b/docs/reporting-provisional-observations.md index 12d8b7ec0..37a279ec7 100644 --- a/docs/reporting-provisional-observations.md +++ b/docs/reporting-provisional-observations.md @@ -78,16 +78,19 @@ snapshot origins and generation schedule recovery. An ordinary snapshot route can keep using its snapshot publisher when the definition lists an unused finality policy. Each scheduled read checks the winner of its own atomic reservation. -A Core producer without observation or checkpoint support can call -`close_elapsed_periods` and then `acquire_obligation` for the first snapshot of -the period it just created. It consumes that allowance before calling the source. -After a retryable fetch failure, the same producer can retry when that slice's -recorded retry becomes due. A manual replay cannot claim the scheduled retry. -If another publisher reserves the period, the ledger's atomic revision commit -rejects the snapshot. With a selected policy and no account publication lock, -additional reads and later periods need durable evidence ruling out a competing -close. Recovery checks immutable official revisions as well as retained -observations and reservations when the stabilization lookup returns no schedule. +When a definition carries an unused finality policy, a Core producer without +observation or checkpoint support can call `close_elapsed_periods` and then +`acquire_obligation` for the first snapshot of the period it just created. +After a retryable fetch failure, an interrupted staged object read, or caller +cancellation, the same producer can resume its frozen scheduled request with +a fresh deadline. It checks its recorded slice +retry and any shared backoff before dispatch. Only a scheduled read for that +obligation can claim the retry. The ledger rejects a competing reservation or a +second snapshot root during the atomic revision commit. With a selected policy +and no account publication lock, additional reads and later periods need durable +evidence ruling out a competing close. Recovery checks immutable official +revisions as well as retained observations and reservations when the stabilization +lookup returns no schedule. ## Persistence and compatibility diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py index 17be1ebc1..ecc45025a 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py @@ -1534,6 +1534,19 @@ async def commit_revision( ) if revision.supersedes_reporting_revision_id: await self._require_current_leaf(connection, revision) + elif ( + revision.finality == "snapshot" + and await ( + await connection.execute( + "SELECT 1 FROM reporting_revisions WHERE account_id=%s" + " AND reporting_obligation_id=%s LIMIT 1", + (revision.account_id, revision.reporting_obligation_id), + ) + ).fetchone() + ): + raise LedgerConflictError( + "SUPERSEDES_STALE", "the obligation already has a snapshot root" + ) write_error = None try: await connection.execute( diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 8c2e5faf8..86c774c76 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -428,6 +428,19 @@ class _StabilizedPublication: rows: Sequence[dict[str, Any]] +@dataclass +class _FirstCoreDelivery: + """The scheduled first read remains one claim through dispatch and retry bookkeeping.""" + + state: Literal["scheduled", "retry", "consumed"] + retry: RetryScheduleEntry | None = None + request: ReportingSourceSliceRequestV1 | None = None + + def interrupted(self) -> None: + if self.state == "scheduled": + self.state = "retry" + + def _implements_store_method(store: object, stub: Callable[..., Any]) -> bool: name = stub.__name__ if inspect.getattr_static(store, name, None) is None: @@ -498,7 +511,7 @@ def __init__( ] = {} self._first_core_deliveries: dict[ tuple[ReportingConfigurationGenerationKey, str], - Literal["ready", "attempted"] | RetryScheduleEntry, + Literal["ready", "attempted"] | _FirstCoreDelivery, ] = {} @property @@ -1804,7 +1817,7 @@ async def _acquire_obligation( finality = target_finality or obligation.required_finality selected_policy = obligation.definition.finality_policy if obligation.definition else None if finality == "official" and isinstance(selected_policy, ReportingStabilizedPolicy): - await self._claim_first_core_delivery(obligation, now=now, retry=False) + await self._claim_first_core_delivery(obligation, now=now, scheduled=False) return await self._acquire_stabilized( configuration, obligation, @@ -1847,10 +1860,51 @@ async def _acquire_obligation( offering_id=retry_offering_id, ) return None - first_core_obligation = await self._claim_first_core_delivery( + first_core = await self._claim_first_core_delivery( obligation, now=now, - retry=finality == "snapshot" and not (manual_replay or restate or track_settling), + scheduled=finality == "snapshot" and not (manual_replay or restate or track_settling), + ) + try: + return await self._acquire_due_obligation( + configuration, + obligation, + turn=turn, + now=now, + finality=finality, + restate=restate, + track_settling=track_settling, + manual_replay=manual_replay, + admit_tracked_stabilization=admit_tracked_stabilization, + first_core=first_core, + ) + except (OSError, asyncio.CancelledError): + if first_core is not None: + first_core.interrupted() + raise + finally: + if first_core is not None and first_core.state == "retry": + self._first_core_deliveries[ + (obligation.generation_key, obligation.reporting_obligation_id) + ] = first_core + + async def _acquire_due_obligation( + self, + configuration: ReportingConfiguration, + obligation: ReportingObligationRecord, + *, + turn: WorkerTurn, + now: datetime, + finality: str, + restate: bool, + track_settling: bool, + manual_replay: bool, + admit_tracked_stabilization: bool, + first_core: _FirstCoreDelivery | None, + ) -> ReportingRevisionRecord | None: + selected_policy = obligation.definition.finality_policy if obligation.definition else None + first_core_obligation = ( + obligation.reporting_obligation_id if first_core is not None else None ) revisions = await self._store.list_revisions( account_id=obligation.account_id, @@ -1961,17 +2015,24 @@ async def _acquire_obligation( return None if checkpoint_store is None: observation = await self._cutoff_observation_ordinal(obligation, observation) - request = self._build_slice( - configuration, - obligation, - offering_id, - finality=finality, - now=now, - observation=observation, - constituents=constituents, - trigger="manual_replay" if manual_observation else "scheduled_poll", - invocation_id=uuid4().hex if manual_snapshot else None, - ) + if first_core is not None and first_core.request is not None: + request = first_core.request.model_copy( + update={"deadline_at": _utc(now) + self._offerings.slice_timeout} + ) + else: + request = self._build_slice( + configuration, + obligation, + offering_id, + finality=finality, + now=now, + observation=observation, + constituents=constituents, + trigger="manual_replay" if manual_observation else "scheduled_poll", + invocation_id=uuid4().hex if manual_snapshot else None, + ) + if first_core is not None and first_core.state == "scheduled": + first_core.request = request.model_copy(deep=True) acquisition = pending # An explicitly tracked first invocation must survive a failed fetch and restart. track_first_manual = manual_snapshot and track_settling and not restate and not revisions @@ -2040,7 +2101,7 @@ async def _acquire_obligation( turn, now=now, replayed=manual_replay, - first_core_delivery=first_core_obligation is not None, + first_core_delivery=first_core, ) self._note_escalation( obligation, @@ -2076,7 +2137,7 @@ async def _acquire_obligation( retry_after_seconds=error.retry_after_seconds, blocked=error.retry == "terminal", replayed=manual_replay, - first_core_delivery=first_core_obligation is not None, + first_core_delivery=first_core, ) self._note_escalation( obligation, @@ -2107,7 +2168,14 @@ async def _acquire_obligation( cutoff_ceiling=now, acquisition=acquisition, ) - rows = await self._read_rows(request, manifest, require_staged=full_authority) + try: + rows = await self._read_rows(request, manifest, require_staged=full_authority) + except LedgerConflictError: + raise + except Exception: + if first_core is not None: + first_core.interrupted() + raise # ``now`` freezes dispatch/lease/cutoff decisions, not publication. # A conforming source can observe finality while acquisition is running. # Every successful scheduled read reaches the atomic observation/checkpoint commit. @@ -2196,6 +2264,8 @@ async def _acquire_obligation( retry_after_seconds=acquisition.policy.cadence.total_seconds(), ) raise + if first_core is not None: + first_core.state = "consumed" await self._clear_retry(obligation, turn=turn, now=published_at) return committed @@ -2290,24 +2360,29 @@ def _retry_keys(self, obligation: ReportingObligationRecord) -> tuple[str, str, ) async def _claim_first_core_delivery( - self, obligation: ReportingObligationRecord, *, now: datetime, retry: bool - ) -> str | None: - """Consume a first attempt or its own due slice retry before source dispatch.""" + self, obligation: ReportingObligationRecord, *, now: datetime, scheduled: bool + ) -> _FirstCoreDelivery | None: + """Claim the first attempt or resume that scheduled read on its own retry state.""" key = (obligation.generation_key, obligation.reporting_obligation_id) state = self._first_core_deliveries.get(key) - if isinstance(state, RetryScheduleEntry): - if not retry or _utc(now) < _utc(state.retry_not_before): + if isinstance(state, _FirstCoreDelivery): + if not scheduled or ( + state.retry is not None and _utc(now) < _utc(state.retry.retry_not_before) + ): return None - retained = await self._get_retry(state.scope_key) + retained = await self._get_retry(self._retry_keys(obligation)[0]) if self._first_core_deliveries.get(key) is not state: return None - if retained != state: + if retained != state.retry: self._first_core_deliveries[key] = "attempted" return None - elif state != "ready": + delivery = replace(state, state="scheduled") + elif state == "ready": + delivery = _FirstCoreDelivery("scheduled" if scheduled else "consumed") + else: return None self._first_core_deliveries[key] = "attempted" - return obligation.reporting_obligation_id + return delivery async def _get_retry(self, key: str) -> RetryScheduleEntry | None: return await self._retry_store.get_retry_schedule(scope_key=key) @@ -2358,11 +2433,13 @@ async def _schedule_retry( retry_after_seconds: float | None = None, blocked: bool = False, replayed: bool = False, - first_core_delivery: bool = False, + first_core_delivery: _FirstCoreDelivery | None = None, ) -> None: # Dispatch ``now`` can be synthetic in tests and integrations. A slow # source may finish long after it, so anchor the cooldown at completion. completed_at = max(_utc(now), _utc(self._clock())) + if first_core_delivery is not None and blocked: + first_core_delivery.state = "consumed" keys = self._retry_keys(obligation) shared = keys[1 if scope == "account" else 2] affected = keys[:1] if scope == "slice" else ((shared,) if blocked else (keys[0], shared)) @@ -2405,10 +2482,16 @@ async def _schedule_retry( ) stored = await self._record_retry(entry) turn._retry_entries[key] = stored - if first_core_delivery and key == keys[0] and not blocked and stored == entry: - self._first_core_deliveries[ - (obligation.generation_key, obligation.reporting_obligation_id) - ] = stored + if ( + first_core_delivery is not None + and first_core_delivery.state == "scheduled" + and key == keys[0] + ): + if stored == entry: + first_core_delivery.retry = stored + first_core_delivery.state = "retry" + else: + first_core_delivery.state = "consumed" if replayed: # A manual probe supersedes an earlier terminal blast radius. The # new error's declared scope governs subsequent automatic work. diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index b13b21f81..122477a79 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -1357,6 +1357,10 @@ async def commit_revision( ) if revision.supersedes_reporting_revision_id: self._require_current_leaf(revision, siblings) + elif revision.finality == "snapshot" and siblings: + raise LedgerConflictError( + "SUPERSEDES_STALE", "the obligation already has a snapshot root" + ) self._revisions[revision.reporting_revision_id] = revision self._revision_identity[revision.reporting_revision_id] = identity self._rows[revision.reporting_revision_id] = tuple(dict(row) for row in rows) diff --git a/tests/conformance/reporting/test_reporting_notification_security.py b/tests/conformance/reporting/test_reporting_notification_security.py index 8a5a74dd5..2ad8d9a2d 100644 --- a/tests/conformance/reporting/test_reporting_notification_security.py +++ b/tests/conformance/reporting/test_reporting_notification_security.py @@ -165,7 +165,7 @@ async def test_every_bound_column_swap_fails_before_any_external_effect( notification_harness, column ): h = notification_harness - obligation, _, _ = await seed(h) + obligation, first, _ = await seed(h) await h.worker().expand_one(account_id="acct_a") (row,) = await h.outbox.list_deliveries(account_id="acct_a") original = row.delivery @@ -173,6 +173,7 @@ async def test_every_bound_column_swap_fails_before_any_external_effect( value = current + 1 if isinstance(current, int) else "swapped-value" if column == "notification_id": revision, rows = revision_for(obligation, suffix="another") + revision = replace(revision, supersedes_reporting_revision_id=first.reporting_revision_id) await h.reliable.store.commit_revision(revision, rows) value = next( event.notification_id diff --git a/tests/conformance/reporting/test_reporting_notification_transactions.py b/tests/conformance/reporting/test_reporting_notification_transactions.py index 176faeaee..b2ce68b6e 100644 --- a/tests/conformance/reporting/test_reporting_notification_transactions.py +++ b/tests/conformance/reporting/test_reporting_notification_transactions.py @@ -181,6 +181,7 @@ async def prepare(h: NotificationHarness, operation: str): return lambda: store.commit_obligation(other) if operation == "revision": changed, rows = revision_for(obligation, suffix="second") + changed = replace(changed, supersedes_reporting_revision_id=revision.reporting_revision_id) return lambda: store.commit_revision(changed, rows) if operation == "readability": return lambda: store.set_revision_readable( diff --git a/tests/conformance/reporting/test_reporting_reconciliation_changes.py b/tests/conformance/reporting/test_reporting_reconciliation_changes.py index 693950323..e9b48242d 100644 --- a/tests/conformance/reporting/test_reporting_reconciliation_changes.py +++ b/tests/conformance/reporting/test_reporting_reconciliation_changes.py @@ -508,11 +508,13 @@ async def test_distinct_concurrent_writes_take_dense_unique_feed_sequences( ) ).rows attempts = [] + predecessor = s.revision for index in range(4): identifier = f"concurrent-revision-{index}" revision = replace( s.revision, reporting_revision_id=identifier, + supersedes_reporting_revision_id=predecessor.reporting_revision_id, finality="snapshot", finality_basis=None, finality_policy_id=None, @@ -525,7 +527,7 @@ async def test_distinct_concurrent_writes_take_dense_unique_feed_sequences( control_total_evidence=s.revision.managed_control_totals, ), ) - await store.commit_revision(revision, rows) + predecessor = await store.commit_revision(revision, rows) # Each attempt is the first for its own revision, so validity does not # depend on which of them the store happens to serialize first. attempts.append( diff --git a/tests/conformance/reporting/test_reporting_row_storage_pg.py b/tests/conformance/reporting/test_reporting_row_storage_pg.py index e8c7d4fce..db9b5fb70 100644 --- a/tests/conformance/reporting/test_reporting_row_storage_pg.py +++ b/tests/conformance/reporting/test_reporting_row_storage_pg.py @@ -225,7 +225,11 @@ async def test_a_retained_pair_hashed_revision_stores_rows_only_and_verifies_its walked, _ = await walk(store, obligation.account_id, revision.reporting_revision_id, 500) assert walked == rows - forged = replace(revision, reporting_revision_id="rpr_forged") + forged = replace( + revision, + reporting_revision_id="rpr_forged", + supersedes_reporting_revision_id=revision.reporting_revision_id, + ) with pytest.raises(LedgerConflictError) as conflict: await store.commit_revision(forged, rows) assert conflict.value.code == "REVISION_CONTENT_MISMATCH" diff --git a/tests/test_reporting_official_acquisition.py b/tests/test_reporting_official_acquisition.py index 6bb3665b1..c2389ed44 100644 --- a/tests/test_reporting_official_acquisition.py +++ b/tests/test_reporting_official_acquisition.py @@ -36,7 +36,9 @@ ReportingStabilizedPolicy, RetryScheduleEntry, WorkerTurn, + project_obligation_health, revision_content_sha256, + select_reporting_revision, ) from adcp.reporting.ledger.finality import manifest_finality_proof from adcp.reporting.ledger.provisional import ( @@ -344,6 +346,142 @@ async def test_failed_first_core_delivery_retries_on_its_recorded_schedule( assert calls == [clock[0]] +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +@pytest.mark.parametrize( + "interruption", ["read_error", "reader_unavailable", "cancel_fetch", "cancel_read"] +) +@pytest.mark.parametrize("competitor", ["none", "slice_retry", "snapshot"]) +async def test_interrupted_first_core_delivery_retains_its_scheduled_request( + cutoff_backend: Any, interruption: str, competitor: str, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, obligation, _, clock = await _core_snapshot_harness( + cutoff_backend[0], ReportingContractualCutoffPolicy("unused", "P2D") + ) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + source = producer._source + execute = source.execute + requests = [] + sealed = None + interrupted = asyncio.Event() + blocked = asyncio.Event() + + async def dispatch(request: Any, **kwargs: Any) -> Any: + nonlocal sealed + requests.append(request) + if sealed is None: + sealed = await execute(request, **kwargs) + if interruption == "cancel_fetch": + interrupted.set() + await blocked.wait() + return sealed + + async def read(**kwargs: Any) -> bytes: + if interruption == "read_error": + raise OSError("staged object temporarily unavailable") + if interruption == "reader_unavailable": + raise RuntimeError("staged reader unavailable") + interrupted.set() + await blocked.wait() + pytest.fail("cancelled read resumed") + + monkeypatch.setattr(source, "execute", dispatch) + with monkeypatch.context() as patch: + patch.setattr(producer._object_reader, "read", read) + delivery = asyncio.create_task(producer.acquire_obligation(config, obligation)) + if interruption in {"read_error", "reader_unavailable"}: + with pytest.raises((OSError, RuntimeError), match="staged"): + await delivery + else: + try: + await asyncio.wait_for(interrupted.wait(), timeout=5) + finally: + delivery.cancel() + with pytest.raises(asyncio.CancelledError): + await delivery + assert await store.list_revisions(**owner) == () + (original,) = requests + publication_id = source.manifest.publication_id + clock[0] += timedelta(hours=2) + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.acquire_obligation(config, obligation, manual_replay=True) + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.close_elapsed_periods(config, now=clock[0] + timedelta(days=1)) + assert requests == [original] + + if competitor == "slice_retry": + retry_at = clock[0] + timedelta(minutes=10) + await store.record_retry_schedule( + RetryScheduleEntry( + producer._retry_keys(obligation)[0], retry_at, 4, recorded_at=clock[0] + ) + ) + assert await producer.acquire_obligation(config, obligation) is None + clock[0] = retry_at + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.acquire_obligation(config, obligation) + assert await store.list_revisions(**owner) == () + assert requests == [original] + elif competitor == "snapshot": + peer = _restart_producer(producer, CoreStore(store), clock) + with monkeypatch.context() as patch: + patch.setattr(source, "execute", execute) + winner = await peer.acquire_obligation(config, obligation, manual_replay=True) + assert await producer.acquire_obligation(config, obligation) is None + assert await store.list_revisions(**owner) == (winner,) + assert requests == [original] + else: + recovered = await producer.acquire_obligation(config, obligation) + assert recovered.source_publication_id == publication_id + assert recovered.reporting_revision_id == f"rpr_{publication_id[4:44]}" + assert await store.list_revisions(**owner) == (recovered,) + first, retry = requests + assert retry.deadline_at > first.deadline_at + assert retry == first.model_copy(update={"deadline_at": retry.deadline_at}) + assert await producer.acquire_obligation(config, obligation) is None + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.acquire_obligation(config, obligation, restate=True, manual_replay=True) + assert len(requests) == 2 + + +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +async def test_first_core_delivery_keeps_its_claim_until_shared_retry_is_recorded( + cutoff_backend: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, obligation, calls, clock = await _core_snapshot_harness( + cutoff_backend[0], ReportingContractualCutoffPolicy("unused", "P2D") + ) + slice_key, account_key, _ = producer._retry_keys(obligation) + writing, release = asyncio.Event(), asyncio.Event() + record = store.record_retry_schedule + + async def record_shared(entry: RetryScheduleEntry) -> RetryScheduleEntry: + if entry.scope_key == account_key: + writing.set() + await release.wait() + return await record(entry) + + monkeypatch.setattr(store, "record_retry_schedule", record_shared) + failure = FailingSource(scope="account") + with monkeypatch.context() as patch: + patch.setattr(producer._source, "execute", failure.execute) + first = asyncio.create_task(producer.acquire_obligation(config, obligation)) + await asyncio.wait_for(writing.wait(), timeout=5) + try: + clock[0] = (await store.get_retry_schedule(scope_key=slice_key)).retry_not_before + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.acquire_obligation(config, obligation) + finally: + release.set() + assert await first is None + assert failure.calls == 1 and calls == [] + shared = await store.get_retry_schedule(scope_key=account_key) + clock[0] = max(clock[0], shared.retry_not_before) + recovered = await producer.acquire_obligation(config, obligation) + assert recovered.finality == "snapshot" and calls == [clock[0]] + + @pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) @pytest.mark.parametrize("competing", ["persisting_retry", "retry", "reservation"]) async def test_core_retry_checks_the_durable_winner( @@ -532,6 +670,75 @@ async def delayed(candidate: Any) -> Any: assert calls == [] +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +@pytest.mark.parametrize("manual_wins", [False, True]) +async def test_competing_manual_and_scheduled_core_snapshots_commit_one_root( + cutoff_backend: Any, manual_wins: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + template, store, config, _, calls, clock = await _harness( + cutoff_backend[0], finality_policy=ReportingContractualCutoffPolicy("unused", "P2D") + ) + config = replace(config, delivery_config_id="concurrent-first-snapshot") + await store.put_configuration(config) + lagging = _restart_producer(template, UnlockedCore(store), clock) + leading = _restart_producer(template, UnlockedCore(store), clock) + entered, release = asyncio.Event(), asyncio.Event() + + async def delayed_close(candidate: Any) -> Any: + entered.set() + await release.wait() + return await store.commit_obligation(candidate) + + monkeypatch.setattr(lagging.store, "commit_obligation", delayed_close) + closing = asyncio.create_task(lagging.close_elapsed_periods(config)) + try: + await asyncio.wait_for(entered.wait(), timeout=5) + (obligation,) = await leading.close_elapsed_periods(config) + finally: + release.set() + closed = await closing + assert closed == [obligation] + + entered, release = asyncio.Event(), asyncio.Event() + candidates: list[ReportingRevisionRecord] = [] + + async def delayed_root(revision: ReportingRevisionRecord, rows: Any) -> Any: + candidates.append(revision) + entered.set() + await release.wait() + return await store.commit_revision(revision, rows) + + monkeypatch.setattr(lagging.store, "commit_revision", delayed_root) + publishing = asyncio.create_task( + lagging.acquire_obligation(config, obligation, manual_replay=not manual_wins) + ) + try: + await asyncio.wait_for(entered.wait(), timeout=5) + winner = await leading.acquire_obligation(config, obligation, manual_replay=manual_wins) + finally: + release.set() + (conflict,) = await asyncio.gather(publishing, return_exceptions=True) + assert isinstance(conflict, LedgerConflictError) and conflict.code == "SUPERSEDES_STALE" + assert len(calls) == 2 + (loser,) = candidates + assert winner is not None + assert loser.reporting_revision_id != winner.reporting_revision_id + assert loser.supersedes_reporting_revision_id is None + assert winner.supersedes_reporting_revision_id is None + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + history = await store.list_revisions(**owner) + assert history == (winner,) + assert await store.commit_revision(winner, ROWS) == winner + selection = select_reporting_revision(history, **owner, required_finality="snapshot") + assert selection.kind == "selected" and selection.revision == winner + health = project_obligation_health( + obligation, history, ledger_as_of=clock[0], scope_closed=True + ) + assert health.current_revision == winner and not health.issues + + @pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) async def test_first_core_snapshot_checks_other_periods_for_a_close(cutoff_backend: Any) -> None: producer, store, config, obligation, calls, clock = await _core_snapshot_harness( From 2371c444f1f42c8844e39b25f5d38e9a2096513b Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 20:44:01 +0530 Subject: [PATCH 27/39] fix(reporting): resume failed first Core snapshots on their frozen identity --- docs/reporting-provisional-observations.md | 12 ++- src/adcp/reporting/ledger/producer.py | 37 ++++--- tests/test_reporting_official_acquisition.py | 104 ++++++++++++++++++- 3 files changed, 131 insertions(+), 22 deletions(-) diff --git a/docs/reporting-provisional-observations.md b/docs/reporting-provisional-observations.md index 37a279ec7..01cbc0a1b 100644 --- a/docs/reporting-provisional-observations.md +++ b/docs/reporting-provisional-observations.md @@ -81,11 +81,13 @@ finality policy. Each scheduled read checks the winner of its own atomic reserva When a definition carries an unused finality policy, a Core producer without observation or checkpoint support can call `close_elapsed_periods` and then `acquire_obligation` for the first snapshot of the period it just created. -After a retryable fetch failure, an interrupted staged object read, or caller -cancellation, the same producer can resume its frozen scheduled request with -a fresh deadline. It checks its recorded slice -retry and any shared backoff before dispatch. Only a scheduled read for that -obligation can claim the retry. The ledger rejects a competing reservation or a +If a manual or scheduled first snapshot encounters a retryable fetch failure, +the same producer can resume the frozen request with a fresh deadline. It +checks the recorded slice retry and any shared backoff before dispatch. An +interrupted staged read, caller cancellation, or an unconfirmed inline storage +operation also preserves the execution identity so the source can replay an +existing seal. Only a scheduled read for that obligation can claim the retry. +The ledger rejects a competing reservation or a second snapshot root during the atomic revision commit. With a selected policy and no account publication lock, additional reads and later periods need durable evidence ruling out a competing close. Recovery checks immutable official diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 86c774c76..dbecf33ae 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -430,14 +430,14 @@ class _StabilizedPublication: @dataclass class _FirstCoreDelivery: - """The scheduled first read remains one claim through dispatch and retry bookkeeping.""" + """The first snapshot remains one claim through dispatch and retry bookkeeping.""" - state: Literal["scheduled", "retry", "consumed"] + state: Literal["dispatching", "retry", "consumed"] retry: RetryScheduleEntry | None = None request: ReportingSourceSliceRequestV1 | None = None def interrupted(self) -> None: - if self.state == "scheduled": + if self.state == "dispatching": self.state = "retry" @@ -1817,7 +1817,7 @@ async def _acquire_obligation( finality = target_finality or obligation.required_finality selected_policy = obligation.definition.finality_policy if obligation.definition else None if finality == "official" and isinstance(selected_policy, ReportingStabilizedPolicy): - await self._claim_first_core_delivery(obligation, now=now, scheduled=False) + await self._claim_first_core_delivery(obligation, now=now, trigger=None) return await self._acquire_stabilized( configuration, obligation, @@ -1860,11 +1860,10 @@ async def _acquire_obligation( offering_id=retry_offering_id, ) return None - first_core = await self._claim_first_core_delivery( - obligation, - now=now, - scheduled=finality == "snapshot" and not (manual_replay or restate or track_settling), - ) + trigger: Literal["scheduled", "manual"] | None = None + if finality == "snapshot" and not track_settling: + trigger = "manual" if manual_replay or restate else "scheduled" + first_core = await self._claim_first_core_delivery(obligation, now=now, trigger=trigger) try: return await self._acquire_due_obligation( configuration, @@ -2031,7 +2030,7 @@ async def _acquire_due_obligation( trigger="manual_replay" if manual_observation else "scheduled_poll", invocation_id=uuid4().hex if manual_snapshot else None, ) - if first_core is not None and first_core.state == "scheduled": + if first_core is not None and first_core.state == "dispatching": first_core.request = request.model_copy(deep=True) acquisition = pending # An explicitly tracked first invocation must survive a failed fetch and restart. @@ -2117,6 +2116,8 @@ async def _acquire_due_obligation( # Return closed data from the executor task: a raw driver exception # must not survive through a context manager or Task wakeup frame. + if result.code == "RESOURCE_UNAVAILABLE" and first_core is not None: + first_core.interrupted() raise InlineStorageError(result.code) if not result.ok: @@ -2360,13 +2361,17 @@ def _retry_keys(self, obligation: ReportingObligationRecord) -> tuple[str, str, ) async def _claim_first_core_delivery( - self, obligation: ReportingObligationRecord, *, now: datetime, scheduled: bool + self, + obligation: ReportingObligationRecord, + *, + now: datetime, + trigger: Literal["scheduled", "manual"] | None, ) -> _FirstCoreDelivery | None: - """Claim the first attempt or resume that scheduled read on its own retry state.""" + """Claim the first attempt or let the scheduler resume its own recorded retry.""" key = (obligation.generation_key, obligation.reporting_obligation_id) state = self._first_core_deliveries.get(key) if isinstance(state, _FirstCoreDelivery): - if not scheduled or ( + if trigger != "scheduled" or ( state.retry is not None and _utc(now) < _utc(state.retry.retry_not_before) ): return None @@ -2376,9 +2381,9 @@ async def _claim_first_core_delivery( if retained != state.retry: self._first_core_deliveries[key] = "attempted" return None - delivery = replace(state, state="scheduled") + delivery = replace(state, state="dispatching") elif state == "ready": - delivery = _FirstCoreDelivery("scheduled" if scheduled else "consumed") + delivery = _FirstCoreDelivery("dispatching" if trigger is not None else "consumed") else: return None self._first_core_deliveries[key] = "attempted" @@ -2484,7 +2489,7 @@ async def _schedule_retry( turn._retry_entries[key] = stored if ( first_core_delivery is not None - and first_core_delivery.state == "scheduled" + and first_core_delivery.state == "dispatching" and key == keys[0] ): if stored == entry: diff --git a/tests/test_reporting_official_acquisition.py b/tests/test_reporting_official_acquisition.py index c2389ed44..dea74568b 100644 --- a/tests/test_reporting_official_acquisition.py +++ b/tests/test_reporting_official_acquisition.py @@ -13,7 +13,11 @@ from typing import Any import pytest -from adcp_reporting_storage import PgReportingLedgerStore +from adcp_reporting_storage import ( + PgReportingLedgerStore, + PgReportingSealStore, + PgReportingStagingStore, +) from adcp.reporting.fixtures import ( OFFICIAL_OFFERING_ID, @@ -26,6 +30,7 @@ InMemorySealStore, InMemoryStagingStore, ) +from adcp.reporting.inline_storage import InlineStorageError from adcp.reporting.ledger import ( InMemoryReportingLedgerStore, LedgerConflictError, @@ -346,6 +351,103 @@ async def test_failed_first_core_delivery_retries_on_its_recorded_schedule( assert calls == [clock[0]] +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +async def test_manual_first_core_snapshot_releases_only_its_due_scheduled_retry( + cutoff_backend: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, obligation, calls, clock = await _core_snapshot_harness( + cutoff_backend[0], ReportingContractualCutoffPolicy("unused", "P2D") + ) + failure = FailingSource(code="RATE_LIMITED", retry_after_seconds=120) + with monkeypatch.context() as patch: + patch.setattr(producer._source, "execute", failure.execute) + assert await producer.acquire_obligation(config, obligation, manual_replay=True) is None + (manual,) = failure.requests + retry = await store.get_retry_schedule(scope_key=producer._retry_keys(obligation)[0]) + assert retry is not None and retry.attempt == 1 + clock[0] = retry.retry_not_before - timedelta(microseconds=1) + assert await producer.acquire_obligation(config, obligation) is None + clock[0] = retry.retry_not_before + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.acquire_obligation(config, obligation, manual_replay=True) + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.close_elapsed_periods(config, now=clock[0] + timedelta(days=1)) + assert calls == [] + recovered = await producer.acquire_obligation(config, obligation) + assert recovered is not None and recovered.finality == "snapshot" + (scheduled,) = producer._source.requests + assert scheduled == manual.model_copy(update={"deadline_at": scheduled.deadline_at}) + assert scheduled.deadline_at > manual.deadline_at + assert await producer.acquire_obligation(config, obligation) is None + assert calls == [clock[0]] + assert (await store.get_retry_schedule(scope_key=retry.scope_key)).attempt == 0 + assert await store.list_revisions( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) == (recovered,) + + +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +async def test_first_core_snapshot_recovers_a_committed_inline_seal( + cutoff_backend: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, obligation, _, clock = await _core_snapshot_harness( + cutoff_backend[0], ReportingContractualCutoffPolicy("unused", "P2D") + ) + if isinstance(store, PgReportingLedgerStore): + staging = PgReportingStagingStore(pool=store._pool) + seals = PgReportingSealStore(pool=store._pool) + await staging.create_schema() + else: + staging = InMemoryStagingStore() + seals = InMemorySealStore() + fetched = [] + + def fetch(request: Any) -> InlineFetchResult: + fetched.append(request) + return InlineFetchResult(rows=ROWS, data_through=request.period.end) + + producer._source = InlineReportingSource( + capabilities=producer._source.capabilities, + fetch=fetch, + staging=staging, + seals=seals, + clock=lambda: clock[0], + ) + producer._object_reader = staging + put = seals.put + + async def lose_acknowledgment(**kwargs: Any) -> None: + await put(**kwargs) + raise InlineStorageError("RESOURCE_UNAVAILABLE") + + with monkeypatch.context() as patch: + patch.setattr(seals, "put", lose_acknowledgment) + with pytest.raises(InlineStorageError) as failure: + await producer.acquire_obligation(config, obligation) + assert failure.value.code == "RESOURCE_UNAVAILABLE" + (first,) = fetched + sealed = await seals.get( + account_id=config.account_id, source_execution_key=first.identity.source_execution_key + ) + assert sealed is not None + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + assert await store.list_revisions(**owner) == () + clock[0] += timedelta(hours=2) + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.acquire_obligation(config, obligation, manual_replay=True) + with pytest.raises(LedgerConflictError, match="publication lock"): + await producer.close_elapsed_periods(config, now=clock[0] + timedelta(days=1)) + recovered = await producer.acquire_obligation(config, obligation) + assert recovered is not None and recovered.finality == "snapshot" + manifest = SourceBatchManifestV1.model_validate_json(sealed.manifest_bytes) + assert recovered.source_publication_id == manifest.publication_id + assert await producer.acquire_obligation(config, obligation) is None + assert fetched == [first] + assert await store.list_revisions(**owner) == (recovered,) + + @pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) @pytest.mark.parametrize( "interruption", ["read_error", "reader_unavailable", "cancel_fetch", "cancel_read"] From 12426ecd2c1bab3551a7be024b1d0fc6d523b67d Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 20:46:37 +0530 Subject: [PATCH 28/39] style(reporting): align finality examples and fixtures with formatter checks --- ...nition-driven-official-reporting-finality.md | 17 +++++++++++------ ...driven-official-reporting-finality-design.md | 3 ++- src/adcp/reporting/ledger/store.py | 7 ++++--- tests/test_reliable_reporting_service.py | 5 ++--- tests/test_reporting_official_acquisition.py | 15 ++++++--------- 5 files changed, 25 insertions(+), 22 deletions(-) diff --git a/docs/superpowers/plans/2026-11-18-definition-driven-official-reporting-finality.md b/docs/superpowers/plans/2026-11-18-definition-driven-official-reporting-finality.md index c089057f1..d665fe10c 100644 --- a/docs/superpowers/plans/2026-11-18-definition-driven-official-reporting-finality.md +++ b/docs/superpowers/plans/2026-11-18-definition-driven-official-reporting-finality.md @@ -176,7 +176,10 @@ async def test_official_configuration_routes_by_selected_policy(policy, expected configuration = official_configuration(policy) await service.configure(configuration) binding = service._bindings[configuration.generation_key] - assert binding.producer._offerings.offering_for(finality_policy_source_class(policy)) == expected_offering + assert ( + binding.producer._offerings.offering_for(finality_policy_source_class(policy)) + == expected_offering + ) async def test_official_configuration_without_policy_fails_before_obligation(): @@ -210,7 +213,9 @@ Expected: missing-policy admission succeeds or the service insists on an authori - [ ] **Step 3: Centralize the source-finality requirement** ```python -def finality_policy_source_class(policy: ReportingFinalityPolicy) -> Literal["snapshot", "official"]: +def finality_policy_source_class( + policy: ReportingFinalityPolicy, +) -> Literal["snapshot", "official"]: return "official" if isinstance(policy, ReportingSourceFinalPolicy) else "snapshot" ``` @@ -346,7 +351,9 @@ git commit -m "feat(reporting): expose retained observation history" ```python async def test_source_final_emits_exact_selected_policy() -> None: policy = ReportingSourceFinalPolicy("delivery-final-v1", "delivery.is_final") - producer, store, configuration, obligation, authoritative_manifest = await official_harness(policy) + producer, store, configuration, obligation, authoritative_manifest = await official_harness( + policy + ) revision = await producer.acquire_obligation(configuration, obligation) @@ -549,9 +556,7 @@ When proven, reuse the qualifying manifest and rows to commit official. If the m ```python def _derived_official_revision_id(manifest, proof: FinalityProof) -> str: digest = hashlib.sha256( - canonical_json_utf8_v1( - [manifest.publication_id, "official", proof.finality_policy_id] - ) + canonical_json_utf8_v1([manifest.publication_id, "official", proof.finality_policy_id]) ).hexdigest() return f"rpr_{digest[:40]}" ``` diff --git a/docs/superpowers/specs/2026-11-18-definition-driven-official-reporting-finality-design.md b/docs/superpowers/specs/2026-11-18-definition-driven-official-reporting-finality-design.md index 3efe27c82..e03d10505 100644 --- a/docs/superpowers/specs/2026-11-18-definition-driven-official-reporting-finality-design.md +++ b/docs/superpowers/specs/2026-11-18-definition-driven-official-reporting-finality-design.md @@ -1,6 +1,7 @@ # Definition-Driven Official Reporting Finality -Status: approved for implementation +Status: approved for implementation + Tracking: AI-22659 ## Context diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index 122477a79..f543d3ae9 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -978,9 +978,10 @@ def __init__( self._restatement_checkpoints: dict[str, RestatementCheckpoint] = {} self._retry_schedules: dict[str, RetryScheduleEntry] = {} self._provisional_acquisitions: dict[tuple[str, str, int], ProvisionalAcquisition] = {} - self._provisional_observations: dict[tuple[str, str], dict[int, ProvisionalObservation]] = ( - {} - ) + self._provisional_observations: dict[ + tuple[str, str], + dict[int, ProvisionalObservation], + ] = {} self._adjustments: dict[str, ReportingAdjustmentRecord] = {} self._statuses: dict[tuple[str, str, str], ConsumerStatusRecord] = {} self._status_identity: dict[tuple[str, str, str], str] = {} diff --git a/tests/test_reliable_reporting_service.py b/tests/test_reliable_reporting_service.py index ec44336fc..a1c3c5783 100644 --- a/tests/test_reliable_reporting_service.py +++ b/tests/test_reliable_reporting_service.py @@ -801,9 +801,8 @@ async def test_official_configuration_must_match_the_declared_public_definition( ) config = replace(_configuration(), required_finality="official", definition=definition) raw = redacted_capabilities().model_dump(mode="json") - raw["offerings"][0]["contract"][ - "report_definition_sha256" - ] = definition.report_definition_sha256 + contract = raw["offerings"][0]["contract"] + contract["report_definition_sha256"] = definition.report_definition_sha256 raw["capabilities_sha256"] = reporting_source_capabilities_sha256_v1(raw) public = {**_capability_offering("gam"), "supported_finality": ["official"]} context = replace(_account_context(config), capability_offering=public) diff --git a/tests/test_reporting_official_acquisition.py b/tests/test_reporting_official_acquisition.py index dea74568b..94cf7ddff 100644 --- a/tests/test_reporting_official_acquisition.py +++ b/tests/test_reporting_official_acquisition.py @@ -254,9 +254,8 @@ async def cutoff_backend(request: Any) -> AsyncIterator[Any]: async with isolated_reporting_pool() as pool: yield partial(PgReportingLedgerStore, pool=pool), False else: - yield ( - ProgressStore if request.param == "progress" else InMemoryReportingLedgerStore - ), request.param == "core" + factory = ProgressStore if request.param == "progress" else InMemoryReportingLedgerStore + yield factory, request.param == "core" async def _core_snapshot_harness(factory: Any, policy: Any) -> tuple[Any, ...]: @@ -1688,12 +1687,10 @@ async def test_invalid_source_namespace_leaves_first_reservation_available(route if route == "bound_snapshot": from adcp.reporting.production.contracts import ReportingProductionSourceBinding - producer._source.configuration_binding = ( - lambda configuration: ReportingProductionSourceBinding.for_configuration( - configuration, - capabilities_sha256=producer._source.capabilities.capabilities_sha256, - media_buy_products=(("media-buy-redacted", "fixture-product"),), - ) + producer._source.configuration_binding = partial( + ReportingProductionSourceBinding.for_configuration, + capabilities_sha256=producer._source.capabilities.capabilities_sha256, + media_buy_products=(("media-buy-redacted", "fixture-product"),), ) correct = producer._offerings producer._offerings = replace(correct, publication_namespace="other-namespace") From 92708c37fc42766da470f11d3dca06b1c7907729 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 21:03:18 +0530 Subject: [PATCH 29/39] refactor(reporting): keep verified snapshot rows bound to their revision --- src/adcp/reporting/ledger/producer.py | 10 ++++++---- src/adcp/reporting/ledger/provisional.py | 2 +- tests/test_reporting_official_acquisition.py | 4 ++++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index dbecf33ae..683e858f7 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -1626,7 +1626,8 @@ async def _stabilized_publication( ) verified: list[tuple[ProvisionalObservation, str, bytes]] = [] previous: ProvisionalObservation | None = None - revision = None + retained: tuple[ReportingRevisionRecord, bytes] | None = None + manifest: SourceBatchManifestV1 | None = None rows: list[dict[str, Any]] = [] for observation in history: acquisition = observation.acquisition @@ -1645,7 +1646,7 @@ async def _stabilized_publication( "HISTORY_UNAVAILABLE", "stabilization history acquisition differs" ) previous = observation - if revision is None or revision.reporting_revision_id != observation.revision_id: + if retained is None or retained[0].reporting_revision_id != observation.revision_id: revision = await self._store.get_revision( account_id=obligation.account_id, reporting_revision_id=observation.revision_id ) @@ -1660,7 +1661,8 @@ async def _stabilized_publication( "HISTORY_UNAVAILABLE", "stabilization history requires a readable snapshot" ) rows = await self._retained_snapshot_rows(revision) - row_binding = hashlib.sha256(rfc8785.dumps(rows)).digest() + retained = revision, hashlib.sha256(rfc8785.dumps(rows)).digest() + revision, row_binding = retained try: manifest = SourceBatchManifestV1.model_validate_json(observation.manifest_json) self._validate_source_manifest( @@ -1711,7 +1713,7 @@ async def _stabilized_publication( ) from error verified.append((observation, manifest.content_fingerprint, row_binding)) proof = prove_stabilized_finality(obligation, policy, verified) - if proof is None: + if proof is None or manifest is None: return None return _StabilizedPublication(proof, manifest, rows) diff --git a/src/adcp/reporting/ledger/provisional.py b/src/adcp/reporting/ledger/provisional.py index 893f3e46f..b1b4ec70d 100644 --- a/src/adcp/reporting/ledger/provisional.py +++ b/src/adcp/reporting/ledger/provisional.py @@ -402,7 +402,7 @@ async def list_provisional_observations( A scheduled snapshot filter applies before the limit. ``limit`` must be a positive integer. """ - pass + ... @runtime_checkable diff --git a/tests/test_reporting_official_acquisition.py b/tests/test_reporting_official_acquisition.py index 94cf7ddff..1c7634e1b 100644 --- a/tests/test_reporting_official_acquisition.py +++ b/tests/test_reporting_official_acquisition.py @@ -29,6 +29,8 @@ InlineReportingSource, InMemorySealStore, InMemoryStagingStore, + ReportingSealStore, + ReportingStagingStore, ) from adcp.reporting.inline_storage import InlineStorageError from adcp.reporting.ledger import ( @@ -399,6 +401,8 @@ async def test_first_core_snapshot_recovers_a_committed_inline_seal( else: staging = InMemoryStagingStore() seals = InMemorySealStore() + assert isinstance(staging, ReportingStagingStore) + assert isinstance(seals, ReportingSealStore) fetched = [] def fetch(request: Any) -> InlineFetchResult: From fe591d134990c5730f6fe8526ebcff9689f67e68 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 21:16:27 +0530 Subject: [PATCH 30/39] fix(reporting): preserve historical PostgreSQL checkpoint contracts --- .../src/adcp_reporting_storage/ledger/pg.py | 129 ++++++++++++------ .../ledger/reporting_ledger.sql | 4 - .../reporting_provisional_observations.sql | 11 ++ .../test_reporting_notification_migration.py | 17 ++- tests/test_reporting_official_finality.py | 49 ++++++- 5 files changed, 158 insertions(+), 52 deletions(-) diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py index ecc45025a..37dfebc75 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py @@ -800,6 +800,36 @@ async def _require_provisional_schema(self, connection: Any) -> None: await validate_provisional_schema(connection) + async def _provisional_tables_installed(self, connection: Any) -> bool: + row = await ( + await connection.execute( + "SELECT to_regclass(quote_ident(current_schema())" + " || '.reporting_provisional_acquisitions') IS NOT NULL" + " OR to_regclass(quote_ident(current_schema())" + " || '.reporting_provisional_observations') IS NOT NULL" + ) + ).fetchone() + return bool(row and row[0]) + + async def _stabilization_checkpoint_join(self, connection: Any) -> str: + """Join private policy metadata when reading a checkpoint on either schema version.""" + row = await ( + await connection.execute( + "SELECT to_regclass(quote_ident(current_schema())" + " || '.adcp_reporting_stabilization_checkpoints') IS NOT NULL" + ) + ).fetchone() + if row and row[0]: + return ( + " LEFT JOIN adcp_reporting_stabilization_checkpoints s" + " ON s.reporting_obligation_id=c.reporting_obligation_id" + " AND s.next_observation=c.next_observation AND s.checked_at=c.checked_at" + ) + return ( + " CROSS JOIN (SELECT NULL::jsonb AS stabilization_policy," + " NULL::timestamptz AS next_stabilization_due_at) s" + ) + async def _notification_now(self, connection: Any) -> datetime: from adcp_reporting_storage.outbox.pg import database_now @@ -1504,20 +1534,21 @@ async def commit_revision( "this period is being retired; no revision may be added to it", ) validate_revision_currency(_obligation_from_row(obligation), revision, rows) - pending = await ( - await connection.execute( - "SELECT 1 FROM reporting_provisional_acquisitions a" - " LEFT JOIN reporting_restatement_checkpoints c" - " USING (account_id, reporting_obligation_id)" - " WHERE a.account_id=%s AND a.reporting_obligation_id=%s" - " AND a.ordinal >= COALESCE(c.next_observation, 0) LIMIT 1", - (revision.account_id, revision.reporting_obligation_id), - ) - ).fetchone() - if pending is not None: - raise LedgerConflictError( - "OBSERVATION_CONFLICT", "a source acquisition is reserved" - ) + if await self._provisional_tables_installed(connection): + pending = await ( + await connection.execute( + "SELECT 1 FROM reporting_provisional_acquisitions a" + " LEFT JOIN reporting_restatement_checkpoints c" + " USING (account_id, reporting_obligation_id)" + " WHERE a.account_id=%s AND a.reporting_obligation_id=%s" + " AND a.ordinal >= COALESCE(c.next_observation, 0) LIMIT 1", + (revision.account_id, revision.reporting_obligation_id), + ) + ).fetchone() + if pending is not None: + raise LedgerConflictError( + "OBSERVATION_CONFLICT", "a source acquisition is reserved" + ) if ( revision.finality == "snapshot" and await ( @@ -1871,25 +1902,20 @@ async def get_stabilization_schedule( ) -> ProvisionalPolicy | None: async with self._connection() as connection: # configure() can precede schema creation on a brand-new installation. - installed = await ( - await connection.execute( - "SELECT to_regclass('reporting_provisional_acquisitions') IS NOT NULL" - " OR to_regclass('reporting_provisional_observations') IS NOT NULL" - ) - ).fetchone() - if installed is not None and not installed[0]: + if not await self._provisional_tables_installed(connection): return None await self._require_provisional_schema(connection) - # The checkpoint's frozen policy can also be absent on pre-upgrade tables. + policy_join = await self._stabilization_checkpoint_join(connection) row = await ( await connection.execute( - "SELECT a.payload->'policy' FROM reporting_obligations o" + "SELECT a.payload->'policy' FROM reporting_obligations o" # noqa: S608 # nosec B608 " JOIN reporting_provisional_acquisitions a" " USING(account_id, reporting_obligation_id)" " LEFT JOIN reporting_provisional_observations h" " USING(account_id, reporting_obligation_id, ordinal)" " LEFT JOIN reporting_restatement_checkpoints c" " USING(account_id, reporting_obligation_id)" + f"{policy_join}" " WHERE o.account_id=%s AND o.consumer_id=%s AND o.delivery_config_id=%s" " AND o.delivery_config_version=%s" " AND a.payload #>> '{request,trigger}' = 'scheduled_poll'" @@ -1900,7 +1926,7 @@ async def get_stabilization_schedule( " AND a.payload #>> '{policy,until_stabilized}' IS DISTINCT FROM 'true'))" " AND (h.ordinal IS NOT NULL OR c.next_observation IS NULL" " OR c.next_observation=a.ordinal OR (c.next_observation>a.ordinal" - " AND to_jsonb(c)->'stabilization_policy'=a.payload->'policy'))" + " AND s.stabilization_policy=a.payload->'policy'))" " AND (o.required_finality='official'" " OR a.payload #>> '{policy,until_stabilized}' = 'true'" " OR a.payload #>> '{policy,official_close_lag_us}' IS NOT NULL)" @@ -2154,13 +2180,14 @@ async def get_restatement_checkpoint( self, *, account_id: str, reporting_obligation_id: str ) -> RestatementCheckpoint | None: async with self._connection() as connection: + policy_join = await self._stabilization_checkpoint_join(connection) row = await ( await connection.execute( - "SELECT account_id, reporting_obligation_id, checked_at," - " next_observation, provisional_until, stabilization_policy," - " next_stabilization_due_at" - " FROM reporting_restatement_checkpoints" - " WHERE account_id = %s AND reporting_obligation_id = %s", + "SELECT c.account_id, c.reporting_obligation_id, c.checked_at," # noqa: S608 # nosec B608 + " c.next_observation, c.provisional_until, s.stabilization_policy," + " s.next_stabilization_due_at" + f" FROM reporting_restatement_checkpoints c{policy_join}" + " WHERE c.account_id = %s AND c.reporting_obligation_id = %s", (account_id, reporting_obligation_id), ) ).fetchone() @@ -2179,7 +2206,7 @@ async def get_restatement_checkpoint( async def record_restatement_checkpoint( self, checkpoint: RestatementCheckpoint ) -> RestatementCheckpoint: - async with self._connection() as connection: + async with self._connection() as connection, connection.transaction(): obligation = await ( await connection.execute( "SELECT 1 FROM reporting_obligations" @@ -2196,14 +2223,12 @@ async def record_restatement_checkpoint( await connection.execute( "INSERT INTO reporting_restatement_checkpoints" " (account_id, reporting_obligation_id, checked_at, next_observation," - " provisional_until, stabilization_policy, next_stabilization_due_at)" - " VALUES (%s, %s, %s, %s, %s, %s::jsonb, %s)" + " provisional_until)" + " VALUES (%s, %s, %s, %s, %s)" " ON CONFLICT (reporting_obligation_id) DO UPDATE SET" " checked_at = EXCLUDED.checked_at," " next_observation = EXCLUDED.next_observation," - " provisional_until = EXCLUDED.provisional_until," - " stabilization_policy = EXCLUDED.stabilization_policy," - " next_stabilization_due_at = EXCLUDED.next_stabilization_due_at" + " provisional_until = EXCLUDED.provisional_until" " WHERE reporting_restatement_checkpoints.account_id = EXCLUDED.account_id" " AND (reporting_restatement_checkpoints.next_observation" " < EXCLUDED.next_observation" @@ -2212,23 +2237,41 @@ async def record_restatement_checkpoint( " AND reporting_restatement_checkpoints.checked_at" " < EXCLUDED.checked_at))" " RETURNING account_id, reporting_obligation_id, checked_at," - " next_observation, provisional_until, stabilization_policy," - " next_stabilization_due_at", + " next_observation, provisional_until", ( checkpoint.account_id, checkpoint.reporting_obligation_id, checkpoint.checked_at, checkpoint.next_observation, checkpoint.provisional_until, + ), + ) + ).fetchone() + if row and ( + checkpoint.stabilization_policy is not None + or checkpoint.next_stabilization_due_at is not None + ): + await connection.execute( + "INSERT INTO adcp_reporting_stabilization_checkpoints" + " (reporting_obligation_id, next_observation, checked_at," + " stabilization_policy, next_stabilization_due_at)" + " VALUES (%s, %s, %s, %s::jsonb, %s)" + " ON CONFLICT (reporting_obligation_id) DO UPDATE SET" + " next_observation=EXCLUDED.next_observation, checked_at=EXCLUDED.checked_at," + " stabilization_policy=EXCLUDED.stabilization_policy," + " next_stabilization_due_at=EXCLUDED.next_stabilization_due_at", + ( + checkpoint.reporting_obligation_id, + checkpoint.next_observation, + checkpoint.checked_at, ( json.dumps(checkpoint.stabilization_policy.to_wire()) - if checkpoint.stabilization_policy + if checkpoint.stabilization_policy is not None else None ), checkpoint.next_stabilization_due_at, ), ) - ).fetchone() if row is None: stored = await self.get_restatement_checkpoint( account_id=checkpoint.account_id, @@ -2246,8 +2289,12 @@ async def record_restatement_checkpoint( checked_at=_utc(row[2]), next_observation=int(row[3]), provisional_until=_utc(row[4]) if row[4] else None, - stabilization_policy=ProvisionalPolicy.from_wire(row[5]) if row[5] else None, - next_stabilization_due_at=_utc(row[6]) if row[6] else None, + stabilization_policy=checkpoint.stabilization_policy, + next_stabilization_due_at=( + _utc(checkpoint.next_stabilization_due_at) + if checkpoint.next_stabilization_due_at is not None + else None + ), ) async def get_revision( diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger.sql b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger.sql index 224a727e1..cb57362cc 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger.sql +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_ledger.sql @@ -199,10 +199,6 @@ CREATE TABLE IF NOT EXISTS reporting_restatement_checkpoints ( CREATE INDEX IF NOT EXISTS reporting_restatement_checkpoints_account_idx ON reporting_restatement_checkpoints (account_id, checked_at); -ALTER TABLE reporting_restatement_checkpoints - ADD COLUMN IF NOT EXISTS stabilization_policy JSONB, - ADD COLUMN IF NOT EXISTS next_stabilization_due_at TIMESTAMPTZ; - -- Mutable scheduling state; it does not enter the immutable ledger change feed. CREATE TABLE IF NOT EXISTS adcp_reporting_producer_retry_schedules ( scope_key TEXT COLLATE "C" PRIMARY KEY, diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_provisional_observations.sql b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_provisional_observations.sql index 78fc6368a..6618949ae 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_provisional_observations.sql +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_provisional_observations.sql @@ -52,3 +52,14 @@ DROP TRIGGER IF EXISTS reporting_provisional_observation_immutable CREATE TRIGGER reporting_provisional_observation_immutable BEFORE UPDATE OR DELETE ON reporting_provisional_observations FOR EACH ROW EXECUTE FUNCTION reporting_provisional_immutable(); + +-- Keep the historical reporting_* checkpoint catalog stable during rolling upgrades. +-- The ordinal and clock bind this optional policy to the exact checkpoint that wrote it. +CREATE TABLE IF NOT EXISTS adcp_reporting_stabilization_checkpoints ( + reporting_obligation_id TEXT COLLATE "C" PRIMARY KEY + REFERENCES reporting_restatement_checkpoints (reporting_obligation_id) ON DELETE CASCADE, + next_observation BIGINT NOT NULL, + checked_at TIMESTAMPTZ NOT NULL, + stabilization_policy JSONB, + next_stabilization_due_at TIMESTAMPTZ +); diff --git a/tests/conformance/reporting/test_reporting_notification_migration.py b/tests/conformance/reporting/test_reporting_notification_migration.py index 0f34c401d..e0fe06bbe 100644 --- a/tests/conformance/reporting/test_reporting_notification_migration.py +++ b/tests/conformance/reporting/test_reporting_notification_migration.py @@ -12,13 +12,13 @@ from adcp_reporting_storage.outbox._schema import schema_contract, validate_schema from adcp_reporting_storage.outbox.status_schema import REQUIRED_STATUS_OBJECTS +from adcp.reporting.ledger import ReportingAdjustmentRecord from adcp.reporting.outbox import ReportingNotificationError from ._generation_support import ( NOW, isolated_reporting_pool, require_rolling_database, - revision_for, ) from ._reconciliation_support import scenario from ._reliable_support import Barrier @@ -146,8 +146,19 @@ async def test_populated_pre_outbox_upgrade_never_rewrites_or_backfills(autocomm # Repeated installation neither rebuilds constraints nor changes MVCC rows. await store.create_schema() assert await retained_physical_rows(pool) == before - revision, rows = revision_for(s.obligation, suffix="post-upgrade") - await store.commit_revision(revision, rows) + await store.commit_adjustment( + ReportingAdjustmentRecord( + "post-upgrade", + s.obligation.account_id, + s.revision.reporting_revision_id, + "source_correction", + s.obligation.period.start, + s.obligation.period.end, + (("impressions", "-1"),), + NOW, + NOW, + ) + ) restarted = PgReportingOutbox(pool=pool, clock=lambda: NOW) assert len(await restarted.list_events(account_id="acct_a")) == 1 diff --git a/tests/test_reporting_official_finality.py b/tests/test_reporting_official_finality.py index 26a69fdf0..5c56c6c10 100644 --- a/tests/test_reporting_official_finality.py +++ b/tests/test_reporting_official_finality.py @@ -38,6 +38,7 @@ ReportingStabilizedPolicy, ReportingStatusCaller, ReportingStatusHandler, + RestatementCheckpoint, revision_content_sha256, ) from adcp.reporting.ledger.finality import ( @@ -166,16 +167,56 @@ async def test_previous_checkpoint_schema_recovers_schedule_before_initialize() ) await producer.run_configuration(config) async with pool.connection() as connection: - await connection.execute( - "ALTER TABLE reporting_restatement_checkpoints DROP COLUMN stabilization_policy," - " DROP COLUMN next_stabilization_due_at" - ) + await connection.execute("DROP TABLE adcp_reporting_stabilization_checkpoints") policy = await producer._recover_stabilization_schedule(config) assert policy is not None and policy.cadence == timedelta(hours=6) await store.create_schema() assert await store.get_stabilization_schedule(config.generation_key) == policy +@pytest.mark.parametrize("later_clock", [False, True]) +async def test_older_writer_advancing_checkpoint_retires_its_stabilization_metadata( + later_clock: bool, +) -> None: + async with isolated_reporting_pool() as pool: + _, store, config, obligation, _, clock = await _stabilized_harness( + partial(PgReportingLedgerStore, pool=pool) + ) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + policy = ProvisionalPolicy(timedelta(days=1), timedelta(hours=6), until_stabilized=True) + first = RestatementCheckpoint( + account_id=config.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + checked_at=clock[0], + next_observation=1, + stabilization_policy=policy, + next_stabilization_due_at=clock[0] + policy.cadence, + ) + assert await store.record_restatement_checkpoint(first) == first + assert await store.get_restatement_checkpoint(**owner) == first + advanced = replace( + first, + checked_at=first.checked_at + timedelta(minutes=1) if later_clock else first.checked_at, + next_observation=1 if later_clock else 2, + stabilization_policy=None, + next_stabilization_due_at=None, + ) + async with pool.connection() as connection: + await connection.execute( + "UPDATE reporting_restatement_checkpoints SET checked_at=%s, next_observation=%s" + " WHERE reporting_obligation_id=%s AND account_id=%s", + ( + advanced.checked_at, + advanced.next_observation, + obligation.reporting_obligation_id, + config.account_id, + ), + ) + assert await store.get_restatement_checkpoint(**owner) == advanced + + @pytest.mark.parametrize("tracked", [False, True]) async def test_manual_change_between_equal_scheduled_reads_preserves_stabilization( store_factory: Any, monkeypatch: pytest.MonkeyPatch, tracked: bool From ff1c8825fa58cedcce114fb34021d52fcb4f4cfb Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 21:18:26 +0530 Subject: [PATCH 31/39] docs(reporting): describe stabilization checkpoint persistence --- docs/reporting-provisional-observations.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/reporting-provisional-observations.md b/docs/reporting-provisional-observations.md index 01cbc0a1b..619708839 100644 --- a/docs/reporting-provisional-observations.md +++ b/docs/reporting-provisional-observations.md @@ -96,11 +96,14 @@ lookup returns no schedule. ## Persistence and compatibility -`PgReportingLedgerStore.create_schema()` installs two additive private tables: -`reporting_provisional_acquisitions` and `reporting_provisional_observations`. -They retain immutable reservation and observation metadata. The existing -restatement checkpoint remains the progress cursor. The in-memory store has the -same atomic operations. Custom producer stores must implement +`PgReportingLedgerStore.create_schema()` installs +`reporting_provisional_acquisitions` and `reporting_provisional_observations` +for immutable reservations and observations. The existing restatement checkpoint +tracks progress. Its stabilization policy and next due time live in +`adcp_reporting_stabilization_checkpoints`, keyed to the checkpoint's obligation, +ordinal and timestamp. An older writer can advance the checkpoint without +inheriting the previous policy or deadline. The in-memory store has the same +atomic operations. Custom producer stores must implement `ProvisionalObservationStore` from `adcp.reporting.ledger.provisional` together with `RestatementCheckpointStore`. From 324a20de96bdc95a4641ab8e88f6bd3dce3c8a54 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 21:34:09 +0530 Subject: [PATCH 32/39] test(reporting): exercise readiness after removing checkpoint storage --- .../reporting/test_reporting_notification_migration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conformance/reporting/test_reporting_notification_migration.py b/tests/conformance/reporting/test_reporting_notification_migration.py index e0fe06bbe..ee06111bd 100644 --- a/tests/conformance/reporting/test_reporting_notification_migration.py +++ b/tests/conformance/reporting/test_reporting_notification_migration.py @@ -286,7 +286,7 @@ async def install_then_rollback(): "CREATE OR REPLACE FUNCTION reporting_notification_immutable() RETURNS TRIGGER" " LANGUAGE plpgsql AS $$ BEGIN RETURN NEW; END $$" ), - "DROP TABLE reporting_restatement_checkpoints", + "DROP TABLE reporting_restatement_checkpoints CASCADE", "ALTER TABLE reporting_restatement_checkpoints ALTER COLUMN next_observation DROP NOT NULL", "DROP INDEX reporting_restatement_checkpoints_account_idx", ( From 6be29d1bc807b7622c9ef23c6d27c7d312b2213d Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 21:44:52 +0530 Subject: [PATCH 33/39] test(reporting): bind late-account producers to their declared finality signal --- tests/conformance/reporting/_late_account_support.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/conformance/reporting/_late_account_support.py b/tests/conformance/reporting/_late_account_support.py index 0fd2dfa24..e47eb0da5 100644 --- a/tests/conformance/reporting/_late_account_support.py +++ b/tests/conformance/reporting/_late_account_support.py @@ -10,8 +10,9 @@ from adcp.reporting.inline_source import InlineFetchResult from adcp.reporting.materializer import ReportingRevisionVerifier, reference_verifier from adcp.reporting.materializer.contracts import failure +from adcp.reporting.source import ReportingFinalityEvidenceV1 -from ._production_support import Source, SQLiteDestination +from ._production_support import Source, SQLiteDestination, reference_source_final ACCOUNTS = {"usd": ("USD", 503), "eur": ("EUR", 3)} @@ -35,7 +36,12 @@ async def fetch(self, request): account = str(request.identity.account_id) self.requests.append(request) return InlineFetchResult( - rows_for(account), data_through=request.period.end, currency=ACCOUNTS[account][0] + rows_for(account), + data_through=request.period.end, + currency=ACCOUNTS[account][0], + finality_evidence=ReportingFinalityEvidenceV1( + basis="provider_declared", observed_at=self.clock(), evidence_ref="fixture_closed" + ), ) @@ -55,6 +61,8 @@ def resolve(self, request, *, phase, context): def verifier_for(currency, capability): base = reference_verifier(capability) + definition_binding = replace(base.key.definition, finality_policy=reference_source_final()) + base = replace(base, key=replace(base.key, definition=definition_binding)) if currency == "USD": return base assert currency == "EUR" From 88f43c49782d21796287e71d133f1affee78c201 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 21:53:36 +0530 Subject: [PATCH 34/39] test(reporting): isolate finality policy tests from frozen SDK controls --- tests/test_reporting_finality_policy.py | 122 ++++++++++++++++++++++++ tests/test_reporting_ledger.py | 113 +--------------------- 2 files changed, 123 insertions(+), 112 deletions(-) create mode 100644 tests/test_reporting_finality_policy.py diff --git a/tests/test_reporting_finality_policy.py b/tests/test_reporting_finality_policy.py new file mode 100644 index 000000000..35638b7ac --- /dev/null +++ b/tests/test_reporting_finality_policy.py @@ -0,0 +1,122 @@ +"""Private selected finality policies validate and round-trip through ledger storage.""" + +from __future__ import annotations + +from dataclasses import replace + +import pytest + +from adcp.reporting.ledger import ( + ReportingContractualCutoffPolicy, + ReportingDefinitionBinding, + ReportingFinalityPolicy, + ReportingSourceFinalPolicy, + ReportingStabilizedPolicy, + reporting_finality_policy_from_storage, +) +from adcp.reporting.ledger._record_rows import _definition_from_payload + +DEFINITION = ReportingDefinitionBinding( + report_definition_uri="https://contracts.example.test/reporting/daily-delivery-v1", + report_definition_sha256="a" * 64, + schema_version="1.0.0", + schema_uri="https://contracts.example.test/reporting/rows-v1.json", + schema_sha256="b" * 64, +) + + +def test_definition_binding_retains_selected_finality_policy_off_wire() -> None: + policy = ReportingStabilizedPolicy( + finality_policy_id="daily-stable-v1", + minimum_age="P2D", + unchanged_for="PT12H", + ) + binding = replace(DEFINITION, finality_policy=policy) + + assert binding.to_wire() == DEFINITION.to_wire() + assert binding.to_storage()["finality_policy"] == { + "finality_policy_id": "daily-stable-v1", + "basis": "stabilized", + "minimum_age": "P2D", + "unchanged_for": "PT12H", + } + assert _definition_from_payload(binding.to_storage()) == binding + + +@pytest.mark.parametrize("value", ["P1M", "P1Y", "not-a-duration", "PT1H\n"]) +def test_finality_policy_refuses_unsupported_duration(value: str) -> None: + with pytest.raises(ValueError, match="supported reporting duration"): + ReportingContractualCutoffPolicy("cutoff-v1", value) + + +@pytest.mark.parametrize( + ("policy", "payload"), + [ + ( + ReportingSourceFinalPolicy("daily.source:v1", "source_closed"), + { + "finality_policy_id": "daily.source:v1", + "basis": "source_final", + "source_signal": "source_closed", + }, + ), + ( + ReportingContractualCutoffPolicy("cutoff-v1", "PT4H"), + { + "finality_policy_id": "cutoff-v1", + "basis": "contractual_cutoff", + "duration_after_period_end": "PT4H", + }, + ), + ], +) +def test_finality_policy_storage_round_trip( + policy: ReportingFinalityPolicy, payload: dict[str, str] +) -> None: + assert policy.to_storage() == payload + assert reporting_finality_policy_from_storage(payload) == policy + + +@pytest.mark.parametrize( + "payload", + [ + {"finality_policy_id": "daily-v1", "basis": "unknown"}, + {"finality_policy_id": "daily-v1", "basis": "source_final"}, + {"finality_policy_id": "daily-v1", "basis": "source_final", "source_signal": ["close"]}, + { + "finality_policy_id": "daily-v1", + "basis": "source_final", + "source_signal": "close", + "duration_after_period_end": "PT4H", + }, + None, + ], +) +def test_definition_binding_rejects_invalid_finality_policy_storage(payload: object) -> None: + with pytest.raises(ValueError, match="finality policy"): + _definition_from_payload({**DEFINITION.to_storage(), "finality_policy": payload}) + + +def test_definition_binding_reads_storage_without_a_finality_policy() -> None: + assert DEFINITION.to_storage() == DEFINITION.to_wire() + assert _definition_from_payload(DEFINITION.to_storage()) == DEFINITION + + +def test_definition_binding_rejects_finality_policy_subclass() -> None: + class CustomPolicy(ReportingSourceFinalPolicy): + pass + + with pytest.raises(ValueError, match="finality_policy"): + replace(DEFINITION, finality_policy=CustomPolicy("daily-v1", "close")) + + +@pytest.mark.parametrize("policy_id", ["", "a" * 256, "contains spaces", "close\n"]) +def test_finality_policy_refuses_invalid_identifier(policy_id: str) -> None: + with pytest.raises(ValueError, match="finality_policy_id"): + ReportingSourceFinalPolicy(policy_id, "close") + + +@pytest.mark.parametrize("source_signal", ["", "a" * 513]) +def test_source_finality_policy_refuses_invalid_signal(source_signal: str) -> None: + with pytest.raises(ValueError, match="source_signal must contain 1 to 512 characters"): + ReportingSourceFinalPolicy("daily-v1", source_signal) diff --git a/tests/test_reporting_ledger.py b/tests/test_reporting_ledger.py index 5daccc635..e6dc209e4 100644 --- a/tests/test_reporting_ledger.py +++ b/tests/test_reporting_ledger.py @@ -8,7 +8,6 @@ from __future__ import annotations -from dataclasses import replace from datetime import datetime, timedelta, timezone from typing import Any, cast @@ -22,16 +21,11 @@ ProducerOfferings, ReportingAdjustmentRecord, ReportingConfiguration, - ReportingContractualCutoffPolicy, - ReportingDefinitionBinding, ReportingDeliveryEscalation, - ReportingFinalityPolicy, ReportingObligationRecord, ReportingProducer, ReportingRevisionRecord, ReportingScheduleSpec, - ReportingSourceFinalPolicy, - ReportingStabilizedPolicy, ReportingStatusCaller, ReportingStatusHandler, aggregate_reporting_health, @@ -39,11 +33,9 @@ derive_period, iso_duration_to_timedelta, project_obligation_health, - reporting_finality_policy_from_storage, revision_content_sha256, ) from adcp.reporting.ledger import ReportingStatusCaller as OwnershipCaller -from adcp.reporting.ledger._record_rows import _definition_from_payload ACCOUNT = "acct_1" CALLER = ReportingStatusCaller(account_id=ACCOUNT, consumer_id="buyer_1") @@ -70,13 +62,6 @@ } HOURLY = ReportingScheduleSpec(period_duration="PT1H", delivery_sla="PT1H", alignment="utc") -DEFINITION = ReportingDefinitionBinding( - report_definition_uri="https://contracts.example.test/reporting/daily-delivery-v1", - report_definition_sha256="a" * 64, - schema_version="1.0.0", - schema_uri="https://contracts.example.test/reporting/rows-v1.json", - schema_sha256="b" * 64, -) def _configuration(**overrides) -> ReportingConfiguration: @@ -176,103 +161,6 @@ def _revision(obligation: ReportingObligationRecord, **overrides) -> ReportingRe return ReportingRevisionRecord(**defaults), rows # type: ignore[return-value,arg-type] -def test_definition_binding_retains_selected_finality_policy_off_wire() -> None: - policy = ReportingStabilizedPolicy( - finality_policy_id="daily-stable-v1", - minimum_age="P2D", - unchanged_for="PT12H", - ) - binding = replace(DEFINITION, finality_policy=policy) - - assert binding.to_wire() == DEFINITION.to_wire() - assert binding.to_storage()["finality_policy"] == { - "finality_policy_id": "daily-stable-v1", - "basis": "stabilized", - "minimum_age": "P2D", - "unchanged_for": "PT12H", - } - assert _definition_from_payload(binding.to_storage()) == binding - - -@pytest.mark.parametrize("value", ["P1M", "P1Y", "not-a-duration", "PT1H\n"]) -def test_finality_policy_refuses_unsupported_duration(value: str) -> None: - with pytest.raises(ValueError, match="supported reporting duration"): - ReportingContractualCutoffPolicy("cutoff-v1", value) - - -@pytest.mark.parametrize( - ("policy", "payload"), - [ - ( - ReportingSourceFinalPolicy("daily.source:v1", "source_closed"), - { - "finality_policy_id": "daily.source:v1", - "basis": "source_final", - "source_signal": "source_closed", - }, - ), - ( - ReportingContractualCutoffPolicy("cutoff-v1", "PT4H"), - { - "finality_policy_id": "cutoff-v1", - "basis": "contractual_cutoff", - "duration_after_period_end": "PT4H", - }, - ), - ], -) -def test_finality_policy_storage_round_trip( - policy: ReportingFinalityPolicy, payload: dict[str, str] -) -> None: - assert policy.to_storage() == payload - assert reporting_finality_policy_from_storage(payload) == policy - - -@pytest.mark.parametrize( - "payload", - [ - {"finality_policy_id": "daily-v1", "basis": "unknown"}, - {"finality_policy_id": "daily-v1", "basis": "source_final"}, - {"finality_policy_id": "daily-v1", "basis": "source_final", "source_signal": ["close"]}, - { - "finality_policy_id": "daily-v1", - "basis": "source_final", - "source_signal": "close", - "duration_after_period_end": "PT4H", - }, - None, - ], -) -def test_definition_binding_rejects_invalid_finality_policy_storage(payload: object) -> None: - with pytest.raises(ValueError, match="finality policy"): - _definition_from_payload({**DEFINITION.to_storage(), "finality_policy": payload}) - - -def test_definition_binding_reads_storage_without_a_finality_policy() -> None: - assert DEFINITION.to_storage() == DEFINITION.to_wire() - assert _definition_from_payload(DEFINITION.to_storage()) == DEFINITION - - -def test_definition_binding_rejects_finality_policy_subclass() -> None: - class CustomPolicy(ReportingSourceFinalPolicy): - pass - - with pytest.raises(ValueError, match="finality_policy"): - replace(DEFINITION, finality_policy=CustomPolicy("daily-v1", "close")) - - -@pytest.mark.parametrize("policy_id", ["", "a" * 256, "contains spaces", "close\n"]) -def test_finality_policy_refuses_invalid_identifier(policy_id: str) -> None: - with pytest.raises(ValueError, match="finality_policy_id"): - ReportingSourceFinalPolicy(policy_id, "close") - - -@pytest.mark.parametrize("source_signal", ["", "a" * 513]) -def test_source_finality_policy_refuses_invalid_signal(source_signal: str) -> None: - with pytest.raises(ValueError, match="source_signal must contain 1 to 512 characters"): - ReportingSourceFinalPolicy("daily-v1", source_signal) - - # -- period derivation ------------------------------------------------------ @@ -443,6 +331,7 @@ async def test_an_unreadable_revision_is_action_required_not_complete() -> None: configuration = _configuration() obligation = _obligation(configuration) revision, _ = _revision(obligation) + from dataclasses import replace projection = project_obligation_health( obligation, From 87a318f2e3a1c1f467a8af18341f4bda4ab47396 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 22:43:09 +0530 Subject: [PATCH 35/39] fix(reporting): resolve provisional storage against the selected ledger --- .../src/adcp_reporting_storage/ledger/pg.py | 61 ++++++++---- .../adcp_reporting_storage/outbox/_schema.py | 57 ++++++----- tests/test_reporting_official_finality.py | 99 ++++++++++++++++++- 3 files changed, 171 insertions(+), 46 deletions(-) diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py index 37dfebc75..28da00942 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py @@ -211,6 +211,11 @@ _ROW_MIGRATION_DDL_PATH = Path(__file__).parent / "reporting_row_migration.sql" _CHANGES_DDL_PATH = Path(__file__).parent / "reporting_changes.sql" _RETIREMENT_DDL_PATH = Path(__file__).parent / "reporting_period_retirement.sql" +_PROVISIONAL_HISTORY_TABLES = ( + "reporting_provisional_acquisitions", + "reporting_provisional_observations", +) +_STABILIZATION_TABLE = "adcp_reporting_stabilization_checkpoints" _BIGINT_MAX = 2**63 - 1 __all__ = ["PG_AVAILABLE", "PgReportingLedgerStore"] @@ -800,26 +805,36 @@ async def _require_provisional_schema(self, connection: Any) -> None: await validate_provisional_schema(connection) - async def _provisional_tables_installed(self, connection: Any) -> bool: - row = await ( + @staticmethod + async def _provisional_tables(connection: Any) -> set[str]: + """Resolve the same tables as the ledger statements and reject split schemas.""" + names = [ + "reporting_restatement_checkpoints", + _STABILIZATION_TABLE, + *_PROVISIONAL_HISTORY_TABLES, + ] + rows = await ( await connection.execute( - "SELECT to_regclass(quote_ident(current_schema())" - " || '.reporting_provisional_acquisitions') IS NOT NULL" - " OR to_regclass(quote_ident(current_schema())" - " || '.reporting_provisional_observations') IS NOT NULL" + "SELECT c.relname, c.relnamespace = (" + " SELECT ledger.relnamespace FROM pg_catalog.pg_class ledger" + " WHERE ledger.oid=pg_catalog.to_regclass('reporting_obligations'))" + " FROM unnest(%s::text[]) AS requested(name)" + " JOIN pg_catalog.pg_class c ON c.oid=pg_catalog.to_regclass(requested.name)", + (names,), ) - ).fetchone() - return bool(row and row[0]) + ).fetchall() + for name, in_ledger in rows: + if not in_ledger: + raise LedgerConflictError( + "PROVISIONAL_SCHEMA_UNREADY", + f"provisional_schema_unready:outside_ledger:{name}", + ) + return {name for name, _ in rows} - async def _stabilization_checkpoint_join(self, connection: Any) -> str: + @staticmethod + def _stabilization_checkpoint_join(tables: set[str]) -> str: """Join private policy metadata when reading a checkpoint on either schema version.""" - row = await ( - await connection.execute( - "SELECT to_regclass(quote_ident(current_schema())" - " || '.adcp_reporting_stabilization_checkpoints') IS NOT NULL" - ) - ).fetchone() - if row and row[0]: + if _STABILIZATION_TABLE in tables: return ( " LEFT JOIN adcp_reporting_stabilization_checkpoints s" " ON s.reporting_obligation_id=c.reporting_obligation_id" @@ -1534,7 +1549,8 @@ async def commit_revision( "this period is being retired; no revision may be added to it", ) validate_revision_currency(_obligation_from_row(obligation), revision, rows) - if await self._provisional_tables_installed(connection): + tables = await self._provisional_tables(connection) + if not tables.isdisjoint(_PROVISIONAL_HISTORY_TABLES): pending = await ( await connection.execute( "SELECT 1 FROM reporting_provisional_acquisitions a" @@ -1792,6 +1808,7 @@ async def get_provisional_acquisition( self, *, account_id: str, reporting_obligation_id: str, ordinal: int ) -> ProvisionalAcquisition | None: async with self._connection() as connection: + await self._provisional_tables(connection) row = await ( await connection.execute( "SELECT payload FROM reporting_provisional_acquisitions" @@ -1877,6 +1894,7 @@ async def get_provisional_observation( async def verify_snapshot_origin(self, revision: ReportingRevisionRecord) -> None: async with self._connection() as connection: + await self._provisional_tables(connection) witness = await ( await connection.execute( "SELECT origin FROM (SELECT COALESCE(" @@ -1902,10 +1920,11 @@ async def get_stabilization_schedule( ) -> ProvisionalPolicy | None: async with self._connection() as connection: # configure() can precede schema creation on a brand-new installation. - if not await self._provisional_tables_installed(connection): + tables = await self._provisional_tables(connection) + if tables.isdisjoint(_PROVISIONAL_HISTORY_TABLES): return None await self._require_provisional_schema(connection) - policy_join = await self._stabilization_checkpoint_join(connection) + policy_join = self._stabilization_checkpoint_join(tables) row = await ( await connection.execute( "SELECT a.payload->'policy' FROM reporting_obligations o" # noqa: S608 # nosec B608 @@ -2180,7 +2199,8 @@ async def get_restatement_checkpoint( self, *, account_id: str, reporting_obligation_id: str ) -> RestatementCheckpoint | None: async with self._connection() as connection: - policy_join = await self._stabilization_checkpoint_join(connection) + tables = await self._provisional_tables(connection) + policy_join = self._stabilization_checkpoint_join(tables) row = await ( await connection.execute( "SELECT c.account_id, c.reporting_obligation_id, c.checked_at," # noqa: S608 # nosec B608 @@ -2207,6 +2227,7 @@ async def record_restatement_checkpoint( self, checkpoint: RestatementCheckpoint ) -> RestatementCheckpoint: async with self._connection() as connection, connection.transaction(): + await self._provisional_tables(connection) obligation = await ( await connection.execute( "SELECT 1 FROM reporting_obligations" diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/outbox/_schema.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/outbox/_schema.py index 86276fd36..2af228ab9 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/outbox/_schema.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/outbox/_schema.py @@ -46,6 +46,7 @@ async def _catalog_objects( table_names: tuple[str, ...] | None = None, function_identity: tuple[str, str] | None = None, name_prefix: str = "reporting_", + ledger_schema: bool = False, ) -> dict[str, dict[str, Any]]: """Read fresh catalog state with the same fingerprints for either scope. @@ -57,25 +58,32 @@ async def _catalog_objects( if name_prefix not in _CATALOG_PREFIXES: raise ValueError("unsupported catalog prefix") - def scoped(query: str) -> str: - return query.replace("'reporting_'", f"'{name_prefix}'") + # An adopter may put a separate application schema before its installed ledger. + # Catalog fingerprints normalize names against the same schema as the table lookup. + schema = ( + "(SELECT n.nspname FROM pg_catalog.pg_namespace n" + " JOIN pg_catalog.pg_class ledger ON ledger.relnamespace=n.oid" + " WHERE ledger.oid=pg_catalog.to_regclass('reporting_obligations'))" + if ledger_schema + else "current_schema()" + ) + + async def execute(query: str, parameters: tuple[object, ...] | None = None) -> Any: + query = query.replace("{schema}", schema).replace("'reporting_'", f"'{name_prefix}'") + return await connection.execute(query, parameters) table_query = ( "SELECT c.oid, c.relname, c.relkind, c.relpersistence FROM pg_class c" - " JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = current_schema()" + " JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = {schema}" " AND c.relkind IN ('r','p') AND starts_with(c.relname, 'reporting_')" - " ORDER BY c.relname" ) + if ledger_schema: + table_query += " AND pg_catalog.pg_table_is_visible(c.oid)" if table_names is not None: - table_query = ( - "SELECT c.oid, c.relname, c.relkind, c.relpersistence FROM pg_class c" - " JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = current_schema()" - " AND c.relkind IN ('r','p') AND starts_with(c.relname, 'reporting_')" - " AND c.relname = ANY(%s::text[]) ORDER BY c.relname" - ) + table_query += " AND c.relname = ANY(%s::text[])" tables = await ( - await connection.execute( - scoped(table_query), + await execute( + table_query + " ORDER BY c.relname", (list(table_names),) if table_names is not None else None, ) ).fetchall() @@ -92,10 +100,10 @@ def remember(key: str, value: object, *, enabled: bool = True) -> None: # DDL drift. Round trips no longer grow with the number of reporting tables. oids = list(names) columns = await ( - await connection.execute( + await execute( "SELECT a.attrelid, a.attname, format_type(a.atttypid, a.atttypmod)," " a.attnotnull, co.collname, replace(pg_get_expr(d.adbin, d.adrelid)," - " quote_ident(current_schema()) || '.', '')," + " quote_ident({schema}) || '.', '')," " a.attidentity, a.attgenerated FROM pg_attribute a" " LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum" " LEFT JOIN pg_collation co ON co.oid = a.attcollation" @@ -107,9 +115,9 @@ def remember(key: str, value: object, *, enabled: bool = True) -> None: for oid, *row in columns: remember(f"column:{names[oid]}.{row[0]}", row[1:]) constraints = await ( - await connection.execute( + await execute( "SELECT conrelid, conname, contype, replace(pg_get_constraintdef(oid)," - " quote_ident(current_schema()) || '.', '')," + " quote_ident({schema}) || '.', '')," " convalidated, condeferrable, condeferred" " FROM pg_constraint WHERE conrelid = ANY(%s::oid[]) ORDER BY conrelid, conname", (oids,), @@ -118,9 +126,9 @@ def remember(key: str, value: object, *, enabled: bool = True) -> None: for oid, *row in constraints: remember(f"constraint:{names[oid]}.{row[0]}", row[1:], enabled=bool(row[3])) indexes = await ( - await connection.execute( + await execute( "SELECT i.indrelid, c.relname, i.indisunique, i.indisvalid, i.indisready, i.indislive," - " replace(pg_get_indexdef(i.indexrelid), quote_ident(current_schema()) || '.', '')" + " replace(pg_get_indexdef(i.indexrelid), quote_ident({schema}) || '.', '')" " FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid" " WHERE i.indrelid = ANY(%s::oid[]) ORDER BY i.indrelid, c.relname", (oids,), @@ -129,9 +137,9 @@ def remember(key: str, value: object, *, enabled: bool = True) -> None: for oid, *row in indexes: remember(f"index:{names[oid]}.{row[0]}", row[1:], enabled=all(row[2:5])) triggers = await ( - await connection.execute( + await execute( "SELECT tgrelid, tgname, tgenabled, replace(pg_get_triggerdef(oid)," - " quote_ident(current_schema()) || '.', '') FROM pg_trigger" + " quote_ident({schema}) || '.', '') FROM pg_trigger" " WHERE tgrelid = ANY(%s::oid[]) AND NOT tgisinternal ORDER BY tgrelid, tgname", (oids,), ) @@ -143,7 +151,7 @@ def remember(key: str, value: object, *, enabled: bool = True) -> None: " l.lanname, p.provolatile, p.proisstrict, p.prosecdef, p.proconfig," " pg_get_function_result(p.oid), p.proparallel, p.proleakproof FROM pg_proc p" " JOIN pg_namespace n ON n.oid = p.pronamespace JOIN pg_language l ON l.oid = p.prolang" - " WHERE n.nspname = current_schema() AND starts_with(p.proname, 'reporting_')" + " WHERE n.nspname = {schema} AND starts_with(p.proname, 'reporting_')" " ORDER BY p.proname, pg_get_function_identity_arguments(p.oid)" ) if function_identity is not None: @@ -152,13 +160,11 @@ def remember(key: str, value: object, *, enabled: bool = True) -> None: " l.lanname, p.provolatile, p.proisstrict, p.prosecdef, p.proconfig," " pg_get_function_result(p.oid), p.proparallel, p.proleakproof FROM pg_proc p" " JOIN pg_namespace n ON n.oid = p.pronamespace JOIN pg_language l ON l.oid = p.prolang" - " WHERE n.nspname = current_schema() AND starts_with(p.proname, 'reporting_')" + " WHERE n.nspname = {schema} AND starts_with(p.proname, 'reporting_')" " AND p.proname=%s AND pg_get_function_identity_arguments(p.oid)=%s" " ORDER BY p.proname, pg_get_function_identity_arguments(p.oid)" ) - functions = await ( - await connection.execute(scoped(function_query), function_identity) - ).fetchall() + functions = await (await execute(function_query, function_identity)).fetchall() for row in functions: remember(f"function:{row[0]}({row[1]})", row[2:]) return result @@ -251,6 +257,7 @@ async def validate_provisional_schema(connection: Any) -> None: "reporting_provisional_observations", ), function_identity=("reporting_provisional_immutable", ""), + ledger_schema=True, ) except Exception: raise LedgerConflictError( diff --git a/tests/test_reporting_official_finality.py b/tests/test_reporting_official_finality.py index 5c56c6c10..cae457401 100644 --- a/tests/test_reporting_official_finality.py +++ b/tests/test_reporting_official_finality.py @@ -61,7 +61,7 @@ source_batch_object_set_sha256_v1, ) from adcp.reporting.testing import DeterministicReportingClock -from tests.conformance.reporting._generation_support import isolated_reporting_pool +from tests.conformance.reporting._generation_support import isolated_reporting_pool, revision_for from tests.test_reporting_source_availability import ( END, PROVIDER_ROW, @@ -174,6 +174,103 @@ async def test_previous_checkpoint_schema_recovers_schedule_before_initialize() assert await store.get_stabilization_schedule(config.generation_key) == policy +@pytest.fixture +async def prefixed_reporting_ledger() -> AsyncIterator[tuple[Any, ...]]: + async with isolated_reporting_pool() as ledger, isolated_reporting_pool() as application: + from psycopg_pool import AsyncConnectionPool + + producer, store, config, obligation, _, clock = await _stabilized_harness( + partial(PgReportingLedgerStore, pool=ledger) + ) + async with application.connection() as app, ledger.connection() as db: + app_row = await (await app.execute("SELECT current_schema()")).fetchone() + ledger_row = await (await db.execute("SELECT current_schema()")).fetchone() + assert app_row is not None and ledger_row is not None + pool: AsyncConnectionPool = AsyncConnectionPool( + ledger.conninfo, + kwargs={"options": f"-csearch_path={app_row[0]},{ledger_row[0]}"}, + min_size=1, + open=False, + ) + async with pool: + yield PgReportingLedgerStore(pool=pool), store, producer, config, obligation, clock[0] + + +async def test_prefixed_search_path_round_trips_checkpoint_policy_and_due_time( + prefixed_reporting_ledger: tuple[Any, ...], +) -> None: + prefixed, installed, _, config, obligation, now = prefixed_reporting_ledger + policy = ProvisionalPolicy(timedelta(days=1), timedelta(hours=6), until_stabilized=True) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + checkpoint = RestatementCheckpoint( + **owner, + checked_at=now, + next_observation=1, + stabilization_policy=policy, + next_stabilization_due_at=now + policy.cadence, + ) + assert await prefixed.record_restatement_checkpoint(checkpoint) == checkpoint + assert await installed.get_restatement_checkpoint(**owner) == checkpoint + assert await prefixed.get_restatement_checkpoint(**owner) == checkpoint + + +async def test_prefixed_search_path_refuses_checkpoint_metadata_outside_the_ledger( + prefixed_reporting_ledger: tuple[Any, ...], +) -> None: + from psycopg import sql + + prefixed, installed, _, config, obligation, now = prefixed_reporting_ledger + async with prefixed._connection() as connection: + (application,) = await (await connection.execute("SELECT current_schema()")).fetchone() + await connection.execute( + sql.SQL("ALTER TABLE adcp_reporting_stabilization_checkpoints SET SCHEMA {}").format( + sql.Identifier(application) + ) + ) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + checkpoint = RestatementCheckpoint( + **owner, + checked_at=now, + next_observation=1, + next_stabilization_due_at=now + timedelta(hours=6), + ) + with pytest.raises(LedgerConflictError) as conflict: + await prefixed.record_restatement_checkpoint(checkpoint) + assert conflict.value.code == "PROVISIONAL_SCHEMA_UNREADY" + assert await installed.get_restatement_checkpoint(**owner) is None + + +@pytest.mark.parametrize("operation", ["recover_schedule", "commit_revision"]) +async def test_prefixed_search_path_respects_a_reserved_acquisition( + prefixed_reporting_ledger: tuple[Any, ...], + operation: str, +) -> None: + prefixed, installed, producer, config, obligation, now = prefixed_reporting_ledger + policy = ProvisionalPolicy(timedelta(days=1), timedelta(hours=6), until_stabilized=True) + request = producer._build_slice(config, obligation, SNAPSHOT_OFFERING_ID, now=now) + await installed.reserve_provisional_acquisition( + ProvisionalAcquisition(request.model_dump_json(), 0, policy) + ) + if operation == "recover_schedule": + assert await prefixed.get_stabilization_schedule(config.generation_key) == policy + else: + revision, rows = revision_for(obligation) + with pytest.raises(LedgerConflictError) as conflict: + await prefixed.commit_revision(revision, rows) + assert conflict.value.code == "OBSERVATION_CONFLICT" + assert ( + await installed.list_revisions( + account_id=config.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + == () + ) + + @pytest.mark.parametrize("later_clock", [False, True]) async def test_older_writer_advancing_checkpoint_retires_its_stabilization_metadata( later_clock: bool, From 6b32a6c52e0744918946723b7374e61adc645f75 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 23:23:40 +0530 Subject: [PATCH 36/39] fix(reporting): revalidate official finality against active snapshot evidence --- docs/reporting-provisional-observations.md | 13 +- src/adcp/reporting/ledger/finality.py | 54 ++- src/adcp/reporting/ledger/producer.py | 264 ++++++++++--- src/adcp/reporting/ledger/store.py | 4 + tests/test_reporting_finality_policy.py | 11 + tests/test_reporting_ledger.py | 22 +- tests/test_reporting_official_acquisition.py | 106 +++++ tests/test_reporting_official_finality.py | 387 ++++++++++++++++++- tests/test_reporting_source_availability.py | 2 + 9 files changed, 774 insertions(+), 89 deletions(-) diff --git a/docs/reporting-provisional-observations.md b/docs/reporting-provisional-observations.md index 619708839..c3f40259d 100644 --- a/docs/reporting-provisional-observations.md +++ b/docs/reporting-provisional-observations.md @@ -56,14 +56,21 @@ configuring an explicit snapshot-to-official close. To migrate unfinished official work with no policy, configure a new generation with the selected policy. Retained revisions and exact replays keep their original evidence. +During a rolling deployment, upgrade every process that reads configuration +or obligation definitions before any writer persists a definition containing +`finality_policy`. Earlier readers reject the additional definition field. + - A `source_final` policy uses the authoritative offering. Its `provider_declared` or `provider_job_terminal` evidence must carry the exact selected signal and the upstream finalization time. - A `contractual_cutoff` policy uses the snapshot offering. Its full, verified observation must reach the declared period-end cutoff. -- A `stabilized` policy uses the snapshot offering and durable scheduled - observation history. Changed content or a missed cadence restarts the - unchanged interval. Manual reads do not establish a run. +- A `stabilized` policy uses full-period batch, constituent and metric + watermarks from durable scheduled snapshots. A poll may arrive after its + due time within the same cadence slot. Missing the entire slot restarts the + unchanged interval. A manual publication that changes the active snapshot + also restarts the interval at the next scheduled observation. Only scheduled + observations can establish the interval. `stable_since` is a source observation and `official_close_lag` is scheduling metadata. Snapshot-only generations need no selection and remain snapshots diff --git a/src/adcp/reporting/ledger/finality.py b/src/adcp/reporting/ledger/finality.py index b88fc86e2..0fa729489 100644 --- a/src/adcp/reporting/ledger/finality.py +++ b/src/adcp/reporting/ledger/finality.py @@ -39,6 +39,13 @@ def _validate_policy_id(value: str) -> None: raise ValueError("finality_policy_id must match [A-Za-z0-9_.:-]{1,255}") +def _policy_duration(value: str) -> timedelta: + duration = timedelta(milliseconds=iso_duration_milliseconds_v1(value)) + if duration > datetime.max - datetime.min: + raise ValueError("finality policy duration exceeds the supported datetime range") + return duration + + @dataclass(frozen=True) class ReportingSourceFinalPolicy: """Close on an authoritative source publication with the declared signal.""" @@ -70,7 +77,7 @@ class ReportingContractualCutoffPolicy: def __post_init__(self) -> None: _validate_policy_id(self.finality_policy_id) - iso_duration_milliseconds_v1(self.duration_after_period_end) + _policy_duration(self.duration_after_period_end) def to_storage(self) -> dict[str, str]: return { @@ -91,8 +98,8 @@ class ReportingStabilizedPolicy: def __post_init__(self) -> None: _validate_policy_id(self.finality_policy_id) - iso_duration_milliseconds_v1(self.minimum_age) - iso_duration_milliseconds_v1(self.unchanged_for) + _policy_duration(self.minimum_age) + _policy_duration(self.unchanged_for) def to_storage(self) -> dict[str, str]: return { @@ -159,6 +166,23 @@ def stabilization_history_limit(policy: ReportingStabilizedPolicy, cadence: time return limit +def stabilization_interval_elapsed( + policy: ReportingStabilizedPolicy, period_end: datetime, first: datetime, latest: datetime +) -> bool: + return latest - period_end >= _policy_duration(policy.minimum_age) and ( + latest - first >= _policy_duration(policy.unchanged_for) + ) + + +def consecutive_stabilization_slots( + prior: ProvisionalObservation, current: ProvisionalObservation +) -> bool: + dispatch_at = current.acquisition.request().period.source_read_cutoff_at + return prior.next_due_at is not None and ( + dispatch_at - prior.next_due_at < prior.acquisition.policy.cadence + ) + + def prove_stabilized_finality( obligation: ReportingObligationRecord, policy: ReportingStabilizedPolicy, @@ -172,19 +196,15 @@ def prove_stabilized_finality( for (prior, prior_fingerprint, prior_rows), (current, fingerprint, current_rows) in zip( reversed(history[:-1]), reversed(history[1:]) ): - dispatch_at = current.acquisition.request().period.source_read_cutoff_at if ( - prior.next_due_at is None - or dispatch_at > prior.next_due_at + not consecutive_stabilization_slots(prior, current) or prior_rows != current_rows or prior_fingerprint != fingerprint ): break run_start = prior.checked_at - age_ms = (latest.checked_at - obligation.period.end) // timedelta(milliseconds=1) - stable_ms = (latest.checked_at - run_start) // timedelta(milliseconds=1) - if age_ms < iso_duration_milliseconds_v1(policy.minimum_age) or ( - stable_ms < iso_duration_milliseconds_v1(policy.unchanged_for) + if not stabilization_interval_elapsed( + policy, obligation.period.end, run_start, latest.checked_at ): return None return FinalityProof(policy.finality_policy_id, policy.basis, latest.checked_at) @@ -193,9 +213,16 @@ def prove_stabilized_finality( def contractual_cutoff_at( obligation: ReportingObligationRecord, policy: ReportingContractualCutoffPolicy ) -> datetime: - return obligation.period.end.astimezone(timezone.utc) + timedelta( - milliseconds=iso_duration_milliseconds_v1(policy.duration_after_period_end) - ) + from adcp.reporting.ledger.store import LedgerConflictError + + try: + return obligation.period.end.astimezone(timezone.utc) + _policy_duration( + policy.duration_after_period_end + ) + except OverflowError as error: + raise LedgerConflictError( + "FINALITY_POLICY_UNPROVEN", "contractual cutoff exceeds the supported datetime range" + ) from error def source_finality_proof( @@ -240,6 +267,7 @@ def manifest_finality_proof( manifest.publication_class != "PROVISIONAL_SNAPSHOT" or not cutoff <= manifest.period.source_read_cutoff_at <= published_at or manifest.observed_at < cutoff + or manifest.acquired_at > published_at ): raise LedgerConflictError( "FINALITY_POLICY_UNPROVEN", "contractual cutoff requires a post-cutoff snapshot" diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 683e858f7..3e026ba13 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -66,11 +66,13 @@ ReportingContractualCutoffPolicy, ReportingSourceFinalPolicy, ReportingStabilizedPolicy, + consecutive_stabilization_slots, contractual_cutoff_at, finality_policy_source_class, manifest_finality_proof, prove_stabilized_finality, stabilization_history_limit, + stabilization_interval_elapsed, ) from adcp.reporting.ledger.models import ( ReportingConfiguration, @@ -426,6 +428,8 @@ class _StabilizedPublication: proof: FinalityProof manifest: SourceBatchManifestV1 rows: Sequence[dict[str, Any]] + history: tuple[ProvisionalObservation, ...] + snapshots: Mapping[str, ReportingRevisionRecord] @dataclass @@ -721,11 +725,14 @@ def _official_close_requested( return ( configuration.required_finality == "official" or configuration.generation_key in self._recovered_stabilization - or any( - isinstance(offering, ProvisionalSnapshotOfferingV1) - and offering.offering_id == self._offerings.snapshot_offering_id - and offering.official_close_lag is not None - for offering in self._source.capabilities.offerings + or ( + self._offerings.snapshot_offering_id is not None + and any( + isinstance(offering, ProvisionalSnapshotOfferingV1) + and offering.offering_id == self._offerings.snapshot_offering_id + and offering.official_close_lag is not None + for offering in self._source.capabilities.offerings + ) ) ) @@ -762,6 +769,10 @@ async def _close_elapsed_periods( ) after = boundary.end continue + if selected is None and self._official_close_requested(configuration): + raise LedgerConflictError( + "FINALITY_POLICY_UNPROVEN", "official finality requires a selected policy" + ) if isinstance(selected, ReportingStabilizedPolicy) and self._stabilization_requested( configuration ): @@ -1181,8 +1192,18 @@ async def _acquire_with_settling_policy( reporting_obligation_id=obligation.reporting_obligation_id, ) scheduled = await self._latest_scheduled_snapshot(obligation, latest) - if scheduled is not None: - frozen = scheduled.acquisition.policy + checkpoint = await checkpoint_store.get_restatement_checkpoint( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + frozen = ( + scheduled.acquisition.policy + if scheduled is not None + else self._checkpoint_snapshot_policy( + latest, checkpoint.provisional_until if checkpoint else None + ) + ) + if frozen is not None: policy = _SettlingPolicy(frozen.window, frozen.cadence, frozen.official_close_lag) if isinstance(selected, ReportingStabilizedPolicy) and ( obligation.required_finality == "official" @@ -1203,10 +1224,6 @@ async def _acquire_with_settling_policy( policy.official_close_lag is not None and self._offerings.offering_for(_source_finality(obligation, "official")) is not None ) - checkpoint = await checkpoint_store.get_restatement_checkpoint( - account_id=obligation.account_id, - reporting_obligation_id=obligation.reporting_obligation_id, - ) ordinal = checkpoint.next_observation if checkpoint is not None else len(revisions) pending = await self._pending_acquisition(obligation, ordinal=ordinal) if pending is not None: @@ -1254,7 +1271,7 @@ async def _acquire_with_settling_policy( ) return False - if latest is not None and scheduled is None: + if latest is not None and frozen is None: # A tracked manual retry advances the durable ordinal only. The # first automatic source read keeps its configured phase and window. checkpoint = None @@ -1266,7 +1283,11 @@ async def _acquire_with_settling_policy( if scheduled is not None: next_due = scheduled.next_due_at elif checkpoint is not None: - next_due = min(_utc(checkpoint.checked_at) + policy.restatement_cadence, settles_at) + next_due = ( + min(_utc(checkpoint.checked_at) + policy.restatement_cadence, settles_at) + if _utc(checkpoint.checked_at) < settles_at + else None + ) else: # A direct revision has no scheduled observation time. next_due = self._scheduled_source_ready_at(configuration, obligation) @@ -1375,6 +1396,23 @@ async def _latest_scheduled_snapshot( raise return history[-1] if history else None + def _checkpoint_snapshot_policy( + self, latest: ProvisionalObservation | None, boundary: datetime | None + ) -> ProvisionalPolicy | None: + if ( + latest is not None + and latest.acquisition.ordinal > 0 + and latest.acquisition.request().trigger == "manual_replay" + and boundary is not None + and not _implements_store_method( + self._store, ProvisionalObservationStore.list_provisional_observations + ) + ): + # The manual reservation carries the frozen schedule. The checkpoint + # retains the preceding scheduled read's anchor and boundary. + return latest.acquisition.policy + return None + async def _recover_stabilization_schedule( self, configuration: ReportingConfiguration, @@ -1550,6 +1588,14 @@ async def _acquire_stabilized( obligation, await store.get_provisional_observation(**owner) ) if latest is not None and pending is None: + if ( + not manual + and now < latest.next_stabilization_due_at + and not stabilization_interval_elapsed( + policy, obligation.period.end, obligation.period.end, latest.checked_at + ) + ): + return None try: official = await self._promote_stabilized(obligation, policy, latest, turn) except LedgerConflictError as error: @@ -1610,6 +1656,7 @@ async def _promote_stabilized( turn=turn, acquisition=None, verified_source_rows=True, + stabilized_publication=evidence, ) async def _stabilized_publication( @@ -1624,11 +1671,23 @@ async def _stabilized_publication( raise LedgerConflictError( "OBSERVATION_CONFLICT", "stabilization history changed during proof" ) - verified: list[tuple[ProvisionalObservation, str, bytes]] = [] + owner = dict( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + snapshots = await self._stabilization_snapshots(obligation) + leaf = select_reporting_revision( + tuple(snapshots.values()), **owner, required_finality="snapshot" + ) + if leaf.kind != "selected" or not leaf.revision.readable: + raise LedgerConflictError( + "HISTORY_UNAVAILABLE", "stabilization history requires a readable snapshot leaf" + ) + candidates: list[ + tuple[ProvisionalObservation, SourceBatchManifestV1, ReportingRevisionRecord] + ] = [] previous: ProvisionalObservation | None = None - retained: tuple[ReportingRevisionRecord, bytes] | None = None - manifest: SourceBatchManifestV1 | None = None - rows: list[dict[str, Any]] = [] + revision: ReportingRevisionRecord | None = None for observation in history: acquisition = observation.acquisition if ( @@ -1646,7 +1705,7 @@ async def _stabilized_publication( "HISTORY_UNAVAILABLE", "stabilization history acquisition differs" ) previous = observation - if retained is None or retained[0].reporting_revision_id != observation.revision_id: + if revision is None or revision.reporting_revision_id != observation.revision_id: revision = await self._store.get_revision( account_id=obligation.account_id, reporting_revision_id=observation.revision_id ) @@ -1656,13 +1715,11 @@ async def _stabilized_publication( or revision.finality != "snapshot" or revision.account_id != obligation.account_id or revision.reporting_obligation_id != obligation.reporting_obligation_id + or revision != snapshots.get(observation.revision_id) ): raise LedgerConflictError( "HISTORY_UNAVAILABLE", "stabilization history requires a readable snapshot" ) - rows = await self._retained_snapshot_rows(revision) - retained = revision, hashlib.sha256(rfc8785.dumps(rows)).digest() - revision, row_binding = retained try: manifest = SourceBatchManifestV1.model_validate_json(observation.manifest_json) self._validate_source_manifest( @@ -1687,7 +1744,7 @@ async def _stabilized_publication( ): raise ValueError("originating snapshot differs from its publication") validate_snapshot_observation_binding(revision, observation) - if not verified or verified[-1][0].revision_id != observation.revision_id: + if not candidates or candidates[-1][0].revision_id != observation.revision_id: await self._stabilization_store().verify_snapshot_origin(revision) if ( revision.managed_control_totals is not None @@ -1705,17 +1762,96 @@ async def _stabilized_publication( and observation.revision_id == latest.revision_id ): raise - verified.clear() + candidates.clear() continue raise LedgerConflictError( "HISTORY_UNAVAILABLE", "stabilization history differs from its snapshot publication", ) from error + try: + manifest.require_full_period_watermarks() + except ValueError: + # A provisional read may be complete enough to publish a snapshot + # while still lacking full-window evidence for official finality. + candidates.clear() + continue + if candidates and ( + not consecutive_stabilization_slots(candidates[-1][0], observation) + or candidates[-1][1].content_fingerprint != manifest.content_fingerprint + or not self._matching_snapshot_lineage( + snapshots, revision, candidates[-1][0].revision_id, manifest + ) + ): + candidates.clear() + candidates.append((observation, manifest, revision)) + if not candidates or not stabilization_interval_elapsed( + policy, obligation.period.end, candidates[0][0].checked_at, latest.checked_at + ): + return None + if not self._matching_snapshot_lineage( + snapshots, leaf.revision, latest.revision_id, candidates[-1][1] + ): + return None + # Only a time-eligible run needs its immutable row bodies. Distinct + # manual revisions in the chain must bind those same verified rows. + verified: list[tuple[ProvisionalObservation, str, bytes]] = [] + rows: list[dict[str, Any]] = [] + row_binding = b"" + for observation, manifest, revision in candidates: + if not verified or verified[-1][0].revision_id != observation.revision_id: + rows = await self._retained_snapshot_rows(revision) + row_binding = hashlib.sha256(rfc8785.dumps(rows)).digest() + if verified and not self._matching_snapshot_lineage( + snapshots, revision, verified[-1][0].revision_id, manifest, rows + ): + verified.clear() verified.append((observation, manifest.content_fingerprint, row_binding)) proof = prove_stabilized_finality(obligation, policy, verified) - if proof is None or manifest is None: + if proof is None or not self._matching_snapshot_lineage( + snapshots, leaf.revision, latest.revision_id, manifest, rows + ): return None - return _StabilizedPublication(proof, manifest, rows) + return _StabilizedPublication(proof, manifest, rows, history, snapshots) + + async def _stabilization_snapshots( + self, obligation: ReportingObligationRecord + ) -> dict[str, ReportingRevisionRecord]: + revisions = await self._store.list_revisions( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + return {r.reporting_revision_id: r for r in revisions if r.finality == "snapshot"} + + @staticmethod + def _matching_snapshot_lineage( + snapshots: Mapping[str, ReportingRevisionRecord], + revision: ReportingRevisionRecord, + ancestor_id: str, + manifest: SourceBatchManifestV1, + rows: Sequence[dict[str, Any]] | None = None, + ) -> bool: + # The selected snapshot chain has one root and no forks or cycles. + # Manual publications can preserve a scheduled run only when each + # intervening immutable revision binds both its fingerprint and rows. + ancestor = snapshots[ancestor_id] + while revision.reporting_revision_id != ancestor_id: + if ( + not revision.readable + or revision.row_count != ancestor.row_count + or revision.control_totals != ancestor.control_totals + or revision.managed_control_totals != ancestor.managed_control_totals + or revision.source_manifest_sha256 + != manifest.content_fingerprint.removeprefix("sha256:") + or ( + rows is not None + and recompute_revision_content_sha256(revision, rows) + != revision.revision_content_sha256 + ) + or revision.supersedes_reporting_revision_id is None + ): + return False + revision = snapshots[revision.supersedes_reporting_revision_id] + return True async def _retained_snapshot_rows( self, revision: ReportingRevisionRecord @@ -2848,9 +2984,13 @@ async def _reserve_observation( scheduled = await self._latest_scheduled_snapshot(obligation, latest) selected = obligation.definition.finality_policy if obligation.definition else None if scheduled is not None: - frozen_policy = scheduled.acquisition.policy previous_boundary = scheduled.provisional_until - else: + frozen_policy = ( + scheduled.acquisition.policy + if scheduled is not None + else self._checkpoint_snapshot_policy(latest, previous_boundary) + ) + if frozen_policy is None: if latest is not None: previous_boundary = None policy = self._settling_policy(configuration, obligation, cutoff_reservation=True) @@ -3144,6 +3284,7 @@ async def _commit_revision_from_manifest( turn: WorkerTurn | None, acquisition: ProvisionalAcquisition | None, verified_source_rows: bool, + stabilized_publication: _StabilizedPublication | None = None, first_core_obligation: str | None = None, suppress_untracked_unchanged: bool = False, admit_tracked_stabilization: bool = True, @@ -3370,6 +3511,28 @@ async def _commit_revision_from_manifest( created_at = now = completed_at if finality == "official" and not stabilized: finality_proof = manifest_finality_proof(obligation, manifest, created_at) + if stabilized and isinstance(selected_policy, ReportingStabilizedPolicy): + if stabilized_publication is None: + scheduled = await self._scheduled_snapshot_history(obligation, 1) + stabilized_publication = ( + await self._stabilized_publication(obligation, selected_policy, scheduled[-1]) + if scheduled + else None + ) + if ( + stabilized_publication is None + or stabilized_publication.proof != finality_proof + or stabilized_publication.manifest != manifest + or ( + rows is not stabilized_publication.rows + and rfc8785.dumps(list(rows)) + != rfc8785.dumps(list(stabilized_publication.rows)) + ) + ): + raise LedgerConflictError( + "REVISION_IMMUTABLE" if retained_official else "FINALITY_POLICY_UNPROVEN", + "the retained history does not prove the official evidence", + ) revision = ReportingRevisionRecord( reporting_revision_id=revision_id, account_id=obligation.account_id, @@ -3447,6 +3610,7 @@ async def _commit_revision_from_manifest( now, acquisition, candidate_proof=candidate_proof, + stabilized_publication=stabilized_publication, origin_store=origin_store, retained_revision_id=( retained.reporting_revision_id if retained is not None else None @@ -3712,6 +3876,7 @@ async def _commit_publication( acquisition: ProvisionalAcquisition | None, *, candidate_proof: FinalityProof | None, + stabilized_publication: _StabilizedPublication | None, origin_store: SnapshotOriginStore | None, retained_revision_id: str | None = None, ) -> tuple[ReportingRevisionRecord, bool]: @@ -3720,23 +3885,21 @@ async def _commit_publication( ): replay = None if finality == "official": - selected = obligation.definition.finality_policy if obligation.definition else None - if candidate_proof is not None and isinstance(selected, ReportingStabilizedPolicy): - scheduled = await self._scheduled_snapshot_history(obligation, 1) - evidence = ( - await self._stabilized_publication(obligation, selected, scheduled[-1]) - if scheduled - else None + if stabilized_publication is not None: + history = stabilized_publication.history + frozen = history[-1].acquisition.policy + selected = ( + obligation.definition.finality_policy if obligation.definition else None ) - if ( - evidence is None - or evidence.manifest != manifest - or evidence.proof != candidate_proof - or rfc8785.dumps(list(rows)) != rfc8785.dumps(list(evidence.rows)) + assert isinstance(selected, ReportingStabilizedPolicy) + limit = stabilization_history_limit(selected, frozen.cadence) + snapshots = await self._stabilization_snapshots(obligation) + if snapshots != stabilized_publication.snapshots or ( + await self._scheduled_snapshot_history(obligation, limit) != history ): raise LedgerConflictError( - "FINALITY_POLICY_UNPROVEN", - "retained snapshot history does not prove this official publication", + "OBSERVATION_CONFLICT", + "the active snapshot or retained history changed during stabilization", ) replay = await self._store.get_revision( account_id=obligation.account_id, @@ -3822,7 +3985,7 @@ async def _official_replay_revision( raise LedgerConflictError( "REVISION_IMMUTABLE", "the winning official source evidence is unavailable" ) - if isinstance(policy, ReportingSourceFinalPolicy): + if isinstance(policy, (ReportingSourceFinalPolicy, ReportingContractualCutoffPolicy)): try: proof = manifest_finality_proof(obligation, manifest, retained.created_at) except LedgerConflictError as error: @@ -3831,7 +3994,8 @@ async def _official_replay_revision( "the manifest contradicts the retained source finality", ) from error if ( - manifest.offering_id != self._offerings.official_offering_id + manifest.offering_id + != self._offerings.offering_for(finality_policy_source_class(policy)) or proof.finality_policy_id != retained.finality_policy_id or proof.basis != retained.finality_basis or proof.finalized_at != retained.finalized_at @@ -3843,20 +4007,6 @@ async def _official_replay_revision( acquisition is not None and observation.acquisition != acquisition ): raise LedgerConflictError("REVISION_IMMUTABLE", "official source observation differs") - if isinstance(policy, ReportingStabilizedPolicy) and not proved_candidate: - assert observation is not None - evidence = await self._stabilized_publication(obligation, policy, observation) - if ( - evidence is None - or evidence.proof.basis != retained.finality_basis - or evidence.proof.finality_policy_id != retained.finality_policy_id - or evidence.proof.finalized_at != retained.finalized_at - or evidence.manifest.publication_id != retained.source_publication_id - ): - raise LedgerConflictError( - "REVISION_IMMUTABLE", - "the retained history does not prove the official evidence", - ) # Creation and contractual finality are sampled by each producer. A # source-final timestamp belongs to the provider's immutable evidence. replay = replace( diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index f543d3ae9..aa64bd777 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -194,6 +194,10 @@ def after_acquisition( if acquisition.request().trigger == "manual_replay": if previous is not None: policy, due = previous.stabilization_policy, previous.next_stabilization_due_at + checked_at = previous.checked_at + provisional_until = previous.provisional_until + else: + provisional_until = None elif retired_stabilized: policy = acquisition.policy if not acquisition.legacy_stabilized_close: diff --git a/tests/test_reporting_finality_policy.py b/tests/test_reporting_finality_policy.py index 35638b7ac..bb7d26fe7 100644 --- a/tests/test_reporting_finality_policy.py +++ b/tests/test_reporting_finality_policy.py @@ -49,6 +49,17 @@ def test_finality_policy_refuses_unsupported_duration(value: str) -> None: ReportingContractualCutoffPolicy("cutoff-v1", value) +@pytest.mark.parametrize("field", ["duration_after_period_end", "minimum_age", "unchanged_for"]) +def test_finality_policy_rejects_durations_beyond_the_datetime_range(field: str) -> None: + with pytest.raises(ValueError, match="duration"): + if field == "duration_after_period_end": + ReportingContractualCutoffPolicy("cutoff", "P99999999D") + elif field == "minimum_age": + ReportingStabilizedPolicy("stable", "P99999999D", "PT1H") + else: + ReportingStabilizedPolicy("stable", "PT1H", "P99999999D") + + @pytest.mark.parametrize( ("policy", "payload"), [ diff --git a/tests/test_reporting_ledger.py b/tests/test_reporting_ledger.py index e6dc209e4..46e24a6a7 100644 --- a/tests/test_reporting_ledger.py +++ b/tests/test_reporting_ledger.py @@ -21,11 +21,13 @@ ProducerOfferings, ReportingAdjustmentRecord, ReportingConfiguration, + ReportingDefinitionBinding, ReportingDeliveryEscalation, ReportingObligationRecord, ReportingProducer, ReportingRevisionRecord, ReportingScheduleSpec, + ReportingSourceFinalPolicy, ReportingStatusCaller, ReportingStatusHandler, aggregate_reporting_health, @@ -62,6 +64,14 @@ } HOURLY = ReportingScheduleSpec(period_duration="PT1H", delivery_sla="PT1H", alignment="utc") +_OFFICIAL_DEFINITION = ReportingDefinitionBinding( + report_definition_uri=_OFFERING["report_definition_uri"], + report_definition_sha256=_OFFERING["report_definition_sha256"], + schema_version=_OFFERING["reporting_profile"]["version"], + schema_uri=_OFFERING["reporting_profile"]["schema_uri"], + schema_sha256=_OFFERING["reporting_profile"]["schema_sha256"], + finality_policy=ReportingSourceFinalPolicy("daily-close", "delivery.closed"), +) def _configuration(**overrides) -> ReportingConfiguration: @@ -220,9 +230,13 @@ def test_periods_track_wall_clock_across_a_dst_transition() -> None: # -- obligations before reports --------------------------------------------- -async def test_an_obligation_is_committed_before_any_source_data_exists() -> None: +@pytest.mark.parametrize("finality", ["snapshot", "official"]) +async def test_an_obligation_is_committed_before_any_source_data_exists(finality: str) -> None: store = InMemoryReportingLedgerStore() - configuration = _configuration() + configuration = _configuration( + required_finality=finality, + definition=_OFFICIAL_DEFINITION if finality == "official" else None, + ) await store.put_configuration(configuration) producer = ReportingProducer(source=_NullSource(), offerings=ProducerOfferings(), store=store) committed = await producer.close_elapsed_periods( @@ -244,7 +258,7 @@ async def test_an_obligation_is_committed_before_any_source_data_exists() -> Non async def test_closing_periods_twice_commits_each_obligation_once() -> None: store = InMemoryReportingLedgerStore() - configuration = _configuration() + configuration = _configuration(definition=_OFFICIAL_DEFINITION) await store.put_configuration(configuration) producer = ReportingProducer(source=_NullSource(), offerings=ProducerOfferings(), store=store) now = datetime(2026, 9, 1, 5, 0, tzinfo=timezone.utc) @@ -255,7 +269,7 @@ async def test_closing_periods_twice_commits_each_obligation_once() -> None: async def test_a_period_that_has_not_closed_is_not_obligated() -> None: store = InMemoryReportingLedgerStore() - configuration = _configuration() + configuration = _configuration(definition=_OFFICIAL_DEFINITION) await store.put_configuration(configuration) producer = ReportingProducer(source=_NullSource(), offerings=ProducerOfferings(), store=store) # Exactly at the boundary: the obligation appears in the first snapshot diff --git a/tests/test_reporting_official_acquisition.py b/tests/test_reporting_official_acquisition.py index 1c7634e1b..a817e88ec 100644 --- a/tests/test_reporting_official_acquisition.py +++ b/tests/test_reporting_official_acquisition.py @@ -72,6 +72,7 @@ PROVIDER_ROW, EvidenceSource, ProgressStore, + _capabilities, _closing_capabilities, _harness, ) @@ -186,6 +187,111 @@ async def test_core_source_final_replay_requires_retained_signal_and_timestamp( ) +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +@pytest.mark.parametrize( + "change", ["offering", "early", "late", "acquired", "constituent", "metric"] +) +async def test_core_contractual_replay_proves_the_retained_publication( + cutoff_backend: Any, change: str +) -> None: + factory, _ = cutoff_backend + producer, store, config, obligation, _, clock = await _harness( + factory, official=True, finality_policy=ReportingContractualCutoffPolicy("cutoff", "PT2H") + ) + producer._store = CoreStore(store) + clock[0] = END + timedelta(hours=3) + first = await producer.acquire_obligation(config, obligation) + manifest = producer._source.manifest + clock[0] += timedelta(hours=1) + assert ( + await producer.commit_revision_from_manifest( + obligation, manifest, rows=ROWS, finality="official" + ) + == first + ) + raw = manifest.model_dump() + if change == "offering": + producer._offerings = replace(producer._offerings, snapshot_offering_id="different") + elif change in {"early", "late"}: + raw["period"]["source_read_cutoff_at"] = END + timedelta( + hours=1 if change == "early" else 4 + ) + elif change == "acquired": + raw["acquired_at"] = clock[0] + elif change == "constituent": + raw["coverage"]["constituents"][0]["data_through"] = END - timedelta(milliseconds=1) + else: + raw["metric_availability"][0]["data_through"] = END - timedelta(milliseconds=1) + with pytest.raises(LedgerConflictError) as conflict: + await producer.commit_revision_from_manifest( + obligation, SourceBatchManifestV1.model_validate(raw), rows=ROWS, finality="official" + ) + assert conflict.value.code == "REVISION_IMMUTABLE" + + +@pytest.mark.parametrize("cutoff_backend", ["memory", "postgres"], indirect=True) +async def test_checkpoint_preserves_scheduled_cadence_without_history_listing( + cutoff_backend: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + factory, _ = cutoff_backend + producer, store, config, obligation, calls, clock = await _harness( + factory, + restatement_window="PT12H", + restatement_cadence="PT6H", + finality_policy=ReportingContractualCutoffPolicy("unused", "P2D"), + ) + observations = store.list_provisional_observations + + async def latest_observation(**owner: Any) -> Any: + found = await observations(**owner, limit=1) + return found[-1] if found else None + + monkeypatch.setattr(store, "get_provisional_observation", latest_observation) + monkeypatch.setattr(store, "list_provisional_observations", None) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + clock[0] = END + timedelta(hours=1) + await producer.run_configuration(config) + first = await store.get_provisional_observation(**owner) + clock[0] += timedelta(hours=1) + manual = producer._build_slice( + config, + obligation, + SNAPSHOT_OFFERING_ID, + finality="snapshot", + now=clock[0], + observation=1, + trigger="manual_replay", + ) + await store.reserve_provisional_acquisition( + ProvisionalAcquisition( + manual.model_dump_json(), + 1, + first.acquisition.policy, + first.revision_id, + first.provisional_until, + ) + ) + await producer.acquire_obligation( + config, obligation, target_finality="snapshot", restate=True, manual_replay=True + ) + assert len(calls) == 2 + producer._source.capabilities = _capabilities( + restatement_window="P2D", restatement_cadence="PT1H" + ) + restarted = _restart_producer(producer, store, clock) + clock[0] = END + timedelta(hours=7, milliseconds=-1) + await restarted.run_configuration(config) + assert len(calls) == 2 + clock[0] += timedelta(milliseconds=1) + await restarted.run_configuration(config) + latest = await store.get_provisional_observation(**owner) + assert len(calls) == 3 and latest.acquisition.ordinal == 2 + assert latest.acquisition.policy == first.acquisition.policy + assert latest.next_due_at == END + timedelta(hours=12) + + @pytest.mark.parametrize("change", ["prefix", "suffix"]) @pytest.mark.parametrize("origin_lookup", [True, False]) @pytest.mark.parametrize( diff --git a/tests/test_reporting_official_finality.py b/tests/test_reporting_official_finality.py index cae457401..0b60a586e 100644 --- a/tests/test_reporting_official_finality.py +++ b/tests/test_reporting_official_finality.py @@ -9,6 +9,7 @@ from dataclasses import replace from datetime import datetime, timedelta from functools import partial +from pathlib import Path from typing import Any import pytest @@ -43,6 +44,7 @@ ) from adcp.reporting.ledger.finality import ( FinalityProof, + contractual_cutoff_at, manifest_finality_proof, source_finality_proof, ) @@ -57,7 +59,9 @@ ReportingSourceExecutorResult, SourceBatchManifestV1, coverage_denominator_fingerprint_v1, + encode_source_batch_manifest_v1, publication_content_fingerprint_v1, + source_batch_manifest_reference_v1, source_batch_object_set_sha256_v1, ) from adcp.reporting.testing import DeterministicReportingClock @@ -315,7 +319,7 @@ async def test_older_writer_advancing_checkpoint_retires_its_stabilization_metad @pytest.mark.parametrize("tracked", [False, True]) -async def test_manual_change_between_equal_scheduled_reads_preserves_stabilization( +async def test_manual_change_between_equal_scheduled_reads_restarts_stabilization( store_factory: Any, monkeypatch: pytest.MonkeyPatch, tracked: bool ) -> None: producer, store, config, obligation, calls, clock = await _stabilized_harness(store_factory) @@ -351,11 +355,303 @@ async def test_manual_change_between_equal_scheduled_reads_preserves_stabilizati config, obligation, restate=True, target_finality="snapshot", manual_replay=True ) assert manual is not None and manual.reporting_revision_id != prior.revision_id - clock[0] = END + timedelta(hours=13) + for hour in (13, 19, 25): + clock[0] = END + timedelta(hours=hour) + await producer.acquire_obligation(config, obligation) + revisions = await store.list_revisions(**owner) + assert [r.finalized_at for r in revisions if r.finality == "official"] == ( + [clock[0]] if hour == 25 else [] + ) + assert len(calls) == 6 + + +async def test_stabilized_promotion_rechecks_the_active_leaf_under_lock( + store_factory: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, obligation, _, clock = await _stabilized_harness(store_factory) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + for hour in (1, 7, 13): + clock[0] = END + timedelta(hours=hour) + await producer.acquire_obligation( + config, obligation, restate=True, target_finality="snapshot", track_settling=True + ) + prepared, resume = asyncio.Event(), asyncio.Event() + publication = producer._revision_publication + + @asynccontextmanager + async def pause_official(*args: Any, official: bool, **kwargs: Any) -> AsyncIterator[Any]: + if official: + prepared.set() + await resume.wait() + async with publication(*args, official=official, **kwargs) as result: + yield result + + monkeypatch.setattr(producer, "_revision_publication", pause_official) + closing = asyncio.create_task(producer.run_configuration(config)) + try: + await asyncio.wait_for(prepared.wait(), timeout=5) + monkeypatch.setattr( + "tests.test_reporting_source_availability.PROVIDER_ROW", + PROVIDER_ROW.replace("10", "11"), + ) + manual = await producer.acquire_obligation( + config, obligation, target_finality="snapshot", restate=True, manual_replay=True + ) + assert manual and manual.finality == "snapshot" + finally: + resume.set() + await asyncio.wait_for(closing, timeout=5) + assert all(r.finality == "snapshot" for r in await store.list_revisions(**owner)) + await producer.acquire_obligation(config, obligation) + assert all(r.finality == "snapshot" for r in await store.list_revisions(**owner)) + + +@pytest.mark.parametrize("missed_slot", [False, True]) +async def test_stabilized_cadence_accepts_jitter_and_resets_after_a_whole_slot( + store_factory: Any, missed_slot: bool +) -> None: + producer, store, config, obligation, _, clock = await _stabilized_harness(store_factory) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) await producer.run_configuration(config) - revisions = await store.list_revisions(**owner) - assert [r.finalized_at for r in revisions if r.finality == "official"] == [clock[0]] - assert len(calls) == 4 + clock[0] += timedelta(hours=12 if missed_slot else 6, milliseconds=1) + await producer.acquire_obligation(config, obligation) + clock[0] += timedelta(hours=6, milliseconds=1) + await producer.acquire_obligation(config, obligation) + official = [r for r in await store.list_revisions(**owner) if r.finality == "official"] + if missed_slot: + assert not official + clock[0] += timedelta(hours=6, milliseconds=1) + await producer.acquire_obligation(config, obligation) + official = [r for r in await store.list_revisions(**owner) if r.finality == "official"] + assert [r.finalized_at for r in official] == [clock[0]] + + +@pytest.mark.parametrize("stale", ["batch", "constituent", "metric"]) +async def test_stabilized_watermarks_cover_the_complete_period( + store_factory: Any, monkeypatch: pytest.MonkeyPatch, stale: str +) -> None: + producer, store, config, obligation, _, clock = await _stabilized_harness(store_factory) + owner = dict( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + + def partial_result(request: Any, **kwargs: Any) -> Any: + if stale == "batch": + kwargs["data_through"] = END - timedelta(milliseconds=1) + _, manifest, reader = redacted_completed_result(request, **kwargs) + raw = manifest.model_dump() + through = END - timedelta(milliseconds=1) + if stale == "constituent": + raw["coverage"]["constituents"][0]["data_through"] = through + if stale == "metric": + raw["metric_availability"][0]["data_through"] = through + manifest = SourceBatchManifestV1.model_validate(raw) + body = encode_source_batch_manifest_v1(manifest) + return ( + ReportingSourceExecutorResult.completed( + request=request, + manifest=source_batch_manifest_reference_v1("watermarks", body), + manifest_bytes=body, + ), + manifest, + reader, + ) + + with monkeypatch.context() as patch: + patch.setattr( + "tests.test_reporting_source_availability.redacted_completed_result", partial_result + ) + for hour in (1, 7, 13): + clock[0] = END + timedelta(hours=hour) + await producer.acquire_obligation(config, obligation) + assert len(await store.list_provisional_observations(**owner, limit=4)) == 3 + assert all(r.finality == "snapshot" for r in await store.list_revisions(**owner)) + for hour in (19, 25, 31): + clock[0] = END + timedelta(hours=hour) + await producer.acquire_obligation(config, obligation) + assert any(r.finality == "official" for r in await store.list_revisions(**owner)) + + +@pytest.mark.parametrize("idle", [False, True]) +async def test_stabilization_reads_rows_only_for_possible_proof_outside_publication( + store_factory: Any, monkeypatch: pytest.MonkeyPatch, idle: bool +) -> None: + producer, store, config, obligation, _, clock = await _stabilized_harness( + store_factory, minimum_age="PT13H" + ) + await producer.run_configuration(config) + locked = False + reads: list[bool] = [] + publication = store.source_publication + read = store.read_revision_rows + + @asynccontextmanager + async def observe_publication(*args: Any, **kwargs: Any) -> AsyncIterator[Any]: + nonlocal locked + async with publication(*args, **kwargs) as result: + locked = True + try: + yield result + finally: + locked = False + + async def observe_read(**kwargs: Any) -> Any: + reads.append(locked) + return await read(**kwargs) + + monkeypatch.setattr(store, "source_publication", observe_publication) + monkeypatch.setattr(store, "read_revision_rows", observe_read) + for hour in (2, 7, 8): + clock[0] = END + timedelta(hours=hour) + await producer.acquire_obligation(config, obligation) + if idle: + assert not reads + reads.clear() + clock[0] = END + timedelta(hours=13) + revision = await producer.acquire_obligation(config, obligation) + assert revision and revision.finality == "official" + assert reads == [False] + reads.clear() + assert ( + await producer.commit_revision_from_manifest( + obligation, producer._source.manifest, rows=ROWS, finality="official" + ) + == revision + ) + assert reads == [False] + + +async def test_stabilized_publication_reads_object_rows_outside_the_account_transaction( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from tests.conformance.reporting.test_reporting_row_storage_object_pg import lease, options + + async with isolated_reporting_pool() as pool: + storage = options(tmp_path) + producer, store, config, obligation, _, clock = await _stabilized_harness( + lambda **kwargs: PgReportingLedgerStore(pool=pool, row_storage=storage, **kwargs) + ) + await lease(pool) + for hour in (1, 7, 13): + clock[0] = END + timedelta(hours=hour) + await producer.acquire_obligation( + config, obligation, restate=True, target_finality="snapshot", track_settling=True + ) + + in_publication = False + get = type(storage.providers["filesystem"]).get + publication = store.source_publication + reads = [] + + @asynccontextmanager + async def locked(*args: Any, **kwargs: Any) -> Any: + nonlocal in_publication + async with publication(*args, **kwargs) as publisher: + in_publication = True + try: + yield publisher + finally: + in_publication = False + + async def read(*args: Any, **kwargs: Any) -> Any: + reads.append(in_publication) + return await get(*args, **kwargs) + + monkeypatch.setattr(type(storage.providers["filesystem"]), "get", read) + monkeypatch.setattr(store, "source_publication", locked) + official = await producer.acquire_obligation(config, obligation) + assert official and official.finality == "official" + assert reads and not any(reads) + reads.clear() + assert ( + await producer.commit_revision_from_manifest( + obligation, producer._source.manifest, rows=ROWS, finality="official" + ) + == official + ) + assert reads and not any(reads) + + +async def test_idle_after_changed_scheduled_content_skips_snapshot_bodies( + store_factory: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + producer, store, config, obligation, _, clock = await _stabilized_harness(store_factory) + for hour in (1, 7, 13): + clock[0] = END + timedelta(hours=hour) + if hour == 13: + monkeypatch.setattr( + "tests.test_reporting_source_availability.PROVIDER_ROW", + PROVIDER_ROW.replace("10", "11"), + ) + await producer.acquire_obligation(config, obligation) + + async def unexpected_read(**kwargs: Any) -> Any: + pytest.fail("an idle poll hydrated the body of an ineligible stability run") + + monkeypatch.setattr(store, "read_revision_rows", unexpected_read) + clock[0] += timedelta(minutes=10) + assert await producer.acquire_obligation(config, obligation) is None + + +@pytest.mark.parametrize("change", ["none", "rows", "totals"]) +async def test_same_fingerprint_manual_leaf_requires_immutable_row_binding( + store_factory: Any, change: str +) -> None: + producer, store, config, obligation, _, clock = await _stabilized_harness(store_factory) + for hour in (1, 7, 13): + clock[0] = END + timedelta(hours=hour) + snapshot = await producer.acquire_obligation( + config, obligation, target_finality="snapshot", restate=True, track_settling=True + ) + clock[0] += timedelta(minutes=10) + request = producer._build_slice( + config, + obligation, + SNAPSHOT_OFFERING_ID, + finality="snapshot", + now=clock[0], + trigger="manual_replay", + observation=3, + ) + _, manifest, _ = redacted_completed_result(request, page_bodies=[PROVIDER_ROW]) + if change == "totals": + revision_id = f"rpr_{manifest.publication_id[4:44]}" + totals = (("impressions", "11"),) + typed = (ReportingControlTotalRecord("impressions", "11", "integer"),) + manual = await store.commit_revision( + replace( + snapshot, + reporting_revision_id=revision_id, + supersedes_reporting_revision_id=snapshot.reporting_revision_id, + source_publication_id=manifest.publication_id, + control_totals=totals, + managed_control_totals=typed, + revision_content_sha256=revision_content_sha256( + reporting_revision_id=revision_id, + row_count=len(ROWS), + control_totals=totals, + control_total_evidence=typed, + reporting_rows=ROWS, + ), + created_at=clock[0], + ), + ROWS, + ) + else: + rows = [{**ROWS[0], "impressions": 11}] if change == "rows" else ROWS + manual = await producer.commit_revision_from_manifest( + obligation, manifest, rows=rows, finality="snapshot" + ) + assert manual.supersedes_reporting_revision_id + await producer.acquire_obligation(config, obligation) + revisions = await store.list_revisions( + account_id=config.account_id, reporting_obligation_id=obligation.reporting_obligation_id + ) + assert any(r.finality == "official" for r in revisions) == (change == "none") async def test_unbound_reused_stabilization_history_recovers_after_fresh_evidence( @@ -1267,12 +1563,12 @@ async def interrupted(revision: Any, rows: Any) -> Any: @pytest.mark.parametrize( ("reset", "hours"), - [("changed", (1, 7, 13, 19)), ("gap", (1, 8, 14, 20)), ("age", (1, 7, 13, 19))], + [("changed", (1, 7, 13, 19)), ("gap", (1, 13, 19, 25)), ("age", (1, 7, 13, 19))], ) async def test_stabilized_requires_a_complete_unchanged_run( reset: str, hours: tuple[int, ...], monkeypatch: pytest.MonkeyPatch ) -> None: - producer, store, config, obligation, calls, clock = await _stabilized_harness( + producer, store, config, obligation, _, clock = await _stabilized_harness( ProgressStore, minimum_age="PT19H" if reset == "age" else "PT0S" ) owner = dict( @@ -1289,7 +1585,13 @@ async def test_stabilized_requires_a_complete_unchanged_run( revisions = await store.list_revisions(**owner) assert any(r.finality == "official" for r in revisions) == (idx == 3) assert (obligation.reporting_obligation_id in store.pending) == (idx != 3) - assert len(calls) == 4 + assert ( + sum( + req.identity.reporting_obligation_id == obligation.reporting_obligation_id + for req in producer._source.requests + ) + == 4 + ) assert sum(r.finality == "snapshot" for r in revisions) == (2 if reset == "changed" else 1) @@ -2760,8 +3062,65 @@ async def test_direct_official_commit_checks_staged_size() -> None: assert conflict.value.code == "STAGED_OBJECT_MISMATCH" -async def test_policyless_official_does_not_dispatch() -> None: - producer, _, config, obligation, calls, _ = await _harness(official=True) +@pytest.mark.parametrize("required", ["official", "snapshot"]) +async def test_standalone_new_close_requires_a_selected_policy( + store_factory: Any, required: str +) -> None: + producer, store, config, obligation, _, clock = await _harness( + store_factory, finality_policy=ReportingSourceFinalPolicy("source", "delivery.is_final") + ) + if required == "snapshot": + producer._source.capabilities = _closing_capabilities() + config = replace(config, delivery_config_version=2, required_finality=required, definition=None) + await store.put_configuration(config) + with pytest.raises(LedgerConflictError) as conflict: + await producer.close_elapsed_periods(config, now=clock[0]) + assert conflict.value.code == "FINALITY_POLICY_UNPROVEN" + assert ( + await store.find_obligation( + account_id=config.account_id, + consumer_id=config.consumer_id, + delivery_config_id=config.delivery_config_id, + delivery_config_version=2, + period_start=obligation.period.start, + period_end=obligation.period.end, + ) + is None + ) + + +async def _retained_policyless_official(store_factory: Any) -> tuple[Any, ...]: + producer, store, config, obligation, calls, clock = await _harness(store_factory) + config = replace(config, delivery_config_version=2, required_finality="official") + await store.put_configuration(config) + obligation = await store.commit_obligation( + replace( + obligation, + delivery_config_version=2, + reporting_obligation_id="archived-official", + required_finality="official", + ) + ) + return producer, store, config, obligation, calls, clock + + +async def test_contractual_cutoff_at_datetime_boundary_fails_with_a_ledger_error() -> None: + _, _, _, obligation, _, _ = await _harness() + end = END.replace(year=9999, month=12, day=31) + period = replace(obligation.period, start=end - timedelta(days=1), end=end, expected_at=end) + with pytest.raises(LedgerConflictError) as error: + contractual_cutoff_at( + replace( + obligation, period=period, scope_resolved_at=end, automated_recovery_deadline_at=end + ), + ReportingContractualCutoffPolicy("cutoff", "P1D"), + ) + assert error.value.code == "FINALITY_POLICY_UNPROVEN" + + +async def test_policyless_official_does_not_dispatch(store_factory: Any) -> None: + producer, _, config, obligation, calls, _ = await _retained_policyless_official(store_factory) + assert await producer.close_elapsed_periods(config) == [] with pytest.raises(LedgerConflictError) as error: await producer.acquire_obligation(config, obligation) assert error.value.code == "FINALITY_POLICY_UNPROVEN" @@ -2910,8 +3269,12 @@ async def test_official_construction_rejects_proof_for_another_policy() -> None: assert conflict.value.code == "FINALITY_POLICY_UNPROVEN" -async def test_retained_official_replay_preserves_original_policy_fields() -> None: - producer, store, config, obligation, _, clock = await _harness(official=True) +async def test_retained_official_replay_preserves_original_policy_fields( + store_factory: Any, +) -> None: + producer, store, config, obligation, _, clock = await _retained_policyless_official( + store_factory + ) request = producer._build_slice(config, obligation, OFFICIAL_OFFERING_ID, now=clock[0]) _, manifest, _ = redacted_completed_result(request, page_bodies=[PROVIDER_ROW]) revision_id = f"rpr_{manifest.publication_id[4:44]}" diff --git a/tests/test_reporting_source_availability.py b/tests/test_reporting_source_availability.py index f778056e3..01ac93e94 100644 --- a/tests/test_reporting_source_availability.py +++ b/tests/test_reporting_source_availability.py @@ -417,6 +417,7 @@ async def test_authoritative_local_readiness_is_conservative_across_clock_change producer, _, _, obligation, _, _ = await _harness( official=True, expected_availability_lag="PT0S", + finality_policy=ReportingSourceFinalPolicy("delivery-v1", "delivery.is_final"), source_timezone="America/New_York", days_after_period_end=days_after, source_local_ready_time=ready_time, @@ -439,6 +440,7 @@ async def test_authoritative_readiness_uses_period_timezone_when_offering_omits_ producer, _, _, obligation, _, _ = await _harness( official=True, source_timezone=None, + finality_policy=ReportingSourceFinalPolicy("delivery-v1", "delivery.is_final"), days_after_period_end=1, source_local_ready_time="06:00", ) From 9f0bcacbb2f11207fea82fb06e69557972040196 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Wed, 18 Nov 2026 23:43:17 +0530 Subject: [PATCH 37/39] fix(reporting): satisfy finality code quality checks --- src/adcp/reporting/ledger/finality.py | 3 +-- src/adcp/reporting/ledger/provisional.py | 8 +++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/adcp/reporting/ledger/finality.py b/src/adcp/reporting/ledger/finality.py index 0fa729489..e286dc749 100644 --- a/src/adcp/reporting/ledger/finality.py +++ b/src/adcp/reporting/ledger/finality.py @@ -312,5 +312,4 @@ def reporting_finality_policy_from_storage(value: Mapping[str, Any]) -> Reportin **extra, } if not extra: return ReportingStabilizedPolicy(policy_id, minimum_age, unchanged_for) - case _: - raise ValueError("invalid reporting finality policy storage payload") + raise ValueError("invalid reporting finality policy storage payload") diff --git a/src/adcp/reporting/ledger/provisional.py b/src/adcp/reporting/ledger/provisional.py index b1b4ec70d..49829aa7d 100644 --- a/src/adcp/reporting/ledger/provisional.py +++ b/src/adcp/reporting/ledger/provisional.py @@ -341,7 +341,8 @@ def from_wire(cls, value: dict[str, Any]) -> ProvisionalObservation: class SnapshotOriginStore(Protocol): """Check a revision against the first retained witness of its full publication ID.""" - async def verify_snapshot_origin(self, revision: ReportingRevisionRecord) -> None: ... + async def verify_snapshot_origin(self, revision: ReportingRevisionRecord) -> None: + pass @runtime_checkable @@ -350,7 +351,8 @@ class StabilizationScheduleStore(Protocol): async def get_stabilization_schedule( self, generation: ReportingConfigurationGenerationKey - ) -> ProvisionalPolicy | None: ... + ) -> ProvisionalPolicy | None: + pass @runtime_checkable @@ -402,7 +404,7 @@ async def list_provisional_observations( A scheduled snapshot filter applies before the limit. ``limit`` must be a positive integer. """ - ... + pass @runtime_checkable From 46fe1bb27b85f81a44406a2041f4ec5ba882f972 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Thu, 19 Nov 2026 00:00:07 +0530 Subject: [PATCH 38/39] fix(test): isolate historical reporting fixtures --- .../test_reporting_activity_schema_proof.py | 4 ++-- tests/reporting_fixtures.py | 24 +++++++++++++++++++ tests/test_reporting_capability_models.py | 4 ++-- tests/test_reporting_ledger.py | 22 ++--------------- tests/test_reporting_wire_defaults.py | 4 ++-- 5 files changed, 32 insertions(+), 26 deletions(-) create mode 100644 tests/reporting_fixtures.py diff --git a/tests/conformance/reporting/test_reporting_activity_schema_proof.py b/tests/conformance/reporting/test_reporting_activity_schema_proof.py index 175cdea91..3c20cd847 100644 --- a/tests/conformance/reporting/test_reporting_activity_schema_proof.py +++ b/tests/conformance/reporting/test_reporting_activity_schema_proof.py @@ -33,8 +33,8 @@ def _storage_split(): ReportingStatusSupport, ) from adcp.types import ReportingDeliveryCapabilities +from tests.reporting_fixtures import REPORTING_OFFERING from tests.test_decisioning_capabilities_projection import _SalesPlatform -from tests.test_reporting_ledger import _OFFERING from ._reliable_support import NotificationHarness, reliable_factory from .test_reporting_webhook_activity import worker_for @@ -47,7 +47,7 @@ def mounted_activity(support): "supported": True, "configuration_task": "sync_accounts", "status_task": "get_reporting_status", - "offerings": [_OFFERING], + "offerings": [REPORTING_OFFERING], "automated_recovery_window_seconds": 3600, "status_retention_days": 30, } diff --git a/tests/reporting_fixtures.py b/tests/reporting_fixtures.py new file mode 100644 index 000000000..9d6c3e15d --- /dev/null +++ b/tests/reporting_fixtures.py @@ -0,0 +1,24 @@ +"""Wire fixtures shared by reporting tests across installed SDK versions.""" + +from typing import Any + +REPORTING_OFFERING: dict[str, Any] = { + "offering_id": "HOURLY_SNAPSHOT_V1", + "feed_purpose": "analytics", + "report_definition_id": "daily_delivery_v1", + "report_definition_uri": "https://contracts.example.test/reporting/daily-delivery-v1", + "report_definition_sha256": "a" * 64, + "reporting_profile": { + "id": "paid_media_delivery", + "version": "1.0.0", + "schema_uri": "https://contracts.example.test/reporting/rows-v1.json", + "schema_sha256": "b" * 64, + "schema_dialect": "https://json-schema.org/draft/2020-12/schema", + "schema_ref_policy": "local_fragment_only", + "grain": "one row per media buy per period", + "primary_keys": ["media_buy_id"], + }, + "schedule": {"period_duration": "PT1H", "delivery_sla": "PT1H", "alignment": "utc"}, + "supported_finality": ["snapshot"], + "reconciliation_mode": "delivery_only", +} diff --git a/tests/test_reporting_capability_models.py b/tests/test_reporting_capability_models.py index 6410070af..6a234f75e 100644 --- a/tests/test_reporting_capability_models.py +++ b/tests/test_reporting_capability_models.py @@ -12,7 +12,7 @@ from pydantic import BaseModel, ValidationError from adcp.types import GetAdcpCapabilitiesResponse, ReportingDeliveryCapabilities -from tests.test_reporting_ledger import _OFFERING +from tests.reporting_fixtures import REPORTING_OFFERING PROMISES = { "receipt_task": "sync_reporting_receipts", @@ -25,7 +25,7 @@ def capability(**fields): return { "supported": True, - "offerings": [_OFFERING], + "offerings": [REPORTING_OFFERING], "automated_recovery_window_seconds": 0, "status_retention_days": 7, **fields, diff --git a/tests/test_reporting_ledger.py b/tests/test_reporting_ledger.py index 46e24a6a7..f69c1a123 100644 --- a/tests/test_reporting_ledger.py +++ b/tests/test_reporting_ledger.py @@ -38,30 +38,12 @@ revision_content_sha256, ) from adcp.reporting.ledger import ReportingStatusCaller as OwnershipCaller +from tests.reporting_fixtures import REPORTING_OFFERING ACCOUNT = "acct_1" CALLER = ReportingStatusCaller(account_id=ACCOUNT, consumer_id="buyer_1") -_OFFERING: dict[str, Any] = { - "offering_id": "HOURLY_SNAPSHOT_V1", - "feed_purpose": "analytics", - "report_definition_id": "daily_delivery_v1", - "report_definition_uri": "https://contracts.example.test/reporting/daily-delivery-v1", - "report_definition_sha256": "a" * 64, - "reporting_profile": { - "id": "paid_media_delivery", - "version": "1.0.0", - "schema_uri": "https://contracts.example.test/reporting/rows-v1.json", - "schema_sha256": "b" * 64, - "schema_dialect": "https://json-schema.org/draft/2020-12/schema", - "schema_ref_policy": "local_fragment_only", - "grain": "one row per media buy per period", - "primary_keys": ["media_buy_id"], - }, - "schedule": {"period_duration": "PT1H", "delivery_sla": "PT1H", "alignment": "utc"}, - "supported_finality": ["snapshot"], - "reconciliation_mode": "delivery_only", -} +_OFFERING = REPORTING_OFFERING HOURLY = ReportingScheduleSpec(period_duration="PT1H", delivery_sla="PT1H", alignment="utc") _OFFICIAL_DEFINITION = ReportingDefinitionBinding( diff --git a/tests/test_reporting_wire_defaults.py b/tests/test_reporting_wire_defaults.py index fe2425163..d9f6bfe87 100644 --- a/tests/test_reporting_wire_defaults.py +++ b/tests/test_reporting_wire_defaults.py @@ -8,7 +8,7 @@ from adcp.decisioning.capabilities import MediaBuy from adcp.types import NotificationConfig from adcp.types.base import AdCPBaseModel -from tests.test_reporting_ledger import _OFFERING +from tests.reporting_fixtures import REPORTING_OFFERING @pytest.mark.parametrize("model_type", [MediaBuy, MediaBuy.__bases__[0]]) @@ -18,7 +18,7 @@ def test_reporting_defaults_are_explicit_in_actual_bundled_and_canonical_models( ): supplied = { "supported": True, - "offerings": [_OFFERING], + "offerings": [REPORTING_OFFERING], "automated_recovery_window_seconds": 0, "status_retention_days": 30, } From af67b2357bc13ed2fde0bc9f45e6f9a9ead17bdd Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Thu, 19 Nov 2026 00:21:25 +0530 Subject: [PATCH 39/39] fix(reporting): keep period closure source independent --- src/adcp/reporting/ledger/producer.py | 15 +++++---------- src/adcp/reporting/service.py | 17 ++++++++++++++++- tests/test_reporting_official_finality.py | 6 +++--- tests/test_reporting_settling.py | 6 ++++++ tests/test_reporting_source_availability.py | 7 +++++++ 5 files changed, 37 insertions(+), 14 deletions(-) diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 3e026ba13..51ff04573 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -368,7 +368,9 @@ class ProducerOfferings: Snapshot and official source publications use different offerings. A source-declared settling policy may use them sequentially for one ledger obligation, so the producer needs both named explicitly rather than - inferring one from the other. + inferring one from the other. ``snapshot_requests_official_close`` records + an explicit source offering declaration at admission so period closure + remains independent of source availability. """ snapshot_offering_id: str | None = None @@ -379,6 +381,7 @@ class ProducerOfferings: currency: str = "USD" source_scope: dict[str, Any] = field(default_factory=dict) slice_timeout: timedelta = timedelta(minutes=10) + snapshot_requests_official_close: bool = False def offering_for(self, finality: str) -> str | None: return self.official_offering_id if finality == "official" else self.snapshot_offering_id @@ -725,15 +728,7 @@ def _official_close_requested( return ( configuration.required_finality == "official" or configuration.generation_key in self._recovered_stabilization - or ( - self._offerings.snapshot_offering_id is not None - and any( - isinstance(offering, ProvisionalSnapshotOfferingV1) - and offering.offering_id == self._offerings.snapshot_offering_id - and offering.official_close_lag is not None - for offering in self._source.capabilities.offerings - ) - ) + or self._offerings.snapshot_requests_official_close ) async def _close_elapsed_periods( diff --git a/src/adcp/reporting/service.py b/src/adcp/reporting/service.py index 025831a72..c8412ee64 100644 --- a/src/adcp/reporting/service.py +++ b/src/adcp/reporting/service.py @@ -14,7 +14,7 @@ import logging import re from collections.abc import Awaitable, Callable, Mapping, Sequence -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from datetime import datetime, timedelta, timezone from types import MappingProxyType from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable @@ -762,6 +762,21 @@ async def _configure( offerings = context.producer_offerings() if not isinstance(offerings, ProducerOfferings): raise TypeError("producer_offerings must return ProducerOfferings") + snapshot = next( + ( + offering + for offering in registration.executor.capabilities.offerings + if offering.offering_id == offerings.snapshot_offering_id + ), + None, + ) + offerings = replace( + offerings, + snapshot_requests_official_close=( + isinstance(snapshot, ProvisionalSnapshotOfferingV1) + and snapshot.official_close_lag is not None + ), + ) except Exception as error: if retrying or (recovering and isinstance(error, _MissingFinalityPolicyError)): raise _AccountContextResolutionError(error, permanent=True) from error diff --git a/tests/test_reporting_official_finality.py b/tests/test_reporting_official_finality.py index 0b60a586e..b0e07e432 100644 --- a/tests/test_reporting_official_finality.py +++ b/tests/test_reporting_official_finality.py @@ -3067,10 +3067,10 @@ async def test_standalone_new_close_requires_a_selected_policy( store_factory: Any, required: str ) -> None: producer, store, config, obligation, _, clock = await _harness( - store_factory, finality_policy=ReportingSourceFinalPolicy("source", "delivery.is_final") + store_factory, + capabilities=_closing_capabilities() if required == "snapshot" else None, + finality_policy=ReportingSourceFinalPolicy("source", "delivery.is_final"), ) - if required == "snapshot": - producer._source.capabilities = _closing_capabilities() config = replace(config, delivery_config_version=2, required_finality=required, definition=None) await store.put_configuration(config) with pytest.raises(LedgerConflictError) as conflict: diff --git a/tests/test_reporting_settling.py b/tests/test_reporting_settling.py index 7a8f3d118..c9cbea845 100644 --- a/tests/test_reporting_settling.py +++ b/tests/test_reporting_settling.py @@ -187,6 +187,12 @@ async def _harness( offerings=ProducerOfferings( snapshot_offering_id=SNAPSHOT_OFFERING_ID, official_offering_id=OFFICIAL_OFFERING_ID, + snapshot_requests_official_close=any( + isinstance(offering, ProvisionalSnapshotOfferingV1) + and offering.offering_id == SNAPSHOT_OFFERING_ID + and offering.official_close_lag is not None + for offering in capabilities.offerings + ), publication_namespace="reporting-source:fixture", requested_metrics=("impressions", "spend"), source_scope=dict(capabilities.source_scope), diff --git a/tests/test_reporting_source_availability.py b/tests/test_reporting_source_availability.py index 01ac93e94..f928cfacb 100644 --- a/tests/test_reporting_source_availability.py +++ b/tests/test_reporting_source_availability.py @@ -36,6 +36,7 @@ ) from adcp.reporting.source import ( MediaBuyConstituentV1, + ProvisionalSnapshotOfferingV1, ReportingFinalityEvidenceV1, ReportingSourceCapabilitiesV1, ReportingSourceExecutorResult, @@ -225,6 +226,12 @@ def fetch(request: Any) -> InlineFetchResult: offerings = ProducerOfferings( snapshot_offering_id=SNAPSHOT_OFFERING_ID, official_offering_id=OFFICIAL_OFFERING_ID, + snapshot_requests_official_close=any( + isinstance(offering, ProvisionalSnapshotOfferingV1) + and offering.offering_id == SNAPSHOT_OFFERING_ID + and offering.official_close_lag is not None + for offering in capabilities.offerings + ), source_scope=dict(capabilities.source_scope), publication_namespace="reporting-source:fixture", )