diff --git a/docs/reporting-source-stable-finality.md b/docs/reporting-source-stable-finality.md deleted file mode 100644 index 9ae083d6a..000000000 --- a/docs/reporting-source-stable-finality.md +++ /dev/null @@ -1,36 +0,0 @@ -# Source stable package finality - -ESA's paid media package definition requires a 90 day source requery window, -content change publication, and official close only after one day of unchanged -source data at or beyond the 90 day age. Configure the SDK's provisional -snapshot offering with `restatement_window="P90D"`, a source safe -`restatement_cadence`, `official_close_lag="P90D"`, and -`official_stability_duration="P1D"`. Keep the provisional offering and add a -distinct `AuthoritativeOfferingV1` for the official read. Set its -`correction_policy="immutable_correction"` and choose a correction window that -matches the definition. Wire both offering IDs into `ProducerOfferings`. - -The snapshot adapter should return the complete package rows and control totals -on every scheduled requery, including an explicit zero-row result when the -package has no activity. An unchanged read advances the durable observation, -checkpoint and next due time; it does not create a revision, ledger change or -notification. A changed read publishes a snapshot that supersedes the current -snapshot. The final snapshot recheck at the 90 day boundary is inclusive. - -For an official read, the source adapter must return `InlineFetchResult` with -`finality_evidence_ref` identifying authoritative source evidence and -`finality_stable_since` naming when that source content last changed. The -producer requires both, checks that the source has been stable for the declared -duration, and waits for the period age, provisional boundary and last changed -snapshot's stability duration. A provisional observation alone never establishes -official finality. An official attempt with missing or premature evidence records -a nonpublishing checkpoint, so its retry uses a fresh source execution key and -read cutoff. The official offering's readiness fields should reflect when -the authoritative source can answer. An official correction after close must -be published through the existing adjustment flow, not as another snapshot or -replacement official revision. - -ESA should adopt the first SDK release containing this change, expected to be -`adcp==10.0.0b21` after its release. The PostgreSQL producer also needs the -matching `adcp-reporting-storage` release containing the observation constraint -migration. Do not deploy the SDK change with an earlier storage package. diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py index 4dc86119d..6e6c4932e 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py @@ -148,9 +148,11 @@ configuration_lifecycle, encode_cursor, issue_is_retirable, + provisional_observation_reuse_candidate, reject_reserved_authoritative_party, validate_adjustment_currency, validate_managed_revision_rows, + validate_retained_provisional_observation, validate_revision_currency, ) from adcp_reporting_storage.ledger.row_storage import ( @@ -1810,10 +1812,13 @@ async def commit_provisional_observation( ) -> ReportingRevisionRecord: acquisition = observation.acquisition key = (acquisition.account_id, acquisition.obligation_id, acquisition.ordinal) + reuse_candidate = provisional_observation_reuse_candidate( + acquisition, observation, revision + ) if revision is not None and ( revision.account_id != acquisition.account_id or revision.reporting_obligation_id != acquisition.obligation_id - or revision.reporting_revision_id != observation.revision_id + or (revision.reporting_revision_id != observation.revision_id and not reuse_candidate) ): raise LedgerConflictError("OBSERVATION_CONFLICT", "observation identity differs") async with self.transaction(), self._connection() as connection: @@ -1839,7 +1844,14 @@ async def commit_provisional_observation( retained_observation = ProvisionalObservation.from_wire(existing[1]) if ( retained_observation.acquisition != acquisition - or existing[0] != observation.revision_id + or ( + existing[0] != observation.revision_id + and ( + not reuse_candidate + or revision is None + or existing[0] != revision.reporting_revision_id + ) + ) or retained_observation.manifest_json != observation.manifest_json ): raise LedgerConflictError("OBSERVATION_CONFLICT", "observation replay differs") @@ -1852,11 +1864,10 @@ async def commit_provisional_observation( ) # Reuse the immutable publication replay checks while holding # the observation's account lock and transaction. - return ( - await self.commit_revision(revision, rows) - if revision is not None - else retained_revision - ) + if revision is None or (reuse_candidate and existing[0] == observation.revision_id): + validate_retained_provisional_observation(retained_revision, observation, rows) + return retained_revision + return await self.commit_revision(revision, rows) reserved = await ( await connection.execute( "SELECT payload FROM reporting_provisional_acquisitions" @@ -1872,7 +1883,7 @@ async def commit_provisional_observation( ) if checkpoint is not None and checkpoint.next_observation != acquisition.ordinal: raise LedgerConflictError("OBSERVATION_CONFLICT", "observation ordinal changed") - if revision is None: + if revision is None or reuse_candidate: if acquisition.predecessor_revision_id != observation.revision_id: raise LedgerConflictError( "OBSERVATION_CONFLICT", "snapshot predecessor differs" @@ -1902,6 +1913,16 @@ async def commit_provisional_observation( raise LedgerConflictError( "OBSERVATION_CONFLICT", "snapshot predecessor is no longer current" ) + validate_retained_provisional_observation( + committed, observation, rows, require_readable=False + ) + if not committed.readable: + if revision is None: + raise LedgerConflictError( + "HISTORY_UNAVAILABLE", "snapshot predecessor is not readable" + ) + committed = await self.commit_revision(revision, rows) + observation = replace(observation, revision_id=committed.reporting_revision_id) else: committed = await self.commit_revision(revision, rows) await self.record_restatement_checkpoint( diff --git a/src/adcp/reporting/inline_source.py b/src/adcp/reporting/inline_source.py index 35c0f0e8d..87c4721a7 100644 --- a/src/adcp/reporting/inline_source.py +++ b/src/adcp/reporting/inline_source.py @@ -284,12 +284,6 @@ class InlineFetchResult: scheduling correctly after a restart. """ - finality_evidence_ref: str | None = field(default=None, kw_only=True) - """Source evidence reference for an authoritative close.""" - - finality_stable_since: datetime | None = field(default=None, kw_only=True) - """Source attestation of when authoritative content last changed.""" - cell_availability: Mapping[str, Mapping[str, MetricEvidence]] | None = field( default=None, kw_only=True ) @@ -825,17 +819,6 @@ async def execute( unavailable_status="delayed", ) - if request.require_finality_evidence and ( - not answer.finality_evidence_ref or answer.finality_stable_since is None - ): - return ReportingSourceExecutorResult.failed( - ReportingSourceErrorV1( - code="PROVIDER_TRANSIENT", - retry="retryable", - safe_message="source has not supplied official finality evidence", - ) - ) - return await self._publish(request, answer, cancel=cancel) # -- fetch ---------------------------------------------------------- @@ -1038,8 +1021,6 @@ async def _publish( control_totals=control_totals, warnings=warnings, provisional_until=result.provisional_until, - finality_evidence_ref=result.finality_evidence_ref, - finality_stable_since=result.finality_stable_since, ) manifest_bytes = encode_source_batch_manifest_v1(manifest) @@ -1215,8 +1196,6 @@ def _seal_manifest( control_totals: list[SourceControlTotalV1], warnings: list[str], provisional_until: datetime | None, - finality_evidence_ref: str | None, - finality_stable_since: datetime | None, ) -> SourceBatchManifestV1: coverage = SourceBatchCoverageV1( denominator_fingerprint=request.coverage.denominator_fingerprint, @@ -1251,19 +1230,9 @@ def _seal_manifest( basis=( "provisional_observation" if request.publication_class == "PROVISIONAL_SNAPSHOT" - else ( - "provider_declared" - if finality_evidence_ref - else "elapsed_settlement_window" - ) + else "elapsed_settlement_window" ), observed_at=finality_at, - evidence_ref=finality_evidence_ref - if request.publication_class == "AUTHORITATIVE" - else None, - stable_since=finality_stable_since - if request.publication_class == "AUTHORITATIVE" - else None, provisional_until=( provisional_until if request.publication_class == "PROVISIONAL_SNAPSHOT" diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 18be4965c..3505ec34d 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -336,7 +336,6 @@ class _SettlingPolicy: restatement_window: timedelta restatement_cadence: timedelta official_close_lag: timedelta | None - official_stability_duration: timedelta | None = None class ReportingProducer: @@ -938,13 +937,6 @@ def _settling_policy( ), restatement_cadence=cadence, official_close_lag=close_lag, - official_stability_duration=( - timedelta( - milliseconds=iso_duration_milliseconds_v1(offering.official_stability_duration) - ) - if offering.official_stability_duration is not None - else None - ), ) async def _acquire_with_settling_policy( @@ -969,12 +961,7 @@ async def _acquire_with_settling_policy( ) if latest is not None: frozen = latest.acquisition.policy - policy = _SettlingPolicy( - frozen.window, - frozen.cadence, - frozen.official_close_lag, - frozen.official_stability_duration, - ) + policy = _SettlingPolicy(frozen.window, frozen.cadence, frozen.official_close_lag) explicit_close = ( policy.official_close_lag is not None and self._offerings.official_offering_id is not None @@ -1058,11 +1045,7 @@ async def _acquire_with_settling_policy( # Preserve explicitly configured official-close boundary precedence. # Snapshot-only policies retain their final inclusive due read through # downtime; expiry alone is never an official publication. - if ( - not explicit_close - or _utc(now) < settles_at - or (policy.official_stability_duration is not None and next_due is not None) - ): + if not explicit_close or _utc(now) < settles_at: if next_due is None: return not explicit_close if _utc(now) >= _utc(next_due): @@ -1090,13 +1073,6 @@ async def _acquire_with_settling_policy( _utc(obligation.period.end) + policy.official_close_lag, self._source_available_at(obligation, target_finality="official"), ) - if policy.official_stability_duration is not None: - snapshot = self._current_snapshot(revisions) - if snapshot is None: - raise LedgerConflictError("HISTORY_UNAVAILABLE", "snapshot predecessor is missing") - closes_at = max( - closes_at, _utc(snapshot.created_at) + policy.official_stability_duration - ) if _utc(now) >= closes_at: await self.acquire_obligation( configuration, @@ -1250,10 +1226,7 @@ async def acquire_obligation( latest.acquisition.policy if latest is not None else ProvisionalPolicy( - policy.restatement_window, - policy.restatement_cadence, - policy.official_close_lag, - policy.official_stability_duration, + policy.restatement_window, policy.restatement_cadence, policy.official_close_lag ) ) leaf = self._current_snapshot(revisions) @@ -1299,14 +1272,6 @@ async def acquire_obligation( # cancellation of this producer (notably on Python 3.10). Retain # the execution explicitly until even synchronous work has settled. await cancel_and_settle(execution) - if ( - acquisition is not None - and finality == "official" - and acquisition.policy.official_stability_duration is not None - ): - await self._record_unpublished_official_attempt( - obligation, acquisition, now=now, manifest_json='{"outcome":"not_ready"}' - ) turn.slices_failed.append(obligation.reporting_obligation_id) await self._schedule_retry(obligation, turn, now=now, replayed=manual_replay) self._note_escalation( @@ -1334,15 +1299,6 @@ async def acquire_obligation( error.code, error.retry, ) - if ( - acquisition is not None - and finality == "official" - and acquisition.policy.official_stability_duration is not None - and error.retry == "retryable" - ): - await self._record_unpublished_official_attempt( - obligation, acquisition, now=now, manifest_json='{"outcome":"not_ready"}' - ) turn.slices_failed.append(obligation.reporting_obligation_id) await self._schedule_retry( obligation, @@ -1370,29 +1326,6 @@ async def acquire_obligation( ) self._validate_manifest_currency(obligation, manifest) rows = await self._read_rows(request, manifest) - if ( - acquisition is not None - and finality == "official" - and acquisition.policy.official_stability_duration is not None - and ( - manifest.finality_evidence.basis - not in ("provider_declared", "provider_job_terminal") - or manifest.finality_evidence.evidence_ref is None - or manifest.finality_evidence.stable_since is None - or _utc(manifest.finality_evidence.observed_at) - < _utc(manifest.finality_evidence.stable_since) - + acquisition.policy.official_stability_duration - or _utc(request.period.source_read_cutoff_at) - < _utc(manifest.finality_evidence.stable_since) - + acquisition.policy.official_stability_duration - ) - ): - await self._record_unpublished_official_attempt( - obligation, acquisition, now=now, manifest_json=manifest.model_dump_json() - ) - turn.slices_failed.append(obligation.reporting_obligation_id) - await self._schedule_retry(obligation, turn, now=now) - return None # ``now`` freezes dispatch/lease/cutoff decisions, not publication. # A conforming source can observe finality while acquisition is running. # Every successful scheduled read reaches the atomic observation/checkpoint commit. @@ -1709,31 +1642,6 @@ def _observation_store(self) -> ProvisionalObservationStore: ) return self._store - async def _record_unpublished_official_attempt( - self, - obligation: ReportingObligationRecord, - acquisition: ProvisionalAcquisition, - *, - now: datetime, - manifest_json: str, - ) -> None: - predecessor = acquisition.predecessor_revision_id - if predecessor is None: - raise LedgerConflictError("HISTORY_UNAVAILABLE", "snapshot predecessor is missing") - boundary = acquisition.previous_boundary or ( - _utc(obligation.period.end) + acquisition.policy.window - ) - observation = ProvisionalObservation( - acquisition, - predecessor, - _utc(now), - _utc(boundary), - None, - manifest_json, - ) - async with self._revision_publication(obligation, acquisition.request().offering_id): - await self._observation_store().commit_provisional_observation(observation, None, ()) - async def _pending_acquisition( self, obligation: ReportingObligationRecord, *, ordinal: int ) -> ProvisionalAcquisition | None: @@ -1870,27 +1778,6 @@ async def commit_revision_from_manifest( ) if selection.kind == "corrupt" or leaf.kind == "corrupt": raise LedgerConflictError("HISTORY_UNAVAILABLE", "the revision history requires repair") - if acquisition is not None and finality == "official": - stable = acquisition.policy.official_stability_duration - if stable is not None: - if ( - leaf.kind != "selected" - or _utc(now) < _utc(obligation.period.end) + acquisition.policy.window - or _utc(now) < _utc(leaf.revision.created_at) + stable - or manifest.finality_evidence.evidence_ref is None - or manifest.finality_evidence.stable_since is None - or _utc(manifest.finality_evidence.observed_at) - < _utc(manifest.finality_evidence.stable_since) + stable - or _utc(acquisition.request().period.source_read_cutoff_at) - < _utc(manifest.finality_evidence.stable_since) + stable - or manifest.finality_evidence.basis - not in ("provider_declared", "provider_job_terminal") - ): - raise LedgerConflictError( - "OFFICIAL_FINALITY_UNPROVEN", - "official close requires the declared age, stable snapshot, " - "and source evidence", - ) supersedes = ( leaf.revision.reporting_revision_id if finality == "snapshot" and leaf.kind == "selected" @@ -1990,18 +1877,30 @@ async def commit_revision_from_manifest( ) if predecessor is None: raise LedgerConflictError("HISTORY_UNAVAILABLE", "snapshot predecessor is missing") - if predecessor.source_manifest_sha256 == manifest.content_fingerprint.split(":", 1)[-1]: + predecessor_content_sha256 = revision_content_sha256( + reporting_revision_id=predecessor.reporting_revision_id, + row_count=revision.row_count, + control_totals=revision.control_totals, + reporting_rows=rows, + control_total_evidence=( + revision.managed_control_totals + if predecessor.managed_control_totals is not None + else None + ), + ) + if ( + predecessor.readable + and predecessor.source_manifest_sha256 + == manifest.content_fingerprint.split(":", 1)[-1] + and predecessor.revision_content_sha256 == predecessor_content_sha256 + ): retained = predecessor - staged = ( - staging is not None - and retained is None - and bool(await staging.stage_revision_rows(revision, rows)) - ) + staged = staging is not None and bool(await staging.stage_revision_rows(revision, rows)) try: committed = await self._commit_publication( obligation, manifest, - revision if retained is None else None, + revision, rows, finality, now, @@ -2013,7 +1912,7 @@ async def commit_revision_from_manifest( finally: if staged and staging is not None: staging.discard_staged_revision_rows(revision) - if retained is None: + if retained is None or committed.reporting_revision_id != retained.reporting_revision_id: turn.revisions_committed.append(committed.reporting_revision_id) return committed @@ -2213,11 +2112,6 @@ def _build_slice( publication_class: ReportingPublicationClass = ( "AUTHORITATIVE" if resolved_finality == "official" else "PROVISIONAL_SNAPSHOT" ) - snapshot_offering = ( - self._source.capabilities.offering(self._offerings.snapshot_offering_id) - if self._offerings.snapshot_offering_id is not None - else None - ) return ReportingSourceSliceRequestV1( identity=ReportingSourceIdentityV1( account_id=obligation.account_id, @@ -2255,11 +2149,6 @@ def _build_slice( ), revision_kind="authoritative" if publication_class == "AUTHORITATIVE" else "snapshot", trigger=trigger, - require_finality_evidence=( - publication_class == "AUTHORITATIVE" - and isinstance(snapshot_offering, ProvisionalSnapshotOfferingV1) - and snapshot_offering.official_stability_duration is not None - ), coverage=ReportingSourceCoverageRequestV1( expected="full", constituents=resolved_constituents, diff --git a/src/adcp/reporting/ledger/provisional.py b/src/adcp/reporting/ledger/provisional.py index 410714697..027f984d8 100644 --- a/src/adcp/reporting/ledger/provisional.py +++ b/src/adcp/reporting/ledger/provisional.py @@ -29,18 +29,12 @@ class ProvisionalPolicy: window: timedelta cadence: timedelta official_close_lag: timedelta | None = None - official_stability_duration: timedelta | None = None def __post_init__(self) -> None: if self.window < timedelta(0) or self.cadence <= timedelta(0): raise ValueError("provisional window must be nonnegative and cadence positive") if self.official_close_lag is not None and self.official_close_lag < timedelta(0): raise ValueError("official close lag must be nonnegative") - if ( - self.official_stability_duration is not None - and self.official_stability_duration <= timedelta(0) - ): - raise ValueError("official stability duration must be positive") def to_wire(self) -> dict[str, int | None]: return { @@ -51,22 +45,15 @@ def to_wire(self) -> dict[str, int | None]: if self.official_close_lag is not None else None ), - "official_stability_duration_us": ( - self.official_stability_duration // timedelta(microseconds=1) - if self.official_stability_duration is not None - else None - ), } @classmethod def from_wire(cls, value: dict[str, Any]) -> ProvisionalPolicy: lag = value["official_close_lag_us"] - stable = value.get("official_stability_duration_us") return cls( timedelta(microseconds=value["window_us"]), timedelta(microseconds=value["cadence_us"]), timedelta(microseconds=lag) if lag is not None else None, - timedelta(microseconds=stable) if stable is not None else None, ) @@ -134,8 +121,10 @@ def to_wire(self) -> dict[str, Any]: @classmethod def from_wire(cls, value: dict[str, Any]) -> ProvisionalAcquisition: + request = dict(value["request"]) + request.pop("require_finality_evidence", None) return cls( - json.dumps(value["request"], sort_keys=True, separators=(",", ":")), + json.dumps(request, sort_keys=True, separators=(",", ":")), value["ordinal"], ProvisionalPolicy.from_wire(value["policy"]), value["predecessor_revision_id"], @@ -161,6 +150,11 @@ def __post_init__(self) -> None: utc(self.provisional_until) if self.next_due_at is not None: utc(self.next_due_at) + object.__setattr__( + self, + "manifest_json", + json.dumps(json.loads(self.manifest_json), sort_keys=True, separators=(",", ":")), + ) def to_wire(self) -> dict[str, Any]: return { diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index d5cb5cd6e..f55a83a8d 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -700,6 +700,48 @@ def validate_managed_revision_rows( ) +def validate_retained_provisional_observation( + revision: ReportingRevisionRecord, + observation: ProvisionalObservation, + rows: Sequence[dict[str, Any]], + *, + require_readable: bool = True, +) -> None: + """Validate an unchanged observation against immutable predecessor content.""" + manifest = json.loads(observation.manifest_json) + if ( + revision.source_manifest_sha256 is None + or manifest.get("content_fingerprint") != f"sha256:{revision.source_manifest_sha256}" + ): + raise LedgerConflictError( + "REVISION_IMMUTABLE", "observation content differs from the snapshot predecessor" + ) + from adcp.reporting.ledger.producer import recompute_revision_content_sha256 + + if len(rows) != revision.row_count: + raise LedgerConflictError("ROW_COUNT_MISMATCH", "row count differs from revision metadata") + if recompute_revision_content_sha256(revision, rows) != revision.revision_content_sha256: + raise LedgerConflictError( + "REVISION_IMMUTABLE", "observation rows differ from the snapshot predecessor" + ) + if require_readable and not revision.readable: + raise LedgerConflictError("HISTORY_UNAVAILABLE", "snapshot predecessor is not readable") + + +def provisional_observation_reuse_candidate( + acquisition: ProvisionalAcquisition, + observation: ProvisionalObservation, + revision: ReportingRevisionRecord | None, +) -> bool: + """Whether a candidate can replace its unchanged predecessor if reuse becomes unsafe.""" + return ( + revision is not None + and revision.reporting_revision_id != observation.revision_id + and acquisition.predecessor_revision_id == observation.revision_id + and revision.supersedes_reporting_revision_id == observation.revision_id + ) + + def _consumer_status_identity(status: ConsumerStatusRecord) -> str: payload: dict[str, Any] = { "chain": list(status.chain_key), @@ -1293,10 +1335,13 @@ async def commit_provisional_observation( ) -> ReportingRevisionRecord: acquisition = observation.acquisition key = (acquisition.account_id, acquisition.obligation_id, acquisition.ordinal) + reuse_candidate = provisional_observation_reuse_candidate( + acquisition, observation, revision + ) if revision is not None and ( revision.account_id != acquisition.account_id or revision.reporting_obligation_id != acquisition.obligation_id - or revision.reporting_revision_id != observation.revision_id + or (revision.reporting_revision_id != observation.revision_id and not reuse_candidate) ): raise LedgerConflictError("OBSERVATION_CONFLICT", "observation identity differs") async with self._mutation(): @@ -1310,7 +1355,14 @@ async def commit_provisional_observation( if existing is not None: if ( existing.acquisition != acquisition - or existing.revision_id != observation.revision_id + or ( + existing.revision_id != observation.revision_id + and ( + not reuse_candidate + or revision is None + or existing.revision_id != revision.reporting_revision_id + ) + ) or existing.manifest_json != observation.manifest_json ): raise LedgerConflictError("OBSERVATION_CONFLICT", "observation replay differs") @@ -1320,18 +1372,21 @@ async def commit_provisional_observation( ) # A matching observation identity does not make conflicting # publication content an idempotent replay. - return ( - await self.commit_revision(revision, rows) - if revision is not None - else self._revisions[existing.revision_id] - ) + if revision is None or ( + reuse_candidate and existing.revision_id == observation.revision_id + ): + validate_retained_provisional_observation( + self._revisions[existing.revision_id], observation, rows + ) + return self._revisions[existing.revision_id] + return await self.commit_revision(revision, rows) retained = self._provisional_acquisitions.get(key) if retained != acquisition: raise LedgerConflictError("OBSERVATION_CONFLICT", "acquisition was not reserved") checkpoint = self._restatement_checkpoints.get(acquisition.obligation_id) if checkpoint is not None and checkpoint.next_observation != acquisition.ordinal: raise LedgerConflictError("OBSERVATION_CONFLICT", "observation ordinal changed") - if revision is None: + if revision is None or reuse_candidate: if acquisition.predecessor_revision_id != observation.revision_id: raise LedgerConflictError( "OBSERVATION_CONFLICT", "snapshot predecessor differs" @@ -1356,6 +1411,16 @@ async def commit_provisional_observation( raise LedgerConflictError( "OBSERVATION_CONFLICT", "snapshot predecessor is no longer current" ) + validate_retained_provisional_observation( + committed, observation, rows, require_readable=False + ) + if not committed.readable: + if revision is None: + raise LedgerConflictError( + "HISTORY_UNAVAILABLE", "snapshot predecessor is not readable" + ) + committed = await self.commit_revision(revision, rows) + observation = replace(observation, revision_id=committed.reporting_revision_id) else: committed = await self.commit_revision(revision, rows) await self.record_restatement_checkpoint( diff --git a/src/adcp/reporting/source.py b/src/adcp/reporting/source.py index 25159a1b3..3963c500d 100644 --- a/src/adcp/reporting/source.py +++ b/src/adcp/reporting/source.py @@ -696,17 +696,12 @@ class ProvisionalSnapshotOfferingV1(_OfferingBase): restatement_window: IsoDuration | None = None restatement_cadence: IsoDuration | None = None official_close_lag: IsoDuration | None = None - official_stability_duration: IsoDuration | None = None @model_validator(mode="after") def _lag_range_is_ordered(self) -> ProvisionalSnapshotOfferingV1: _validate_lag_range(self.expected_availability_lag, self.worst_case_availability_lag) if self.restatement_window is None: - if ( - self.restatement_cadence is not None - or self.official_close_lag is not None - or self.official_stability_duration is not None - ): + if self.restatement_cadence is not None or self.official_close_lag is not None: raise ValueError( "restatement_cadence and official_close_lag require restatement_window" ) @@ -721,11 +716,6 @@ def _lag_range_is_ordered(self) -> ProvisionalSnapshotOfferingV1: raise ValueError("restatement_cadence must not be faster than fastest_safe_cadence") if self.official_close_lag is not None: iso_duration_milliseconds_v1(self.official_close_lag) - if self.official_stability_duration is not None: - if self.official_close_lag is None: - raise ValueError("official_stability_duration requires official_close_lag") - if iso_duration_milliseconds_v1(self.official_stability_duration) <= 0: - raise ValueError("official_stability_duration must be positive") return self @@ -888,7 +878,6 @@ class ReportingSourceSliceRequestV1(_Frozen): revision_kind: ReportingRevisionKind supersedes_publication_id: OpaqueReference | None = None trigger: ReportingTriggerKind = "scheduled_poll" - require_finality_evidence: bool = False coverage: ReportingSourceCoverageRequestV1 requested_metrics: list[MetricName] = Field(min_length=1, max_length=1_000) requested_dimensions: list[MetricName] = Field(default_factory=list, max_length=1_000) @@ -1017,16 +1006,14 @@ class ReportingFinalityEvidenceV1(_Frozen): ] observed_at: datetime evidence_ref: ExternalId | None = None - stable_since: datetime | None = None + stable_since: datetime | None = Field( + default=None, + description="Source evidence timestamp preserved without deriving SDK finality policy", + ) provisional_until: datetime | None = None @model_validator(mode="after") def _provisional_signal_matches_basis(self) -> ReportingFinalityEvidenceV1: - if self.stable_since is not None: - if self.basis not in ("provider_declared", "provider_job_terminal"): - raise ValueError("stable_since requires authoritative source evidence") - if _aware(self.stable_since, "stable_since") > _aware(self.observed_at, "observed_at"): - raise ValueError("stable_since must not follow finality observation") if self.provisional_until is None: return self if self.basis != "provisional_observation": diff --git a/tests/conformance/reporting/test_reporting_core_lifecycle.py b/tests/conformance/reporting/test_reporting_core_lifecycle.py index 88a45b68c..563400f9f 100644 --- a/tests/conformance/reporting/test_reporting_core_lifecycle.py +++ b/tests/conformance/reporting/test_reporting_core_lifecycle.py @@ -436,12 +436,11 @@ async def test_reporting_core_lifecycle(ledger: PgReportingLedgerStore) -> None: assert len(turn.obligations_committed) == 1 # The source omits an explicit restatement window, so the SDK's default # policy also schedules another read of the first period on this turn. + # Identical content advances the observation without creating a revision. first_revisions = await ledger.list_revisions( account_id=ACCOUNT, reporting_obligation_id=obligation.reporting_obligation_id ) - assert len(first_revisions) == 2 - automatic = next(item for item in first_revisions if item.reporting_revision_id != revision_id) - assert automatic.supersedes_reporting_revision_id == revision_id + assert [item.reporting_revision_id for item in first_revisions] == [revision_id] zero_obligation = await ledger.find_obligation( account_id=ACCOUNT, @@ -456,10 +455,7 @@ async def test_reporting_core_lifecycle(ledger: PgReportingLedgerStore) -> None: account_id=ACCOUNT, reporting_obligation_id=zero_obligation.reporting_obligation_id ) assert [item.row_count for item in zero_revisions] == [0] - assert set(turn.revisions_committed) == { - automatic.reporting_revision_id, - zero_revisions[0].reporting_revision_id, - } + assert turn.revisions_committed == [zero_revisions[0].reporting_revision_id] settled = await _reconcile(ledger, expected=_expected_periods(2)) assert settled.definitive is True, settled.obligations @@ -503,14 +499,12 @@ async def test_reporting_core_lifecycle(ledger: PgReportingLedgerStore) -> None: ) == () ) - # At this later turn the first period's next hourly observation is due - # too. Preserve that automatic revision as the next restatement's parent. + # At this later turn the first period's next hourly observation is due too. + # Repeated observations of the same content retain the published revision. refreshed_first = await ledger.list_revisions( account_id=ACCOUNT, reporting_obligation_id=obligation.reporting_obligation_id ) - assert len(refreshed_first) == 3 - automatic_at_ready = max(refreshed_first, key=lambda item: item.created_at) - assert automatic_at_ready.supersedes_reporting_revision_id == automatic.reporting_revision_id + assert [item.reporting_revision_id for item in refreshed_first] == [revision_id] # The zero-row period and the silent period are distinguishable, which is # the entire operational point of the tier. @@ -560,8 +554,8 @@ async def test_reporting_core_lifecycle(ledger: PgReportingLedgerStore) -> None: chain = await ledger.list_revisions( account_id=ACCOUNT, reporting_obligation_id=obligation.reporting_obligation_id ) - assert len(chain) == 4 - assert restated.supersedes_reporting_revision_id == automatic_at_ready.reporting_revision_id + assert len(chain) == 2 + assert restated.supersedes_reporting_revision_id == revision_id # The superseded revision is still retained and still readable: a consumer # that already cited it must be able to fetch exactly what it cited. superseded = await ledger.read_revision_rows( diff --git a/tests/conformance/reporting/test_reporting_provisional_catalog_scope.py b/tests/conformance/reporting/test_reporting_provisional_catalog_scope.py index 5dde7569d..43ce63d73 100644 --- a/tests/conformance/reporting/test_reporting_provisional_catalog_scope.py +++ b/tests/conformance/reporting/test_reporting_provisional_catalog_scope.py @@ -46,7 +46,7 @@ def assert_narrow_scope(self): assert parameters == (oids,) assert self.calls[5][1] == FUNCTION assert [(row[0], row[1]) for row in self.calls[5][2]] == [FUNCTION] - assert [len(rows) for _, _, rows in self.calls] == [2, 10, 8, 4, 2, 1] + assert [len(rows) for _, _, rows in self.calls] == [2, 10, 7, 3, 2, 1] @pytest.mark.parametrize("autocommit", [False, True]) @@ -68,14 +68,14 @@ async def test_exact_catalog_scope_preserves_full_contract_and_excludes_overload .read_text() ) ) - assert len(expected) == 794 and len(PROVISIONAL_REQUIRED_OBJECTS) == 27 + assert len(expected) == 792 and len(PROVISIONAL_REQUIRED_OBJECTS) == 25 async with isolated_reporting_pool(autocommit=autocommit) as pool: await PgReportingReceiptStore(pool=pool).create_schema() async with pool.connection() as connection: full = CatalogProbe(connection) assert await schema_objects(full) == expected assert len(full.calls) == 6 - assert sum(len(rows) for _, _, rows in full.calls) == 794 + assert sum(len(rows) for _, _, rows in full.calls) == 792 await connection.execute( "CREATE TABLE reporting_unrelated_catalog_scope (value integer);" " CREATE FUNCTION reporting_provisional_immutable(integer) RETURNS integer" diff --git a/tests/test_reporting_provisional_observations.py b/tests/test_reporting_provisional_observations.py index 1d984b47f..07c9993be 100644 --- a/tests/test_reporting_provisional_observations.py +++ b/tests/test_reporting_provisional_observations.py @@ -160,6 +160,28 @@ def unavailable(request): assert retained.request_json == legacy.request_json +async def test_retained_acquisition_decodes_finality_evidence_flag_without_exposing_it( + make_harness, +): + producer, store, _, _ = await make_harness(_capabilities(restatement_window="P3D")) + producer._source._fetch = lambda request: None + await producer.run_worker() + obligation = await _only_obligation(store) + reserved = await store.get_provisional_acquisition( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ordinal=0, + ) + assert reserved is not None + wire = reserved.to_wire() + wire["request"]["require_finality_evidence"] = False + + decoded = ProvisionalAcquisition.from_wire(wire) + + assert decoded.request() == reserved.request() + assert "require_finality_evidence" not in decoded.request().model_dump() + + async def _publication_effect_counts(store): if isinstance(store, PgReportingLedgerStore): async with store._pool.connection() as connection: @@ -246,6 +268,69 @@ async def test_unchanged_scheduled_read_keeps_snapshot_and_advances_observation( assert len(fetch.calls) == 2 +async def test_unchanged_content_repairs_an_unreadable_snapshot(make_harness): + producer, store, _, clock = await make_harness(_capabilities(restatement_window=None)) + await producer.run_worker() + original = (await _revisions(store))[0] + clock[0] += timedelta(hours=1) + await producer.run_worker() + observation = await latest(store) + rows = ( + await store.read_revision_rows( + account_id=ACCOUNT, + reporting_revision_id=original.reporting_revision_id, + ) + ).rows + await store.set_revision_readable( + account_id=ACCOUNT, + reporting_revision_id=original.reporting_revision_id, + readable=False, + ) + + with pytest.raises(LedgerConflictError) as conflict: + await store.commit_provisional_observation(observation, None, rows) + assert conflict.value.code == "HISTORY_UNAVAILABLE" + + assert observation.next_due_at is not None + clock[0] = observation.next_due_at + await producer.run_worker() + history = await _revisions(store) + assert len(history) == 2 + assert not history[0].readable + assert history[1].readable + assert history[1].supersedes_reporting_revision_id == original.reporting_revision_id + assert (await latest(store)).revision_id == history[1].reporting_revision_id + + +async def test_readability_change_during_unchanged_commit_publishes_repair( + make_harness, monkeypatch +): + producer, store, _, clock = await make_harness(_capabilities(restatement_window=None)) + await producer.run_worker() + original = (await _revisions(store))[0] + commit_observation = store.commit_provisional_observation + + async def make_predecessor_unreadable(observation, revision, rows): + await store.set_revision_readable( + account_id=ACCOUNT, + reporting_revision_id=original.reporting_revision_id, + readable=False, + ) + return await commit_observation(observation, revision, rows) + + monkeypatch.setattr(store, "commit_provisional_observation", make_predecessor_unreadable) + clock[0] += timedelta(hours=1) + turn = await producer.run_worker() + + history = await _revisions(store) + assert len(history) == 2 + assert not history[0].readable + assert history[1].readable + assert history[1].supersedes_reporting_revision_id == original.reporting_revision_id + assert turn.revisions_committed == [history[1].reporting_revision_id] + assert (await latest(store)).revision_id == history[1].reporting_revision_id + + @pytest.mark.parametrize("offset", [timedelta(minutes=30), timedelta(days=4)]) async def test_source_boundary_shortens_or_extends_fallback(make_harness, offset): producer, store, fetch, clock = await make_harness(_capabilities(restatement_window=None)) @@ -704,6 +789,7 @@ async def test_recorded_observation_replay_validates_content_and_keeps_original_ for submitted_revision, submitted_rows, code in ( (changed_revision, changed_rows, "REVISION_IMMUTABLE"), (revision, (), "ROW_COUNT_MISMATCH"), + (None, changed_rows, "REVISION_IMMUTABLE"), ): with pytest.raises(LedgerConflictError) as conflict: await store.commit_provisional_observation( diff --git a/tests/test_reporting_settling.py b/tests/test_reporting_settling.py index 85c1dfffa..d7289f35e 100644 --- a/tests/test_reporting_settling.py +++ b/tests/test_reporting_settling.py @@ -47,7 +47,6 @@ def _capabilities( restatement_window: str | None, restatement_cadence: str | None = None, official_close_lag: str | None = None, - official_stability_duration: str | None = None, ) -> ReportingSourceCapabilitiesV1: base = redacted_capabilities() snapshot = base.offering(SNAPSHOT_OFFERING_ID) @@ -58,7 +57,6 @@ def _capabilities( "restatement_window": restatement_window, "restatement_cadence": restatement_cadence, "official_close_lag": official_close_lag, - "official_stability_duration": official_stability_duration, # These tests exercise restatement policy with already-ready data. "expected_availability_lag": "PT0S", "worst_case_availability_lag": "PT0S", @@ -92,8 +90,6 @@ class _MutableFetch: def __init__(self) -> None: self.impressions = 10 self.provisional_until: datetime | None = None - self.finality_evidence_ref: str | None = None - self.finality_stable_since: datetime | None = None self.calls: list[str] = [] self.requests: list[Any] = [] @@ -114,8 +110,6 @@ def __call__(self, request: Any) -> InlineFetchResult: if request.publication_class == "PROVISIONAL_SNAPSHOT" else None ), - finality_evidence_ref=self.finality_evidence_ref, - finality_stable_since=self.finality_stable_since, ) @@ -250,53 +244,6 @@ async def test_unchanged_refresh_honors_cadence_and_commits_an_observation() -> assert checkpoint is not None and checkpoint.next_observation == 2 -async def test_source_stable_close_requires_age_stability_and_evidence() -> None: - producer, store, fetch, clock = await _harness( - _capabilities( - restatement_window="P90D", - restatement_cadence="P1D", - official_close_lag="P90D", - official_stability_duration="P1D", - ) - ) - await producer.run_worker() - obligation = await _only_obligation(store) - clock[0] = obligation.period.end + timedelta(days=89) - fetch.impressions = 20 - await producer.run_worker() - clock[0] = obligation.period.end + timedelta(days=90) - await producer.run_worker() - assert all(item.finality == "snapshot" for item in await _revisions(store)) - assert len(await _revisions(store)) == 2 - - clock[0] += timedelta(seconds=1) - await producer.run_worker() - assert all(item.finality == "snapshot" for item in await _revisions(store)) - first_official = fetch.requests[-1] - - fetch.finality_evidence_ref = "source-ledger:close-1" - fetch.finality_stable_since = obligation.period.end + timedelta(days=90, hours=12) - clock[0] += timedelta(days=1) - await producer.run_worker() - assert all(item.finality == "snapshot" for item in await _revisions(store)) - assert ( - fetch.requests[-1].identity.source_execution_key - != first_official.identity.source_execution_key - ) - assert ( - fetch.requests[-1].period.source_read_cutoff_at - > first_official.period.source_read_cutoff_at - ) - early_evidence = fetch.requests[-1] - clock[0] += timedelta(hours=12) - await producer.run_worker() - assert ( - fetch.requests[-1].identity.source_execution_key - != early_evidence.identity.source_execution_key - ) - assert (await _revisions(store))[-1].finality == "official" - - async def test_changed_refresh_supersedes_snapshot_and_window_expiry_closes_officially() -> None: producer, store, fetch, clock = await _harness( _capabilities( @@ -402,7 +349,17 @@ def test_settling_options_require_a_declared_window_and_safe_cadence() -> None: "restatement_cadence": "PT1M", } ) - with pytest.raises(ValidationError, match="requires official_close_lag"): + + +def test_snapshot_offering_rejects_official_stability_policy() -> None: + snapshot = redacted_capabilities().offering(SNAPSHOT_OFFERING_ID) + assert isinstance(snapshot, ProvisionalSnapshotOfferingV1) + with pytest.raises(ValidationError, match="official_stability_duration"): ProvisionalSnapshotOfferingV1.model_validate( - {**payload, "restatement_window": "P90D", "official_stability_duration": "P1D"} + { + **snapshot.model_dump(mode="python"), + "restatement_window": "P90D", + "official_close_lag": "P90D", + "official_stability_duration": "P1D", + } ) diff --git a/tests/test_reporting_source_contract.py b/tests/test_reporting_source_contract.py index 57e6d886f..0814b4328 100644 --- a/tests/test_reporting_source_contract.py +++ b/tests/test_reporting_source_contract.py @@ -15,6 +15,7 @@ import pytest from pydantic import ValidationError +from adcp.reporting.canonical_json import canonical_json_utf8_v1 from adcp.reporting.conformance import ( ReportingSourceConformanceError, run_reporting_source_replay_conformance, @@ -83,6 +84,21 @@ def _rebind(manifest: SourceBatchManifestV1, **updates) -> SourceBatchManifestV1 return SourceBatchManifestV1.model_validate(payload) +def test_manifest_preserves_declared_source_stability_evidence() -> None: + _, manifest, _ = redacted_completed_result(redacted_authoritative_request()) + payload = manifest.model_dump(mode="json", exclude_none=True) + payload["finality_evidence"].update( + basis="provider_declared", + evidence_ref="source-ledger:close-1", + stable_since=payload["finality_evidence"]["observed_at"], + ) + manifest_bytes = canonical_json_utf8_v1(payload) + + retained = SourceBatchManifestV1.model_validate_json(manifest_bytes) + + assert encode_source_batch_manifest_v1(retained) == manifest_bytes + + # -- capability admission ---------------------------------------------------