From 772f3df408fa6536def299a1f29a399cfe3158c6 Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Tue, 17 Nov 2026 04:18:31 +0530 Subject: [PATCH 1/3] fix(reporting): suppress unchanged restatement revisions --- .../src/adcp_reporting_storage/ledger/pg.py | 36 +++- src/adcp/reporting/ledger/producer.py | 3 +- src/adcp/reporting/ledger/provisional.py | 6 + src/adcp/reporting/ledger/store.py | 48 ++++- .../test_reporting_production_settling.py | 10 +- ...test_reporting_provisional_observations.py | 201 ++++++++++++++---- tests/test_reporting_settling.py | 6 +- 7 files changed, 246 insertions(+), 64 deletions(-) diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py index f22c48ed0e..11639c68e5 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/pg.py @@ -152,6 +152,7 @@ validate_adjustment_currency, validate_managed_revision_rows, validate_revision_currency, + validate_revision_rows_binding, ) from adcp_reporting_storage.ledger.row_storage import ( REPORTING_ROW_STORAGE_DDL_PATH, @@ -1837,10 +1838,7 @@ async def commit_provisional_observation( ).fetchone() if existing is not None: retained_observation = ProvisionalObservation.from_wire(existing[1]) - if ( - retained_observation.acquisition != acquisition - or existing[0] != observation.revision_id - ): + if retained_observation.acquisition != acquisition: raise LedgerConflictError("OBSERVATION_CONFLICT", "observation replay differs") retained_revision = await self.get_revision( account_id=acquisition.account_id, reporting_revision_id=existing[0] @@ -1849,6 +1847,18 @@ async def commit_provisional_observation( raise LedgerConflictError( "HISTORY_UNAVAILABLE", "observation revision is missing" ) + if existing[0] != observation.revision_id: + if ( + revision.finality != "snapshot" + or revision.supersedes_reporting_revision_id != existing[0] + or revision.source_manifest_sha256 + != retained_revision.source_manifest_sha256 + ): + raise LedgerConflictError( + "OBSERVATION_CONFLICT", "observation replay differs" + ) + validate_revision_rows_binding(retained_revision, rows) + revision = retained_revision # Reuse the immutable publication replay checks while holding # the observation's account lock and transaction. return await self.commit_revision(revision, rows) @@ -1867,6 +1877,24 @@ async def commit_provisional_observation( ) if checkpoint is not None and checkpoint.next_observation != acquisition.ordinal: raise LedgerConflictError("OBSERVATION_CONFLICT", "observation ordinal changed") + predecessor_id = revision.supersedes_reporting_revision_id + if revision.finality == "snapshot" and predecessor_id is not None: + await self._require_current_leaf(connection, revision) + predecessor = await self.get_revision( + account_id=acquisition.account_id, + reporting_revision_id=predecessor_id, + ) + if predecessor is None: + raise LedgerConflictError( + "HISTORY_UNAVAILABLE", "observation predecessor is missing" + ) + if ( + predecessor.readable + and predecessor.source_manifest_sha256 == revision.source_manifest_sha256 + ): + validate_revision_rows_binding(predecessor, rows) + revision = predecessor + observation = replace(observation, revision_id=predecessor_id) committed = await self.commit_revision(revision, rows) await self.record_restatement_checkpoint( RestatementCheckpoint( diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 449acde584..5a242e642d 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -1874,7 +1874,8 @@ async def commit_revision_from_manifest( finally: if staged and staging is not None: staging.discard_staged_revision_rows(revision) - turn.revisions_committed.append(committed.reporting_revision_id) + if committed.reporting_revision_id == revision.reporting_revision_id: + turn.revisions_committed.append(committed.reporting_revision_id) return committed async def _commit_publication( diff --git a/src/adcp/reporting/ledger/provisional.py b/src/adcp/reporting/ledger/provisional.py index 589b185783..7e1088d782 100644 --- a/src/adcp/reporting/ledger/provisional.py +++ b/src/adcp/reporting/ledger/provisional.py @@ -205,4 +205,10 @@ async def commit_provisional_observation( revision: ReportingRevisionRecord, rows: Sequence[dict[str, Any]], ) -> ReportingRevisionRecord: + """Atomically publish an observation and advance its checkpoint. + + An unchanged snapshot must reuse its readable current predecessor instead + of emitting a revision. Implementations make that decision under the same + account lock that commits changed content and the observation. + """ pass diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index b91762212b..bc03a995bf 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -700,6 +700,18 @@ def validate_managed_revision_rows( ) +def validate_revision_rows_binding( + revision: ReportingRevisionRecord, rows: Sequence[dict[str, Any]] +) -> None: + """Verify submitted rows against an already retained revision's Core binding.""" + from adcp.reporting.ledger.producer import recompute_revision_content_sha256 + + if recompute_revision_content_sha256(revision, rows) != revision.revision_content_sha256: + raise LedgerConflictError( + "REVISION_CONTENT_MISMATCH", "revision row content does not match its immutable binding" + ) + + def _consumer_status_identity(status: ConsumerStatusRecord) -> str: payload: dict[str, Any] = { "chain": list(status.chain_key), @@ -1308,15 +1320,25 @@ async def commit_provisional_observation( "observation replay differs: acquisition does not bind this obligation", ) if existing is not None: - if ( - existing.acquisition != acquisition - or existing.revision_id != observation.revision_id - ): + if existing.acquisition != acquisition: raise LedgerConflictError("OBSERVATION_CONFLICT", "observation replay differs") - if existing.revision_id not in self._revisions: + retained_revision = self._revisions.get(existing.revision_id) + if retained_revision is None: raise LedgerConflictError( "HISTORY_UNAVAILABLE", "observation revision is missing" ) + if existing.revision_id != observation.revision_id: + if ( + revision.finality != "snapshot" + or revision.supersedes_reporting_revision_id != existing.revision_id + or revision.source_manifest_sha256 + != retained_revision.source_manifest_sha256 + ): + raise LedgerConflictError( + "OBSERVATION_CONFLICT", "observation replay differs" + ) + validate_revision_rows_binding(retained_revision, rows) + revision = retained_revision # A matching observation identity does not make conflicting # publication content an idempotent replay. return await self.commit_revision(revision, rows) @@ -1326,6 +1348,22 @@ async def commit_provisional_observation( checkpoint = self._restatement_checkpoints.get(acquisition.obligation_id) if checkpoint is not None and checkpoint.next_observation != acquisition.ordinal: raise LedgerConflictError("OBSERVATION_CONFLICT", "observation ordinal changed") + predecessor_id = revision.supersedes_reporting_revision_id + if revision.finality == "snapshot" and predecessor_id is not None: + siblings = [ + item + for item in self._revisions.values() + if item.reporting_obligation_id == revision.reporting_obligation_id + ] + self._require_current_leaf(revision, siblings) + predecessor = self._revisions[predecessor_id] + if ( + predecessor.readable + and predecessor.source_manifest_sha256 == revision.source_manifest_sha256 + ): + validate_revision_rows_binding(predecessor, rows) + revision = predecessor + observation = replace(observation, revision_id=predecessor_id) committed = await self.commit_revision(revision, rows) await self.record_restatement_checkpoint( RestatementCheckpoint( diff --git a/tests/conformance/reporting/test_reporting_production_settling.py b/tests/conformance/reporting/test_reporting_production_settling.py index c1fb07fa59..e1c5f49431 100644 --- a/tests/conformance/reporting/test_reporting_production_settling.py +++ b/tests/conformance/reporting/test_reporting_production_settling.py @@ -96,7 +96,7 @@ async def test_progress_retains_policy_until_terminal_publication( assert len(source.requests) == 1 h.source_clock.now = END + timedelta(hours=1) unchanged = await source_turn(h.production) - assert len(unchanged.revisions_committed) == 1 + assert not unchanged.revisions_committed assert len(source.requests) == 2 checkpoint = await h.store.get_restatement_checkpoint( account_id=h.item.config.account_id, @@ -111,9 +111,8 @@ async def test_progress_retains_policy_until_terminal_publication( changed = await source_turn(h.production) assert len(changed.revisions_committed) == 1 history = await revisions(h) - assert len(history) == 3 + assert len(history) == 2 assert history[1].supersedes_reporting_revision_id == history[0].reporting_revision_id - assert history[2].supersedes_reporting_revision_id == history[1].reporting_revision_id h.source_clock.now = END + timedelta(hours=3) await source_turn(h.production) @@ -135,7 +134,6 @@ async def test_progress_retains_policy_until_terminal_publication( completed = await source_turn(h.production) assert len(completed.revisions_committed) == 1 assert [r.finality for r in await revisions(h)] == [ - "snapshot", "snapshot", "snapshot", "official", @@ -163,7 +161,7 @@ async def test_settling_checkpoint_and_pending_work_survive_fresh_service(backen assert len(initial.revisions_committed) == 1 h.source_clock.now = END + timedelta(hours=1) noop = await source_turn(h.production) - assert len(noop.revisions_committed) == 1 + assert not noop.revisions_committed original_source = h.production.offerings[0].producer._source assert len(original_source.requests) == 2 prior_keys = {r.identity.source_execution_key for r in original_source.requests} @@ -199,6 +197,6 @@ async def test_settling_checkpoint_and_pending_work_survive_fresh_service(backen reporting_obligation_id=fresh.item.obligation.reporting_obligation_id, ) assert checkpoint is not None and checkpoint.next_observation == 3 - assert len(await revisions(fresh)) == 3 + assert len(await revisions(fresh)) == 2 unfinished = await pending(fresh) assert unfinished == (fresh.item.obligation.reporting_obligation_id,) diff --git a/tests/test_reporting_provisional_observations.py b/tests/test_reporting_provisional_observations.py index bd13fc29d6..1d106ff7aa 100644 --- a/tests/test_reporting_provisional_observations.py +++ b/tests/test_reporting_provisional_observations.py @@ -184,6 +184,52 @@ async def _publication_effect_counts(store): ) +class _ReadabilityRaceStore(InMemoryReportingLedgerStore): + mark_predecessor_unreadable = False + + async def commit_provisional_observation(self, observation, revision, rows): + if self.mark_predecessor_unreadable: + self.mark_predecessor_unreadable = False + predecessor_id = revision.supersedes_reporting_revision_id + assert predecessor_id is not None + await self.set_revision_readable( + account_id=revision.account_id, + reporting_revision_id=predecessor_id, + readable=False, + ) + return await super().commit_provisional_observation(observation, revision, rows) + + +class _LeafRaceStore(InMemoryReportingLedgerStore): + advance_predecessor = False + + async def commit_provisional_observation(self, observation, revision, rows): + if self.advance_predecessor: + self.advance_predecessor = False + predecessor_id = revision.supersedes_reporting_revision_id + assert predecessor_id is not None + predecessor = await self.get_revision( + account_id=revision.account_id, + reporting_revision_id=predecessor_id, + ) + assert predecessor is not None + concurrent_id = f"{predecessor.reporting_revision_id}-concurrent" + concurrent = replace( + predecessor, + reporting_revision_id=concurrent_id, + supersedes_reporting_revision_id=predecessor.reporting_revision_id, + revision_content_sha256=revision_content_sha256( + reporting_revision_id=concurrent_id, + row_count=predecessor.row_count, + control_totals=predecessor.control_totals, + reporting_rows=rows, + control_total_evidence=predecessor.managed_control_totals, + ), + ) + await self.commit_revision(concurrent, rows) + return await super().commit_provisional_observation(observation, revision, rows) + + async def test_adapter_without_window_rereads_with_sdk_default(make_harness): producer, store, fetch, clock = await make_harness(_capabilities(restatement_window=None)) first = await producer.run_worker() @@ -196,56 +242,116 @@ async def test_adapter_without_window_rereads_with_sdk_default(make_harness): refreshed = await producer.run_worker() assert fetch.calls == ["PROVISIONAL_SNAPSHOT", "PROVISIONAL_SNAPSHOT"] - assert len(refreshed.revisions_committed) == 1 - assert len(await _revisions(store)) == 2 + assert refreshed.revisions_committed == [] + assert len(await _revisions(store)) == 1 assert all(item.finality == "snapshot" for item in await _revisions(store)) -async def test_unchanged_scheduled_read_is_a_distinct_immutable_observation(make_harness): +async def test_unchanged_scheduled_read_advances_observation_without_a_revision(make_harness): producer, store, fetch, clock = await make_harness( _capabilities(restatement_window="P3D", restatement_cadence="PT1H") ) await producer.run_worker() original = (await _revisions(store))[0] first = await latest(store) - original_rows = await store.read_revision_rows( - account_id=ACCOUNT, reporting_revision_id=original.reporting_revision_id - ) + effects_before = await _publication_effect_counts(store) clock[0] += timedelta(hours=1) refreshed = await producer.run_worker() assert fetch.calls == ["PROVISIONAL_SNAPSHOT", "PROVISIONAL_SNAPSHOT"] - assert len(refreshed.revisions_committed) == 1 + assert refreshed.revisions_committed == [] history = await _revisions(store) - assert len(history) == 2 and history[0] == original - assert history[1].reporting_revision_id != original.reporting_revision_id - assert history[1].supersedes_reporting_revision_id == original.reporting_revision_id - assert history[1].observed_at > original.observed_at - assert history[1].source_manifest_sha256 == original.source_manifest_sha256 - assert history[1].revision_content_sha256 != original.revision_content_sha256 + assert history == (original,) second = await latest(store) assert second.acquisition.ordinal == first.acquisition.ordinal + 1 assert second.checked_at > first.checked_at assert second.next_due_at == clock[0] + timedelta(hours=1) - assert second.revision_id == history[1].reporting_revision_id - current_rows = await store.read_revision_rows( - account_id=ACCOUNT, reporting_revision_id=second.revision_id - ) - assert current_rows.rows == original_rows.rows - # The legacy SQL layout retains every new revision's rows as well. - if isinstance(store, PgReportingLedgerStore): - async with store._pool.connection() as connection: - counts = await ( - await connection.execute( - "SELECT reporting_revision_id,count(*) FROM reporting_revision_rows" - " GROUP BY reporting_revision_id ORDER BY reporting_revision_id" - ) - ).fetchall() - assert sorted(count for _, count in counts) == [1, 1] + assert second.revision_id == original.reporting_revision_id + effects_after = await _publication_effect_counts(store) + assert effects_after[:2] == effects_before[:2] + assert effects_after[2] == effects_before[2] + 1 + assert effects_after[3:] == effects_before[3:] await producer.run_worker() assert len(fetch.calls) == 2 +async def test_unreadable_predecessor_race_commits_a_readable_repair(): + producer, store, _, clock = await _harness( + _capabilities(restatement_window="P3D", restatement_cadence="PT1H"), + store_factory=lambda now: _ReadabilityRaceStore(clock=now), + ) + await producer.run_worker() + store.mark_predecessor_unreadable = True + clock[0] += timedelta(hours=1) + repaired = await producer.run_worker() + + history = await _revisions(store) + assert len(repaired.revisions_committed) == 1 + assert len(history) == 2 + assert not history[0].readable + assert history[1].readable + assert history[1].supersedes_reporting_revision_id == history[0].reporting_revision_id + assert (await latest(store)).revision_id == history[1].reporting_revision_id + + +async def test_suppressed_observation_replay_rejects_conflicting_rows(make_harness): + producer, store, _, clock = await make_harness( + _capabilities(restatement_window="P3D", restatement_cadence="PT1H") + ) + await producer.run_worker() + clock[0] += timedelta(hours=1) + await producer.run_worker() + observation = await latest(store) + revision = (await _revisions(store))[0] + rows = ( + await store.read_revision_rows( + account_id=ACCOUNT, + reporting_revision_id=revision.reporting_revision_id, + ) + ).rows + changed_rows = (dict(rows[0], impressions=rows[0]["impressions"] + 1),) + manifest = SourceBatchManifestV1.model_validate_json(observation.manifest_json) + obligation = await _only_obligation(store) + effects = await _publication_effect_counts(store) + + with pytest.raises(LedgerConflictError) as conflict: + await producer.commit_revision_from_manifest( + obligation, + manifest, + rows=changed_rows, + finality="snapshot", + now=clock[0], + acquisition=observation.acquisition, + ) + + assert conflict.value.code == "REVISION_CONTENT_MISMATCH" + assert await _publication_effect_counts(store) == effects + assert (await _revisions(store)) == (revision,) + assert await latest(store) == observation + + +async def test_concurrent_leaf_advance_cannot_attach_an_observation_to_a_stale_revision(): + producer, store, _, clock = await _harness( + _capabilities(restatement_window="P3D", restatement_cadence="PT1H"), + store_factory=lambda now: _LeafRaceStore(clock=now), + ) + await producer.run_worker() + obligation = await _only_obligation(store) + first = await latest(store) + store.advance_predecessor = True + clock[0] += timedelta(hours=1) + with pytest.raises(LedgerConflictError) as conflict: + await producer.run_worker() + + assert conflict.value.code == "SUPERSEDES_STALE" + assert (await latest(store)) == first + checkpoint = await store.get_restatement_checkpoint( + account_id=ACCOUNT, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + assert checkpoint is not None and checkpoint.next_observation == 1 + + @pytest.mark.parametrize("offset", [timedelta(minutes=30), timedelta(days=4)]) async def test_source_boundary_shortens_or_extends_fallback(make_harness, offset): producer, store, fetch, clock = await make_harness(_capabilities(restatement_window=None)) @@ -259,11 +365,13 @@ async def test_source_boundary_shortens_or_extends_fallback(make_harness, offset clock[0] = first.next_due_at await producer.run_worker() assert len(fetch.calls) == 2 - assert len(await _revisions(store)) == 2 + assert len(await _revisions(store)) == 1 -async def test_final_read_at_boundary_and_overdue_restart_never_promote(make_harness): - producer, store, fetch, clock = await make_harness(_capabilities(restatement_window=None)) +async def test_snapshot_only_final_read_never_requires_an_official_close(make_harness): + producer, store, fetch, clock = await make_harness( + _capabilities(restatement_window="P90D", restatement_cadence="P1D") + ) await producer.run_worker() first = await latest(store) clock[0] = first.provisional_until + timedelta(days=1) @@ -271,7 +379,7 @@ async def test_final_read_at_boundary_and_overdue_restart_never_promote(make_har final = await latest(store) assert final.next_due_at is None assert len(fetch.calls) == 2 - assert len(await _revisions(store)) == 2 + assert len(await _revisions(store)) == 1 assert all(item.finality == "snapshot" for item in await _revisions(store)) await producer.run_worker() assert len(fetch.calls) == 2 @@ -388,7 +496,7 @@ async def test_concurrent_same_acquisition_commits_once(make_harness): for _ in range(2) ] ) - assert len(await _revisions(store)) == 2 + assert len(await _revisions(store)) == 1 assert (await latest(store)).acquisition.ordinal == 1 checkpoint = await store.get_restatement_checkpoint( account_id=ACCOUNT, reporting_obligation_id=obligation.reporting_obligation_id @@ -486,7 +594,7 @@ async def cancelled(checkpoint): assert fetch.calls == ["PROVISIONAL_SNAPSHOT"] -async def test_watermark_and_history_advance_with_unchanged_rows(make_harness): +async def test_watermark_advances_without_revising_unchanged_rows(make_harness): producer, store, fetch, clock = await make_harness(_capabilities(restatement_window=None)) source = producer._source original_fetch = source._fetch @@ -506,10 +614,12 @@ def observed(request): clock[0] += timedelta(hours=1) await producer.run_worker() history = await _revisions(store) - assert len(history) == 2 - assert history[1].data_through > first.data_through - assert history[1].source_manifest_sha256 == first.source_manifest_sha256 - assert (await latest(store)).revision_id == history[1].reporting_revision_id + assert history == (first,) + observation = await latest(store) + assert SourceBatchManifestV1.model_validate_json(observation.manifest_json).data_through > ( + first.data_through + ) + assert observation.revision_id == first.reporting_revision_id async def test_schema_reinstallation_preserves_observations_and_row_layout(make_harness): @@ -763,10 +873,10 @@ def not_ready_once(request): await producer.run_worker() assert requests[1].identity == requests[0].identity assert requests[1].publication_class == "PROVISIONAL_SNAPSHOT" - assert [revision.finality for revision in await _revisions(store)] == ["snapshot", "snapshot"] + assert [revision.finality for revision in await _revisions(store)] == ["snapshot"] await producer.run_worker() history = await _revisions(store) - assert sorted(revision.finality for revision in history) == ["official", "snapshot", "snapshot"] + assert sorted(revision.finality for revision in history) == ["official", "snapshot"] official = next(revision for revision in history if revision.finality == "official") assert (await latest(store)).revision_id == official.reporting_revision_id assert (await latest(store)).acquisition.ordinal == 2 @@ -830,7 +940,7 @@ def not_ready_once(request): if pending_lookup: assert requests[1].identity == requests[0].identity assert requests[1].publication_class == "PROVISIONAL_SNAPSHOT" - assert len(await _revisions(store)) == (2 if pending_lookup else 1) + assert len(await _revisions(store)) == 1 await restarted.run_worker() assert len(requests) == (2 if pending_lookup else 1) # Waiting for the next offering must not freeze an early source cutoff. @@ -852,10 +962,10 @@ def not_ready_once(request): await restarted.run_worker() assert requests[-1].publication_class == "AUTHORITATIVE" assert requests[-1].period.source_read_cutoff_at == clock[0] - assert len(await _revisions(store)) == 3 + assert len(await _revisions(store)) == 2 -async def test_empty_successful_reads_still_append_observations(make_harness): +async def test_empty_successful_reads_advance_observations_without_revisions(make_harness): producer, store, fetch, clock = await make_harness(_capabilities(restatement_window=None)) source = producer._source original_fetch = source._fetch @@ -864,6 +974,7 @@ async def test_empty_successful_reads_still_append_observations(make_harness): clock[0] += timedelta(hours=1) await producer.run_worker() history = await _revisions(store) - assert len(history) == 2 + assert len(history) == 1 assert all(revision.row_count == 0 and revision.readable for revision in history) - assert history[1].supersedes_reporting_revision_id == history[0].reporting_revision_id + assert (await latest(store)).acquisition.ordinal == 1 + assert (await latest(store)).revision_id == history[0].reporting_revision_id diff --git a/tests/test_reporting_settling.py b/tests/test_reporting_settling.py index b7110e99a9..6c249d6621 100644 --- a/tests/test_reporting_settling.py +++ b/tests/test_reporting_settling.py @@ -226,9 +226,9 @@ async def test_unchanged_refresh_honors_cadence_and_commits_an_observation() -> clock[0] += timedelta(minutes=1) unchanged = await producer.run_worker() - assert len(unchanged.revisions_committed) == 1 + assert unchanged.revisions_committed == [] assert fetch.calls == ["PROVISIONAL_SNAPSHOT", "PROVISIONAL_SNAPSHOT"] - assert len(await _revisions(store)) == 2 + assert len(await _revisions(store)) == 1 # The successful observation advances both the cadence clock and execution # ordinal, so an immediate worker turn neither polls nor replays it. @@ -323,7 +323,7 @@ async def test_source_without_a_window_retains_its_final_default_read() -> None: clock[0] = obligation.period.end + timedelta(days=10) await producer.run_worker() assert fetch.calls == ["PROVISIONAL_SNAPSHOT", "PROVISIONAL_SNAPSHOT"] - assert len(await _revisions(store)) == 2 + assert len(await _revisions(store)) == 1 checkpoint = await store.get_restatement_checkpoint( account_id=ACCOUNT, reporting_obligation_id=obligation.reporting_obligation_id, From 4af04fa142c807a656c84067ad6561a7961d404c Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Tue, 17 Nov 2026 08:41:41 +0530 Subject: [PATCH 2/3] fix(reporting): allow repeated revision observations --- .../reporting_provisional_observations.sql | 8 ++++++- .../ledger/required_provisional_schema.json | 8 ------- .../reporting/_provisional_catalog.py | 2 +- .../test_reporting_notification_packaging.py | 2 +- .../test_reporting_provisional_catalog.py | 24 +++++++++++++++++++ 5 files changed, 33 insertions(+), 11 deletions(-) diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_provisional_observations.sql b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_provisional_observations.sql index 5206ffc65c..30bb0c705e 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_provisional_observations.sql +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/reporting_provisional_observations.sql @@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS reporting_provisional_observations ( account_id TEXT COLLATE "C" NOT NULL, reporting_obligation_id TEXT COLLATE "C" NOT NULL, ordinal BIGINT NOT NULL, - reporting_revision_id TEXT COLLATE "C" NOT NULL UNIQUE, + reporting_revision_id TEXT COLLATE "C" NOT NULL, payload JSONB NOT NULL, PRIMARY KEY (account_id, reporting_obligation_id, ordinal), CONSTRAINT provisional_observation_revision_fk @@ -32,6 +32,12 @@ CREATE TABLE IF NOT EXISTS reporting_provisional_observations ( (account_id, reporting_obligation_id, ordinal) ); +-- Source observations and immutable revisions have many-to-one cardinality: +-- successful observations with identical content retain one revision. +ALTER TABLE reporting_provisional_observations + DROP CONSTRAINT IF EXISTS + reporting_provisional_observations_reporting_revision_id_key; + CREATE OR REPLACE FUNCTION reporting_provisional_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN diff --git a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/required_provisional_schema.json b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/required_provisional_schema.json index 08e6eafe8d..925cbdb4a2 100644 --- a/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/required_provisional_schema.json +++ b/packages/adcp-reporting-storage/src/adcp_reporting_storage/ledger/required_provisional_schema.json @@ -67,10 +67,6 @@ "enabled": true, "fingerprint": "6e0009dfd0c8c76adc077b6f9e8bc866b476342946a107c8cba342ee3c6ec602" }, - "constraint:reporting_provisional_observations.reporting_provisional_observations_reporting_revision_id_key": { - "enabled": true, - "fingerprint": "dface19184e3b1196ac3f05865ce9fd2c1fde32593a604fd8e59e0e7b37525e7" - }, "function:reporting_provisional_immutable()": { "enabled": true, "fingerprint": "5f46da14407d6b8acab20bf770940bcae96698b27a941eab2d1f4aa5c271484b" @@ -87,10 +83,6 @@ "enabled": true, "fingerprint": "cc1a138288cf600afcffe140e7001d4e44d1a762ba2b6feeea6c5615c2e1c204" }, - "index:reporting_provisional_observations.reporting_provisional_observations_reporting_revision_id_key": { - "enabled": true, - "fingerprint": "423b8e84d59c9f7c8a580f1b8f97058e964b191a9c4a2dfd0e1e95e00b1872c2" - }, "table:reporting_provisional_acquisitions": { "enabled": true, "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" diff --git a/tests/conformance/reporting/_provisional_catalog.py b/tests/conformance/reporting/_provisional_catalog.py index 801def976b..4c29a0e3ea 100644 --- a/tests/conformance/reporting/_provisional_catalog.py +++ b/tests/conformance/reporting/_provisional_catalog.py @@ -19,4 +19,4 @@ def _storage_split(): PROVISIONAL_REQUIRED_OBJECTS as PROVISIONAL_OBJECTS, ) -assert len(PROVISIONAL_OBJECTS) == 27 +assert len(PROVISIONAL_OBJECTS) == 25 diff --git a/tests/conformance/reporting/test_reporting_notification_packaging.py b/tests/conformance/reporting/test_reporting_notification_packaging.py index c5a1bdadce..a1ffc7afb3 100644 --- a/tests/conformance/reporting/test_reporting_notification_packaging.py +++ b/tests/conformance/reporting/test_reporting_notification_packaging.py @@ -601,7 +601,7 @@ async def test_installed_pg_extra_migrates_commits_and_restarts(installed_distri values = json.load(sys.stdin) assert len(values["waiver_objects"]) == 10 -assert len(values["provisional_objects"]) == 27 +assert len(values["provisional_objects"]) == 25 assert json.loads(files("adcp_reporting_storage.ledger").joinpath( "required_provisional_schema.json").read_text()) == values["provisional_objects"] expected_objects = { diff --git a/tests/conformance/reporting/test_reporting_provisional_catalog.py b/tests/conformance/reporting/test_reporting_provisional_catalog.py index c1e97a845b..e49cfdae5d 100644 --- a/tests/conformance/reporting/test_reporting_provisional_catalog.py +++ b/tests/conformance/reporting/test_reporting_provisional_catalog.py @@ -120,6 +120,30 @@ async def test_both_observation_tables_reject_update_and_delete(): assert await latest(store) == before +async def test_observation_schema_upgrade_removes_revision_uniqueness(): + async with isolated_reporting_pool() as pool: + store = PgReportingLedgerStore(pool=pool) + await store.create_schema() + async with pool.connection() as connection: + await connection.execute( + "ALTER TABLE reporting_provisional_observations " + "ADD CONSTRAINT " + "reporting_provisional_observations_reporting_revision_id_key " + "UNIQUE (reporting_revision_id)" + ) + await store.create_schema() + async with pool.connection() as connection: + retained = await ( + await connection.execute( + "SELECT 1 FROM pg_constraint " + "WHERE conrelid='reporting_provisional_observations'::regclass " + "AND conname=" + "'reporting_provisional_observations_reporting_revision_id_key'" + ) + ).fetchone() + assert retained is None + + async def test_observation_foreign_keys_bind_both_obligation_and_revision_accounts(): psycopg = pytest.importorskip("psycopg") async with isolated_reporting_pool() as pool: From 121835d4d9e9d3f75c0e87266109ce79cec6085b Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Tue, 17 Nov 2026 11:19:51 +0530 Subject: [PATCH 3/3] test(reporting): align lifecycle with unchanged observations --- .../test_reporting_core_lifecycle.py | 45 +++++++++++++------ ...est_reporting_provisional_catalog_scope.py | 6 +-- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/tests/conformance/reporting/test_reporting_core_lifecycle.py b/tests/conformance/reporting/test_reporting_core_lifecycle.py index 88a45b68c9..2bc97d1daa 100644 --- a/tests/conformance/reporting/test_reporting_core_lifecycle.py +++ b/tests/conformance/reporting/test_reporting_core_lifecycle.py @@ -439,9 +439,19 @@ async def test_reporting_core_lifecycle(ledger: PgReportingLedgerStore) -> None: first_revisions = await ledger.list_revisions( account_id=ACCOUNT, reporting_obligation_id=obligation.reporting_obligation_id ) - assert len(first_revisions) == 2 - automatic = next(item for item in first_revisions if item.reporting_revision_id != revision_id) - assert automatic.supersedes_reporting_revision_id == revision_id + assert first_revisions == (revision,) + second_observation = await ledger.get_provisional_observation( + account_id=ACCOUNT, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + assert second_observation is not None + assert second_observation.acquisition.ordinal == 1 + assert second_observation.revision_id == revision.reporting_revision_id + second_checkpoint = await ledger.get_restatement_checkpoint( + account_id=ACCOUNT, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + assert second_checkpoint is not None and second_checkpoint.next_observation == 2 zero_obligation = await ledger.find_obligation( account_id=ACCOUNT, @@ -456,10 +466,7 @@ async def test_reporting_core_lifecycle(ledger: PgReportingLedgerStore) -> None: account_id=ACCOUNT, reporting_obligation_id=zero_obligation.reporting_obligation_id ) assert [item.row_count for item in zero_revisions] == [0] - assert set(turn.revisions_committed) == { - automatic.reporting_revision_id, - zero_revisions[0].reporting_revision_id, - } + assert turn.revisions_committed == [zero_revisions[0].reporting_revision_id] settled = await _reconcile(ledger, expected=_expected_periods(2)) assert settled.definitive is True, settled.obligations @@ -503,14 +510,24 @@ async def test_reporting_core_lifecycle(ledger: PgReportingLedgerStore) -> None: ) == () ) - # At this later turn the first period's next hourly observation is due - # too. Preserve that automatic revision as the next restatement's parent. + # At this later turn the first period's next hourly observation is due too. + # Its unchanged content keeps the existing revision as the restatement parent. refreshed_first = await ledger.list_revisions( account_id=ACCOUNT, reporting_obligation_id=obligation.reporting_obligation_id ) - assert len(refreshed_first) == 3 - automatic_at_ready = max(refreshed_first, key=lambda item: item.created_at) - assert automatic_at_ready.supersedes_reporting_revision_id == automatic.reporting_revision_id + assert refreshed_first == (revision,) + ready_observation = await ledger.get_provisional_observation( + account_id=ACCOUNT, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + assert ready_observation is not None + assert ready_observation.acquisition.ordinal == 2 + assert ready_observation.revision_id == revision.reporting_revision_id + ready_checkpoint = await ledger.get_restatement_checkpoint( + account_id=ACCOUNT, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + assert ready_checkpoint is not None and ready_checkpoint.next_observation == 3 # The zero-row period and the silent period are distinguishable, which is # the entire operational point of the tier. @@ -560,8 +577,8 @@ async def test_reporting_core_lifecycle(ledger: PgReportingLedgerStore) -> None: chain = await ledger.list_revisions( account_id=ACCOUNT, reporting_obligation_id=obligation.reporting_obligation_id ) - assert len(chain) == 4 - assert restated.supersedes_reporting_revision_id == automatic_at_ready.reporting_revision_id + assert len(chain) == 2 + assert restated.supersedes_reporting_revision_id == revision.reporting_revision_id # The superseded revision is still retained and still readable: a consumer # that already cited it must be able to fetch exactly what it cited. superseded = await ledger.read_revision_rows( diff --git a/tests/conformance/reporting/test_reporting_provisional_catalog_scope.py b/tests/conformance/reporting/test_reporting_provisional_catalog_scope.py index 5dde7569de..43ce63d732 100644 --- a/tests/conformance/reporting/test_reporting_provisional_catalog_scope.py +++ b/tests/conformance/reporting/test_reporting_provisional_catalog_scope.py @@ -46,7 +46,7 @@ def assert_narrow_scope(self): assert parameters == (oids,) assert self.calls[5][1] == FUNCTION assert [(row[0], row[1]) for row in self.calls[5][2]] == [FUNCTION] - assert [len(rows) for _, _, rows in self.calls] == [2, 10, 8, 4, 2, 1] + assert [len(rows) for _, _, rows in self.calls] == [2, 10, 7, 3, 2, 1] @pytest.mark.parametrize("autocommit", [False, True]) @@ -68,14 +68,14 @@ async def test_exact_catalog_scope_preserves_full_contract_and_excludes_overload .read_text() ) ) - assert len(expected) == 794 and len(PROVISIONAL_REQUIRED_OBJECTS) == 27 + assert len(expected) == 792 and len(PROVISIONAL_REQUIRED_OBJECTS) == 25 async with isolated_reporting_pool(autocommit=autocommit) as pool: await PgReportingReceiptStore(pool=pool).create_schema() async with pool.connection() as connection: full = CatalogProbe(connection) assert await schema_objects(full) == expected assert len(full.calls) == 6 - assert sum(len(rows) for _, _, rows in full.calls) == 794 + assert sum(len(rows) for _, _, rows in full.calls) == 792 await connection.execute( "CREATE TABLE reporting_unrelated_catalog_scope (value integer);" " CREATE FUNCTION reporting_provisional_immutable(integer) RETURNS integer"