diff --git a/src/adcp/reporting/conformance.py b/src/adcp/reporting/conformance.py index c7b000ee4..c56e571ea 100644 --- a/src/adcp/reporting/conformance.py +++ b/src/adcp/reporting/conformance.py @@ -34,25 +34,33 @@ import asyncio import hashlib +import json from collections.abc import Awaitable, Callable, Sequence from datetime import datetime, timezone from typing import Any, Literal, TypeVar +from pydantic import ValidationError + from adcp.reporting.source import ( AuthoritativeOfferingV1, ProvisionalSnapshotOfferingV1, ReportingSourceCapabilitiesV1, + ReportingSourceCapabilitiesV2, ReportingSourceErrorCode, ReportingSourceErrorV1, ReportingSourceExecutor, ReportingSourceExecutorResult, ReportingSourceSliceRequestV1, + ReportingSourceSliceRequestV2, ReportingSourceStagedObjectReader, + SourceBatchManifestError, SourceBatchManifestV1, + SourceBatchManifestV2, _validate_metric_applicability, deterministic_source_publication_id_v1, iso_duration_milliseconds_v1, parse_verified_source_batch_manifest_v1, + parse_verified_source_batch_manifest_v2, source_batch_object_set_sha256_v1, ) @@ -152,9 +160,20 @@ async def _with_deadline( def _validate_request_against_capabilities( - capabilities: ReportingSourceCapabilitiesV1, request: ReportingSourceSliceRequestV1 + capabilities: ReportingSourceCapabilitiesV1 | ReportingSourceCapabilitiesV2, + request: ReportingSourceSliceRequestV1 | ReportingSourceSliceRequestV2, ) -> ProvisionalSnapshotOfferingV1 | AuthoritativeOfferingV1: """Is this frozen slice an exact atomic offering of this executor build?""" + supported_versions = ( + capabilities.supported_contract_versions + if isinstance(capabilities, ReportingSourceCapabilitiesV2) + else (capabilities.contract_version,) + ) + if request.contract_version not in supported_versions: + raise _fail( + "CAPABILITY_MISMATCH", + "the executor must explicitly declare the request contract version", + ) if capabilities.scope != "effective_account": raise _fail( "CAPABILITY_MISMATCH", @@ -200,6 +219,15 @@ def _validate_request_against_capabilities( _validate_window_bounds(offering, request) + if ( + isinstance(request, ReportingSourceSliceRequestV2) + and "package_item" not in offering.constituent_kinds + ): + raise _fail( + "CAPABILITY_MISMATCH", + "a V2 package slice requires package_item support in the selected offering", + ) + for constituent in request.coverage.constituents: if constituent.product_id not in offering.product_ids: raise _fail( @@ -291,9 +319,11 @@ def _source_calendar_elapsed_ms(request: ReportingSourceSliceRequestV1) -> int: def _validate_manifest_against_request( offering: ProvisionalSnapshotOfferingV1 | AuthoritativeOfferingV1, - request: ReportingSourceSliceRequestV1, - manifest: SourceBatchManifestV1, + request: ReportingSourceSliceRequestV1 | ReportingSourceSliceRequestV2, + manifest: SourceBatchManifestV1 | SourceBatchManifestV2, ) -> None: + if manifest.manifest_version != request.contract_version: + raise _fail("MANIFEST_MISMATCH", "the manifest version must match the request version") if request.coverage.expected == "full" and manifest.coverage.status != "full": raise _fail( "MANIFEST_MISMATCH", @@ -376,6 +406,22 @@ def _validate_manifest_against_request( request.coverage.denominator_fingerprint, ), ] + if isinstance(request, ReportingSourceSliceRequestV2): + assert isinstance(manifest, SourceBatchManifestV2) + echoed.extend( + [ + ( + "coverage.media_buy_ids", + manifest.coverage.media_buy_ids, + request.coverage.media_buy_ids, + ), + ( + "coverage.constituents", + [item.constituent for item in manifest.coverage.constituents], + request.coverage.constituents, + ), + ] + ) for field, actual, expected in echoed: if actual != expected: raise _fail( @@ -445,13 +491,13 @@ def _validate_manifest_against_request( async def validate_reporting_source_execution( *, - capabilities: ReportingSourceCapabilitiesV1, - request: ReportingSourceSliceRequestV1, + capabilities: ReportingSourceCapabilitiesV1 | ReportingSourceCapabilitiesV2, + request: ReportingSourceSliceRequestV1 | ReportingSourceSliceRequestV2, result: ReportingSourceExecutorResult, object_reader: ReportingSourceStagedObjectReader, cancel: asyncio.Event | None = None, clock: Callable[[], datetime] | None = None, -) -> SourceBatchManifestV1: +) -> SourceBatchManifestV1 | SourceBatchManifestV2: """Validate one execution end to end and return its verified manifest. Reads every staged object the manifest names and checks its bytes against @@ -470,12 +516,32 @@ async def validate_reporting_source_execution( manifest_bytes = result.manifest_bytes assert manifest_bytes is not None + if response.contract_version != request.contract_version: + raise _fail("MANIFEST_MISMATCH", "the execution response version must match the request") + if response.identity != request.identity: raise _fail( "IDENTITY_MISMATCH", "the response must echo the complete frozen request identity" ) - manifest = parse_verified_source_batch_manifest_v1(response.manifest, manifest_bytes) + manifest: SourceBatchManifestV1 | SourceBatchManifestV2 + if isinstance(request, ReportingSourceSliceRequestV2): + try: + manifest = parse_verified_source_batch_manifest_v2(response.manifest, manifest_bytes) + except SourceBatchManifestError as error: + if error.code == "SOURCE_MANIFEST_VERSION_MISMATCH": + raise _fail("MANIFEST_MISMATCH", "the manifest must declare version 2.0") from error + raise + else: + try: + manifest = parse_verified_source_batch_manifest_v1(response.manifest, manifest_bytes) + except ValidationError as error: + payload = json.loads(manifest_bytes) + if isinstance(payload, dict) and payload.get("manifest_version") == "2.0": + raise _fail( + "MANIFEST_MISMATCH", "the manifest version must match the request" + ) from error + raise _validate_manifest_against_request(offering, request, manifest) cancel = cancel or asyncio.Event() diff --git a/src/adcp/reporting/source.py b/src/adcp/reporting/source.py index 4661c3058..90d6265ad 100644 --- a/src/adcp/reporting/source.py +++ b/src/adcp/reporting/source.py @@ -88,18 +88,22 @@ __all__ = [ "REPORTING_EVIDENCE_REASON_MAX_LENGTH_V1", "REPORTING_SOURCE_CONTRACT_VERSION_V1", + "REPORTING_SOURCE_CONTRACT_VERSION_V2", "SOURCE_BATCH_MANIFEST_MAX_BYTES_V1", "SOURCE_BATCH_MANIFEST_MAX_METRIC_AVAILABILITY_V1", "SOURCE_BATCH_MANIFEST_MAX_PROVIDER_CALLS_V1", "AuthoritativeOfferingV1", "MediaBuyConstituentV1", "PackageItemConstituentV1", + "PackageItemConstituentV2", "ProductConstituentV1", "ProvisionalSnapshotOfferingV1", "ReportingAdapterBuildIdentityV1", "ReportingAvailabilityStatus", "ReportingConstituent", + "ReportingConstituentV2", "ReportingConstituentCoverageV1", + "ReportingConstituentCoverageV2", "ReportingContractIdentityV1", "ReportingCoverageStatus", "ReportingFinalityEvidenceV1", @@ -111,10 +115,13 @@ "ReportingPublicationClass", "ReportingRevisionKind", "ReportingSourceCapabilitiesV1", + "ReportingSourceCapabilitiesV2", + "ReportingSourceCoverageRequestV2", "ReportingSourceError", "ReportingSourceErrorCode", "ReportingSourceErrorV1", "ReportingSourceExecutionResponseV1", + "ReportingSourceExecutionResponseV2", "ReportingSourceExecutor", "ReportingSourceExecutorResult", "ReportingSourceIdentityV1", @@ -122,25 +129,37 @@ "ReportingSourcePeriodV1", "ReportingSourceRetryClass", "ReportingSourceSliceRequestV1", + "ReportingSourceSliceRequestV2", "ReportingSourceStagedObjectReader", "ReportingSourceWindowingV1", "ReportingTriggerKind", "SourceBatchManifestReferenceV1", "SourceBatchManifestV1", + "SourceBatchManifestV2", + "SourceBatchCoverageV2", "SourceBatchObjectV1", "SourceControlTotalV1", "completed_reporting_source_response_v1", "coverage_denominator_fingerprint_v1", + "coverage_denominator_fingerprint_v2", "deterministic_source_publication_id_v1", "encode_source_batch_manifest_v1", + "encode_source_batch_manifest_v2", + "parse_verified_source_batch_manifest_v2", "publication_content_fingerprint_v1", + "publication_content_fingerprint_v2", "reporting_source_capabilities_sha256_v1", + "reporting_source_capabilities_sha256_v2", "source_batch_coverage_fingerprint_v1", + "source_batch_coverage_fingerprint_v2", "source_batch_manifest_reference_v1", "source_batch_object_set_sha256_v1", + "validate_reporting_source_rows_v2", + "validate_reporting_source_staged_rows_v2", ] REPORTING_SOURCE_CONTRACT_VERSION_V1: Final[Literal["1.0"]] = "1.0" +REPORTING_SOURCE_CONTRACT_VERSION_V2: Final[Literal["2.0"]] = "2.0" #: A retained manifest is evidence, not a payload. Rows belong in staged #: objects; the manifest that binds them stays small enough to keep forever. @@ -515,6 +534,16 @@ class PackageItemConstituentV1(_Frozen): package_id: ExternalId +class PackageItemConstituentV2(_Frozen): + """A package constituent scoped to exactly one media buy.""" + + constituent_kind: Literal["package_item"] = "package_item" + constituent_id: ExternalId + product_id: ExternalId + package_id: ExternalId + media_buy_id: ExternalId + + class MediaBuyConstituentV1(_Frozen): """A media-buy constituent: a product inside a named media buy.""" @@ -529,6 +558,21 @@ class MediaBuyConstituentV1(_Frozen): Field(discriminator="constituent_kind"), ] +ReportingConstituentV2 = Annotated[ + ProductConstituentV1 | PackageItemConstituentV2 | MediaBuyConstituentV1, + Field(discriminator="constituent_kind"), +] + + +def _require_unique_package_pairs(constituents: Sequence[ReportingConstituentV2]) -> None: + pairs = [ + (item.media_buy_id, item.package_id) + for item in constituents + if isinstance(item, PackageItemConstituentV2) + ] + if len(set(pairs)) != len(pairs): + raise ValueError("a package pair may appear only once in the coverage denominator") + def coverage_denominator_fingerprint_v1(constituents: Sequence[Any]) -> str: """Fingerprint the *requested* constituent set, excluding publication status. @@ -554,6 +598,35 @@ def coverage_denominator_fingerprint_v1(constituents: Sequence[Any]) -> str: ) +def coverage_denominator_fingerprint_v2( + constituents: Sequence[Any], *, media_buy_ids: Sequence[str] | None = None +) -> str: + """Fingerprint the frozen buy selector and its exact package pair set.""" + if media_buy_ids is None: + media_buy_ids = sorted({item.media_buy_id for item in constituents}) + ordered = sorted( + ( + { + key: value + for key, value in _constituent_identity(constituent).items() + if value is not None + } + for constituent in constituents + ), + key=lambda item: str(item["constituent_id"]).encode("utf-16-be", errors="surrogatepass"), + ) + return reporting_fingerprint_v1( + { + "kind": "reporting_coverage_denominator_v2", + "media_buy_ids": sorted( + media_buy_ids, + key=lambda value: str(value).encode("utf-16-be", errors="surrogatepass"), + ), + "constituents": ordered, + } + ) + + def _constituent_identity(constituent: Any) -> dict[str, Any]: """The identity half of a constituent, with mutable publication status dropped.""" if isinstance(constituent, BaseModel): @@ -800,6 +873,22 @@ def offering(self, offering_id: str) -> ProvisionalSnapshotOfferingV1 | Authorit raise KeyError(offering_id) +class ReportingSourceCapabilitiesV2(ReportingSourceCapabilitiesV1): + """An executor declaration with an explicit set of supported source versions.""" + + contract_version: Literal["2.0"] # type: ignore[assignment] + supported_contract_versions: tuple[Literal["1.0", "2.0"], ...] = Field( + min_length=1, max_length=2 + ) + + @model_validator(mode="after") + def _supported_versions_are_unique(self) -> ReportingSourceCapabilitiesV2: + versions = self.supported_contract_versions + if len(set(versions)) != len(versions) or "2.0" not in versions: + raise ValueError("supported_contract_versions must uniquely include version 2.0") + return self + + def reporting_source_capabilities_sha256_v1( capabilities: ReportingSourceCapabilitiesV1 | Mapping[str, Any], ) -> str: @@ -814,6 +903,13 @@ def reporting_source_capabilities_sha256_v1( return canonical_json_sha256_v1(payload) +def reporting_source_capabilities_sha256_v2( + capabilities: ReportingSourceCapabilitiesV2 | Mapping[str, Any], +) -> str: + """Bind the complete V2 capability declaration, including its version.""" + return reporting_source_capabilities_sha256_v1(capabilities) + + # -------------------------------------------------------------------------- # Slice request # -------------------------------------------------------------------------- @@ -848,6 +944,35 @@ def _denominator_binds_the_constituents(self) -> ReportingSourceCoverageRequestV return self +class ReportingSourceCoverageRequestV2(ReportingSourceCoverageRequestV1): + """The frozen buy selector and its requested package pairs.""" + + media_buy_ids: tuple[ExternalId, ...] = Field(min_length=1, max_length=100_000) + constituents: list[PackageItemConstituentV2] = Field( # type: ignore[assignment] + max_length=SOURCE_BATCH_MANIFEST_MAX_METRIC_AVAILABILITY_V1 + ) + + @model_validator(mode="after") + def _denominator_binds_the_constituents(self) -> ReportingSourceCoverageRequestV2: + ids = [constituent.constituent_id for constituent in self.constituents] + if len(set(ids)) != len(ids): + raise ValueError("constituent_id must be unique within a coverage denominator") + if len(set(self.media_buy_ids)) != len(self.media_buy_ids): + raise ValueError("media_buy_ids must be unique") + if any(item.media_buy_id not in self.media_buy_ids for item in self.constituents): + raise ValueError("package pairs must belong to the frozen media_buy_ids") + _require_unique_package_pairs(self.constituents) + expected = coverage_denominator_fingerprint_v2( + self.constituents, media_buy_ids=self.media_buy_ids + ) + if expected != self.denominator_fingerprint: + raise ValueError( + "denominator_fingerprint must bind the complete constituent tuple set; " + f"expected {expected}" + ) + return self + + class ReportingSourceCheckpointV1(_Frozen): """Producer-owned cursor evidence an executor may read but never advance.""" @@ -942,6 +1067,92 @@ def package_ids(self) -> tuple[str, ...]: ) +class ReportingSourceSliceRequestV2(ReportingSourceSliceRequestV1): + """A package-aware slice sent only to a V2-capable executor.""" + + contract_version: Literal["2.0"] # type: ignore[assignment] + coverage: ReportingSourceCoverageRequestV2 + + @property + def package_ids(self) -> tuple[str, ...]: + """The unique packages represented by the V2 denominator.""" + return tuple( + dict.fromkeys( + constituent.package_id + for constituent in self.coverage.constituents + if isinstance(constituent, PackageItemConstituentV2) + ) + ) + + @property + def media_buy_ids(self) -> tuple[str, ...]: + """The frozen media buys, including those with no package pairs.""" + return self.coverage.media_buy_ids + + +def validate_reporting_source_rows_v2( + request: ReportingSourceSliceRequestV2, + manifest: SourceBatchManifestV2, + rows: Sequence[Mapping[str, Any]], +) -> None: + """Reject normalized rows outside a frozen V2 coverage denominator before revision staging.""" + if manifest.coverage.denominator_fingerprint != request.coverage.denominator_fingerprint: + raise ValueError("manifest coverage differs from the requested coverage") + matched = _matched_reporting_source_rows_v2(request, rows) + statuses = {item.constituent_id: item.status for item in manifest.coverage.constituents} + for index, constituent in enumerate(matched): + if statuses.get(constituent.constituent_id) == "explicit_zero": + raise ValueError(f"row {index} contradicts explicit zero coverage") + + +def validate_reporting_source_staged_rows_v2( + request: ReportingSourceSliceRequestV2, rows: Sequence[Mapping[str, Any]] +) -> None: + """Fence normalized package rows against the frozen request before staging.""" + _matched_reporting_source_rows_v2(request, rows) + + +def _matched_reporting_source_rows_v2( + request: ReportingSourceSliceRequestV2, rows: Sequence[Mapping[str, Any]] +) -> list[PackageItemConstituentV2]: + packages = { + (item.media_buy_id, item.package_id): item for item in request.coverage.constituents + } + matched: list[PackageItemConstituentV2] = [] + seen_grains: set[tuple[str, str, bytes]] = set() + for index, row in enumerate(rows): + package_id = row.get("package_id") + media_buy_id = row.get("media_buy_id") + product_id = row.get("product_id") + constituent_id = row.get("constituent_id") + if any( + value is not None and not isinstance(value, str) + for value in (package_id, media_buy_id, product_id, constituent_id) + ): + raise ValueError(f"row {index} is outside the requested coverage") + constituent = ( + packages.get((media_buy_id, package_id)) + if isinstance(media_buy_id, str) and isinstance(package_id, str) + else None + ) + if ( + constituent is None + or (product_id is not None and product_id != constituent.product_id) + or (constituent_id is not None and constituent_id != constituent.constituent_id) + ): + raise ValueError(f"row {index} is outside the requested coverage") + grain = ( + constituent.media_buy_id, + constituent.package_id, + canonical_json_utf8_v1([row.get(name) for name in request.requested_dimensions]), + ) + if grain in seen_grains: + raise ValueError(f"row {index} is a duplicate row at the requested dimension grain") + seen_grains.add(grain) + matched.append(constituent) + return matched + + # -------------------------------------------------------------------------- # Manifest # -------------------------------------------------------------------------- @@ -1040,6 +1251,25 @@ def constituent_id(self) -> str: return self.constituent.constituent_id +class ReportingConstituentCoverageV2(_Frozen): + """Coverage evidence for a constituent with a complete package identity.""" + + constituent: PackageItemConstituentV2 + status: ReportingAvailabilityStatus + data_through: datetime | None = None + reason: EvidenceReason | None = None + + @model_validator(mode="after") + def _unavailable_coverage_is_explained(self) -> ReportingConstituentCoverageV2: + if self.status not in _AVAILABLE_STATUSES and not self.reason: + raise ValueError(f"constituent status {self.status!r} needs a stable reason") + return self + + @property + def constituent_id(self) -> str: + return self.constituent.constituent_id + + class ReportingMetricAvailabilityV1(_Frozen): """One ``(constituent, metric)`` cell's publication evidence. @@ -1114,6 +1344,49 @@ def _status_matches_the_evidence(self) -> SourceBatchCoverageV1: return self +class SourceBatchCoverageV2(SourceBatchCoverageV1): + """Coverage evidence for the frozen buy selector and package pairs.""" + + media_buy_ids: tuple[ExternalId, ...] = Field(min_length=1, max_length=100_000) + constituents: list[ReportingConstituentCoverageV2] = Field( # type: ignore[assignment] + max_length=100_000 + ) + + @model_validator(mode="after") + def _status_matches_the_evidence(self) -> SourceBatchCoverageV2: + ids = [item.constituent_id for item in self.constituents] + if len(set(ids)) != len(ids): + raise ValueError("constituent_id must be unique within manifest coverage") + if len(set(self.media_buy_ids)) != len(self.media_buy_ids): + raise ValueError("media_buy_ids must be unique") + if any( + item.constituent.media_buy_id not in self.media_buy_ids for item in self.constituents + ): + raise ValueError("package pairs must belong to the frozen media_buy_ids") + _require_unique_package_pairs([item.constituent for item in self.constituents]) + covered = sum(1 for item in self.constituents if item.status in _AVAILABLE_STATUSES) + partial = sum(1 for item in self.constituents if item.status == "partial") + all_covered = covered == len(self.constituents) + any_evidence = covered > 0 or partial > 0 + if self.status == "full" and not all_covered: + raise ValueError("full coverage requires every constituent present or explicit zero") + if self.status == "none" and any_evidence: + raise ValueError("none coverage must not contain a covered or partial constituent") + if self.status == "partial" and (not any_evidence or all_covered): + raise ValueError( + "partial coverage requires coverage evidence without every constituent covered" + ) + expected = coverage_denominator_fingerprint_v2( + [item.constituent for item in self.constituents], media_buy_ids=self.media_buy_ids + ) + if expected != self.denominator_fingerprint: + raise ValueError( + "denominator_fingerprint must bind the complete constituent tuple set; " + f"expected {expected}" + ) + return self + + class ReportingProviderPageV1(_Frozen): """One successful upstream result page. @@ -1649,6 +1922,50 @@ def _validate_strictness(self) -> None: ) +class SourceBatchManifestV2(SourceBatchManifestV1): + """A versioned publication that echoes the package and media-buy pairs.""" + + manifest_version: Literal["2.0"] # type: ignore[assignment] + coverage: SourceBatchCoverageV2 + metric_availability: list[ReportingMetricAvailabilityV1] = Field( + max_length=SOURCE_BATCH_MANIFEST_MAX_METRIC_AVAILABILITY_V1 + ) + + def _validate_zero_rows(self) -> None: + if not self.coverage.constituents and self.coverage.status == "full": + if self.row_count != 0 or self.metric_availability or self.explicit_zero: + raise ValueError("complete empty package coverage requires zero rows and cells") + if self.event_time_range is not None or any( + not total.is_zero for total in self.control_totals + ): + raise ValueError("complete empty package coverage cannot declare events or totals") + if self.data_through != min(self.period.end, self.period.source_read_cutoff_at): + raise ValueError("complete empty package coverage needs the current source cutoff") + return + super()._validate_zero_rows() + + @property + def contract_version(self) -> Literal["2.0"]: + """The negotiated source contract version.""" + return self.manifest_version + + @model_validator(mode="after") + def _manifest_is_internally_consistent(self) -> SourceBatchManifestV2: + self._validate_objects() + self._validate_time_evidence() + self._validate_finality() + self._validate_cells() + self._validate_zero_rows() + self._validate_event_range() + self._validate_strictness() + expected = publication_content_fingerprint_v2(self) + if expected != self.content_fingerprint: + raise ValueError( + f"content_fingerprint must bind the publication content; expected {expected}" + ) + return self + + # -------------------------------------------------------------------------- # Fingerprints and sealing # -------------------------------------------------------------------------- @@ -1698,6 +2015,30 @@ def source_batch_coverage_fingerprint_v1( ) +def source_batch_coverage_fingerprint_v2( + constituents: Sequence[ReportingConstituentCoverageV2], + *, + media_buy_ids: Sequence[str], +) -> str: + """Fingerprint package evidence and its frozen media-buy selector.""" + ordered = sorted( + (item.canonical_payload() for item in constituents), + key=lambda item: str(item["constituent"]["constituent_id"]).encode( + "utf-16-be", errors="surrogatepass" + ), + ) + return reporting_fingerprint_v1( + { + "kind": "reporting_coverage_evidence_v2", + "media_buy_ids": sorted( + media_buy_ids, + key=lambda value: str(value).encode("utf-16-be", errors="surrogatepass"), + ), + "constituents": ordered, + } + ) + + def publication_content_fingerprint_v1(manifest: SourceBatchManifestV1 | Mapping[str, Any]) -> str: """Fingerprint the source-neutral *content* of a publication. @@ -1764,6 +2105,16 @@ def publication_content_fingerprint_v1(manifest: SourceBatchManifestV1 | Mapping return reporting_fingerprint_v1(content) +def publication_content_fingerprint_v2(manifest: SourceBatchManifestV2 | Mapping[str, Any]) -> str: + """Bind V2 content and its version, including the pair-bound denominator.""" + return reporting_fingerprint_v1( + { + "kind": "reporting_source_content_v2", + "content_fingerprint": publication_content_fingerprint_v1(manifest), + } + ) + + def deterministic_source_publication_id_v1( *, source_execution_key: str, @@ -1809,6 +2160,19 @@ def encode_source_batch_manifest_v1(manifest: SourceBatchManifestV1) -> bytes: return encoded +def encode_source_batch_manifest_v2(manifest: SourceBatchManifestV2) -> bytes: + """Seal V2 manifest bytes with the retained-evidence size bound.""" + payload = manifest.model_dump(mode="json", exclude_none=True) + encoded = canonical_json_utf8_v1(payload) + if len(encoded) > SOURCE_BATCH_MANIFEST_MAX_BYTES_V1: + raise SourceBatchManifestError( + "SOURCE_MANIFEST_TOO_LARGE", + "the source manifest exceeds the 1 MiB retained-evidence limit; " + "large data belongs in staged objects", + ) + return encoded + + def source_batch_manifest_reference_v1( staged_commit_ref: str, manifest_bytes: bytes ) -> SourceBatchManifestReferenceV1: @@ -1874,6 +2238,45 @@ def parse_verified_source_batch_manifest_v1( return manifest +def parse_verified_source_batch_manifest_v2( + reference: SourceBatchManifestReferenceV1, raw_bytes: bytes +) -> SourceBatchManifestV2: + """Verify the reference, V2 schema, and exact canonical manifest bytes.""" + import hashlib + import json + + if ( + len(raw_bytes) > SOURCE_BATCH_MANIFEST_MAX_BYTES_V1 + or len(raw_bytes) != reference.byte_count + ): + raise SourceBatchManifestError( + "SOURCE_MANIFEST_TOO_LARGE", + "the staged manifest size does not match the bounded reference", + ) + if hashlib.sha256(raw_bytes).hexdigest() != reference.manifest_sha256: + raise SourceBatchManifestError( + "SOURCE_MANIFEST_CHECKSUM_MISMATCH", + "the staged manifest bytes do not match the declared checksum", + ) + try: + payload = json.loads(raw_bytes.decode("utf-8")) + except (UnicodeDecodeError, ValueError) as error: + raise SourceBatchManifestError( + "SOURCE_MANIFEST_JSON_INVALID", "the staged manifest is not valid UTF-8 JSON" + ) from error + if not isinstance(payload, dict) or payload.get("manifest_version") != "2.0": + raise SourceBatchManifestError( + "SOURCE_MANIFEST_VERSION_MISMATCH", "the staged manifest must declare version 2.0" + ) + manifest = SourceBatchManifestV2.model_validate(payload) + if encode_source_batch_manifest_v2(manifest) != raw_bytes: + raise SourceBatchManifestError( + "SOURCE_MANIFEST_CANONICAL_ENCODING_MISMATCH", + "the staged manifest is not canonical_json_utf8_v1", + ) + return manifest + + # -------------------------------------------------------------------------- # Execution result # -------------------------------------------------------------------------- @@ -1888,6 +2291,12 @@ class ReportingSourceExecutionResponseV1(_Frozen): manifest: SourceBatchManifestReferenceV1 +class ReportingSourceExecutionResponseV2(ReportingSourceExecutionResponseV1): + """The V2 success arm with the request identity and manifest reference.""" + + contract_version: Literal["2.0"] # type: ignore[assignment] + + class ReportingSourceErrorV1(_Frozen): """The failure arm: a stable code, a retry classification, and a safe message. @@ -1965,7 +2374,7 @@ def _default_retry_class(code: ReportingSourceErrorCode) -> ReportingSourceRetry class ReportingSourceExecutorResult(_Frozen): """Exactly one of: a completed publication, or a typed failure.""" - response: ReportingSourceExecutionResponseV1 | None = None + response: ReportingSourceExecutionResponseV1 | ReportingSourceExecutionResponseV2 | None = None manifest_bytes: bytes | None = None error: ReportingSourceErrorV1 | None = None @@ -1988,14 +2397,20 @@ def ok(self) -> bool: def completed( cls, *, - request: ReportingSourceSliceRequestV1, + request: ReportingSourceSliceRequestV1 | ReportingSourceSliceRequestV2, manifest: SourceBatchManifestReferenceV1, manifest_bytes: bytes, ) -> ReportingSourceExecutorResult: - return cls( - response=completed_reporting_source_response_v1(request=request, manifest=manifest), - manifest_bytes=manifest_bytes, + response = ( + ReportingSourceExecutionResponseV2( + contract_version=REPORTING_SOURCE_CONTRACT_VERSION_V2, + identity=request.identity, + manifest=manifest, + ) + if isinstance(request, ReportingSourceSliceRequestV2) + else completed_reporting_source_response_v1(request=request, manifest=manifest) ) + return cls(response=response, manifest_bytes=manifest_bytes) @classmethod def failed(cls, error: ReportingSourceErrorV1) -> ReportingSourceExecutorResult: diff --git a/tests/test_reporting_source_contract_v2.py b/tests/test_reporting_source_contract_v2.py new file mode 100644 index 000000000..2e3cc3910 --- /dev/null +++ b/tests/test_reporting_source_contract_v2.py @@ -0,0 +1,756 @@ +"""Versioned package source contracts keep each package bound to its media buy.""" + +from __future__ import annotations + +import hashlib + +import pytest +from pydantic import ValidationError + +from adcp.reporting.conformance import ( + ReportingSourceConformanceError, + validate_reporting_source_execution, +) +from adcp.reporting.fixtures import ( + redacted_capabilities, + redacted_completed_result, + redacted_snapshot_request, +) +from adcp.reporting.source import ( + PackageItemConstituentV1, + PackageItemConstituentV2, + ReportingConstituentCoverageV2, + ReportingSourceCapabilitiesV2, + ReportingSourceCoverageRequestV2, + ReportingSourceExecutionResponseV2, + ReportingSourceExecutorResult, + ReportingSourceSliceRequestV1, + ReportingSourceSliceRequestV2, + ReportingSourceStagedObjectReader, + SourceBatchCoverageV1, + SourceBatchCoverageV2, + SourceBatchManifestError, + SourceBatchManifestV2, + coverage_denominator_fingerprint_v1, + coverage_denominator_fingerprint_v2, + encode_source_batch_manifest_v2, + publication_content_fingerprint_v2, + reporting_source_capabilities_sha256_v1, + reporting_source_capabilities_sha256_v2, + source_batch_coverage_fingerprint_v2, + source_batch_manifest_reference_v1, + validate_reporting_source_rows_v2, + validate_reporting_source_staged_rows_v2, +) + + +def _package_request() -> ReportingSourceSliceRequestV2: + base = redacted_snapshot_request() + constituent = PackageItemConstituentV2( + constituent_id="package-constituent", + product_id="fixture-product", + media_buy_id="buy-1", + package_id="package-1", + ) + payload = base.model_dump(mode="json") + payload["contract_version"] = "2.0" + payload["coverage"] = ReportingSourceCoverageRequestV2( + media_buy_ids=["buy-1"], + constituents=[constituent], + denominator_fingerprint=coverage_denominator_fingerprint_v2([constituent]), + ).model_dump(mode="json") + return ReportingSourceSliceRequestV2.model_validate(payload) + + +def _v2_capabilities() -> ReportingSourceCapabilitiesV2: + payload = redacted_capabilities().model_dump(mode="json") + payload["contract_version"] = "2.0" + payload["supported_contract_versions"] = ["2.0"] + payload["capabilities_sha256"] = reporting_source_capabilities_sha256_v2(payload) + return ReportingSourceCapabilitiesV2.model_validate(payload) + + +def _v2_result( + request: ReportingSourceSliceRequestV2, +) -> tuple[ReportingSourceExecutorResult, SourceBatchManifestV2, ReportingSourceStagedObjectReader]: + base = redacted_snapshot_request() + _, manifest, reader = redacted_completed_result(base) + payload = manifest.model_dump(mode="json") + payload["manifest_version"] = "2.0" + payload["coverage"]["denominator_fingerprint"] = request.coverage.denominator_fingerprint + payload["coverage"]["media_buy_ids"] = list(request.coverage.media_buy_ids) + payload["coverage"]["constituents"][0]["constituent"] = request.coverage.constituents[ + 0 + ].model_dump(mode="json") + for cell in payload["metric_availability"]: + cell["constituent_id"] = request.coverage.constituents[0].constituent_id + payload["content_fingerprint"] = publication_content_fingerprint_v2(payload) + versioned_manifest = SourceBatchManifestV2.model_validate(payload) + manifest_bytes = encode_source_batch_manifest_v2(versioned_manifest) + reference = source_batch_manifest_reference_v1("manifest-package-v2", manifest_bytes) + result = ReportingSourceExecutorResult.completed( + request=request, manifest=reference, manifest_bytes=manifest_bytes + ) + return result, versioned_manifest, reader + + +def _empty_package_request(media_buy_id: str) -> ReportingSourceSliceRequestV2: + payload = redacted_snapshot_request().model_dump(mode="json") + payload["contract_version"] = "2.0" + payload["coverage"] = { + "expected": "full", + "media_buy_ids": [media_buy_id], + "constituents": [], + "denominator_fingerprint": coverage_denominator_fingerprint_v2( + [], media_buy_ids=[media_buy_id] + ), + } + return ReportingSourceSliceRequestV2.model_validate(payload) + + +def test_v2_empty_pairs_keep_distinct_frozen_buy_selectors() -> None: + first = _empty_package_request("buy-1") + second = _empty_package_request("buy-2") + assert first.media_buy_ids == ("buy-1",) + assert second.media_buy_ids == ("buy-2",) + assert first.coverage.denominator_fingerprint != second.coverage.denominator_fingerprint + + +async def test_v2_complete_empty_report_admits_zero_rows_without_metric_cells() -> None: + request = _empty_package_request("buy-1") + base = redacted_snapshot_request() + _, manifest, reader = redacted_completed_result(base, page_bodies=[""], row_counts=[0]) + payload = manifest.model_dump(mode="json") + payload["manifest_version"] = "2.0" + payload["coverage"] = { + "media_buy_ids": ["buy-1"], + "denominator_fingerprint": request.coverage.denominator_fingerprint, + "status": "full", + "constituents": [], + } + payload["metric_availability"] = [] + payload["explicit_zero"] = False + payload["content_fingerprint"] = publication_content_fingerprint_v2(payload) + complete = SourceBatchManifestV2.model_validate(payload) + manifest_bytes = encode_source_batch_manifest_v2(complete) + result = ReportingSourceExecutorResult.completed( + request=request, + manifest=source_batch_manifest_reference_v1("manifest-complete-empty", manifest_bytes), + manifest_bytes=manifest_bytes, + ) + assert ( + await validate_reporting_source_execution( + capabilities=_v2_capabilities(), + request=request, + result=result, + object_reader=reader, + ) + == complete + ) + + +def test_v2_complete_empty_report_requires_current_source_watermark() -> None: + request = _empty_package_request("buy-1") + _, manifest, _ = redacted_completed_result( + redacted_snapshot_request(), page_bodies=[""], row_counts=[0] + ) + payload = manifest.model_dump(mode="json") + payload["manifest_version"] = "2.0" + payload["coverage"] = { + "media_buy_ids": ["buy-1"], + "denominator_fingerprint": request.coverage.denominator_fingerprint, + "status": "full", + "constituents": [], + } + payload["metric_availability"] = [] + payload["explicit_zero"] = False + payload["data_through"] = request.period.start.isoformat() + payload["content_fingerprint"] = publication_content_fingerprint_v2(payload) + with pytest.raises(ValidationError, match="current source cutoff"): + SourceBatchManifestV2.model_validate(payload) + + +def test_v1_package_shape_and_request_bytes_remain_frozen() -> None: + request = redacted_snapshot_request() + assert request.contract_version == "1.0" + encoded = request.model_dump_json() + assert ReportingSourceSliceRequestV1.model_validate_json(encoded).model_dump_json() == encoded + assert hashlib.sha256(encoded.encode()).hexdigest() == ( + "53b59efb8632ac2e314da2d4144ed2295ef1c7e9a49cb1fc3f4d55f9da724585" + ) + with pytest.raises(ValidationError): + PackageItemConstituentV1( + constituent_id="item", + product_id="fixture-product", + package_id="package-1", + media_buy_id="buy-1", # type: ignore[call-arg] + ) + with pytest.raises(ValidationError): + ReportingSourceSliceRequestV1.model_validate( + {**request.model_dump(mode="json"), "contract_version": "2.0"} + ) + + +def test_v1_coverage_rejects_nested_v2_package_instance() -> None: + package = _package_request().coverage.constituents[0] + assert isinstance(package, PackageItemConstituentV2) + base = redacted_snapshot_request() + payload = base.coverage.model_dump(mode="json") + payload["constituents"] = [package] + payload["denominator_fingerprint"] = coverage_denominator_fingerprint_v1([package]) + with pytest.raises(ValidationError): + type(base.coverage).model_validate(payload) + + +def test_v1_manifest_coverage_rejects_completed_v2_constituent_evidence() -> None: + package = _package_request().coverage.constituents[0] + item = ReportingConstituentCoverageV2(constituent=package, status="present") + with pytest.raises(ValidationError): + SourceBatchCoverageV1( + denominator_fingerprint=coverage_denominator_fingerprint_v1([package]), + status="full", + constituents=[item], + ) + + +def test_v1_manifest_coverage_rejects_v2_batch_instance() -> None: + _, manifest, _ = _v2_result(_package_request()) + with pytest.raises(ValidationError): + SourceBatchCoverageV1.model_validate(manifest.coverage) + + +def test_v2_package_requires_both_parent_and_package() -> None: + package = _package_request().coverage.constituents[0] + for absent in ("media_buy_id", "package_id"): + payload = package.model_dump(mode="json") + payload.pop(absent) + with pytest.raises(ValidationError): + PackageItemConstituentV2.model_validate(payload) + + +def test_v2_denominator_fingerprint_binds_both_ids() -> None: + package = _package_request().coverage.constituents[0] + original = coverage_denominator_fingerprint_v2([package]) + assert original != coverage_denominator_fingerprint_v2( + [package.model_copy(update={"media_buy_id": "buy-2"})] + ) + assert original != coverage_denominator_fingerprint_v2( + [package.model_copy(update={"package_id": "package-2"})] + ) + assert _package_request().media_buy_ids == ("buy-1",) + assert _package_request().package_ids == ("package-1",) + + +def test_v2_selector_accepts_the_package_constituent_external_buy_id() -> None: + payload = _package_request().model_dump(mode="json") + package = ( + _package_request() + .coverage.constituents[0] + .model_copy(update={"media_buy_id": "buyer/buy-1"}) + ) + payload["coverage"] = { + "expected": "full", + "media_buy_ids": ["buyer/buy-1"], + "constituents": [package.model_dump(mode="json")], + "denominator_fingerprint": coverage_denominator_fingerprint_v2( + [package], media_buy_ids=["buyer/buy-1"] + ), + } + request = ReportingSourceSliceRequestV2.model_validate(payload) + _, manifest, _ = _v2_result(request) + assert request.media_buy_ids == manifest.coverage.media_buy_ids == ("buyer/buy-1",) + + +def test_v2_request_rejects_duplicate_pair_with_distinct_constituent_ids() -> None: + package = _package_request().coverage.constituents[0] + duplicate = package.model_copy(update={"constituent_id": "package-constituent-2"}) + with pytest.raises(ValidationError): + ReportingSourceCoverageRequestV2( + media_buy_ids=["buy-1"], + constituents=[package, duplicate], + denominator_fingerprint=coverage_denominator_fingerprint_v2([package, duplicate]), + ) + + +def test_v2_manifest_rejects_duplicate_pair_with_distinct_constituent_ids() -> None: + request = _package_request() + _, manifest, _ = _v2_result(request) + package = request.coverage.constituents[0] + duplicate = package.model_copy(update={"constituent_id": "package-constituent-2"}) + payload = manifest.model_dump(mode="json") + item = payload["coverage"]["constituents"][0].copy() + item["constituent"] = duplicate.model_dump(mode="json") + payload["coverage"]["constituents"].append(item) + payload["coverage"]["denominator_fingerprint"] = coverage_denominator_fingerprint_v2( + [package, duplicate] + ) + for cell in list(payload["metric_availability"]): + payload["metric_availability"].append({**cell, "constituent_id": duplicate.constituent_id}) + payload["content_fingerprint"] = publication_content_fingerprint_v2(payload) + with pytest.raises(ValidationError): + SourceBatchManifestV2.model_validate(payload) + + +def test_v2_allows_one_package_id_under_two_buys() -> None: + package = _package_request().coverage.constituents[0] + shared_id = package.model_copy( + update={"constituent_id": "package-constituent-2", "media_buy_id": "buy-2"} + ) + denominator = coverage_denominator_fingerprint_v2([package, shared_id]) + request = ReportingSourceCoverageRequestV2( + media_buy_ids=["buy-1", "buy-2"], + constituents=[package, shared_id], + denominator_fingerprint=denominator, + ) + coverage = SourceBatchCoverageV2( + media_buy_ids=["buy-1", "buy-2"], + denominator_fingerprint=denominator, + status="full", + constituents=[ + ReportingConstituentCoverageV2(constituent=item, status="present") + for item in request.constituents + ], + ) + assert [(item.media_buy_id, item.package_id) for item in request.constituents] == [ + ("buy-1", "package-1"), + ("buy-2", "package-1"), + ] + assert coverage.status == "full" + + +def test_v2_rows_require_an_assigned_package_pair() -> None: + from adcp.reporting.source import validate_reporting_source_rows_v2 + + package = _package_request().coverage.constituents[0] + other_buy = package.model_copy( + update={"constituent_id": "package-constituent-2", "media_buy_id": "buy-2"} + ) + request = _package_request().model_copy( + update={ + "coverage": ReportingSourceCoverageRequestV2( + media_buy_ids=["buy-1", "buy-2"], + constituents=[package, other_buy], + denominator_fingerprint=coverage_denominator_fingerprint_v2([package, other_buy]), + ) + } + ) + _, base_manifest, _ = _v2_result(_package_request()) + manifest_payload = base_manifest.model_dump(mode="json") + manifest_payload["coverage"]["denominator_fingerprint"] = ( + request.coverage.denominator_fingerprint + ) + manifest_payload["coverage"]["media_buy_ids"] = list(request.coverage.media_buy_ids) + other_evidence = dict(manifest_payload["coverage"]["constituents"][0]) + other_evidence["constituent"] = other_buy.model_dump(mode="json") + manifest_payload["coverage"]["constituents"].append(other_evidence) + for cell in list(manifest_payload["metric_availability"]): + manifest_payload["metric_availability"].append( + {**cell, "constituent_id": other_buy.constituent_id} + ) + manifest_payload["content_fingerprint"] = publication_content_fingerprint_v2(manifest_payload) + manifest = SourceBatchManifestV2.model_validate(manifest_payload) + with pytest.raises(ValueError, match="manifest coverage differs"): + validate_reporting_source_rows_v2(_package_request(), manifest, []) + validate_reporting_source_rows_v2( + request, + manifest, + [ + {"media_buy_id": "buy-1", "package_id": "package-1", "impressions": 5}, + {"media_buy_id": "buy-2", "package_id": "package-1", "impressions": 0}, + ], + ) + for row in ( + {"media_buy_id": "buy-3", "package_id": "package-1"}, + {"package_id": "package-1"}, + {"media_buy_id": "buy-1", "package_id": "external-line-1"}, + { + "media_buy_id": "buy-1", + "package_id": "package-1", + "constituent_id": other_buy.constituent_id, + }, + {"line_id": "external-line-1"}, + ): + with pytest.raises(ValueError, match="outside the requested coverage"): + validate_reporting_source_rows_v2(request, manifest, [row]) + + +def test_v2_staged_rows_are_fenced_by_request_before_manifest_exists() -> None: + from adcp.reporting.source import validate_reporting_source_staged_rows_v2 + + request = _package_request() + validate_reporting_source_staged_rows_v2( + request, [{"media_buy_id": "buy-1", "package_id": "package-1"}] + ) + for rows in ( + [{"media_buy_id": "buy-1"}], + [{"media_buy_id": "buy-2", "package_id": "package-1"}], + [{"media_buy_id": "buy-1", "package_id": "package-2"}], + [{"media_buy_id": "buy-2", "package_id": "package-1", "impressions": 0}], + [{"media_buy_id": "buy-1", "package_id": "package-2", "impressions": 0}], + ): + with pytest.raises(ValueError, match="outside the requested coverage"): + validate_reporting_source_staged_rows_v2(request, rows) + + +@pytest.mark.parametrize("stage", ["staged", "final"]) +def test_v2_rows_reject_duplicate_package_and_requested_dimension_values(stage: str) -> None: + request = _package_request() + _, manifest, _ = _v2_result(request) + rows = [ + { + "media_buy_id": "buy-1", + "package_id": "package-1", + "campaign_id": "campaign-1", + "impressions": 5, + }, + { + "media_buy_id": "buy-1", + "package_id": "package-1", + "campaign_id": "campaign-1", + "impressions": 7, + }, + ] + with pytest.raises(ValueError, match="duplicate row"): + if stage == "staged": + validate_reporting_source_staged_rows_v2(request, rows) + else: + validate_reporting_source_rows_v2(request, manifest, rows) + + +@pytest.mark.parametrize("stage", ["staged", "final"]) +def test_v2_rows_preserve_distinct_requested_dimension_values(stage: str) -> None: + request = _package_request() + _, manifest, _ = _v2_result(request) + rows = [ + {"media_buy_id": "buy-1", "package_id": "package-1", "campaign_id": "campaign-1"}, + {"media_buy_id": "buy-1", "package_id": "package-1", "campaign_id": "campaign-2"}, + ] + if stage == "staged": + validate_reporting_source_staged_rows_v2(request, rows) + else: + validate_reporting_source_rows_v2(request, manifest, rows) + + +def test_v2_manifest_keeps_explicit_zero_on_one_package_pair() -> None: + package = _package_request().coverage.constituents[0] + other_buy = package.model_copy( + update={"constituent_id": "package-constituent-2", "media_buy_id": "buy-2"} + ) + _, manifest, _ = _v2_result(_package_request()) + payload = manifest.model_dump(mode="json") + payload["coverage"]["denominator_fingerprint"] = coverage_denominator_fingerprint_v2( + [package, other_buy] + ) + payload["coverage"]["media_buy_ids"] = ["buy-1", "buy-2"] + zero_evidence = dict(payload["coverage"]["constituents"][0]) + zero_evidence["constituent"] = other_buy.model_dump(mode="json") + zero_evidence["status"] = "explicit_zero" + payload["coverage"]["constituents"].append(zero_evidence) + for cell in list(payload["metric_availability"]): + payload["metric_availability"].append( + {**cell, "constituent_id": other_buy.constituent_id, "status": "explicit_zero"} + ) + payload["content_fingerprint"] = publication_content_fingerprint_v2(payload) + versioned = SourceBatchManifestV2.model_validate(payload) + assert [item.status for item in versioned.coverage.constituents] == [ + "present", + "explicit_zero", + ] + from adcp.reporting.source import validate_reporting_source_rows_v2 + + request_payload = _package_request().model_dump(mode="json") + request_payload["coverage"] = ReportingSourceCoverageRequestV2( + media_buy_ids=["buy-1", "buy-2"], + constituents=[package, other_buy], + denominator_fingerprint=payload["coverage"]["denominator_fingerprint"], + ).model_dump(mode="json") + request = ReportingSourceSliceRequestV2.model_validate(request_payload) + with pytest.raises(ValueError, match="explicit zero"): + validate_reporting_source_rows_v2( + request, + versioned, + [{"media_buy_id": "buy-2", "package_id": "package-1"}], + ) + + +def test_v2_request_retains_metric_cell_limit() -> None: + package = _package_request().coverage.constituents[0] + other_buy = package.model_copy( + update={"constituent_id": "package-constituent-2", "media_buy_id": "buy-2"} + ) + payload = _package_request().model_dump(mode="json") + payload["coverage"] = ReportingSourceCoverageRequestV2( + media_buy_ids=["buy-1", "buy-2"], + constituents=[package, other_buy], + denominator_fingerprint=coverage_denominator_fingerprint_v2([package, other_buy]), + ).model_dump(mode="json") + payload["requested_metrics"] = [f"metric_{index}" for index in range(501)] + with pytest.raises(ValidationError, match="at most 1000 constituent-metric cells"): + ReportingSourceSliceRequestV2.model_validate(payload) + + +def test_v2_two_metric_package_limit_is_checked_at_request_validation() -> None: + first = _package_request().coverage.constituents[0] + + def request_with_packages(count: int) -> ReportingSourceSliceRequestV2: + packages = [ + first.model_copy( + update={"constituent_id": f"constituent-{index}", "package_id": f"package-{index}"} + ) + for index in range(count) + ] + payload = _package_request().model_dump(mode="json") + payload["coverage"] = { + "expected": "full", + "media_buy_ids": ["buy-1"], + "constituents": [item.model_dump(mode="json") for item in packages], + "denominator_fingerprint": coverage_denominator_fingerprint_v2( + packages, media_buy_ids=["buy-1"] + ), + } + return ReportingSourceSliceRequestV2.model_validate(payload) + + assert len(request_with_packages(500).coverage.constituents) == 500 + with pytest.raises(ValidationError, match="at most 1000 constituent-metric cells"): + request_with_packages(501) + + +def test_v2_capability_checksum_binds_declared_version() -> None: + payload = redacted_capabilities().model_dump(mode="json") + payload["contract_version"] = "2.0" + assert reporting_source_capabilities_sha256_v1(payload) != payload["capabilities_sha256"] + with pytest.raises(ValidationError): + ReportingSourceCapabilitiesV2.model_validate(payload) + + +def test_v2_contract_versions_are_explicit() -> None: + capabilities_payload = _v2_capabilities().model_dump(mode="json") + capabilities_payload.pop("contract_version") + with pytest.raises(ValidationError): + ReportingSourceCapabilitiesV2.model_validate(capabilities_payload) + capabilities_payload = _v2_capabilities().model_dump(mode="json") + capabilities_payload.pop("supported_contract_versions") + with pytest.raises(ValidationError): + ReportingSourceCapabilitiesV2.model_validate(capabilities_payload) + + request_payload = _package_request().model_dump(mode="json") + request_payload.pop("contract_version") + with pytest.raises(ValidationError): + ReportingSourceSliceRequestV2.model_validate(request_payload) + + result, manifest, _ = _v2_result(_package_request()) + manifest_payload = manifest.model_dump(mode="json") + manifest_payload.pop("manifest_version") + with pytest.raises(ValidationError): + SourceBatchManifestV2.model_validate(manifest_payload) + assert result.response is not None + response_payload = result.response.model_dump(mode="json") + response_payload.pop("contract_version") + with pytest.raises(ValidationError): + ReportingSourceExecutionResponseV2.model_validate(response_payload) + + +def test_supported_contract_versions_cannot_change_after_checksum_validation() -> None: + capabilities = _v2_capabilities() + checksum = capabilities.capabilities_sha256 + with pytest.raises(AttributeError): + capabilities.supported_contract_versions.append("1.0") + assert capabilities.capabilities_sha256 == checksum + assert capabilities.supported_contract_versions == ("2.0",) + + +@pytest.mark.parametrize("versions", [["1.0"], ["2.0", "2.0"]]) +def test_v2_capability_rejects_incoherent_supported_versions(versions: list[str]) -> None: + payload = _v2_capabilities().model_dump(mode="json") + payload["supported_contract_versions"] = versions + payload["capabilities_sha256"] = reporting_source_capabilities_sha256_v2(payload) + with pytest.raises(ValidationError): + ReportingSourceCapabilitiesV2.model_validate(payload) + + +def test_v2_coverage_fingerprint_binds_package_pair() -> None: + package = _package_request().coverage.constituents[0] + item = ReportingConstituentCoverageV2(constituent=package, status="present") + rebound = ReportingConstituentCoverageV2( + constituent=package.model_copy(update={"media_buy_id": "buy-2"}), status="present" + ) + assert source_batch_coverage_fingerprint_v2( + [item], media_buy_ids=["buy-1"] + ) != source_batch_coverage_fingerprint_v2([rebound], media_buy_ids=["buy-2"]) + + +def test_v2_coverage_fingerprint_binds_empty_buy_selector() -> None: + assert source_batch_coverage_fingerprint_v2( + [], media_buy_ids=["buy-1"] + ) != source_batch_coverage_fingerprint_v2([], media_buy_ids=["buy-2"]) + + +async def test_v2_execution_requires_explicit_capability_and_matching_manifest() -> None: + request = _package_request() + result, manifest, reader = _v2_result(request) + assert manifest.manifest_version == "2.0" + assert manifest.contract_version == "2.0" + with pytest.raises(ReportingSourceConformanceError) as error: + await validate_reporting_source_execution( + capabilities=redacted_capabilities(), + request=request, + result=result, + object_reader=reader, + ) + assert error.value.code == "CAPABILITY_MISMATCH" + assert ( + await validate_reporting_source_execution( + capabilities=_v2_capabilities(), + request=request, + result=result, + object_reader=reader, + ) + == manifest + ) + + +async def test_v2_empty_pair_request_requires_package_capability() -> None: + request = _empty_package_request("buy-1") + payload = _v2_capabilities().model_dump(mode="json") + for offering in payload["offerings"]: + offering["constituent_kinds"] = ["media_buy"] + payload["capabilities_sha256"] = reporting_source_capabilities_sha256_v2(payload) + capabilities = ReportingSourceCapabilitiesV2.model_validate(payload) + result, _, reader = _v2_result(_package_request()) + with pytest.raises(ReportingSourceConformanceError) as error: + await validate_reporting_source_execution( + capabilities=capabilities, + request=request, + result=result, + object_reader=reader, + ) + assert error.value.code == "CAPABILITY_MISMATCH" + + +async def test_one_executor_can_negotiate_v1_and_v2_requests() -> None: + payload = _v2_capabilities().model_dump(mode="json") + payload["supported_contract_versions"] = ["1.0", "2.0"] + payload["capabilities_sha256"] = reporting_source_capabilities_sha256_v2(payload) + capabilities = ReportingSourceCapabilitiesV2.model_validate(payload) + + v1_request = redacted_snapshot_request() + v1_result, v1_manifest, v1_reader = redacted_completed_result(v1_request) + assert ( + await validate_reporting_source_execution( + capabilities=capabilities, + request=v1_request, + result=v1_result, + object_reader=v1_reader, + ) + == v1_manifest + ) + + v2_request = _package_request() + v2_result, v2_manifest, v2_reader = _v2_result(v2_request) + assert ( + await validate_reporting_source_execution( + capabilities=capabilities, + request=v2_request, + result=v2_result, + object_reader=v2_reader, + ) + == v2_manifest + ) + + +async def test_v2_only_capability_rejects_v1_request() -> None: + request = redacted_snapshot_request() + result, _, reader = redacted_completed_result(request) + with pytest.raises(ReportingSourceConformanceError) as error: + await validate_reporting_source_execution( + capabilities=_v2_capabilities(), + request=request, + result=result, + object_reader=reader, + ) + assert error.value.code == "CAPABILITY_MISMATCH" + + +async def test_mixed_request_manifest_versions_fail_admission() -> None: + request = _package_request() + _, v1_manifest, reader = redacted_completed_result(redacted_snapshot_request()) + payload = v1_manifest.model_dump(mode="json") + payload["coverage"]["denominator_fingerprint"] = request.coverage.denominator_fingerprint + payload["coverage"]["constituents"][0]["constituent"] = request.coverage.constituents[ + 0 + ].model_dump(mode="json") + for cell in payload["metric_availability"]: + cell["constituent_id"] = request.coverage.constituents[0].constituent_id + payload["content_fingerprint"] = publication_content_fingerprint_v2(payload) + from adcp.reporting.canonical_json import canonical_json_utf8_v1 + + manifest_bytes = canonical_json_utf8_v1(payload) + result = ReportingSourceExecutorResult.completed( + request=request, + manifest=source_batch_manifest_reference_v1("manifest-mixed", manifest_bytes), + manifest_bytes=manifest_bytes, + ) + with pytest.raises(ReportingSourceConformanceError) as error: + await validate_reporting_source_execution( + capabilities=_v2_capabilities(), + request=request, + result=result, + object_reader=reader, + ) + assert error.value.code == "MANIFEST_MISMATCH" + + +async def test_v1_request_rejects_v2_manifest() -> None: + request = redacted_snapshot_request() + versioned_request = _package_request() + versioned_result, _, reader = _v2_result(versioned_request) + assert versioned_result.response is not None + assert versioned_result.manifest_bytes is not None + result = ReportingSourceExecutorResult.completed( + request=request, + manifest=versioned_result.response.manifest, + manifest_bytes=versioned_result.manifest_bytes, + ) + with pytest.raises(ReportingSourceConformanceError) as error: + await validate_reporting_source_execution( + capabilities=redacted_capabilities(), + request=request, + result=result, + object_reader=reader, + ) + assert error.value.code == "MANIFEST_MISMATCH" + + +def test_v2_manifest_rejects_package_without_parent_buy() -> None: + request = _package_request() + _, manifest, _ = _v2_result(request) + payload = manifest.model_dump(mode="json") + payload["coverage"]["constituents"][0]["constituent"].pop("media_buy_id") + with pytest.raises(ValidationError): + SourceBatchManifestV2.model_validate(payload) + + +def test_v2_execution_response_round_trips_without_downgrading() -> None: + result, _, _ = _v2_result(_package_request()) + round_trip = ReportingSourceExecutorResult.model_validate_json(result.model_dump_json()) + assert round_trip.response is not None + assert round_trip.response.contract_version == "2.0" + + +async def test_mixed_version_bytes_must_pass_reference_integrity_first() -> None: + result, _, reader = _v2_result(_package_request()) + assert result.response is not None + bad_reference = result.response.manifest.model_copy(update={"manifest_sha256": "0" * 64}) + mixed = ReportingSourceExecutorResult.completed( + request=redacted_snapshot_request(), + manifest=bad_reference, + manifest_bytes=result.manifest_bytes or b"", + ) + with pytest.raises(SourceBatchManifestError) as error: + await validate_reporting_source_execution( + capabilities=redacted_capabilities(), + request=redacted_snapshot_request(), + result=mixed, + object_reader=reader, + ) + assert error.value.code == "SOURCE_MANIFEST_CHECKSUM_MISMATCH"