From ed3433ac7f4f969ec444f1cc25f7d2e60b9856eb Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Mon, 2 Nov 2026 14:59:08 +0000 Subject: [PATCH] =?UTF-8?q?feat(training-agent):=20experimental=203.3=20co?= =?UTF-8?q?llections=20=E2=80=94=20collection=20cards,=20list=5Fcollection?= =?UTF-8?q?s,=20card=20webhooks,=20creator=20fixtures=20(#8181,=20#8093)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the 3.3-beta.3 preview pin (and in candidate mode) the training agent serves the experimental 3.3 collection surfaces. Stable, major-only and unpinned callers see none of them. - media_buy.collection_cards: cards for every sold collection (images, sample_content, url, talent brand_ref, distribution followers), and include_collections on get_products, list_products, request_proposals and refine_proposals. The cards cover only the referenced collections, one per key, and never appear on unchanged or rejected answers. Feed versions are scoped by the flag with distinct tokens. Creator product cards use the sample_content reference-asset role. - media_buy.collection_catalog: list_collections with key lookup, filters (RFC 4647 languages), pagination and its own conditional fetch. Card webhooks (collection_card.updated / .removed, collection_catalog.bulk_change) are registered through sync_accounts notification_configs; bulk thresholds apply, and a no-op follower refresh sends nothing. The local seed_collection controller scenario drives them. - brand.collection_links: get_brand_identity collections on /brand for the CreatorLoop creators. One brand_ref is deliberately unconfirmed. - adagents.json declares the collections, with anchored facts and collection-scoped authorization, so copies resolve as verified. - Stable tools/list no longer advertises include_collections. - The collection-copies reference grader moved to tests/helpers/collection-copies-grader.cjs. The agent's real outputs are graded with it, with webhooks delivered to a local receiver. Co-Authored-By: Claude Opus 5.5 --- ...test-agent-3-3-preview-collections-docs.md | 5 + docs/reference/3-3-beta.mdx | 9 +- server/src/training-agent/account-handlers.ts | 42 +- server/src/training-agent/brand-handlers.ts | 15 +- .../training-agent/collections/brand-links.ts | 105 +++ .../collections/capabilities.ts | 48 ++ .../collections/card-catalog.ts | 502 ++++++++++++++ .../collections/card-fixtures.ts | 171 +++++ .../collections/card-webhooks.ts | 196 ++++++ .../collections/include-collections.ts | 204 ++++++ .../collections/list-collections.ts | 177 +++++ .../collections/seed-collection.ts | 256 ++++++++ .../training-agent/comply-test-controller.ts | 7 + server/src/training-agent/index.ts | 25 + server/src/training-agent/tenants/router.ts | 26 +- server/src/training-agent/tenants/sales.ts | 6 + .../tenants/served-release-shape.test.ts | 64 +- .../training-agent/tenants/tool-catalog.ts | 3 + .../src/training-agent/v6-sales-platform.ts | 65 +- .../unit/training-agent-collections.test.ts | 611 ++++++++++++++++++ tests/collection-copies-vectors.test.cjs | 290 +-------- tests/helpers/collection-copies-grader.cjs | 307 +++++++++ 22 files changed, 2830 insertions(+), 304 deletions(-) create mode 100644 .changeset/test-agent-3-3-preview-collections-docs.md create mode 100644 server/src/training-agent/collections/brand-links.ts create mode 100644 server/src/training-agent/collections/capabilities.ts create mode 100644 server/src/training-agent/collections/card-catalog.ts create mode 100644 server/src/training-agent/collections/card-fixtures.ts create mode 100644 server/src/training-agent/collections/card-webhooks.ts create mode 100644 server/src/training-agent/collections/include-collections.ts create mode 100644 server/src/training-agent/collections/list-collections.ts create mode 100644 server/src/training-agent/collections/seed-collection.ts create mode 100644 server/tests/unit/training-agent-collections.test.ts create mode 100644 tests/helpers/collection-copies-grader.cjs diff --git a/.changeset/test-agent-3-3-preview-collections-docs.md b/.changeset/test-agent-3-3-preview-collections-docs.md new file mode 100644 index 00000000000..fbee0084d17 --- /dev/null +++ b/.changeset/test-agent-3-3-preview-collections-docs.md @@ -0,0 +1,5 @@ +--- +"adcontextprotocol": patch +--- + +Docs: the 3.3 beta program page lists the experimental collection surfaces the public test agent now serves at its `3.3-beta.3` preview: collection cards through `include_collections`, `list_collections` and the collection card webhooks, `get_brand_identity` collection links, and the `seed_collection` controller scenario that drives the webhooks. diff --git a/docs/reference/3-3-beta.mdx b/docs/reference/3-3-beta.mdx index 255e78372ae..1145eeb4ebb 100644 --- a/docs/reference/3-3-beta.mdx +++ b/docs/reference/3-3-beta.mdx @@ -92,9 +92,14 @@ At the preview pin, `/sales/mcp` serves the experimental 3.3 surfaces the traini - account requests (`account.requests`): `request_account`, and `account_requests` on `list_accounts`; - the caller's `authorization`, with `authorized_at` and `regranted_at`, and `revision` on each `list_accounts` account; - principal-level reporting subscriptions (`media_buy.reporting_subscriptions`): the `reporting_subscriptions` section of `sync_principal` and `get_principal`, and `subscription_coverage` on `get_reporting_status`; -- the `reporting_subscription_probe` controller scenario, which revokes, reduces, and restores grants and advances a per-caller sandbox clock. +- the `reporting_subscription_probe` controller scenario, which revokes, reduces, and restores grants and advances a per-caller sandbox clock; +- collection cards (`media_buy.collection_cards`): `include_collections` on `get_products`, `list_products`, `request_proposals` and `refine_proposals`, with images, samples, links, talent `brand_ref` and follower counts on the CreatorLoop creator channels; +- the collection catalog (`media_buy.collection_catalog`): `list_collections`, and the `collection_card.*` and `collection_catalog.bulk_change` webhooks registered through `sync_accounts`; +- the `seed_collection` controller scenario, which changes the caller's cards (a sample, a removal, a follower sweep) so the card webhooks fire. -`tools/list` includes `request_account` only when the listing is pinned to the preview (`params._meta.adcp_version`, as `@adcp/sdk` sends it). Hold policy, `extend_hold`, and response collections are not served yet ([#8181](https://github.com/adcontextprotocol/adcp/issues/8181)). +At the same pin, `/brand/mcp` returns `collections` on `get_brand_identity` for the creators (`brand.collection_links`). + +`tools/list` includes `request_account` and `list_collections` only when the listing is pinned to the preview (`params._meta.adcp_version`, as `@adcp/sdk` sends it). Hold policy and `extend_hold` are not served yet ([#8181](https://github.com/adcontextprotocol/adcp/issues/8181)). With `@adcp/sdk`, use 17.0.0-beta.11 or later, which negotiates exactly `3.3-beta.3`. Earlier 17.0.0 betas bundle an older beta, so they settle on 3.2 with the test agent. diff --git a/server/src/training-agent/account-handlers.ts b/server/src/training-agent/account-handlers.ts index dd4a20bfe25..7dbb2f29956 100644 --- a/server/src/training-agent/account-handlers.ts +++ b/server/src/training-agent/account-handlers.ts @@ -39,6 +39,7 @@ import { proveAccountWebhookControl, } from './webhook-challenge.js'; import { emitAccountNotificationWebhook } from './webhooks.js'; +import { isCollectionCardEventType, supportsCollections33 } from './collections/card-catalog.js'; import { clearSharedAccountResources } from './shared-account-resources.js'; import { createLogger } from '../logger.js'; import { @@ -921,6 +922,9 @@ async function normalizeNotificationConfigs( existing: NotificationConfigState[]; dryRun: boolean; proofDeadlineMs: number; + /** Collection card events (experimental, 3.3) are registrable only on + * the 3.3 wire, where the seller declares them. */ + servedAdcpVersion?: string; }, ): Promise } | undefined> { if (input.notification_configs === undefined) return undefined; @@ -982,7 +986,8 @@ async function normalizeNotificationConfigs( } seenEventTypes.add(eventType); } - const invalidIndex = config.event_types.findIndex(t => !ACCOUNT_ANCHORED_NOTIFICATION_TYPES.has(t)); + const invalidIndex = config.event_types.findIndex(t => !ACCOUNT_ANCHORED_NOTIFICATION_TYPES.has(t) + && !(isCollectionCardEventType(t) && supportsCollections33(options.servedAdcpVersion))); if (invalidIndex !== -1) { return { error: validationFailure( @@ -1126,6 +1131,39 @@ export function getAccountNotificationSubscribers( return out; } +/** + * Every active subscriber to `notificationType` on any account the principal + * holds, across its session partitions. For seller-wide changes the + * principal can see on every account, such as its collection cards. + */ +export function getPrincipalNotificationSubscribers( + principal: string | undefined, + notificationType: string, +): AccountNotificationSubscriber[] { + const prefix = `${principalScope(principal)}\u001F`; + const out: AccountNotificationSubscriber[] = []; + const seen = new Set(); + for (const [storeKey, accounts] of accountStore) { + if (!storeKey.startsWith(prefix)) continue; + for (const account of accounts.values()) { + for (const config of account.notificationConfigs) { + if (!config.active || !config.eventTypes.includes(notificationType)) continue; + const subscriberKey = `${account.accountId}\u001F${config.subscriberId}`; + if (seen.has(subscriberKey)) continue; + seen.add(subscriberKey); + out.push({ + accountId: account.accountId, + subscriberId: config.subscriberId, + url: config.url, + eventTypes: [...config.eventTypes], + authentication: config.authentication, + }); + } + } + } + return out; +} + /** * Server-internal fan-out for account business changes. Unlike the * request-principal helper above, this enumerates every independently proven @@ -1862,6 +1900,7 @@ export async function handleSyncAccounts(args: ToolArgs, ctx: TrainingContext) { existing: existing.notificationConfigs, dryRun: req.dry_run === true, proofDeadlineMs, + servedAdcpVersion: ctx.servedAdcpVersion, }); if (notificationConfigs && 'error' in notificationConfigs) { results.push(notificationConfigs.error); @@ -2135,6 +2174,7 @@ export async function handleSyncAccounts(args: ToolArgs, ctx: TrainingContext) { existing: existing?.notificationConfigs ?? [], dryRun: req.dry_run === true, proofDeadlineMs, + servedAdcpVersion: ctx.servedAdcpVersion, }); if (notificationConfigs && 'error' in notificationConfigs) { results.push(notificationConfigs.error); diff --git a/server/src/training-agent/brand-handlers.ts b/server/src/training-agent/brand-handlers.ts index 778613bccef..e00d64836cd 100644 --- a/server/src/training-agent/brand-handlers.ts +++ b/server/src/training-agent/brand-handlers.ts @@ -16,6 +16,7 @@ import { resolveGovernanceAgentsForAccount } from './account-handlers.js'; import { getCanonicalBase, getTrainingGovernanceIssuer } from './canonical-base.js'; import { decodeOffsetCursor, encodeOffsetCursor } from './pagination.js'; import { sourceSchemaDocument } from './source-schema.js'; +import { creatorBrandEntries, withCollectionLinks } from './collections/brand-links.js'; async function governedCommitmentRejection( governanceContext: string, @@ -626,6 +627,10 @@ const BRAND_MAP = new Map([ ...ADVERTISER_BRANDS.map(b => [b.brand_id, b] as [string, AnyBrand]), ]); +const CREATOR_BRAND_MAP = new Map( + creatorBrandEntries().map(brand => [brand.brand_id, brand as AnyBrand]), +); + // Fields that are always returned (public, not selectable) const PUBLIC_FIELDS = ['description', 'industries', 'keller_type', 'logos', 'tagline'] as const; const AUTHORIZED_FIELDS = ['colors', 'fonts', 'visual_guidelines', 'tone', 'voice_synthesis', 'assets', 'rights'] as const; @@ -936,7 +941,9 @@ export function handleGetBrandIdentity( const fields = req.fields; const authorized = req.authorized ?? false; - const talent = BRAND_MAP.get(brandId); + // Creator identities behind CreatorLoop collections resolve by id only; + // search_brands keeps its roster. + const talent = BRAND_MAP.get(brandId) ?? CREATOR_BRAND_MAP.get(brandId); if (!talent) { return { errors: [{ code: 'REFERENCE_NOT_FOUND', message: `No brand with id '${brandId}'`, field: 'brand_id' }] }; } @@ -968,7 +975,11 @@ export function handleGetBrandIdentity( response.available_fields = withheld; } - return response; + // Experimental 3.3 brand.collection_links: the creator's channels. Only + // the /brand tenant declares the feature. + return ctx.tenantId === 'brand' + ? withCollectionLinks(response, brandId, fields, ctx.servedAdcpVersion) + : response; } function computeMatchScore( diff --git a/server/src/training-agent/collections/brand-links.ts b/server/src/training-agent/collections/brand-links.ts new file mode 100644 index 00000000000..2ded2a07cdb --- /dev/null +++ b/server/src/training-agent/collections/brand-links.ts @@ -0,0 +1,105 @@ +/** + * Creator identities on the /brand tenant and their collection links + * (experimental `brand.collection_links`, #8141). + * + * A CreatorLoop collection's `talent[].brand_ref` names the creator's brand + * identity under the CreatorLoop Talent house. `get_brand_identity` for that + * brand returns, on the 3.3 wire, `collections`: the creator's channels as + * `{publisher_domain, collection_id}` links. A `brand_ref` is confirmed only + * when the identity lists the same key back + * (get_brand_identity.mdx, "People behind collections"). The fixtures keep + * one creator unconfirmed: the collection claims the identity, and the + * identity lists a different channel. + */ + +import { getAgentUrl } from '../config.js'; +import { CREATOR_PUBLISHER } from '../creators.js'; +import { + CREATOR_CARD_FIXTURES, + CREATOR_TALENT_HOUSE, + OFF_PLATFORM_COLLECTION_LINKS, +} from './card-fixtures.js'; +import { supportsCollections33 } from './card-catalog.js'; + +export interface CreatorBrandEntry { + brand_id: string; + house: { domain: string; name: string }; + names: Array>; + description: string; + industries: string[]; + keller_type: string; + tagline: string; + logos: Array<{ url: string; variant?: string }>; + tone: { voice: string; attributes: string[]; dos: string[]; donts: string[] }; +} + +export interface CollectionLink { + publisher_domain: string; + collection_id: string; + name: string; + role?: string; + seller_agent_url?: string; +} + +function showFor(showId: string) { + return CREATOR_PUBLISHER.shows?.find(show => show.showId === showId); +} + +let entries: CreatorBrandEntry[] | undefined; + +/** Brand identities for the creators whose collections carry a brand_ref. */ +export function creatorBrandEntries(): CreatorBrandEntry[] { + entries ??= CREATOR_CARD_FIXTURES.filter(fixture => fixture.brandRef).map(fixture => { + const show = showFor(fixture.showId); + const name = show?.talent?.find(person => person.role === 'creator')?.name ?? fixture.slug; + return { + brand_id: fixture.slug, + house: { ...CREATOR_TALENT_HOUSE }, + names: [{ en: name }], + description: fixture.bio, + industries: [], + keller_type: 'independent', + tagline: '', + // The person's picture: always public (get_brand_identity.mdx). + logos: [{ url: `https://picsum.photos/seed/creatorloop-${fixture.slug}-avatar/800/800`, variant: 'primary' }], + tone: { voice: '', attributes: [], dos: [], donts: [] }, + }; + }); + return entries; +} + +/** The channels a creator identity lists, or undefined for other brands. */ +export function collectionLinksFor(brandId: string): CollectionLink[] | undefined { + const fixture = CREATOR_CARD_FIXTURES.find(candidate => candidate.slug === brandId && candidate.brandRef); + if (!fixture) return undefined; + if (!fixture.brandConfirms) { + const offPlatform = OFF_PLATFORM_COLLECTION_LINKS[brandId]; + return offPlatform ? [{ ...offPlatform, role: 'creator' }] : []; + } + const show = showFor(fixture.showId); + return [{ + publisher_domain: CREATOR_PUBLISHER.domain, + collection_id: fixture.showId, + name: show?.name ?? fixture.showId, + role: 'creator', + seller_agent_url: `${getAgentUrl()}/sales/mcp`, + }]; +} + +/** + * Add `collections` to a get_brand_identity response for a caller served 3.3 + * or later, when the request did not narrow `fields` to exclude it. Mutates + * and returns the response. + */ +export function withCollectionLinks>( + response: T, + brandId: string, + fields: unknown, + servedAdcpVersion: string | undefined, +): T { + if (!supportsCollections33(servedAdcpVersion)) return response; + if (Array.isArray(fields) && !fields.includes('collections')) return response; + const links = collectionLinksFor(brandId); + if (links && links.length > 0) (response as Record).collections = links; + return response; +} diff --git a/server/src/training-agent/collections/capabilities.ts b/server/src/training-agent/collections/capabilities.ts new file mode 100644 index 00000000000..1c4e1868df3 --- /dev/null +++ b/server/src/training-agent/collections/capabilities.ts @@ -0,0 +1,48 @@ +/** + * Capability declarations for the experimental 3.3 collection surfaces, added + * only when the caller is served 3.3 or later: + * + * - /sales: `media_buy.collection_cards` (cards in product and proposal + * reads), `media_buy.collection_catalog` (list_collections and the card + * webhooks), and the three card event types in + * `wholesale_feed_webhooks.event_types`. + * - /brand: `brand.collection_links` (`collections` on get_brand_identity). + */ + +import { + BRAND_COLLECTION_LINKS_FEATURE, + COLLECTION_CARD_EVENT_TYPES, + COLLECTION_CARDS_FEATURE, + COLLECTION_CATALOG_FEATURE, + supportsCollections33, +} from './card-catalog.js'; + +interface CapabilitiesShape { + experimental_features?: unknown; + wholesale_feed_webhooks?: Record; +} + +function addFeatures(structured: CapabilitiesShape, features: readonly string[]): void { + const current = Array.isArray(structured.experimental_features) + ? structured.experimental_features.filter((feature): feature is string => typeof feature === 'string') + : []; + for (const feature of features) if (!current.includes(feature)) current.push(feature); + structured.experimental_features = current; +} + +export function projectCollectionCapabilities( + structured: CapabilitiesShape, + tenantId: string, + servedVersion: string | undefined, +): void { + if (!supportsCollections33(servedVersion)) return; + if (tenantId === 'sales') { + addFeatures(structured, [COLLECTION_CARDS_FEATURE, COLLECTION_CATALOG_FEATURE]); + const webhooks = structured.wholesale_feed_webhooks; + if (webhooks && Array.isArray(webhooks.event_types)) { + webhooks.event_types = [...new Set([...(webhooks.event_types as string[]), ...COLLECTION_CARD_EVENT_TYPES])]; + } + } else if (tenantId === 'brand') { + addFeatures(structured, [BRAND_COLLECTION_LINKS_FEATURE]); + } +} diff --git a/server/src/training-agent/collections/card-catalog.ts b/server/src/training-agent/collections/card-catalog.ts new file mode 100644 index 00000000000..5568f964d7f --- /dev/null +++ b/server/src/training-agent/collections/card-catalog.ts @@ -0,0 +1,502 @@ +/** + * The training agent's collection card catalog (experimental, #8181). + * + * One card per collection the sales tenant sells, keyed by + * `(publisher_domain, collection_id)`. The same copy is served by + * `include_collections` on product and proposal reads, by `list_collections`, + * and in `collection_card.updated` webhooks + * (list_collections.mdx, "Consistency with include_collections"). + * + * Authority (collections-and-installments.mdx, "Collections in responses"): + * the agent's adagents.json declares every collection with its anchored + * fields (`declaredCollections`), and a card is that declaration plus the + * closed list the seller owns: `images`, `sample_content`, `url`, + * `description`, `season`, `cadence` and `distribution[].followers`. Card + * changes (the `seed_collection` controller scenario) touch only the closed + * list, so a copy never restates an anchored fact differently from the + * declaration. + * + * Sandbox scope: card changes are kept per caller principal, like the rest of + * the training agent's sandbox state, so one caller's edits never change the + * cards another caller sees. Within one principal every account sees the same + * card for a key; no card is account-scoped, so every read and event is + * `cache_scope: "public"`. + */ + +import { createHash } from 'node:crypto'; +import { PUBLISHERS } from '../publishers.js'; +import type { PublisherProfile, ShowDefinition } from '../types.js'; +import { atLeastAdcpVersion } from '../types.js'; +import { sourceSchemaDocument } from '../source-schema.js'; +import { + CREATOR_CARD_FIXTURE_BY_SHOW, + CREATOR_TALENT_HOUSE, + type CreatorCardFixture, +} from './card-fixtures.js'; + +export const COLLECTION_CARDS_FEATURE = 'media_buy.collection_cards'; +export const COLLECTION_CATALOG_FEATURE = 'media_buy.collection_catalog'; +export const BRAND_COLLECTION_LINKS_FEATURE = 'brand.collection_links'; + +export const COLLECTION_CARD_EVENT_TYPES = [ + 'collection_card.updated', + 'collection_card.removed', + 'collection_catalog.bulk_change', +] as const; +export type CollectionCardEventType = typeof COLLECTION_CARD_EVENT_TYPES[number]; + +export function isCollectionCardEventType(value: unknown): value is CollectionCardEventType { + return typeof value === 'string' && (COLLECTION_CARD_EVENT_TYPES as readonly string[]).includes(value); +} + +/** Fields a seller's copy wins on, besides `distribution[].followers`. */ +export const CLOSED_LIST_FIELDS = ['images', 'sample_content', 'url', 'description', 'season', 'cadence'] as const; +export type ClosedListField = typeof CLOSED_LIST_FIELDS[number]; + +/** First wire checkpoint carrying the experimental 3.3 collection surfaces + * (`media_buy.collection_cards`, `media_buy.collection_catalog`, + * `brand.collection_links`). Stable callers (3.2 and below) never see them. */ +export const COLLECTIONS_33_ADCP_VERSION = '3.3-beta' as const; + +export function supportsCollections33(servedVersion: string | undefined): boolean { + return atLeastAdcpVersion(servedVersion, COLLECTIONS_33_ADCP_VERSION); +} + +export type CollectionCard = Record & { + publisher_domain: string; + collection_id: string; + name: string; +}; + +export interface CollectionKey { + publisher_domain: string; + collection_id: string; +} + +export interface AudienceSize { + lower_bound: number; + upper_bound: number; + precision: 'exact' | 'approximate'; + as_of?: string; +} + +export function keyOf(card: CollectionKey): string { + return `${card.publisher_domain}\u0000${card.collection_id}`; +} + +/** Canonical JSON with sorted object keys. */ +export function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; + if (value && typeof value === 'object') { + const record = value as Record; + return `{${Object.keys(record).sort() + .filter(key => record[key] !== undefined) + .map(key => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(',')}}`; + } + return JSON.stringify(value); +} + +function digest(value: string, length: number): string { + return createHash('sha256').update(value).digest('hex').slice(0, length); +} + +const CADENCES = new Set(['daily', 'weekly', 'monthly', 'seasonal', 'event', 'irregular']); +const STATUSES = new Set(['active', 'hiatus', 'ended', 'upcoming']); +const RATING_SYSTEMS = new Set(['tv_parental', 'mpaa', 'podcast', 'esrb', 'bbfc', 'fsk', 'acb', 'chvrs', 'csa', 'pegi', 'custom']); + +// ── Declarations (adagents.json) ───────────────────────────────────── + +function creatorTalent(fixture: CreatorCardFixture | undefined, talent: NonNullable): Array> { + return talent.map(person => ({ + role: person.role, + name: person.name, + ...(fixture?.brandRef && person.role === 'creator' && { + brand_ref: { domain: CREATOR_TALENT_HOUSE.domain, brand_id: fixture.slug }, + }), + })); +} + +/** Identifier types the show fixtures predate, under their 3.x names. */ +const IDENTIFIER_RENAMES: Readonly> = { spotify_show_id: 'spotify_collection_id' }; +let identifierTypes: Set | undefined; + +function wireIdentifiers(identifiers: Array<{ type: string; value: string }>): Array<{ type: string; value: string }> { + identifierTypes ??= new Set((sourceSchemaDocument('enums/distribution-identifier-type.json') as { enum?: string[] }).enum ?? []); + return identifiers + .map(id => ({ type: IDENTIFIER_RENAMES[id.type] ?? id.type, value: id.value })) + .filter(id => identifierTypes!.has(id.type)); +} + +function declaredDistribution(show: ShowDefinition, fixture: CreatorCardFixture | undefined): Array> { + const entries = (show.distribution ?? []) + .map(entry => ({ publisher_domain: entry.publisherDomain, identifiers: wireIdentifiers(entry.identifiers) })) + .filter(entry => entry.identifiers.length > 0); + if (fixture?.crossPost) { + entries.push({ + publisher_domain: fixture.crossPost.publisherDomain, + identifiers: [{ type: fixture.crossPost.identifierType, value: fixture.crossPost.identifierValue }], + }); + } + return entries; +} + +/** The publisher's declaration of one show: the anchored facts, plus the + * declared `description` and `cadence` a card may refresh. */ +function declarationFor(pub: PublisherProfile, show: ShowDefinition): CollectionCard { + const fixture = CREATOR_CARD_FIXTURE_BY_SHOW.get(show.showId); + const rating = show.contentRatings?.find(entry => RATING_SYSTEMS.has(entry.system)); + const distribution = declaredDistribution(show, fixture); + return { + publisher_domain: pub.domain, + collection_id: show.showId, + name: show.name, + ...(show.kind && { kind: show.kind }), + ...(show.genre.length > 0 && { genre: [...show.genre] }), + ...(show.language && { language: show.language }), + ...(rating && { content_rating: { system: rating.system, rating: rating.rating } }), + ...(CADENCES.has(show.cadence) && { cadence: show.cadence }), + ...(STATUSES.has(show.status) && { status: show.status }), + ...(show.description && { description: show.description }), + ...(show.talent?.length && { talent: creatorTalent(fixture, show.talent) }), + ...(distribution.length > 0 && { distribution }), + }; +} + +function showsBySeller(): Array<{ pub: PublisherProfile; show: ShowDefinition }> { + return PUBLISHERS.flatMap(pub => (pub.shows ?? []).map(show => ({ pub, show }))); +} + +let declarationCache: CollectionCard[] | undefined; + +/** Every collection the agent's adagents.json declares, as the `collections` + * array of that file (publisher_domain is required in a cross-origin shared + * document). */ +export function declaredCollections(): CollectionCard[] { + declarationCache ??= showsBySeller().map(({ pub, show }) => declarationFor(pub, show)); + return structuredClone(declarationCache); +} + +/** The collection-scoped selectors that authorize the sales agent, in bulk + * form: every collection each publisher declares. */ +export function collectionAuthorizationSelectors(): Array<{ publisher_domain: string }> { + return [...new Set(showsBySeller().map(({ pub }) => pub.domain))].sort().map(publisher_domain => ({ publisher_domain })); +} + +// ── Base cards ─────────────────────────────────────────────────────── + +function creatorCardExtras(pub: PublisherProfile, show: ShowDefinition, fixture: CreatorCardFixture): Record { + const primary = show.distribution?.[0]; + const platformDomain = primary?.publisherDomain ?? pub.domain; + const shortForm = primary?.identifiers.some(id => id.type === 'tiktok_id') === true; + const samples = show.offer?.sampleContent ?? []; + return { + url: `https://${pub.domain}/creators/${fixture.slug.replace(/_/g, '-')}`, + images: [ + { + asset_type: 'image', + url: `https://picsum.photos/seed/creatorloop-${fixture.slug}-avatar/800/800`, + width: 800, + height: 800, + alt_text: `${show.name} channel avatar`, + }, + { + asset_type: 'image', + url: `https://picsum.photos/seed/creatorloop-${fixture.slug}-banner/1280/720`, + width: 1280, + height: 720, + alt_text: `${show.name} channel banner`, + }, + ], + sample_content: samples.slice(0, 2).map((sample, i) => ({ + asset_type: 'published_post', + post_url: samplePostUrl(platformDomain, fixture.slug, i, shortForm), + platform: platformDomain, + platform_post_id: `${fixture.slug}-${i + 1}`, + published_at: fixture.samplePublishedAt[i], + identity_ref: { handle: fixture.handle }, + })), + }; +} + +function samplePostUrl(platformDomain: string, slug: string, index: number, shortForm: boolean): string { + return `https://${platformDomain}/${shortForm ? 'v' : 'watch'}/${slug.replace(/_/g, '-')}-${index + 1}`; +} + +/** The `sample_content` post URLs of a creator offer, in order, so product + * cards can carry the same samples as the collection card. */ +export function creatorSamplePostUrls(showId: string): string[] { + for (const { pub, show } of showsBySeller()) { + if (show.showId !== showId) continue; + const fixture = CREATOR_CARD_FIXTURE_BY_SHOW.get(showId); + if (!fixture) return []; + const samples = (creatorCardExtras(pub, show, fixture).sample_content as Array<{ post_url: string }>); + return samples.map(sample => sample.post_url); + } + return []; +} + +function baseCardFor(pub: PublisherProfile, show: ShowDefinition): CollectionCard { + const card = declarationFor(pub, show); + const fixture = CREATOR_CARD_FIXTURE_BY_SHOW.get(show.showId); + if (fixture) { + Object.assign(card, creatorCardExtras(pub, show, fixture)); + const distribution = card.distribution as Array> | undefined; + for (const entry of distribution ?? []) { + const count = fixture.followers.find(f => f.publisherDomain === entry.publisher_domain); + if (count) { + entry.followers = { + lower_bound: count.lowerBound, + upper_bound: count.upperBound, + precision: 'approximate', + as_of: count.asOf, + }; + } + } + } else { + card.url = `https://${pub.domain}/shows/${show.showId.replace(/_/g, '-')}`; + if (pub.heroImageUrl) { + card.images = [{ + asset_type: 'image', + url: pub.heroImageUrl, + width: 600, + height: 300, + alt_text: `${show.name} key art`, + }]; + } + } + return card; +} + +let baseCache: Map | undefined; + +function baseCards(): Map { + baseCache ??= new Map(showsBySeller() + .map(({ pub, show }) => baseCardFor(pub, show)) + .sort(compareCards) + .map(card => [keyOf(card), card])); + return baseCache; +} + +function compareCards(a: CollectionKey, b: CollectionKey): number { + return a.publisher_domain.localeCompare(b.publisher_domain) || a.collection_id.localeCompare(b.collection_id); +} + +// ── Per-principal sandbox state ────────────────────────────────────── + +interface CardOverlay { + /** Closed-list field values that replace the base card's; `null` removes. */ + patches: Map>>; + /** key → distribution publisher_domain → follower count. */ + followers: Map>; + /** key → removal reason, for cards this seller no longer offers. */ + removed: Map; +} + +const MAX_OVERLAYS = 1000; +const overlays = new Map(); + +function principalKey(principal: string | undefined): string { + return principal && principal.length > 0 ? principal : 'anonymous'; +} + +function overlayFor(principal: string | undefined, create: true): CardOverlay; +function overlayFor(principal: string | undefined, create?: false): CardOverlay | undefined; +function overlayFor(principal: string | undefined, create = false): CardOverlay | undefined { + const key = principalKey(principal); + let overlay = overlays.get(key); + if (!overlay && create) { + if (overlays.size >= MAX_OVERLAYS) { + const oldest = overlays.keys().next().value; + if (oldest !== undefined) overlays.delete(oldest); + } + overlay = { patches: new Map(), followers: new Map(), removed: new Map() }; + overlays.set(key, overlay); + } + return overlay; +} + +/** Test and reset hook: forget every principal's card changes. */ +export function clearCollectionCardState(principal?: string): void { + if (principal === undefined) overlays.clear(); + else overlays.delete(principalKey(principal)); +} + +function applyOverlay(base: CollectionCard, overlay: CardOverlay | undefined): CollectionCard { + const card = structuredClone(base); + if (!overlay) return card; + const key = keyOf(base); + const patch = overlay.patches.get(key); + if (patch) { + for (const [field, value] of Object.entries(patch)) { + if (value === null) delete card[field]; + else card[field] = structuredClone(value); + } + } + const followers = overlay.followers.get(key); + if (followers && Array.isArray(card.distribution)) { + for (const entry of card.distribution as Array>) { + const count = followers.get(entry.publisher_domain as string); + if (count) entry.followers = structuredClone(count); + } + } + return card; +} + +/** Every card this seller offers the principal, in catalog order + * (publisher_domain, then collection_id). */ +export function cardsFor(principal: string | undefined): CollectionCard[] { + const overlay = overlayFor(principal); + const out: CollectionCard[] = []; + for (const [key, base] of baseCards()) { + if (overlay?.removed.has(key)) continue; + out.push(applyOverlay(base, overlay)); + } + return out; +} + +/** The card for one key, or undefined when the seller does not offer it. */ +export function cardFor(principal: string | undefined, key: CollectionKey): CollectionCard | undefined { + const base = baseCards().get(keyOf(key)); + if (!base) return undefined; + const overlay = overlayFor(principal); + if (overlay?.removed.has(keyOf(key))) return undefined; + return applyOverlay(base, overlay); +} + +/** Whether the catalog has ever had a card for this key (offered or removed). */ +export function catalogHasKey(key: CollectionKey): boolean { + return baseCards().has(keyOf(key)); +} + +export function catalogSize(principal: string | undefined): number { + return cardsFor(principal).length; +} + +// ── Versions ───────────────────────────────────────────────────────── + +export interface CatalogScope { + collections?: Array<{ publisher_domain: string; collection_ids?: string[] }>; + filters?: Record; +} + +/** Selectors and filters with set semantics; the cursor is never part of it. */ +function canonicalScope(scope: CatalogScope): string { + const selectors = scope.collections + ? [...new Set(scope.collections.map(s => canonicalJson({ + publisher_domain: s.publisher_domain, + ...(s.collection_ids ? { collection_ids: [...new Set(s.collection_ids)].sort() } : {}), + })))].sort() + : null; + const filters = scope.filters && Object.keys(scope.filters).length > 0 + ? Object.fromEntries(Object.entries(scope.filters).map(([name, value]) => [ + name, + Array.isArray(value) ? [...new Set(value.map(canonicalJson))].sort() : value, + ])) + : null; + return canonicalJson({ selectors, filters }); +} + +/** `collections_version` for a selection: changes when any selected copy + * changes or the selected set changes, and differs between selections. */ +export function collectionsVersion(scope: CatalogScope, selected: readonly CollectionCard[]): string { + return `cc_${digest(`${canonicalScope(scope)}\u0001${canonicalJson(selected)}`, 24)}`; +} + +/** The unfiltered catalog's version: the one webhooks carry and a mirror + * bootstraps from. */ +export function unfilteredCollectionsVersion(principal: string | undefined): string { + return collectionsVersion({}, cardsFor(principal)); +} + +/** Digest of every card the principal is offered, for the include_collections + * scope of product feed versions. */ +export function cardsDigest(principal: string | undefined): string { + return digest(canonicalJson(cardsFor(principal)), 16); +} + +// ── Mutations (seed_collection) ────────────────────────────────────── + +export interface CardMutationResult { + key: CollectionKey; + /** Closed-list fields whose value changed. */ + changedFields: string[]; + /** The card was removed and enters the catalog again. */ + reentered: boolean; +} + +/** Replace closed-list fields on one card. Returns the fields whose value + * changed, or an empty list when the card is unchanged. */ +export function patchCard( + principal: string | undefined, + key: CollectionKey, + patch: Partial>, +): CardMutationResult | undefined { + const base = baseCards().get(keyOf(key)); + if (!base) return undefined; + const overlay = overlayFor(principal, true); + const wasRemoved = overlay.removed.has(keyOf(key)); + const before = applyOverlay(base, overlay); + const current = { ...(overlay.patches.get(keyOf(key)) ?? {}) }; + for (const field of CLOSED_LIST_FIELDS) { + if (!(field in patch)) continue; + const value = patch[field]; + // A value equal to the publisher's (base) copy needs no override. + if (value !== null && canonicalJson(value) === canonicalJson(base[field])) delete current[field]; + else if (value === null && base[field] === undefined) delete current[field]; + else current[field] = structuredClone(value); + } + if (Object.keys(current).length > 0) overlay.patches.set(keyOf(key), current); + else overlay.patches.delete(keyOf(key)); + overlay.removed.delete(keyOf(key)); + const after = applyOverlay(base, overlay); + const changedFields = CLOSED_LIST_FIELDS.filter(field => canonicalJson(before[field]) !== canonicalJson(after[field])); + // A removed card that comes back re-enters the catalog: an upsert. + return { key, changedFields, reentered: wasRemoved }; +} + +/** Record one follower refresh. A refresh that leaves the bounds and + * precision unchanged is not a change: it keeps the earlier `as_of` and + * returns false (collections-and-installments.mdx, "Follower counts"). */ +export function refreshFollowers( + principal: string | undefined, + key: CollectionKey, + distributionPublisherDomain: string, + count: AudienceSize, + nowIso: string, +): boolean | undefined { + const base = baseCards().get(keyOf(key)); + if (!base) return undefined; + const overlay = overlayFor(principal, true); + const current = applyOverlay(base, overlay); + const entry = (current.distribution as Array> | undefined) + ?.find(candidate => candidate.publisher_domain === distributionPublisherDomain); + if (!entry) return undefined; + const previous = entry.followers as AudienceSize | undefined; + if ( + previous + && previous.lower_bound === count.lower_bound + && previous.upper_bound === count.upper_bound + && previous.precision === count.precision + ) return false; + let followers = overlay.followers.get(keyOf(key)); + if (!followers) { + followers = new Map(); + overlay.followers.set(keyOf(key), followers); + } + followers.set(distributionPublisherDomain, { + lower_bound: count.lower_bound, + upper_bound: count.upper_bound, + precision: count.precision, + as_of: count.as_of ?? nowIso, + }); + return true; +} + +/** Stop offering one card. Returns false when it was already removed. */ +export function removeCard(principal: string | undefined, key: CollectionKey, reason: string): boolean | undefined { + if (!baseCards().has(keyOf(key))) return undefined; + const overlay = overlayFor(principal, true); + if (overlay.removed.has(keyOf(key))) return false; + overlay.removed.set(keyOf(key), reason); + return true; +} diff --git a/server/src/training-agent/collections/card-fixtures.ts b/server/src/training-agent/collections/card-fixtures.ts new file mode 100644 index 00000000000..fdeda9d96ac --- /dev/null +++ b/server/src/training-agent/collections/card-fixtures.ts @@ -0,0 +1,171 @@ +/** + * Collection card fixtures (#8093, #8181): the display facts a selling agent + * serves on its copy of a collection (experimental `media_buy.collection_cards`) + * and the creator identities behind CreatorLoop's channels (experimental + * `brand.collection_links`). + * + * The collections themselves come from the publishers' shows (creators.ts and + * publishers.ts). This module adds only what a card needs on top of them: + * + * - `images`, `sample_content` and `url`: the closed-list display copies the + * seller owns and refreshes without touching the publisher's declaration; + * - `distribution[].followers`: platform-reported follower counts, with + * `as_of` the last time a count changed; + * - extra `distribution` entries for creators who cross-post (anchored, so + * they are declared in adagents.json too); + * - `talent[].brand_ref`: the typed link to each creator's brand identity, + * which the /brand tenant links back through `get_brand_identity` + * `collections`. + * + * Follower counts are rounded the way video platforms display them, so each + * one is an `approximate` interval: a displayed "1.23M" is 1,230,000 to + * 1,239,999. + */ + +/** Talent agency house that represents the CreatorLoop creators' identities. */ +export const CREATOR_TALENT_HOUSE = Object.freeze({ + domain: 'creatorlooptalent.example', + name: 'CreatorLoop Talent', +}); + +export interface FollowerFixture { + /** The distribution entry this count belongs to, by its publisher_domain. */ + publisherDomain: string; + lowerBound: number; + upperBound: number; + asOf: string; +} + +export interface CreatorCardFixture { + /** CreatorLoop show id (`creatorloop_channel_`). */ + showId: string; + slug: string; + /** Display handle on the creator's platforms. */ + handle: string; + followers: FollowerFixture[]; + /** A second platform the creator cross-posts to (anchored). */ + crossPost?: { + publisherDomain: string; + identifierType: 'youtube_channel_id' | 'tiktok_id'; + identifierValue: string; + }; + /** Publication dates of the two sample posts, oldest first. */ + samplePublishedAt: [string, string]; + /** Whether the collection names the creator's brand identity. */ + brandRef: boolean; + /** Whether the creator's brand identity links back to this collection. + * When false, the collection's brand_ref is the publisher's unconfirmed + * claim (collections-and-installments.mdx, "Collection cards"). */ + brandConfirms: boolean; + /** Short creator bio for the brand identity. */ + bio: string; +} + +const VIDEO = 'video.creatorloop.example'; +const SHORTS = 'shorts.creatorloop.example'; + +function approx(publisherDomain: string, displayed: number, unit: 'K' | 'M', asOf: string): FollowerFixture { + // A displayed "1.23M" covers [1,230,000, 1,239,999]; "865K" covers + // [865,000, 865,999]; "3.8M" covers [3,800,000, 3,899,999]. + const scale = unit === 'M' ? 1_000_000 : 1_000; + const decimals = (String(displayed).split('.')[1] ?? '').length; + const step = scale / 10 ** decimals; + const lowerBound = Math.round(displayed * scale); + return { publisherDomain, lowerBound, upperBound: lowerBound + step - 1, asOf }; +} + +export const CREATOR_CARD_FIXTURES: readonly CreatorCardFixture[] = Object.freeze([ + { + showId: 'creatorloop_channel_juniper_vale', slug: 'juniper_vale', handle: '@junipervalecooks', + followers: [approx(VIDEO, 1.23, 'M', '2026-10-12T00:00:00Z')], + samplePublishedAt: ['2026-09-18T15:00:00Z', '2026-10-09T15:00:00Z'], + brandRef: true, brandConfirms: true, + bio: 'Home cook and recipe developer behind Juniper Vale Cooks.', + }, + { + showId: 'creatorloop_channel_tobias_quill', slug: 'tobias_quill', handle: '@quilltechteardown', + followers: [approx(VIDEO, 2.41, 'M', '2026-10-05T00:00:00Z')], + samplePublishedAt: ['2026-09-21T17:00:00Z', '2026-10-14T17:00:00Z'], + brandRef: true, brandConfirms: true, + bio: 'Hardware reviewer who takes gadgets apart on camera.', + }, + { + showId: 'creatorloop_channel_nadia_ferro', slug: 'nadia_ferro', handle: '@nadiaferro', + followers: [ + approx(SHORTS, 3.8, 'M', '2026-10-15T00:00:00Z'), + approx(VIDEO, 412, 'K', '2026-10-01T00:00:00Z'), + ], + crossPost: { publisherDomain: VIDEO, identifierType: 'youtube_channel_id', identifierValue: 'UCcl0nadiaFerro2Lm7pQe4w' }, + samplePublishedAt: ['2026-10-02T12:00:00Z', '2026-10-16T12:00:00Z'], + brandRef: true, brandConfirms: true, + bio: 'Certified trainer making ten-minute workouts for desk workers.', + }, + { + showId: 'creatorloop_channel_kenji_arden', slug: 'kenji_arden', handle: '@ardentrailnotes', + followers: [approx(VIDEO, 865, 'K', '2026-09-28T00:00:00Z')], + samplePublishedAt: ['2026-08-30T14:00:00Z', '2026-09-27T14:00:00Z'], + brandRef: true, brandConfirms: true, + bio: 'Long-distance hiker and gear reviewer.', + }, + { + showId: 'creatorloop_channel_sol_marquez', slug: 'sol_marquez', handle: '@solmarquezplays', + followers: [approx(VIDEO, 1.57, 'M', '2026-10-11T00:00:00Z')], + samplePublishedAt: ['2026-09-25T20:00:00Z', '2026-10-10T20:00:00Z'], + brandRef: true, brandConfirms: true, + bio: 'Variety gamer known for cozy co-op streams.', + }, + { + showId: 'creatorloop_channel_priya_halden', slug: 'priya_halden', handle: '@haldenhomestudio', + followers: [approx(SHORTS, 2.2, 'M', '2026-10-08T00:00:00Z')], + samplePublishedAt: ['2026-09-20T16:00:00Z', '2026-10-04T16:00:00Z'], + brandRef: true, brandConfirms: true, + bio: 'Renter-friendly interiors on a budget.', + }, + { + // No brand_ref: the publisher has not linked this creator's identity. + showId: 'creatorloop_channel_dex_okafor', slug: 'dex_okafor', handle: '@dexmoneyminute', + followers: [approx(SHORTS, 1.9, 'M', '2026-10-03T00:00:00Z')], + samplePublishedAt: ['2026-09-29T13:00:00Z', '2026-10-13T13:00:00Z'], + brandRef: false, brandConfirms: false, + bio: 'Personal-finance explainer.', + }, + { + showId: 'creatorloop_channel_lena_brightwater', slug: 'lena_brightwater', handle: '@brightwaterbeautylab', + followers: [ + approx(VIDEO, 3.12, 'M', '2026-10-14T00:00:00Z'), + approx(SHORTS, 1.4, 'M', '2026-10-14T00:00:00Z'), + ], + crossPost: { publisherDomain: SHORTS, identifierType: 'tiktok_id', identifierValue: 'brightwaterbeautylab' }, + samplePublishedAt: ['2026-09-24T18:00:00Z', '2026-10-15T18:00:00Z'], + brandRef: true, brandConfirms: true, + bio: 'Cosmetic chemist reviewing skincare ingredients.', + }, + { + // The publisher claims this brand_ref, but the creator's identity does + // not list the channel back: an unconfirmed link. + showId: 'creatorloop_channel_ravi_castellan', slug: 'ravi_castellan', handle: '@castellangarage', + followers: [approx(VIDEO, 742, 'K', '2026-09-30T00:00:00Z')], + samplePublishedAt: ['2026-09-06T15:00:00Z', '2026-10-04T15:00:00Z'], + brandRef: true, brandConfirms: false, + bio: 'Mechanic and used-car buying advisor.', + }, + { + showId: 'creatorloop_channel_odette_moreau', slug: 'odette_moreau', handle: '@odettestudiodiaries', + followers: [approx(SHORTS, 4.4, 'M', '2026-10-16T00:00:00Z')], + samplePublishedAt: ['2026-10-01T11:00:00Z', '2026-10-15T11:00:00Z'], + brandRef: true, brandConfirms: true, + bio: 'Fashion designer documenting her studio and capsule wardrobes.', + }, +]); + +export const CREATOR_CARD_FIXTURE_BY_SHOW = new Map(CREATOR_CARD_FIXTURES.map(f => [f.showId, f])); + +/** Channels a creator's brand identity names that this seller does not sell + * (the unconfirmed creator above lists only this one). */ +export const OFF_PLATFORM_COLLECTION_LINKS: Readonly> = Object.freeze({ + ravi_castellan: { + publisher_domain: 'garagenights.example', + collection_id: 'garage_nights', + name: 'Garage Nights', + }, +}); diff --git a/server/src/training-agent/collections/card-webhooks.ts b/server/src/training-agent/collections/card-webhooks.ts new file mode 100644 index 00000000000..897dc6e9455 --- /dev/null +++ b/server/src/training-agent/collections/card-webhooks.ts @@ -0,0 +1,196 @@ +/** + * Collection card webhooks (experimental, `media_buy.collection_catalog`): + * `collection_card.updated`, `collection_card.removed` and + * `collection_catalog.bulk_change`, pushed to account-level subscribers + * registered through `sync_accounts` `notification_configs`. + * + * Envelope (list_collections.mdx, "Envelope"): the wholesale feed webhook, + * with `notification_id` equal to `event.event_id`, `wholesale_feed_version` + * the post-change `collections_version` of the unfiltered catalog and + * `previous_wholesale_feed_version` the one before it, no + * `product_payload_view`, and `cache_scope` equal to `applies_to.scope`. + * Event ids are UUID v7 and strictly increasing, so a mirror can detect an + * out-of-order delivery. Deliveries are RFC 9421-signed by the shared + * emitter (webhooks.ts). + * + * Visibility: every card is public within the principal's sandbox, so every + * subscriber on the principal's accounts may see every card event. + */ + +import { createHash, randomBytes } from 'node:crypto'; +import type { WebhookAuthentication } from '@adcp/sdk/server'; +import { getPrincipalNotificationSubscribers } from '../account-handlers.js'; +import { emitAccountNotificationWebhook } from '../webhooks.js'; +import { keyOf, type CollectionCard, type CollectionCardEventType, type CollectionKey } from './card-catalog.js'; + +export type CardEvent = + | { + type: 'collection_card.updated'; + card: CollectionCard; + changedFields?: string[]; + previousVersion: string; + version: string; + } + | { + type: 'collection_card.removed'; + key: CollectionKey; + removalReason?: string; + previousVersion: string; + version: string; + } + | { + type: 'collection_catalog.bulk_change'; + summary: string; + affectedCount: number; + previousVersion: string; + version: string; + }; + +let lastMs = 0; +let sequence = 0; + +/** A UUID v7 that sorts after every earlier one from this process, also + * within one millisecond (RFC 9562 method 1: a counter in rand_a). */ +export function monotonicUuidV7(now = Date.now()): string { + if (now > lastMs) { + lastMs = now; + sequence = 0; + } else { + sequence += 1; + if (sequence > 0xfff) { + lastMs += 1; + sequence = 0; + } + } + const bytes = randomBytes(16); + const ms = lastMs; + bytes[0] = Math.floor(ms / 2 ** 40) & 0xff; + bytes[1] = Math.floor(ms / 2 ** 32) & 0xff; + bytes[2] = Math.floor(ms / 2 ** 24) & 0xff; + bytes[3] = Math.floor(ms / 2 ** 16) & 0xff; + bytes[4] = Math.floor(ms / 2 ** 8) & 0xff; + bytes[5] = ms & 0xff; + bytes[6] = 0x70 | ((sequence >> 8) & 0x0f); + bytes[7] = sequence & 0xff; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + const hex = bytes.toString('hex'); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20, 32)}`; +} + +/** Opaque entity id; receivers key the mirror on the payload's key. */ +function cardEntityId(key: CollectionKey): string { + return `card_${createHash('sha256').update(keyOf(key)).digest('hex').slice(0, 16)}`; +} + +/** The event (without subscriber fields) for one card change. */ +export function cardEventEnvelope(event: CardEvent, firedAt: string): { + notification_id: string; + notification_type: CollectionCardEventType; + fired_at: string; + wholesale_feed_version: string; + previous_wholesale_feed_version: string; + cache_scope: 'public'; + event: Record; +} { + const eventId = monotonicUuidV7(); + const appliesTo = { scope: 'public' as const }; + const body = event.type === 'collection_card.updated' + ? { + entity_type: 'collection', + entity_id: cardEntityId(event.card), + payload: { + collection: event.card, + ...(event.changedFields && event.changedFields.length > 0 && { changed_fields: event.changedFields }), + applies_to: appliesTo, + }, + } + : event.type === 'collection_card.removed' + ? { + entity_type: 'collection', + entity_id: cardEntityId(event.key), + payload: { + publisher_domain: event.key.publisher_domain, + collection_id: event.key.collection_id, + ...(event.removalReason && { removal_reason: event.removalReason }), + applies_to: appliesTo, + }, + } + : { + entity_type: 'feed', + entity_id: 'collection_catalog', + payload: { + summary: event.summary, + affected_count: event.affectedCount, + recommendation: 'wholesale_resync', + applies_to: appliesTo, + }, + }; + return { + notification_id: eventId, + notification_type: event.type, + fired_at: firedAt, + wholesale_feed_version: event.version, + previous_wholesale_feed_version: event.previousVersion, + cache_scope: 'public', + event: { event_id: eventId, event_type: event.type, ...body, created_at: firedAt }, + }; +} + +function webhookAuthentication( + auth: { schemes: string[]; credentials?: string } | undefined, +): WebhookAuthentication | undefined { + if (!auth?.credentials) return undefined; + const schemes = auth.schemes.map(scheme => scheme.toLowerCase().replace(/-/g, '_')); + if (schemes.includes('bearer')) return { type: 'bearer', token: auth.credentials }; + if (schemes.includes('hmac_sha256')) return { type: 'hmac_sha256', secret: auth.credentials }; + return undefined; +} + +export interface CardDeliveryReport { + events: Array<{ notification_type: string; notification_id: string; wholesale_feed_version: string }>; + subscribers: number; + delivered: number; + failures: string[]; +} + +/** Push the events, in order, to every subscriber of each event's type on + * the principal's accounts. */ +export async function deliverCardEvents(principal: string | undefined, events: readonly CardEvent[]): Promise { + const report: CardDeliveryReport = { events: [], subscribers: 0, delivered: 0, failures: [] }; + const counted = new Set(); + for (const event of events) { + const envelope = cardEventEnvelope(event, new Date().toISOString()); + report.events.push({ + notification_type: envelope.notification_type, + notification_id: envelope.notification_id, + wholesale_feed_version: envelope.wholesale_feed_version, + }); + for (const subscriber of getPrincipalNotificationSubscribers(principal, event.type)) { + const subscriberKey = `${subscriber.accountId}\u001F${subscriber.subscriberId}`; + if (!counted.has(subscriberKey)) { + counted.add(subscriberKey); + report.subscribers += 1; + } + const idempotencyKey = `whk_${monotonicUuidV7()}`; + try { + const result = await emitAccountNotificationWebhook({ + url: subscriber.url, + payload: { + idempotency_key: idempotencyKey, + ...envelope, + subscriber_id: subscriber.subscriberId, + account_id: subscriber.accountId, + }, + operationId: `${subscriber.accountId}:${subscriber.subscriberId}:${envelope.notification_id}`, + notificationType: event.type, + authentication: webhookAuthentication(subscriber.authentication), + }); + if (result.delivered) report.delivered += 1; + else report.failures.push(...result.errors); + } catch (err) { + report.failures.push(err instanceof Error ? err.message : String(err)); + } + } + } + return report; +} diff --git a/server/src/training-agent/collections/include-collections.ts b/server/src/training-agent/collections/include-collections.ts new file mode 100644 index 00000000000..b0a6eba7ffb --- /dev/null +++ b/server/src/training-agent/collections/include-collections.ts @@ -0,0 +1,204 @@ +/** + * `include_collections` on `get_products`, `list_products`, + * `request_proposals` and `refine_proposals` (experimental, + * `media_buy.collection_cards`, #8094/#8141). See + * collections-and-installments.mdx, "Collections in responses". + * + * - Opt-in: `collections` is returned only when the request set + * `include_collections: true`, and only to a caller served 3.3 or later. + * Stable callers never see it, whatever they send. + * - Scope: one card per `(publisher_domain, collection_id)` referenced by a + * returned product's `collections` selectors, only alongside `products` + * (never on a rejected or `unchanged` answer). A `fields` projection that + * drops product `collections` drops the array too. + * - Feed versions: the flag is part of the `wholesale_feed_version` / + * `feed_version` scope. With it set, the token also covers the card copies, + * so a card change moves it; without it, cards are outside the version. + * The two settings never share a token, so a token presented under the + * other setting is a mismatch and gets the full payload. + * + * At the 3.3 wire, creator product cards also carry their samples with the + * `sample_content` reference asset role, as a `url` asset built from the + * collection card's `post_url`. + */ + +import { + cardFor, + cardsDigest, + creatorSamplePostUrls, + keyOf, + supportsCollections33, + type CollectionCard, +} from './card-catalog.js'; +import { buildCatalog } from '../product-factory.js'; + +export type CollectionReadTool = 'get_products' | 'list_products' | 'request_proposals' | 'refine_proposals'; + +interface CardsContext { + servedAdcpVersion?: string; + principal?: string; +} + +/** Marks a token issued with `include_collections: true`. Tokens issued + * without the flag never contain it. */ +const CARDS_SCOPE_MARKER = '~cards.'; + +const CONDITIONAL_FIELD: Partial> = { + get_products: 'if_wholesale_feed_version', + list_products: 'if_feed_version', +}; + +const VERSION_FIELD: Partial> = { + get_products: 'wholesale_feed_version', + list_products: 'feed_version', +}; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export function collectionsRequested(args: Record, ctx: CardsContext): boolean { + return supportsCollections33(ctx.servedAdcpVersion) && args.include_collections === true; +} + +function cardsScopeSuffix(ctx: CardsContext): string { + return `${CARDS_SCOPE_MARKER}${cardsDigest(ctx.principal)}`; +} + +/** + * Map the presented conditional token onto the product feed's own token. + * With the flag, a token is current only when it was issued with the flag + * over the current cards; anything else is forced to mismatch. Without the + * flag, a token issued with it never matches, because product feed tokens + * never carry the marker. + */ +export function prepareCollectionRead( + tool: CollectionReadTool, + args: Record, + ctx: CardsContext, +): Record { + const field = CONDITIONAL_FIELD[tool]; + if (!field || typeof args[field] !== 'string' || !collectionsRequested(args, ctx)) return args; + const presented = args[field] as string; + const suffix = cardsScopeSuffix(ctx); + return { + ...args, + [field]: presented.endsWith(suffix) + ? presented.slice(0, -suffix.length) + : `${presented}${CARDS_SCOPE_MARKER}mismatch`, + }; +} + +function referencedKeys(products: unknown[]): Array<{ publisher_domain: string; collection_id: string }> { + const seen = new Set(); + const keys: Array<{ publisher_domain: string; collection_id: string }> = []; + for (const product of products) { + if (!isRecord(product) || !Array.isArray(product.collections)) continue; + for (const selector of product.collections) { + if (!isRecord(selector) || typeof selector.publisher_domain !== 'string') continue; + if (!Array.isArray(selector.collection_ids)) continue; + for (const id of selector.collection_ids) { + if (typeof id !== 'string') continue; + const key = { publisher_domain: selector.publisher_domain, collection_id: id }; + if (seen.has(keyOf(key))) continue; + seen.add(keyOf(key)); + keys.push(key); + } + } + } + return keys; +} + +/** The cards for the collections the given products reference, deduplicated + * by key, in first-reference order. */ +export function cardsForProducts(products: unknown[], principal: string | undefined): CollectionCard[] { + return referencedKeys(products) + .map(key => cardFor(principal, key)) + .filter((card): card is CollectionCard => card !== undefined); +} + +/** Replaces, never mutates, product cards: products can share nested objects + * with the cached catalog that stable callers read too. */ +function projectSampleContentRole(products: unknown[]): void { + for (const [index, product] of products.entries()) { + if (!isRecord(product)) continue; + const detailed = product.product_card_detailed; + if (!isRecord(detailed) || !Array.isArray(detailed.reference_assets)) continue; + const selector = Array.isArray(product.collections) ? product.collections.find(isRecord) : undefined; + const showId = Array.isArray(selector?.collection_ids) ? selector.collection_ids[0] : undefined; + const postUrls = typeof showId === 'string' ? creatorSamplePostUrls(showId) : []; + if (postUrls.length === 0) continue; + let sample = 0; + const referenceAssets = detailed.reference_assets.map(asset => { + if (!isRecord(asset) || asset.role !== 'other' || asset.role_label !== 'Sample content') return asset; + const postUrl = postUrls[sample]; + sample += 1; + if (!postUrl) return asset; + const { role_label: _label, asset: _video, ...rest } = asset; + return { ...rest, role: 'sample_content', asset: { asset_type: 'url', url: postUrl } }; + }); + products[index] = { ...product, product_card_detailed: { ...detailed, reference_assets: referenceAssets } }; + } +} + +/** + * Finish a product or proposal read for a 3.3 caller: the `sample_content` + * role on creator product cards, the include_collections token scope, and + * the top-level `collections` array. Mutates and returns `result`. A stable + * caller's result is returned untouched. + */ +export function finishCollectionRead( + tool: CollectionReadTool, + args: Record, + result: T, + ctx: CardsContext, +): T { + if (!isRecord(result) || !supportsCollections33(ctx.servedAdcpVersion)) return result; + const record: Record = result; + const products = Array.isArray(record.products) ? record.products : undefined; + if (products) projectSampleContentRole(products); + if (!collectionsRequested(args, ctx)) return result; + const versionField = VERSION_FIELD[tool]; + if (versionField && typeof record[versionField] === 'string') { + record[versionField] = `${record[versionField] as string}${cardsScopeSuffix(ctx)}`; + } + // Only alongside products: never on rejected or unchanged answers. + if (!products || record.unchanged === true || record.outcome === 'unchanged' || record.outcome === 'rejected') { + return result; + } + if (tool === 'request_proposals' || tool === 'refine_proposals') withCatalogSelectors(products); + const cards = cardsForProducts(products, ctx.principal); + if (cards.length > 0) record.collections = cards; + else delete record.collections; + return result; +} + +let catalogSelectors: Map | undefined; + +/** + * Proposal reads return minimal canonical products (`product_id`, `name`). + * Cards are scoped by the returned products' `collections` selectors, so with + * `include_collections` the products carry their catalog selectors and the + * cards they reference can be joined to them. + */ +function withCatalogSelectors(products: unknown[]): void { + catalogSelectors ??= new Map(buildCatalog() + .filter(entry => Array.isArray(entry.product.collections) && entry.product.collections.length > 0) + .map(entry => [entry.product.product_id, entry.product.collections as unknown[]])); + for (const [index, product] of products.entries()) { + if (!isRecord(product) || product.collections !== undefined || typeof product.product_id !== 'string') continue; + const selectors = catalogSelectors.get(product.product_id); + if (selectors) products[index] = { ...product, collections: structuredClone(selectors) }; + } +} + +/** Run one product or proposal read with include_collections applied. */ +export async function runCollectionRead( + tool: CollectionReadTool, + args: Record, + ctx: CardsContext, + run: (args: Record) => Promise, +): Promise { + const result = await run(prepareCollectionRead(tool, args, ctx)); + return finishCollectionRead(tool, args, result, ctx); +} diff --git a/server/src/training-agent/collections/list-collections.ts b/server/src/training-agent/collections/list-collections.ts new file mode 100644 index 00000000000..cf4eb9cb07d --- /dev/null +++ b/server/src/training-agent/collections/list-collections.ts @@ -0,0 +1,177 @@ +/** + * `list_collections` (experimental, `media_buy.collection_catalog`, #8183): + * the sales tenant's catalog of collection cards, by key or with filters, + * with its own `collections_version` conditional fetch. It returns the same + * copies `include_collections` adds to product and proposal reads, and never + * products, pricing or availability. Side-effect free. + * + * See docs/media-buy/task-reference/list_collections.mdx. + */ + +import { TOOL_INPUT_SHAPES } from '@adcp/sdk/schemas'; +import { decodeOffsetCursor, encodeOffsetCursor } from '../pagination.js'; +import type { ToolArgs, TrainingContext } from '../types.js'; +import { customToolFor } from '../tenants/custom-tool-helper.js'; +import { + cardsFor, + collectionsVersion, + supportsCollections33, + type CatalogScope, + type CollectionCard, +} from './card-catalog.js'; + +interface Selector { + publisher_domain: string; + collection_ids?: string[]; +} + +interface Filters { + kinds?: string[]; + genres_include?: string[]; + genres_exclude?: string[]; + genre_taxonomy?: string; + statuses?: string[]; + languages?: string[]; + talent?: Array<{ domain: string; brand_id?: string }>; + platforms?: string[]; + name_contains?: string; +} + +interface ListCollectionsArgs { + collections?: Selector[]; + filters?: Filters; + pagination?: { max_results?: number; cursor?: string }; + account?: unknown; + if_collections_version?: string; +} + +const MAX_COLLECTION_IDS = 100; +const DEFAULT_PAGE = 50; +const MAX_PAGE = 100; + +function failure(code: string, message: string, field?: string): { errors: Array> } { + return { errors: [{ code, message, ...(field && { field }), recovery: 'correctable' }] }; +} + +function selectorMatches(card: CollectionCard, selectors: Selector[] | undefined): boolean { + if (!selectors) return true; + return selectors.some(selector => ( + selector.publisher_domain === card.publisher_domain + && (!selector.collection_ids || selector.collection_ids.includes(card.collection_id)) + )); +} + +/** RFC 4647 basic filtering, case-insensitive: `en` matches `en-US`. */ +export function languageRangeMatches(range: string, tag: unknown): boolean { + if (typeof tag !== 'string' || tag.length === 0) return false; + const [r, t] = [range.toLowerCase(), tag.toLowerCase()]; + return r === '*' || t === r || t.startsWith(`${r}-`); +} + +/** Values within one filter are alternatives (OR); filters combine with AND. */ +export function filterMatches(card: CollectionCard, filters: Filters | undefined): boolean { + if (!filters) return true; + const kind = typeof card.kind === 'string' ? card.kind : 'series'; + if (filters.kinds && !filters.kinds.includes(kind)) return false; + if (filters.statuses && !filters.statuses.includes(card.status as string)) return false; + if (filters.languages && !filters.languages.some(range => languageRangeMatches(range, card.language))) return false; + const genres = Array.isArray(card.genre) ? card.genre as string[] : []; + // Genre values are compared within one taxonomy: a filter that names a + // taxonomy matches only cards declared in it. + const taxonomyOk = !filters.genre_taxonomy || filters.genre_taxonomy === card.genre_taxonomy; + if (filters.genres_include && !(taxonomyOk && filters.genres_include.some(g => genres.includes(g)))) return false; + if (filters.genres_exclude && taxonomyOk && filters.genres_exclude.some(g => genres.includes(g))) return false; + if (filters.talent) { + const talent = Array.isArray(card.talent) ? card.talent as Array<{ brand_ref?: { domain?: string; brand_id?: string } }> : []; + const matched = filters.talent.some(wanted => talent.some(person => ( + person.brand_ref?.domain === wanted.domain + && (!wanted.brand_id || person.brand_ref.brand_id === wanted.brand_id) + ))); + if (!matched) return false; + } + if (filters.platforms) { + const distribution = Array.isArray(card.distribution) ? card.distribution as Array<{ publisher_domain?: string }> : []; + if (!filters.platforms.some(domain => distribution.some(entry => entry.publisher_domain === domain))) return false; + } + if (filters.name_contains && !String(card.name).toLowerCase().includes(filters.name_contains.toLowerCase())) return false; + return true; +} + +/** The cards a request selects, before pagination. */ +export function selectCards(principal: string | undefined, scope: CatalogScope): CollectionCard[] { + return cardsFor(principal).filter(card => ( + selectorMatches(card, scope.collections as Selector[] | undefined) + && filterMatches(card, scope.filters as Filters | undefined) + )); +} + +export async function handleListCollections(args: ToolArgs, ctx: TrainingContext): Promise { + if (!supportsCollections33(ctx.servedAdcpVersion)) { + return failure( + 'UNSUPPORTED_FEATURE', + 'list_collections is an experimental AdCP 3.3 task (media_buy.collection_catalog).', + 'adcp_version', + ); + } + const request = args as unknown as ListCollectionsArgs; + const idCount = (request.collections ?? []).reduce((sum, selector) => sum + (selector.collection_ids?.length ?? 0), 0); + if (idCount > MAX_COLLECTION_IDS) { + return failure( + 'VALIDATION_ERROR', + `At most ${MAX_COLLECTION_IDS} collection IDs in total across all selectors; this request names ${idCount}.`, + 'collections', + ); + } + const offset = decodeOffsetCursor('list_collections', request.pagination?.cursor); + if (offset === null) return failure('INVALID_REQUEST', 'pagination.cursor is malformed.', 'pagination.cursor'); + + const scope: CatalogScope = { + ...(request.collections && { collections: request.collections }), + ...(request.filters && Object.keys(request.filters).length > 0 && { filters: request.filters as Record }), + }; + const selected = selectCards(ctx.principal, scope); + const version = collectionsVersion(scope, selected); + // No card is account-scoped, so every answer is the public layer, with or + // without an account. + const cacheScope = 'public'; + if (request.if_collections_version !== undefined && request.if_collections_version === version) { + return { status: 'completed', outcome: 'unchanged', collections_version: version, cache_scope: cacheScope }; + } + const maxResults = Math.min(Math.max(request.pagination?.max_results ?? DEFAULT_PAGE, 1), MAX_PAGE); + const page = selected.slice(offset, offset + maxResults); + const next = offset + page.length; + const hasMore = next < selected.length; + return { + status: 'completed', + outcome: 'listed', + collections: page, + pagination: { + has_more: hasMore, + ...(hasMore && { cursor: encodeOffsetCursor('list_collections', next) }), + total_count: selected.length, + }, + collections_version: version, + cache_scope: cacheScope, + }; +} + +/** The sales tenant's `list_collections` custom tool. Registered only where + * the 3.3 wire is served (the hosted preview's registry, candidate mode and + * the identity route); the handler refuses a caller below 3.3 itself. */ +export function listCollectionsTool() { + return customToolFor( + 'list_collections', + 'List this seller\'s collection cards (shows, creators, podcasts) by key or with filters, without products. Experimental (media_buy.collection_catalog).', + TOOL_INPUT_SHAPES.list_collections!, + handleListCollections, + { + annotations: { readOnlyHint: true, idempotentHint: true }, + gatesOwnRelease: true, + responseSummary: result => ( + result.outcome === 'unchanged' + ? 'Collection catalog unchanged.' + : `Listed ${Array.isArray(result.collections) ? result.collections.length : 0} collection cards.` + ), + }, + ); +} diff --git a/server/src/training-agent/collections/seed-collection.ts b/server/src/training-agent/collections/seed-collection.ts new file mode 100644 index 00000000000..9b57be4c7df --- /dev/null +++ b/server/src/training-agent/collections/seed-collection.ts @@ -0,0 +1,256 @@ +/** + * `seed_collection` comply_test_controller scenario (training-agent local, + * 3.3 wire only): change the caller's collection cards so card webhooks and + * conditional reads can be exercised. + * + * Only the closed list a selling agent owns can change (`images`, + * `sample_content`, `url`, `description`, `season`, `cadence` and + * `distribution[].followers`); anchored facts stay as adagents.json declares + * them. Each change fires the event the spec calls for: + * + * - `action: "update"`: replace closed-list fields on one or more cards. + * - `action: "remove"` / `"restore"`: the seller stops / resumes offering a + * card (`collection_card.removed` / `collection_card.updated`). + * - `action: "refresh_followers"`: a follower sweep. Counts whose bounds and + * precision are unchanged are not changes and keep their `as_of`; a sweep + * that changes any count goes out as one `collection_catalog.bulk_change`, + * and a sweep that changes none fires nothing and moves no version. + * + * An operation that touches more than 5% of the catalog or more than 100 + * cards, whichever is smaller (and always more than one card), goes out as + * one `collection_catalog.bulk_change` instead of per-card events. + * + * params: + * { action: "update", collections: [{ publisher_domain, collection_id, fields: { ... } }] } + * { action: "remove", collections: [{ publisher_domain, collection_id, removal_reason? }] } + * { action: "restore", collections: [{ publisher_domain, collection_id }] } + * { action: "refresh_followers", followers: [{ publisher_domain, collection_id, + * distribution_publisher_domain, lower_bound, upper_bound, precision, as_of? }] } + */ + +import type { TrainingContext } from '../types.js'; +import { + CLOSED_LIST_FIELDS, + cardFor, + catalogHasKey, + catalogSize, + keyOf, + patchCard, + refreshFollowers, + removeCard, + supportsCollections33, + unfilteredCollectionsVersion, + type AudienceSize, + type ClosedListField, + type CollectionKey, +} from './card-catalog.js'; +import { deliverCardEvents, type CardEvent } from './card-webhooks.js'; + +export const SEED_COLLECTION_SCENARIO = 'seed_collection'; + +const CADENCES = new Set(['daily', 'weekly', 'monthly', 'seasonal', 'event', 'irregular']); +const REMOVAL_REASONS = new Set(['withdrawn', 'cancellation', 'expired', 'depublication', 'policy_takedown']); + +type Failure = { success: false; error: string; error_detail: string }; + +function invalid(detail: string): Failure { + return { success: false, error: 'INVALID_PARAMS', error_detail: detail }; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** Bulk when one operation touches more than this many cards. */ +export function bulkThreshold(catalogCards: number): number { + return Math.max(1, Math.min(Math.floor(catalogCards * 0.05), 100)); +} + +function parseKey(entry: unknown, path: string): CollectionKey | Failure { + if (!isRecord(entry) || typeof entry.publisher_domain !== 'string' || typeof entry.collection_id !== 'string') { + return invalid(`${path} requires publisher_domain and collection_id`); + } + const key = { publisher_domain: entry.publisher_domain, collection_id: entry.collection_id }; + if (!catalogHasKey(key)) return invalid(`${path}: this seller has no collection ${entry.publisher_domain}/${entry.collection_id}`); + return key; +} + +function isHttpsUrl(value: unknown): boolean { + if (typeof value !== 'string') return false; + try { + return new URL(value).protocol === 'https:'; + } catch { + return false; + } +} + +function validateFields(fields: unknown, path: string): Partial> | Failure { + if (!isRecord(fields) || Object.keys(fields).length === 0) return invalid(`${path} must name at least one field`); + const out: Partial> = {}; + for (const [name, value] of Object.entries(fields)) { + if (!(CLOSED_LIST_FIELDS as readonly string[]).includes(name)) { + return invalid(`${path}.${name}: only ${CLOSED_LIST_FIELDS.join(', ')} can change on a seller's copy; other collection facts are anchored to adagents.json (followers change through refresh_followers)`); + } + if (value !== null) { + if ((name === 'images' || name === 'sample_content') + && (!Array.isArray(value) || value.length < 1 || value.length > 10 || !value.every(isRecord))) { + return invalid(`${path}.${name} must be an array of 1 to 10 assets`); + } + if (name === 'images' && !(value as Array>).every(asset => ( + asset.asset_type === 'image' && isHttpsUrl(asset.url) + && Number.isInteger(asset.width) && Number.isInteger(asset.height) + ))) return invalid(`${path}.images entries need asset_type "image", an https url, width and height`); + if (name === 'sample_content' && !(value as Array>).every(asset => ( + asset.asset_type === 'published_post' && (isHttpsUrl(asset.post_url) || typeof asset.platform_post_id === 'string') + ))) return invalid(`${path}.sample_content entries need asset_type "published_post" and a post_url or platform_post_id`); + if (name === 'url' && !isHttpsUrl(value)) return invalid(`${path}.url must be an https URL`); + if ((name === 'description' || name === 'season') && (typeof value !== 'string' || value.length === 0)) { + return invalid(`${path}.${name} must be a non-empty string`); + } + if (name === 'cadence' && !CADENCES.has(value as string)) return invalid(`${path}.cadence is not a collection cadence`); + } + out[name as ClosedListField] = value; + } + return out; +} + +function parseCount(entry: Record, path: string): AudienceSize | Failure { + const { lower_bound: lower, upper_bound: upper, precision, as_of: asOf } = entry; + if (!Number.isInteger(lower) || !Number.isInteger(upper) || (lower as number) < 0 || (upper as number) < (lower as number)) { + return invalid(`${path} needs integer lower_bound <= upper_bound`); + } + if (precision !== 'exact' && precision !== 'approximate') return invalid(`${path}.precision must be exact or approximate`); + if (precision === 'exact' && lower !== upper) return invalid(`${path}: an exact count has lower_bound equal to upper_bound`); + if (asOf !== undefined && (typeof asOf !== 'string' || Number.isNaN(Date.parse(asOf)))) { + return invalid(`${path}.as_of must be a date-time`); + } + return { + lower_bound: lower as number, + upper_bound: upper as number, + precision, + ...(typeof asOf === 'string' && { as_of: asOf }), + }; +} + +function failed(value: unknown): value is Failure { + return isRecord(value) && value.success === false; +} + +export async function handleSeedCollection(rawArgs: Record, ctx: TrainingContext): Promise { + if (!supportsCollections33(ctx.servedAdcpVersion)) { + return { + success: false, + error: 'UNKNOWN_SCENARIO', + error_detail: 'seed_collection drives experimental 3.3 collection cards; pin the 3.3 preview to use it.', + }; + } + const params = isRecord(rawArgs.params) ? rawArgs.params : {}; + const principal = ctx.principal; + const action = params.action; + const startVersion = unfilteredCollectionsVersion(principal); + const startSize = catalogSize(principal); + const events: CardEvent[] = []; + const changedKeys: CollectionKey[] = []; + let previous = startVersion; + const record = (event: Omit & Record, key: CollectionKey) => { + const version = unfilteredCollectionsVersion(principal); + events.push({ ...event, previousVersion: previous, version } as CardEvent); + changedKeys.push(key); + previous = version; + }; + + if (action === 'update' || action === 'remove' || action === 'restore') { + const entries = params.collections; + if (!Array.isArray(entries) || entries.length === 0) return invalid(`seed_collection ${action} requires params.collections`); + const parsed: Array<{ key: CollectionKey; fields?: Partial>; reason?: string }> = []; + for (const [index, entry] of entries.entries()) { + const path = `params.collections[${index}]`; + const key = parseKey(entry, path); + if (failed(key)) return key; + if (action === 'update') { + const fields = validateFields((entry as Record).fields, `${path}.fields`); + if (failed(fields)) return fields; + parsed.push({ key, fields }); + } else if (action === 'remove') { + const reason = (entry as Record).removal_reason; + if (reason !== undefined && !REMOVAL_REASONS.has(reason as string)) return invalid(`${path}.removal_reason is not a removal reason`); + parsed.push({ key, reason: (reason as string | undefined) ?? 'withdrawn' }); + } else { + parsed.push({ key }); + } + } + for (const { key, fields, reason } of parsed) { + if (action === 'remove') { + if (removeCard(principal, key, reason!)) { + record({ type: 'collection_card.removed', key, removalReason: reason }, key); + } + continue; + } + const wasOffered = cardFor(principal, key) !== undefined; + if (action === 'restore' && wasOffered) continue; + const result = patchCard(principal, key, fields ?? {}); + if (!result || (!result.reentered && result.changedFields.length === 0)) continue; + record({ + type: 'collection_card.updated', + card: cardFor(principal, key)!, + ...(!result.reentered && { changedFields: result.changedFields }), + }, key); + } + } else if (action === 'refresh_followers') { + const entries = params.followers; + if (!Array.isArray(entries) || entries.length === 0) return invalid('seed_collection refresh_followers requires params.followers'); + const nowIso = new Date().toISOString(); + const parsed: Array<{ key: CollectionKey; platform: string; count: AudienceSize }> = []; + for (const [index, entry] of entries.entries()) { + const path = `params.followers[${index}]`; + const key = parseKey(entry, path); + if (failed(key)) return key; + const platform = (entry as Record).distribution_publisher_domain; + if (typeof platform !== 'string') return invalid(`${path}.distribution_publisher_domain is required`); + const count = parseCount(entry as Record, path); + if (failed(count)) return count; + parsed.push({ key, platform, count }); + } + const touched = new Map(); + for (const { key, platform, count } of parsed) { + const changed = refreshFollowers(principal, key, platform, count, nowIso); + if (changed === undefined) return invalid(`${keyOf(key).replace('\u0000', '/')} has no distribution entry on ${platform}`); + if (changed) touched.set(keyOf(key), key); + } + if (touched.size > 0) { + // A follower sweep always goes out as one bulk event. + events.push({ + type: 'collection_catalog.bulk_change', + summary: `Follower counts refreshed on ${touched.size} collection card${touched.size === 1 ? '' : 's'}`, + affectedCount: touched.size, + previousVersion: startVersion, + version: unfilteredCollectionsVersion(principal), + }); + changedKeys.push(...touched.values()); + } + } else { + return invalid('seed_collection requires params.action: update, remove, restore or refresh_followers'); + } + + // One operation that touches more cards than the threshold is one bulk event. + let outgoing = events; + if (events.length > bulkThreshold(startSize) + && events.every(event => event.type !== 'collection_catalog.bulk_change')) { + outgoing = [{ + type: 'collection_catalog.bulk_change', + summary: `${events.length} collection cards changed in one operation`, + affectedCount: events.length, + previousVersion: startVersion, + version: unfilteredCollectionsVersion(principal), + }]; + } + const report = await deliverCardEvents(principal, outgoing); + return { + success: report.failures.length === 0, + action, + changed: changedKeys, + collections_version: unfilteredCollectionsVersion(principal), + ...report, + ...(report.failures.length > 0 && { error: 'DELIVERY_FAILED', error_detail: report.failures.slice(0, 3).join('; ') }), + }; +} diff --git a/server/src/training-agent/comply-test-controller.ts b/server/src/training-agent/comply-test-controller.ts index c41ebfd53e4..52c85d2090f 100644 --- a/server/src/training-agent/comply-test-controller.ts +++ b/server/src/training-agent/comply-test-controller.ts @@ -80,6 +80,7 @@ import { mintWrongAudDemoToken, } from './governance-verify.js'; import { emitAccountNotificationWebhook } from './webhooks.js'; +import { handleSeedCollection, SEED_COLLECTION_SCENARIO } from './collections/seed-collection.js'; import { getSharedAccountCreative, removeSharedAccountCreative, @@ -2342,6 +2343,12 @@ export async function handleComplyTestController(args: ToolArgs, ctx: TrainingCo ctx.moduleId, targetsControllerFixtureState ? controllerFixturePrincipal(ctx.principal) : undefined, ); + if (scenario === SEED_COLLECTION_SCENARIO) { + if (ctx.storyboardCompat?.version === '3.0') { + return { success: false, error: 'UNKNOWN_SCENARIO', error_detail: 'seed_collection is an experimental AdCP 3.3 scenario' }; + } + return handleSeedCollection(rawArgs, ctx); + } if (scenario === REPORTING_SUBSCRIPTION_PROBE) { if (ctx.storyboardCompat?.version === '3.0' || !supportsAccountSurfaces33(ctx.servedAdcpVersion)) { return { diff --git a/server/src/training-agent/index.ts b/server/src/training-agent/index.ts index 58aac862e36..14b2edfa808 100644 --- a/server/src/training-agent/index.ts +++ b/server/src/training-agent/index.ts @@ -63,6 +63,7 @@ import { isWorkOSApiKeyFormat } from '../middleware/api-key-format.js'; import { buildGovernanceAgentCredentialAuthenticator } from './governance-agent-credentials.js'; import { createTrainingBuyerOAuthMiddleware, trainingBuyerOAuthConfig, trainingBuyerMetadata, trainingBuyerDocumentation, TRAINING_BUYER_METADATA_PATH } from './buyer-oauth.js'; import { requestPinsPreview } from './candidate-version.js'; +import { collectionAuthorizationSelectors, declaredCollections } from './collections/card-catalog.js'; import { ensurePreviewSchemaBundle } from './schema-compat.js'; const logger = createLogger('training-agent-routes'); @@ -815,6 +816,25 @@ export function createTrainingAgentRouter(options: { })), ), }, + // The collections the sales agent sells, scoped by collection + // selectors: every collection each publisher declares below. Response + // copies of these collections (include_collections, list_collections) + // are verified against this file (#8181). + { + url: `${agentUrl}/sales/mcp`, + authorized_for: 'AdCP training — sales (collections: shows, podcasts, creator channels)', + authorization_type: 'inline_properties', + properties: PUBLISHERS + .filter(pub => (pub.shows?.length ?? 0) > 0) + .flatMap(pub => pub.properties.map(prop => ({ + identifier_type: prop.identifierType, + identifier_value: prop.identifierValue, + name: prop.name, + supported_channels: prop.channels, + tags: prop.tags, + }))), + collections: collectionAuthorizationSelectors(), + }, { url: `${agentUrl}/signals/mcp`, authorized_for: 'AdCP training — signals (marketplace + owned)', @@ -822,6 +842,11 @@ export function createTrainingAgentRouter(options: { signal_tags: SIGNAL_TAG_VALUES, }, ], + // The publishers' collection declarations. This shared file speaks for + // several publishers, so each carries its publisher_domain. Anchored + // facts live here; a selling agent's card copy wins only on its closed + // list of display fields. + collections: declaredCollections(), signals: SIGNAL_PROVIDERS.flatMap(provider => provider.signals.map(signal => ({ id: signal.signalAgentSegmentId, diff --git a/server/src/training-agent/tenants/router.ts b/server/src/training-agent/tenants/router.ts index bf8eddd0cb4..506acca668a 100644 --- a/server/src/training-agent/tenants/router.ts +++ b/server/src/training-agent/tenants/router.ts @@ -74,6 +74,9 @@ import { } from '../reporting-subscriptions.js'; import { ACCOUNT_REQUESTS_CAPABILITY, ACCOUNT_REQUESTS_FEATURE } from '../account-requests.js'; import { grantFor, grantPermits, loadSandbox } from '../account-sandbox.js'; +import { projectCollectionCapabilities } from '../collections/capabilities.js'; +import { supportsCollections33 } from '../collections/card-catalog.js'; +import { SEED_COLLECTION_SCENARIO } from '../collections/seed-collection.js'; const logger = createLogger('training-agent-tenant-router'); const PRODUCT_WHOLESALE_EVENTS = ['product.created', 'product.updated', 'product.priced', 'product.removed'] as const; @@ -280,9 +283,13 @@ function salesComplyScenarios( ? [...current, 'expire_account_change_cursor'] : current; // Experimental 3.3: grants, account timezone, and the sandbox clock. - return supportsAccountSurfaces33(servedVersion) + const withAccounts33 = supportsAccountSurfaces33(servedVersion) ? [...withChangeFeed, 'reporting_subscription_probe'] : withChangeFeed; + // Experimental 3.3 collection cards: change cards to drive card webhooks. + return supportsCollections33(servedVersion) + ? [...withAccounts33, SEED_COLLECTION_SCENARIO] + : withAccounts33; } function salesCapabilityScenarios( @@ -794,6 +801,7 @@ async function tryHandleLocalComplyScenario( && rawArgs.scenario !== 'query_account_governance_binding' && rawArgs.scenario !== 'force_media_buy_confirmation' && rawArgs.scenario !== 'reporting_subscription_probe' + && rawArgs.scenario !== SEED_COLLECTION_SCENARIO && rawArgs.scenario !== 'list_scenarios' && !isCompactLifecycleProbe && !isRejectedGetProductsDirective @@ -815,6 +823,7 @@ async function tryHandleLocalComplyScenario( || rawArgs.scenario === 'query_account_governance_binding' || rawArgs.scenario === 'force_media_buy_confirmation' || rawArgs.scenario === 'reporting_subscription_probe' + || rawArgs.scenario === SEED_COLLECTION_SCENARIO || isRejectedGetProductsDirective ) ) return false; @@ -1033,6 +1042,19 @@ function projectTenantToolDiscovery( }; } + // Experimental 3.3 include_collections (#8181): only a listing served + // at 3.3 or later advertises the flag. The SDK stamps the listing's + // release on the result; under the `all` tool profile it does not shape + // input schemas to it. + const listingMeta = (parsed.result as { _meta?: { adcp_version?: unknown } } | undefined)?._meta; + for (const tool of tools) { + const listedAt = listingMeta?.adcp_version ?? (tool as { _meta?: { adcp_version?: unknown } })._meta?.adcp_version; + const properties = tool.inputSchema?.properties; + if (!properties || !('include_collections' in properties)) continue; + if (supportsCollections33(typeof listedAt === 'string' ? listedAt : undefined)) continue; + const { include_collections: _flag, ...rest } = properties; + tool.inputSchema = { ...tool.inputSchema, properties: rest }; + } } if (storyboardCompat?.version === '3.0') { parsed.result!.tools = tools.filter(tool => tool.name !== 'validate_input'); @@ -1474,6 +1496,8 @@ function projectTenantCapabilities( } } projectWholesaleCapabilities(structured, tenantId, storyboardCompat); + // Experimental 3.3 collection cards, catalog and brand links (#8181). + if (storyboardCompat?.version !== '3.0') projectCollectionCapabilities(structured, tenantId, servedVersion); // Every tenant, last: withhold fields added after the served release // (3.3 account.stable_account_id, governance.supports_budget_periods, // the property-breakdown feature flags, ...), whichever projection above diff --git a/server/src/training-agent/tenants/sales.ts b/server/src/training-agent/tenants/sales.ts index e68b4bbd5c7..2e56d91d1a4 100644 --- a/server/src/training-agent/tenants/sales.ts +++ b/server/src/training-agent/tenants/sales.ts @@ -20,6 +20,7 @@ import { import { getTenantSigningMaterial } from './signing.js'; import { buildSalesComplyConfig } from './comply.js'; import { listAccountChangesTool, listAccountsTool, requestAccountTool, syncGovernanceTool } from './account-tools.js'; +import { listCollectionsTool } from '../collections/list-collections.js'; import { reportUsageTool } from './report-usage-tool.js'; import { validateInputTool } from './validate-input-tool.js'; import { buildCreativeTool, previewCreativeTool } from './creative-tools.js'; @@ -287,6 +288,11 @@ export function buildSalesTenantConfig( ...(options.storyboardCompat?.version !== '3.0' && (options.identity33 || options.preview || TRAINING_AGENT_REGISTERS_ACCOUNT_SURFACES_33) && { request_account: requestAccountTool() }), + // Experimental 3.3 collection catalog (media_buy.collection_catalog), + // registered where the 3.3 wire is served, like request_account. + ...(options.storyboardCompat?.version !== '3.0' + && (options.identity33 || options.preview || TRAINING_AGENT_REGISTERS_ACCOUNT_SURFACES_33) + && { list_collections: listCollectionsTool() }), report_usage: reportUsageTool({ creativeBillsThroughAdcp: false }), ...(options.storyboardCompat?.version !== '3.0' && { get_reporting_status: customToolFor( diff --git a/server/src/training-agent/tenants/served-release-shape.test.ts b/server/src/training-agent/tenants/served-release-shape.test.ts index d41bb479323..2fa3a22b683 100644 --- a/server/src/training-agent/tenants/served-release-shape.test.ts +++ b/server/src/training-agent/tenants/served-release-shape.test.ts @@ -140,7 +140,10 @@ function argsFor(tool: string, label: string, version: string, state: FlowState) const image = { asset_type: 'image', url: 'https://cdn.example.com/served-release.png', width: 300, height: 250 }; switch (tool) { case 'get_products': - return { buying_mode: 'brief', brief: 'Display and video for sports fans in the US' }; + // Every release is asked for collection cards (experimental 3.3, + // #8181): get_products accepts unknown fields, so a stable caller's + // flag is ignored and only the preview answers with `collections`. + return { buying_mode: 'brief', brief: 'Display and video for sports fans in the US', include_collections: true }; case 'create_media_buy': { const product = state.products?.find(candidate => { const option = (candidate.pricing_options as Array> | undefined)?.[0]; @@ -371,6 +374,17 @@ describe('served-release response shape below 3.3 (#8487)', () => { `${entry.instancePath} ${entry.message} ${JSON.stringify(entry.params)}` )))}`); } + if (tool === 'get_products') { + const isPreview = version === TRAINING_AGENT_HOSTED_PREVIEW.wireVersion; + const referencesCollections = (structured.products as Array>) + .some(product => Array.isArray(product.collections) && product.collections.length > 0); + if (!isPreview && structured.collections !== undefined) { + failures.push(`${label} get_products @${version}: collection cards served to a stable caller`); + } + if (isPreview && referencesCollections && !Array.isArray(structured.collections)) { + failures.push(`${label} get_products @${version}: include_collections answered without collections`); + } + } // A 3.0 product card is a format reference plus its manifest; the // inline card replaced it in 3.1. Withholding the card would also // validate, so require the release's own shape on a carded product. @@ -393,6 +407,54 @@ describe('served-release response shape below 3.3 (#8487)', () => { }, 180_000); } + it('shows stable and major-only callers none of the experimental 3.3 collection surfaces (#8181)', async () => { + const preview = TRAINING_AGENT_HOSTED_PREVIEW.wireVersion; + const sales = `${baseUrl}/sales/mcp`; + const brand = `${baseUrl}/brand/mcp`; + const cardFeatures = ['media_buy.collection_cards', 'media_buy.collection_catalog', 'brand.collection_links']; + const cardEvents = ['collection_card.updated', 'collection_card.removed', 'collection_catalog.bulk_change']; + const brief = { buying_mode: 'brief', brief: 'Creator sponsored video integrations for a lifestyle audience', include_collections: true }; + const failures: string[] = []; + // Preview reads first: they must not leave anything behind that a later + // stable read of the shared catalog could see. + const previewProducts = await call(sales, 'get_products', { adcp_version: preview, ...brief }); + expect(Array.isArray(previewProducts.structured?.collections), JSON.stringify(previewProducts.raw).slice(0, 300)).toBe(true); + expect(JSON.stringify(previewProducts.structured)).toContain('"sample_content"'); + await call(`${baseUrl}/mcp`, 'get_products', { adcp_version: preview, ...brief }); + for (const pin of [{ adcp_version: '3.2' }, { adcp_version: '3.1' }, { adcp_version: '3.0' }, { adcp_major_version: 3 }, {}]) { + const label = JSON.stringify(pin); + for (const url of [sales, brand]) { + const { structured } = await call(url, 'get_adcp_capabilities', pin); + const features = (structured?.experimental_features as string[] | undefined) ?? []; + const events = ((structured?.wholesale_feed_webhooks as { event_types?: string[] } | undefined)?.event_types) ?? []; + if (features.some(feature => cardFeatures.includes(feature))) failures.push(`${url} ${label}: declares ${features.join(', ')}`); + if (events.some(event => cardEvents.includes(event))) failures.push(`${url} ${label}: lists card events`); + } + const products = await call(sales, 'get_products', { ...pin, ...brief }); + if (!Array.isArray(products.structured?.products)) failures.push(`get_products ${label}: ${JSON.stringify(products.raw).slice(0, 200)}`); + if (products.structured?.collections !== undefined) failures.push(`get_products ${label}: served collection cards`); + if (JSON.stringify(products.structured ?? {}).includes('"sample_content"')) failures.push(`get_products ${label}: served the sample_content role`); + const legacy = await call(`${baseUrl}/mcp`, 'get_products', { ...pin, ...brief }); + if (legacy.structured?.collections !== undefined) failures.push(`/mcp get_products ${label}: served collection cards`); + const identity = await call(brand, 'get_brand_identity', { ...pin, brand_id: 'juniper_vale' }); + if (identity.structured?.collections !== undefined) failures.push(`get_brand_identity ${label}: served collection links`); + const listing = await rpc(sales, rpcId += 1, 'tools/list', pin.adcp_version ? { _meta: { adcp_version: pin.adcp_version } } : {}); + const listed = (listing.result as { tools?: Array<{ name: string; inputSchema?: { properties?: Record } }> } | undefined)?.tools ?? []; + if (listed.length === 0) failures.push(`tools/list ${label}: ${JSON.stringify(listing).slice(0, 200)}`); + if (listed.some(tool => tool.name === 'list_collections')) failures.push(`tools/list ${label}: lists list_collections`); + const flagged = listed.filter(tool => tool.inputSchema?.properties && 'include_collections' in tool.inputSchema.properties); + if (flagged.length > 0) failures.push(`tools/list ${label}: advertises include_collections on ${flagged.map(tool => tool.name).join(', ')}`); + } + expect(failures).toEqual([]); + // The same calls at the preview pin reach every surface. + const previewIdentity = await call(brand, 'get_brand_identity', { adcp_version: preview, brand_id: 'juniper_vale' }); + expect(Array.isArray(previewIdentity.structured?.collections)).toBe(true); + const previewListing = await rpc(sales, rpcId += 1, 'tools/list', { _meta: { adcp_version: preview } }); + const previewTools = (previewListing.result as { tools: Array<{ name: string; inputSchema?: { properties?: Record } }> }).tools; + expect(previewTools.map(tool => tool.name)).toContain('list_collections'); + expect(previewTools.find(tool => tool.name === 'get_products')?.inputSchema?.properties).toHaveProperty('include_collections'); + }, 120_000); + it('answers /mcp search_brands without a query with the correctable error the request schema requires', async () => { const url = `${baseUrl}/mcp`; for (const version of ['3.2', '3.1']) { diff --git a/server/src/training-agent/tenants/tool-catalog.ts b/server/src/training-agent/tenants/tool-catalog.ts index 0c80ccaddcb..e3caa36666d 100644 --- a/server/src/training-agent/tenants/tool-catalog.ts +++ b/server/src/training-agent/tenants/tool-catalog.ts @@ -47,6 +47,7 @@ export const TOOL_CATALOG: Readonly> = { request_proposals: ['sales'], refine_proposals: ['sales'], decline_proposals: ['sales'], + list_collections: ['sales'], buy_products: ['sales'], accept_proposal: ['sales'], control_media_buy: ['sales'], @@ -149,6 +150,7 @@ export function toolsForTenant( .filter(tool => { const negotiatedVersion = options.storyboardCompat?.version ?? options.adcpVersion; if (tool === 'request_account' && !TRAINING_AGENT_REGISTERS_ACCOUNT_SURFACES_33) return false; + if (tool === 'list_collections' && !TRAINING_AGENT_REGISTERS_ACCOUNT_SURFACES_33) return false; if ( tool === 'list_account_changes' && !supportsAccountChangeFeed(negotiatedVersion ?? '3.2-beta.5') @@ -175,6 +177,7 @@ export function toolsForTenant( || tool === 'control_media_buy' || tool === 'list_account_changes' || tool === 'request_account' + || tool === 'list_collections' ) return false; if (tool === 'validate_input' || tool === 'list_transformers') return false; if ( diff --git a/server/src/training-agent/v6-sales-platform.ts b/server/src/training-agent/v6-sales-platform.ts index 7ee31468b13..f07c429c779 100644 --- a/server/src/training-agent/v6-sales-platform.ts +++ b/server/src/training-agent/v6-sales-platform.ts @@ -102,6 +102,7 @@ import { taskRegistryScopeFromContext, } from './task-registry-scope.js'; import { scopedPrincipal } from './idempotency.js'; +import { finishCollectionRead, prepareCollectionRead, runCollectionRead } from './collections/include-collections.js'; import { SellerManagedControlJobCoordinator, type SellerManagedControlJobContext, @@ -1165,7 +1166,11 @@ export function legacyGetProductsHandler( const servedAdcpVersion = ctx.servedAdcpVersion ?? (versionResolution.ok ? versionResolution.servedVersion : undefined); if (servedAdcpVersion) trainingCtx.servedAdcpVersion = servedAdcpVersion; - const executed = await executeTrainingAgentTool('get_products', normalizedReq, trainingCtx); + const executed = await executeTrainingAgentTool( + 'get_products', + prepareCollectionRead('get_products', normalizedRecord, trainingCtx) as typeof normalizedReq, + trainingCtx, + ); if (!executed.success) throwGetProductsExecutionError(executed.error ?? 'get_products failed'); const rawResponse = executed.data as Record | undefined; if ( @@ -1214,10 +1219,15 @@ export function legacyGetProductsHandler( executed.data, { allowAdvisories: true }, ); - return projectGetProductsCompatibilityWire( - response, - req as unknown as Record, - servedAdcpVersion, + return finishCollectionRead( + 'get_products', + normalizedRecord, + projectGetProductsCompatibilityWire( + response, + req as unknown as Record, + servedAdcpVersion, + ), + trainingCtx, ) as Awaited>>; }; } @@ -1810,31 +1820,42 @@ export class TrainingSalesPlatform return { proposalRefinement: proposalCapabilitiesForProfile(this.proposalNegotiationProfile), - listProducts: async (req, ctx) => translateV5Result( - await executeProductDiscoveryPlatformTool( - 'list_products', - withCurrentAccountScope(req as unknown as Record, ctx.account, ctx.input) as unknown as Record, - buildTrainingCtx(ctx, this.storyboardCompat, this.proposalNegotiationProfile), - ), - { allowAdvisories: true }, - ), + // include_collections (experimental 3.3, collections/include-collections.ts) + // wraps the three split reads that can carry collection cards. + listProducts: async (req, ctx) => { + const trainingCtx = buildTrainingCtx(ctx, this.storyboardCompat, this.proposalNegotiationProfile); + return translateV5Result( + await runCollectionRead( + 'list_products', + withCurrentAccountScope(req as unknown as Record, ctx.account, ctx.input) as unknown as Record, + trainingCtx, + args => executeProductDiscoveryPlatformTool('list_products', args, trainingCtx), + ), + { allowAdvisories: true }, + ); + }, - requestProposals: async (req, ctx) => translateV5Result( - await executeProductDiscoveryPlatformTool( - 'request_proposals', - withCurrentAccountScope(req as unknown as Record, ctx.account, ctx.input) as unknown as Record, - buildTrainingCtx(ctx, this.storyboardCompat, this.proposalNegotiationProfile), - ), - { allowAdvisories: true }, - ), + requestProposals: async (req, ctx) => { + const trainingCtx = buildTrainingCtx(ctx, this.storyboardCompat, this.proposalNegotiationProfile); + return translateV5Result( + await runCollectionRead( + 'request_proposals', + withCurrentAccountScope(req as unknown as Record, ctx.account, ctx.input) as unknown as Record, + trainingCtx, + args => executeProductDiscoveryPlatformTool('request_proposals', args, trainingCtx), + ), + { allowAdvisories: true }, + ); + }, refineProposals: async (req, ctx) => { const trainingCtx = buildTrainingCtx(ctx, this.storyboardCompat, this.proposalNegotiationProfile); return translateV5Result( - await executeProductDiscoveryPlatformTool( + await runCollectionRead( 'refine_proposals', withCurrentAccountScope(req as unknown as Record, ctx.account, ctx.input) as unknown as Record, trainingCtx, + args => executeProductDiscoveryPlatformTool('refine_proposals', args, trainingCtx), ), { allowAdvisories: true }, ); diff --git a/server/tests/unit/training-agent-collections.test.ts b/server/tests/unit/training-agent-collections.test.ts new file mode 100644 index 00000000000..2c3222092bb --- /dev/null +++ b/server/tests/unit/training-agent-collections.test.ts @@ -0,0 +1,611 @@ +/** + * #8181 / #8093: the experimental 3.3 collection surfaces on the training + * agent, exercised over MCP at the hosted 3.3 preview pin and graded with the + * reference grader for collection copies + * (tests/helpers/collection-copies-grader.cjs, the same module that replays + * static/compliance/source/test-vectors/collection-copies/). + * + * - media_buy.collection_cards: include_collections on get_products, + * list_products, request_proposals and refine_proposals; feed versions + * scoped by the flag; card fields and follower counts. + * - media_buy.collection_catalog: list_collections, the card webhooks + * (delivered to a local receiver), the seed_collection controller scenario + * that drives them, and the consistency rule. + * - brand.collection_links: get_brand_identity collections on /brand. + * + * Stable callers never see any of it; tenants/served-release-shape.test.ts + * proves that for the full read flow. + */ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import express from 'express'; +import http from 'node:http'; +import { createRequire } from 'node:module'; + +process.env.PUBLIC_TEST_AGENT_TOKEN = 'collections-test-public-token'; + +import { TRAINING_AGENT_HOSTED_PREVIEW } from '../../src/training-agent/candidate-version.js'; +import { stopSessionCleanup } from '../../src/training-agent/state.js'; +import { clearCollectionCardState } from '../../src/training-agent/collections/card-catalog.js'; +import { bulkThreshold } from '../../src/training-agent/collections/seed-collection.js'; +import { monotonicUuidV7 } from '../../src/training-agent/collections/card-webhooks.js'; + +const grader = createRequire(import.meta.url)('../../../tests/helpers/collection-copies-grader.cjs') as { + CATALOG_FEATURE: string; + CARDS_FEATURE: string; + validate: Record boolean) & { errors?: unknown }>; + canonical: (value: unknown) => string; + mergedView: (vector: Record) => Array>; + gradeResponse: (vector: Record) => string[]; + gradeWebhook: (vector: Record) => string[]; + applyEvents: (vector: Record) => { verdict: string; repair_token: string | null; mirror: { cards: unknown[]; version: string } }; +}; + +const PREVIEW = TRAINING_AGENT_HOSTED_PREVIEW.wireVersion; +/** A caller-unique demo credential: card changes are kept per principal. */ +const TOKEN = 'demo-collection-cards-v1'; +const STABLE_PINS: Array> = [ + {}, { adcp_major_version: 3 }, { adcp_version: '3.2' }, { adcp_version: '3.1' }, { adcp_version: '3.0' }, { adcp_version: '3.3' }, +]; +const CARD_EVENTS = ['collection_card.updated', 'collection_card.removed', 'collection_catalog.bulk_change']; +const CREATORS = 'creatorloop.example'; + +type Structured = Record; +type Card = Record & { publisher_domain: string; collection_id: string }; + +const keyOf = (card: { publisher_domain: string; collection_id: string }) => `${card.publisher_domain}/${card.collection_id}`; + +function referencedKeys(products: Array>): Set { + const keys = new Set(); + for (const product of products) { + for (const selector of product.collections ?? []) { + for (const id of selector.collection_ids ?? []) keys.add(`${selector.publisher_domain}/${id}`); + } + } + return keys; +} + +describe('experimental 3.3 collections on the training agent', () => { + let server: http.Server; + let receiver: http.Server; + let baseUrl: string; + let hookUrl: string; + let rpcId = 0; + const received: Array<{ headers: http.IncomingHttpHeaders; body: Record }> = []; + + const rpc = async (path: string, method: string, params?: Record, token = TOKEN) => { + rpcId += 1; + const response = await fetch(`${baseUrl}${path}`, { + method: 'POST', + headers: { 'content-type': 'application/json', accept: 'application/json, text/event-stream', authorization: `Bearer ${token}` }, + body: JSON.stringify({ jsonrpc: '2.0', id: rpcId, method, ...(params && { params }) }), + }); + return response.json() as Promise>; + }; + const call = async (path: string, name: string, args: Record, token = TOKEN): Promise => { + const envelope = await rpc(path, 'tools/call', { name, arguments: args }, token); + return (envelope.result?.structuredContent ?? { __envelope: envelope }) as Structured; + }; + const preview = (name: string, args: Record = {}, path = '/sales/mcp') => call(path, name, { adcp_version: PREVIEW, ...args }); + const listAll = async (): Promise<{ cards: Card[]; version: string; cacheScope: string }> => { + const cards: Card[] = []; + let cursor: string | undefined; + let version = ''; + let cacheScope = ''; + do { + const page = await preview('list_collections', { pagination: { max_results: 7, ...(cursor && { cursor }) } }); + expect(page.outcome, JSON.stringify(page).slice(0, 300)).toBe('listed'); + if (version) expect(page.collections_version).toBe(version); + version = page.collections_version; + cacheScope = page.cache_scope; + cards.push(...page.collections); + cursor = page.pagination?.has_more ? page.pagination.cursor : undefined; + } while (cursor); + return { cards, version, cacheScope }; + }; + const seed = (params: Record) => preview('comply_test_controller', { + scenario: 'seed_collection', + account: { brand: { domain: 'collections-buyer.example' }, operator: 'collections-buyer.example', sandbox: true }, + params, + }); + /** The grader's seller declaration for this agent. */ + const SELLER = { experimental_features: [] as string[], wholesale_feed_webhook_event_types: [] as string[] }; + + beforeAll(async () => { + clearCollectionCardState(); + receiver = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on('data', chunk => chunks.push(chunk as Buffer)); + req.on('end', () => { + const body = JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}') as Record; + if (body.type === 'webhook.challenge') { + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ challenge: body.challenge })); + return; + } + received.push({ headers: req.headers, body }); + res.writeHead(204); + res.end(); + }); + }); + await new Promise(resolve => receiver.listen(0, '127.0.0.1', resolve)); + const hookAddress = receiver.address(); + if (!hookAddress || typeof hookAddress === 'string') throw new Error('Missing receiver address'); + hookUrl = `http://127.0.0.1:${hookAddress.port}/hooks/collection-cards`; + + const { createTrainingAgentRouter } = await import('../../src/training-agent/index.js'); + const app = express(); + app.use(express.json({ limit: '5mb', verify: (req, _res, body) => { + (req as unknown as { rawBody: string }).rawBody = body.toString('utf8'); + } })); + app.use('/api/training-agent', createTrainingAgentRouter({ disableRateLimit: true })); + server = http.createServer(app); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('Missing test server address'); + baseUrl = `http://127.0.0.1:${address.port}/api/training-agent`; + }, 60_000); + + afterAll(async () => { + clearCollectionCardState(); + stopSessionCleanup(); + for (const s of [server, receiver]) { + if (!s) continue; + s.closeAllConnections(); + await new Promise(resolve => s.close(() => resolve())); + } + }); + + it('declares the collection features and card events at the preview pin only', async () => { + const sales = await preview('get_adcp_capabilities'); + expect(sales.adcp_version).toBe(PREVIEW); + expect(sales.experimental_features).toEqual(expect.arrayContaining([grader.CARDS_FEATURE, grader.CATALOG_FEATURE])); + expect(sales.wholesale_feed_webhooks.event_types).toEqual(expect.arrayContaining(CARD_EVENTS)); + expect(sales.media_buy.lifecycle_tools ?? []).not.toContain('list_collections'); + SELLER.experimental_features = sales.experimental_features; + SELLER.wholesale_feed_webhook_event_types = sales.wholesale_feed_webhooks.event_types; + const brand = await preview('get_adcp_capabilities', {}, '/brand/mcp'); + expect(brand.experimental_features).toContain('brand.collection_links'); + + for (const pin of STABLE_PINS) { + const stable = await call('/sales/mcp', 'get_adcp_capabilities', pin); + expect(stable.adcp_version, JSON.stringify(pin)).not.toBe(PREVIEW); + expect(stable.experimental_features ?? []).not.toContain(grader.CARDS_FEATURE); + expect(stable.experimental_features ?? []).not.toContain(grader.CATALOG_FEATURE); + for (const type of CARD_EVENTS) expect(stable.wholesale_feed_webhooks?.event_types ?? []).not.toContain(type); + const stableBrand = await call('/brand/mcp', 'get_adcp_capabilities', pin); + expect(stableBrand.experimental_features ?? []).not.toContain('brand.collection_links'); + } + }, 60_000); + + it('lists list_collections and seed_collection only at the preview pin', async () => { + const pinned = await rpc('/sales/mcp', 'tools/list', { _meta: { adcp_version: PREVIEW } }); + expect(pinned.result.tools.map((tool: { name: string }) => tool.name)).toContain('list_collections'); + for (const params of [{}, { _meta: { adcp_version: '3.2' } }, { _meta: { adcp_version: '3.3' } }]) { + const stable = await rpc('/sales/mcp', 'tools/list', params); + expect(stable.result.tools.map((tool: { name: string }) => tool.name)).not.toContain('list_collections'); + } + for (const adcp_version of ['3.2', '3.3']) { + const refused = await rpc('/sales/mcp', 'tools/call', { name: 'list_collections', arguments: { adcp_version } }); + expect(refused.result?.isError, JSON.stringify(refused).slice(0, 300)).toBe(true); + } + const scenarios = await preview('comply_test_controller', { scenario: 'list_scenarios' }); + expect(scenarios.scenarios).toContain('seed_collection'); + const stableScenarios = await call('/sales/mcp', 'comply_test_controller', { adcp_version: '3.2', scenario: 'list_scenarios' }); + expect(stableScenarios.scenarios).not.toContain('seed_collection'); + }, 60_000); + + it('serves creator collection cards with images, samples, links, talent brand_refs and follower counts', async () => { + const { cards, version, cacheScope } = await listAll(); + expect(cacheScope).toBe('public'); + expect(version).toMatch(/^cc_/); + const creators = cards.filter(card => card.publisher_domain === CREATORS); + expect(creators).toHaveLength(10); + for (const card of cards) expect(grader.validate.copy(card), `${keyOf(card)} ${JSON.stringify(grader.validate.copy.errors)}`).toBe(true); + for (const card of creators) { + expect(card.kind).toBe('channel'); + expect(card.url).toMatch(/^https:\/\/creatorloop\.example\/creators\//); + expect(card.images.length).toBeGreaterThan(0); + expect(card.sample_content.every((post: Record) => post.asset_type === 'published_post')).toBe(true); + expect(card.talent[0].role).toBe('creator'); + for (const entry of card.distribution) { + expect(entry.followers, keyOf(card)).toMatchObject({ precision: 'approximate' }); + expect(entry.followers.upper_bound).toBeGreaterThanOrEqual(entry.followers.lower_bound); + expect(typeof entry.followers.as_of).toBe('string'); + } + } + // Ten creators; nine name their brand identity, one does not. + expect(creators.filter(card => card.talent[0].brand_ref).length).toBe(9); + // Two creators cross-post, so a card can carry more than one count. + expect(creators.filter(card => card.distribution.length > 1).length).toBe(2); + }, 60_000); + + it('answers include_collections on all four product and proposal reads with the referenced cards only', async () => { + const catalog = new Map((await listAll()).cards.map(card => [keyOf(card), card])); + const checkCards = (label: string, result: Structured) => { + expect(result.adcp_error, `${label}: ${JSON.stringify(result).slice(0, 400)}`).toBeUndefined(); + expect(Array.isArray(result.products), label).toBe(true); + const referenced = referencedKeys(result.products); + expect(referenced.size, label).toBeGreaterThan(0); + const keys = (result.collections as Card[]).map(keyOf); + // Deduplicated, referenced only, and complete. + expect(new Set(keys).size, label).toBe(keys.length); + for (const key of keys) expect(referenced.has(key), `${label}: stray ${key}`).toBe(true); + expect(new Set(keys), label).toEqual(referenced); + // The same copy list_collections serves for the key. + for (const card of result.collections as Card[]) { + expect(grader.canonical(card), `${label}: ${keyOf(card)}`).toBe(grader.canonical(catalog.get(keyOf(card)))); + } + return result.collections as Card[]; + }; + + const brief = 'Creator sponsored video integrations for an everyday lifestyle audience'; + const gp = await preview('get_products', { buying_mode: 'brief', brief, include_collections: true }); + checkCards('get_products', gp); + const creatorProduct = (gp.products as Array>).find(product => (product.collections ?? []) + .some((selector: { publisher_domain: string }) => selector.publisher_domain === CREATORS)); + expect(creatorProduct).toBeDefined(); + // Creator product cards carry their samples with the sample_content role. + const samples = creatorProduct!.product_card_detailed.reference_assets + .filter((asset: { role: string }) => asset.role === 'sample_content'); + expect(samples.length).toBeGreaterThan(0); + expect(samples[0].asset).toMatchObject({ asset_type: 'url' }); + const card = (gp.collections as Card[]).find(entry => entry.collection_id === creatorProduct!.collections[0].collection_ids[0]); + expect(card!.sample_content.map((post: { post_url: string }) => post.post_url)).toContain(samples[0].asset.url); + + const lp = await preview('list_products', { include_collections: true }); + checkCards('list_products', lp); + + const rp = await preview('request_proposals', { + brand: { domain: 'collections-buyer.example' }, + brief, + include_collections: true, + idempotency_key: 'collections-request-proposals-0001', + }); + expect(rp.outcome, JSON.stringify(rp).slice(0, 400)).toBe('proposed'); + checkCards('request_proposals', rp); + + const proposalId = rp.proposals[0].proposal_id; + const refined = await preview('refine_proposals', { + refinements: [{ proposal_id: proposalId, action: 'revise', ask: 'Drop the most expensive creator.' }], + include_collections: true, + idempotency_key: 'collections-refine-proposals-0001', + }); + checkCards('refine_proposals', refined); + + // Without the flag, no cards. + for (const [name, args] of [ + ['get_products', { buying_mode: 'brief', brief }], + ['list_products', {}], + ['get_products', { buying_mode: 'brief', brief, include_collections: false }], + ] as const) { + const plain = await preview(name, args); + expect(plain.collections, name).toBeUndefined(); + } + + // A fields projection without product collections omits the array too. + const projected = await preview('list_products', { include_collections: true, fields: ['name'] }); + expect(projected.outcome).toBe('listed'); + expect(projected.collections).toBeUndefined(); + }, 120_000); + + it('grades include_collections copies against the agent adagents.json: every referenced card is verified', async () => { + const adagents = await (await fetch(`${baseUrl}/.well-known/adagents.json`)).json() as Record; + const declarations: Record = {}; + for (const declared of adagents.collections as Card[]) { + const { publisher_domain: domain, ...rest } = declared; + expect(grader.validate.collection(declared), JSON.stringify(grader.validate.collection.errors)).toBe(true); + (declarations[domain] ??= []).push(rest); + } + const sellerAuthorized = (adagents.authorized_agents as Array>) + .filter(entry => String(entry.url).endsWith('/sales/mcp') && Array.isArray(entry.collections)) + .flatMap(entry => entry.collections); + expect(sellerAuthorized.length).toBeGreaterThan(0); + + const gp = await preview('get_products', { + buying_mode: 'brief', + brief: 'Creator sponsored video integrations for an everyday lifestyle audience', + include_collections: true, + }); + const keys = [...referencedKeys(gp.products)].map(key => { + const [publisher_domain, collection_id] = key.split('/'); + return { publisher_domain, collection_id }; + }); + const merged = grader.mergedView({ keys, declarations, seller_authorized: sellerAuthorized, copies: gp.collections }); + for (const entry of merged) { + expect(entry.status, `${keyOf(entry as Card)}`).toBe('verified'); + expect(entry.carriage_eligible).toBe(true); + // The copy wins on its closed list: the card's follower counts and + // images reach the merged view; anchored facts are the declaration's. + const copy = (gp.collections as Card[]).find(candidate => keyOf(candidate) === keyOf(entry as Card))!; + const effective = entry.effective as Card; + expect(effective.images).toEqual(copy.images); + expect(effective.talent).toEqual(copy.talent); + expect(effective.distribution).toEqual(copy.distribution); + } + }, 60_000); + + it('serves list_collections by key and with filters, graded conformant by the reference grader', async () => { + const { cards } = await listAll(); + const catalog = cards.map(card => ({ card, visible_to: 'public' })); + const grade = async (request: Record, token?: Record) => { + const response = await preview('list_collections', request); + const { adcp_version: _served, ...wire } = response; + const violations = grader.gradeResponse({ + seller: SELLER, + seller_catalog: catalog, + request: { adcp_version: PREVIEW, ...request }, + response: wire, + ...(token && { token_issued_for: token }), + }); + expect(violations, `${JSON.stringify(request)} → ${JSON.stringify(response).slice(0, 400)}`).toEqual([]); + return response; + }; + + const byKey = await grade({ + collections: [ + { publisher_domain: CREATORS, collection_ids: ['creatorloop_channel_juniper_vale', 'creatorloop_channel_unknown'] }, + { publisher_domain: 'unknown-publisher.example', collection_ids: ['nothing'] }, + ], + }); + // Unknown keys are omitted silently, with no per-key error. + expect(byKey.collections.map(keyOf)).toEqual([`${CREATORS}/creatorloop_channel_juniper_vale`]); + + const whole = await grade({ collections: [{ publisher_domain: CREATORS }] }); + expect(whole.collections).toHaveLength(10); + + expect((await grade({ filters: { kinds: ['channel'] } })).collections).toHaveLength(10); + // RFC 4647 basic filtering: "en" matches "en" and "en-US"; "en-GB" does not match "en". + expect((await grade({ filters: { languages: ['en'] } })).collections.length).toBeGreaterThanOrEqual(10); + expect((await grade({ filters: { languages: ['en-GB'] } })).collections).toHaveLength(0); + expect((await grade({ filters: { statuses: ['active'] } })).collections.length).toBeGreaterThan(0); + expect((await grade({ filters: { genres_include: ['fitness'] } })).collections.map(keyOf)) + .toEqual([`${CREATORS}/creatorloop_channel_nadia_ferro`]); + const juniper = await grade({ filters: { talent: [{ domain: 'creatorlooptalent.example', brand_id: 'juniper_vale' }] } }); + expect(juniper.collections.map(keyOf)).toEqual([`${CREATORS}/creatorloop_channel_juniper_vale`]); + const shorts = await grade({ filters: { platforms: ['shorts.creatorloop.example'] } }); + expect(shorts.collections.length).toBe(5); + expect((await grade({ filters: { name_contains: 'garage' } })).collections.map(keyOf)) + .toEqual([`${CREATORS}/creatorloop_channel_ravi_castellan`]); + await grade({ filters: { kinds: ['channel'], platforms: ['video.creatorloop.example'], genres_exclude: ['gaming'] } }); + + // Pagination: the version is stable across pages and the pages tile the selection. + const first = await grade({ filters: { kinds: ['channel'] }, pagination: { max_results: 4 } }); + expect(first.pagination).toMatchObject({ has_more: true }); + const second = await preview('list_collections', { filters: { kinds: ['channel'] }, pagination: { max_results: 4, cursor: first.pagination.cursor } }); + expect(second.collections_version).toBe(first.collections_version); + expect(second.collections.map(keyOf).some((key: string) => first.collections.map(keyOf).includes(key))).toBe(false); + + // Conditional fetch: same selection → unchanged; any other selection → listed. + const filters = { kinds: ['channel'], languages: ['en'] }; + const listed = await grade({ filters }); + const unchanged = await grade( + { filters: { languages: ['en'], kinds: ['channel'] }, if_collections_version: listed.collections_version }, + { filters, cache_scope: 'public' }, + ); + expect(unchanged.outcome).toBe('unchanged'); + expect(unchanged.collections).toBeUndefined(); + const other = await grade({ filters: { kinds: ['channel'] }, if_collections_version: listed.collections_version }); + expect(other.outcome).toBe('listed'); + + // More than 100 collection IDs fails the request. + const tooMany = await preview('list_collections', { + collections: [{ publisher_domain: CREATORS, collection_ids: Array.from({ length: 101 }, (_, i) => `id_${i}`) }], + }); + expect(tooMany.adcp_error?.code, JSON.stringify(tooMany).slice(0, 300)).toBe('VALIDATION_ERROR'); + }, 120_000); + + it('scopes product feed versions by include_collections and moves only the card scope on a card change', async () => { + const plain = await preview('list_products', {}); + const withCards = await preview('list_products', { include_collections: true }); + expect(typeof plain.feed_version).toBe('string'); + expect(withCards.feed_version).not.toBe(plain.feed_version); + // Each token is current under its own setting... + expect((await preview('list_products', { if_feed_version: plain.feed_version })).outcome).toBe('unchanged'); + const unchangedCards = await preview('list_products', { include_collections: true, if_feed_version: withCards.feed_version }); + expect(unchangedCards.outcome).toBe('unchanged'); + expect(unchangedCards.collections).toBeUndefined(); + // ...and a mismatch under the other. + expect((await preview('list_products', { include_collections: true, if_feed_version: plain.feed_version })).outcome).toBe('listed'); + expect((await preview('list_products', { if_feed_version: withCards.feed_version })).outcome).toBe('listed'); + + const gpPlain = await preview('get_products', { buying_mode: 'wholesale' }); + const gpCards = await preview('get_products', { buying_mode: 'wholesale', include_collections: true }); + expect(gpCards.wholesale_feed_version).not.toBe(gpPlain.wholesale_feed_version); + expect((await preview('get_products', { buying_mode: 'wholesale', include_collections: true, if_wholesale_feed_version: gpCards.wholesale_feed_version })).unchanged).toBe(true); + + // A card change moves the card scope only. + const changed = await seed({ + action: 'update', + collections: [{ publisher_domain: CREATORS, collection_id: 'creatorloop_channel_kenji_arden', fields: { season: '2026' } }], + }); + expect(changed.success, JSON.stringify(changed)).toBe(true); + expect((await preview('list_products', { if_feed_version: plain.feed_version })).outcome).toBe('unchanged'); + const moved = await preview('list_products', { include_collections: true, if_feed_version: withCards.feed_version }); + expect(moved.outcome).toBe('listed'); + expect(moved.feed_version).not.toBe(withCards.feed_version); + expect(moved.collections.find((card: Card) => card.collection_id === 'creatorloop_channel_kenji_arden')?.season).toBe('2026'); + }, 120_000); + + it('pushes card webhooks to a local receiver, graded conformant, and a mirror applying them matches list_collections', async () => { + clearCollectionCardState(); + received.length = 0; + const account = { brand: { domain: 'collections-buyer.example' }, operator: 'collections-buyer.example', sandbox: true }; + const subscription = { subscriber_id: 'card-mirror', url: hookUrl, event_types: CARD_EVENTS, active: true }; + // Undeclared below 3.3: the stable wire rejects the card event types. + const stable = await call('/sales/mcp', 'sync_accounts', { + adcp_version: '3.2', + accounts: [{ ...account, billing: 'operator', payment_terms: 'net_30', notification_configs: [subscription] }], + idempotency_key: 'collections-sync-accounts-stable-0001', + }); + expect(stable.accounts?.[0], JSON.stringify(stable).slice(0, 400)).toMatchObject({ + action: 'failed', + status: 'rejected', + errors: [expect.objectContaining({ field: 'notification_configs[0].event_types[0]' })], + }); + const synced = await preview('sync_accounts', { + accounts: [{ ...account, billing: 'operator', payment_terms: 'net_30', notification_configs: [subscription] }], + idempotency_key: 'collections-sync-accounts-0001', + }); + expect(synced.accounts?.[0]?.status, JSON.stringify(synced).slice(0, 400)).toBe('active'); + const accountId = synced.accounts[0].account_id as string; + + // Bootstrap the mirror from an unfiltered read. + const bootstrap = await listAll(); + const mirror = { cache_scope: bootstrap.cacheScope, version: bootstrap.version, last_event_id: null as string | null, cards: bootstrap.cards }; + + // A follower refresh that leaves the bounds and precision unchanged is not a change. + const nadia = bootstrap.cards.find(card => card.collection_id === 'creatorloop_channel_nadia_ferro')!; + const shorts = nadia.distribution.find((entry: Record) => entry.publisher_domain === 'shorts.creatorloop.example'); + const sameCount = await seed({ + action: 'refresh_followers', + followers: [{ + publisher_domain: CREATORS, + collection_id: nadia.collection_id, + distribution_publisher_domain: 'shorts.creatorloop.example', + lower_bound: shorts.followers.lower_bound, + upper_bound: shorts.followers.upper_bound, + precision: 'approximate', + as_of: '2026-11-01T00:00:00Z', + }], + }); + expect(sameCount.events).toEqual([]); + expect(sameCount.collections_version).toBe(bootstrap.version); + expect(received).toHaveLength(0); + const recheck = await preview('list_collections', { collections: [{ publisher_domain: CREATORS, collection_ids: [nadia.collection_id] }] }); + expect(recheck.collections[0].distribution).toEqual(nadia.distribution); + + // One editorial change → one collection_card.updated with the full card. + const newSample = { + asset_type: 'published_post', + post_url: 'https://shorts.creatorloop.example/v/nadia-ferro-3', + platform: 'shorts.creatorloop.example', + published_at: '2026-10-30T12:00:00Z', + }; + const updated = await seed({ + action: 'update', + collections: [{ publisher_domain: CREATORS, collection_id: nadia.collection_id, fields: { sample_content: [newSample, ...nadia.sample_content].slice(0, 3) } }], + }); + expect(updated.success, JSON.stringify(updated)).toBe(true); + expect(updated.events.map((event: { notification_type: string }) => event.notification_type)).toEqual(['collection_card.updated']); + + // One removal and one re-entry. + const removed = await seed({ action: 'remove', collections: [{ publisher_domain: CREATORS, collection_id: 'creatorloop_channel_sol_marquez', removal_reason: 'withdrawn' }] }); + expect(removed.events.map((event: { notification_type: string }) => event.notification_type)).toEqual(['collection_card.removed']); + const restored = await seed({ action: 'restore', collections: [{ publisher_domain: CREATORS, collection_id: 'creatorloop_channel_sol_marquez' }] }); + expect(restored.events.map((event: { notification_type: string }) => event.notification_type)).toEqual(['collection_card.updated']); + + // A follower sweep that moves counts goes out as one bulk event. + const sweep = await seed({ + action: 'refresh_followers', + followers: [ + { publisher_domain: CREATORS, collection_id: nadia.collection_id, distribution_publisher_domain: 'shorts.creatorloop.example', lower_bound: 3_900_000, upper_bound: 3_999_999, precision: 'approximate' }, + { publisher_domain: CREATORS, collection_id: 'creatorloop_channel_odette_moreau', distribution_publisher_domain: 'shorts.creatorloop.example', lower_bound: 4_500_000, upper_bound: 4_599_999, precision: 'approximate' }, + ], + }); + expect(sweep.events.map((event: { notification_type: string }) => event.notification_type)).toEqual(['collection_catalog.bulk_change']); + + // An editorial operation over more cards than the threshold is one bulk event too. + expect(bulkThreshold(17)).toBe(1); + const multi = await seed({ + action: 'update', + collections: [ + { publisher_domain: CREATORS, collection_id: 'creatorloop_channel_juniper_vale', fields: { season: '4' } }, + { publisher_domain: CREATORS, collection_id: 'creatorloop_channel_tobias_quill', fields: { season: '7' } }, + ], + }); + expect(multi.events.map((event: { notification_type: string }) => event.notification_type)).toEqual(['collection_catalog.bulk_change']); + + // Anchored facts cannot change on the seller's copy. + const anchored = await seed({ action: 'update', collections: [{ publisher_domain: CREATORS, collection_id: nadia.collection_id, fields: { status: 'ended' } }] }); + expect(anchored.success).toBe(false); + expect(anchored.error).toBe('INVALID_PARAMS'); + + // Every event was delivered, signed, to the local receiver, in order. + const deliveries = received.map(entry => entry.body); + expect(deliveries.map(body => body.notification_type)).toEqual([ + 'collection_card.updated', 'collection_card.removed', 'collection_card.updated', + 'collection_catalog.bulk_change', 'collection_catalog.bulk_change', + ]); + for (const { headers } of received) { + expect(headers['signature-input'] ?? headers.signature, 'RFC 9421 signature').toBeDefined(); + } + const visibleKeys = (await listAll()).cards.map(card => ({ publisher_domain: card.publisher_domain, collection_id: card.collection_id })); + for (const body of deliveries) { + expect(body.account_id).toBe(accountId); + expect(body.product_payload_view).toBeUndefined(); + const violations = grader.gradeWebhook({ + seller: SELLER, + subscription: { account_id: accountId, event_types: CARD_EVENTS }, + subscriber_visible_keys: visibleKeys, + webhook: body, + }); + expect(violations, JSON.stringify(body).slice(0, 400)).toEqual([]); + } + // The updated event carries exactly the card list_collections serves. + const updatedCard = deliveries[0].event.payload.collection; + expect(deliveries[0].event.payload.changed_fields).toEqual(['sample_content']); + const removedPayload = deliveries[1].event.payload; + expect(removedPayload).toMatchObject({ publisher_domain: CREATORS, collection_id: 'creatorloop_channel_sol_marquez', removal_reason: 'withdrawn' }); + + // Per-card events apply in order to the mirror; the first bulk event asks + // for a full replace from an unfiltered read. + const applied = grader.applyEvents({ mirror, events: deliveries.slice(0, 3).map(webhook => ({ webhook })) }); + expect(applied.verdict).toBe('applied'); + const afterRestore = (applied.mirror.cards as Card[]).find(card => card.collection_id === nadia.collection_id); + expect(grader.canonical(afterRestore)).toBe(grader.canonical(updatedCard)); + const bulk = grader.applyEvents({ mirror: { ...mirror, ...applied.mirror, last_event_id: deliveries[2].event.event_id, cards: applied.mirror.cards }, events: deliveries.slice(3).map(webhook => ({ webhook })) }); + expect(bulk.verdict).toBe('repair_full'); + + // A mirror that applies the per-card events matches list_collections at + // that point: replay the same changes on a fresh sandbox and compare. + clearCollectionCardState(); + await seed({ action: 'update', collections: [{ publisher_domain: CREATORS, collection_id: nadia.collection_id, fields: { sample_content: [newSample, ...nadia.sample_content].slice(0, 3) } }] }); + await seed({ action: 'remove', collections: [{ publisher_domain: CREATORS, collection_id: 'creatorloop_channel_sol_marquez' }] }); + await seed({ action: 'restore', collections: [{ publisher_domain: CREATORS, collection_id: 'creatorloop_channel_sol_marquez' }] }); + const fresh = await listAll(); + expect(fresh.version).toBe(applied.mirror.version); + expect(grader.canonical(fresh.cards)).toBe(grader.canonical(applied.mirror.cards)); + + // A gap repairs with the last applied version, which list_collections + // answers with a full page. + const gap = grader.applyEvents({ mirror, events: [deliveries[1], deliveries[2]].map(webhook => ({ webhook })) }); + expect(gap.verdict).toBe('repair_conditional'); + expect(gap.repair_token).toBe(bootstrap.version); + const repaired = await preview('list_collections', { if_collections_version: gap.repair_token }); + expect(repaired.outcome).toBe('listed'); + }, 120_000); + + it('links creator identities back to their collections on /brand at the preview only', async () => { + const { cards } = await listAll(); + const creators = cards.filter(card => card.publisher_domain === CREATORS && card.talent?.[0]?.brand_ref); + const confirmed: string[] = []; + for (const card of creators) { + const ref = card.talent[0].brand_ref as { domain: string; brand_id: string }; + expect(ref.domain).toBe('creatorlooptalent.example'); + const identity = await preview('get_brand_identity', { brand_id: ref.brand_id, fields: ['description', 'logos', 'collections'] }, '/brand/mcp'); + expect(identity.adcp_error, JSON.stringify(identity).slice(0, 300)).toBeUndefined(); + expect(identity.logos?.length).toBeGreaterThan(0); + const links = identity.collections as Array<{ publisher_domain: string; collection_id: string }>; + expect(links?.length, ref.brand_id).toBeGreaterThan(0); + if (links.some(link => keyOf(link) === keyOf(card))) confirmed.push(ref.brand_id); + } + // Every brand_ref is confirmed by the identity listing the key back, except one. + expect(confirmed).toHaveLength(8); + expect(creators.map(card => card.talent[0].brand_ref.brand_id).filter(id => !confirmed.includes(id))).toEqual(['ravi_castellan']); + + const narrowed = await preview('get_brand_identity', { brand_id: 'juniper_vale', fields: ['description'] }, '/brand/mcp'); + expect(narrowed.collections).toBeUndefined(); + for (const pin of [{ adcp_version: '3.2' }, { adcp_major_version: 3 }, {}]) { + const stable = await call('/brand/mcp', 'get_brand_identity', { ...pin, brand_id: 'juniper_vale' }); + expect(stable.adcp_error, JSON.stringify(stable).slice(0, 300)).toBeUndefined(); + expect(stable.collections, JSON.stringify(pin)).toBeUndefined(); + } + }, 60_000); + + it('issues strictly increasing UUID v7 event ids', () => { + const ids = Array.from({ length: 50 }, () => monotonicUuidV7(1_800_000_000_000)); + expect([...ids].sort()).toEqual(ids); + expect(new Set(ids).size).toBe(ids.length); + for (const id of ids) expect(id).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/); + }); +}); diff --git a/tests/collection-copies-vectors.test.cjs b/tests/collection-copies-vectors.test.cjs index ffeec9f0bc9..dffbac71f73 100644 --- a/tests/collection-copies-vectors.test.cjs +++ b/tests/collection-copies-vectors.test.cjs @@ -7,18 +7,25 @@ * include_collections and list_collections), list_collections seller * responses, collection card webhook payloads, and card mirror application. * - * The grader below is the reference: implementers port it, not the prose. + * The grader in tests/helpers/collection-copies-grader.cjs is the reference: + * implementers port it, not the prose. The training agent's collection tests + * grade the agent's real outputs with the same module. */ const assert = require('node:assert/strict'); const fs = require('node:fs'); const path = require('node:path'); const test = require('node:test'); -const Ajv = require('ajv'); -const addFormats = require('ajv-formats'); -const { ensureBuiltSchemas, readBuiltSchema, builtSchemaFiles } = require('../scripts/built-schemas.cjs'); - -ensureBuiltSchemas(); +const { + CATALOG_FEATURE, + validate, + languageMatches, + belowVersion, + mergedView: gradeMergedView, + gradeResponse: gradeSellerResponse, + gradeWebhook, + applyEvents, +} = require('./helpers/collection-copies-grader.cjs'); const ROOT = path.resolve(__dirname, '..'); const vectors = JSON.parse( @@ -27,275 +34,8 @@ const vectors = JSON.parse( /** A vector's own declarations, seller_authorized or seller_catalog, else the shared fixture. */ const withFixtures = vector => ({ ...vectors.fixtures, ...vector }); -function loadSchemas() { - const ajv = new Ajv({ allErrors: true, strict: false }); - addFormats(ajv); - // The published schemas (dist/schemas/latest), with `/schemas/...` ids. - for (const file of builtSchemaFiles()) { - const schema = readBuiltSchema(file); - if (schema.$id && !ajv.getSchema(schema.$id)) ajv.addSchema(schema); - } - return ajv; -} - -const ajv = loadSchemas(); -const validate = { - collection: ajv.getSchema('/schemas/core/collection.json'), - copy: ajv.getSchema('/schemas/core/response-collection.json'), - selector: ajv.getSchema('/schemas/core/collection-selector.json'), - request: ajv.getSchema('/schemas/media-buy/list-collections-request.json'), - response: ajv.getSchema('/schemas/media-buy/list-collections-response.json'), - error: ajv.getSchema('/schemas/core/error.json'), - webhook: ajv.getSchema('/schemas/core/wholesale-feed-webhook.json'), -}; - -// --------------------------------------------------------------------------- -// Reference grader -// --------------------------------------------------------------------------- - -const CATALOG_FEATURE = 'media_buy.collection_catalog'; -const CARDS_FEATURE = 'media_buy.collection_cards'; -const CLOSED_LIST = ['images', 'sample_content', 'url', 'description', 'season', 'cadence']; -const CARD_EVENTS = new Set(['collection_card.updated', 'collection_card.removed', 'collection_catalog.bulk_change']); - -const clone = value => JSON.parse(JSON.stringify(value)); -const keyOf = c => `${c.publisher_domain}\u0000${c.collection_id}`; - -/** Canonical JSON with sorted object keys, for equality and scope comparison. */ -function canonical(value) { - if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`; - if (value && typeof value === 'object') { - return `{${Object.keys(value).sort().map(k => `${JSON.stringify(k)}:${canonical(value[k])}`).join(',')}}`; - } - return JSON.stringify(value); -} - -function belowVersion(version, floor) { - if (!version) return false; - const parse = v => v.split('-')[0].split('.').map(Number); - const [a, b] = [parse(version), parse(floor)]; - return a[0] < b[0] || (a[0] === b[0] && a[1] < b[1]); -} - -function sameIdentifiers(a, b) { - const set = entry => (entry.identifiers || []).map(canonical).sort().join('|'); - return set(a) === set(b); -} - -/** Declared entry overlaid with the closed list from a verified copy. */ -function overlay(publisherDomain, declared, copy) { - const effective = { publisher_domain: publisherDomain, ...clone(declared) }; - for (const field of CLOSED_LIST) if (field in copy) effective[field] = clone(copy[field]); - if (Array.isArray(effective.distribution) && Array.isArray(copy.distribution)) { - for (const entry of copy.distribution) { - if (!entry.followers) continue; - let matches = effective.distribution.filter(d => d.publisher_domain === entry.publisher_domain); - if (matches.length > 1) matches = matches.filter(d => sameIdentifiers(d, entry)); - if (matches.length === 1) matches[0].followers = clone(entry.followers); - } - } - return effective; -} - -function mergedView(input) { - const vector = withFixtures(input); - return vector.keys.map(k => { - const declared = (vector.declarations[k.publisher_domain] || []).find(c => c.collection_id === k.collection_id); - const authorized = vector.seller_authorized.some(sel => ( - sel.publisher_domain === k.publisher_domain && (!sel.collection_ids || sel.collection_ids.includes(k.collection_id)) - )); - const copies = vector.copies.filter(c => keyOf(c) === keyOf(k)); - const fromDeclaration = declared ? { publisher_domain: k.publisher_domain, ...clone(declared) } : null; - const out = (status, effective, display) => ({ - ...k, status, effective, display, carriage_eligible: Boolean(declared), - }); - if (copies.length > 1) return out('unresolvable', fromDeclaration, fromDeclaration); - if (copies.length === 0) return declared ? out('fallback', fromDeclaration, fromDeclaration) : out('unknown', null, null); - if (declared && authorized) { - const effective = overlay(k.publisher_domain, declared, copies[0]); - return out('verified', effective, effective); - } - return out('unverified', null, clone(copies[0])); - }); -} - -function selectorMatches(card, selectors) { - if (!selectors) return true; - return selectors.some(sel => ( - sel.publisher_domain === card.publisher_domain && (!sel.collection_ids || sel.collection_ids.includes(card.collection_id)) - )); -} - -/** RFC 4647 basic filtering, case-insensitive. */ -function languageMatches(range, tag) { - if (!tag) return false; - const [r, t] = [range.toLowerCase(), tag.toLowerCase()]; - return r === '*' || t === r || t.startsWith(`${r}-`); -} - -function filterMatches(card, filters) { - if (!filters) return true; - const any = (values, test) => values.some(test); - if (filters.kinds && !filters.kinds.includes(card.kind || 'series')) return false; - if (filters.statuses && !filters.statuses.includes(card.status)) return false; - if (filters.languages && !any(filters.languages, r => languageMatches(r, card.language))) return false; - const taxonomyOk = !filters.genre_taxonomy || filters.genre_taxonomy === card.genre_taxonomy; - const genres = card.genre || []; - if (filters.genres_include && !(taxonomyOk && any(filters.genres_include, g => genres.includes(g)))) return false; - if (filters.genres_exclude && taxonomyOk && any(filters.genres_exclude, g => genres.includes(g))) return false; - if (filters.talent && !any(filters.talent, t => (card.talent || []).some(p => ( - p.brand_ref && p.brand_ref.domain === t.domain && (!t.brand_id || p.brand_ref.brand_id === t.brand_id) - )))) return false; - if (filters.platforms && !any(filters.platforms, d => (card.distribution || []).some(e => e.publisher_domain === d))) return false; - if (filters.name_contains && !(card.name || '').toLowerCase().includes(filters.name_contains.toLowerCase())) return false; - return true; -} - -/** Selectors and filters as sets; cursor excluded; account only for account-scoped tokens. */ -function scopeOf({ collections, filters, accountId, cacheScope }) { - const selectors = collections - ? [...new Set(collections.map(s => canonical({ - publisher_domain: s.publisher_domain, - ...(s.collection_ids ? { collection_ids: [...new Set(s.collection_ids)].sort() } : {}), - })))].sort() - : null; - const normalizedFilters = filters - ? Object.fromEntries(Object.entries(filters).map(([k, v]) => [ - k, Array.isArray(v) ? [...new Set(v.map(canonical))].sort() : v, - ])) - : null; - return canonical({ - selectors, - filters: normalizedFilters, - account: cacheScope === 'account' ? accountId || null : null, - }); -} - -function idCount(request) { - return (request.collections || []).reduce((sum, s) => sum + (s.collection_ids || []).length, 0); -} - -function gradeResponse(input) { - const vector = withFixtures(input); - const violations = new Set(); - const { seller, seller_catalog: catalog, request, response, task_error: taskError } = vector; - const schemaOk = validate.request(request) - && (taskError ? !response && validate.error(taskError) : validate.response(response)); - if (!schemaOk) return ['schema_invalid']; - - const features = seller.experimental_features || []; - if (response && (!features.includes(CATALOG_FEATURE) || !features.includes(CARDS_FEATURE))) { - violations.add('feature_not_declared'); - } - if (belowVersion(request.adcp_version, '3.3')) { - if (response || taskError.code !== 'UNSUPPORTED_FEATURE') violations.add('version_gate'); - return [...violations]; - } - if (idCount(request) > 100) { - if (response) violations.add('id_limit_not_enforced'); - return [...violations]; - } - if (taskError) { - if (taskError.code === 'REFERENCE_NOT_FOUND') violations.add('per_key_error'); - return [...violations]; - } - - const accountId = request.account && request.account.account_id; - if (!request.account && response.cache_scope === 'account') violations.add('cache_scope_without_account'); - - if (response.outcome === 'unchanged') { - const token = vector.token_issued_for; - const presented = scopeOf({ - collections: token.collections, filters: token.filters, accountId: token.account_id, cacheScope: token.cache_scope, - }); - const current = scopeOf({ - collections: request.collections, filters: request.filters, accountId, cacheScope: token.cache_scope, - }); - if (presented !== current) violations.add('token_scope_mismatch'); - return [...violations]; - } - - const visible = entry => entry.visible_to === 'public' || (accountId && entry.visible_to.includes(accountId)); - const byKey = new Map(catalog.map(entry => [keyOf(entry.card), entry])); - const seen = new Set(); - for (const card of response.collections) { - const key = keyOf(card); - if (seen.has(key)) violations.add('duplicate_key'); - seen.add(key); - const entry = byKey.get(key); - if (!entry || !visible(entry)) violations.add('invisible_entry_returned'); - else if (canonical(card) !== canonical(entry.card)) violations.add('copy_inconsistent'); - if (!selectorMatches(card, request.collections)) violations.add('selector_mismatch'); - if (!filterMatches(card, request.filters)) violations.add('filter_mismatch'); - } - const firstPage = !(request.pagination && request.pagination.cursor); - const lastPage = !(response.pagination && response.pagination.has_more); - if (firstPage && lastPage) { - for (const entry of catalog) { - if (!visible(entry) || seen.has(keyOf(entry.card))) continue; - if (selectorMatches(entry.card, request.collections) && filterMatches(entry.card, request.filters)) { - violations.add('missing_entry'); - } - } - } - return [...violations]; -} - -function gradeWebhook(vector) { - const violations = new Set(); - const { webhook, seller, subscription } = vector; - if (!validate.webhook(webhook)) return ['schema_invalid']; - const type = webhook.notification_type; - if (webhook.notification_id !== webhook.event.event_id) violations.add('notification_id_mismatch'); - if (CARD_EVENTS.has(type) && !(seller.experimental_features || []).includes(CATALOG_FEATURE)) { - violations.add('feature_not_declared'); - } - if (!(seller.wholesale_feed_webhook_event_types || []).includes(type)) violations.add('undeclared_event_type'); - if (!subscription.event_types.includes(type)) violations.add('unsubscribed_event_type'); - if (webhook.account_id !== subscription.account_id) violations.add('account_mismatch'); - if (type === 'collection_card.updated') { - const visible = new Set(vector.subscriber_visible_keys.map(keyOf)); - if (!visible.has(keyOf(webhook.event.payload.collection))) violations.add('visibility_leak'); - } - return [...violations]; -} - -function snapshot(mirror, cards) { - return { - cache_scope: mirror.cache_scope, - version: mirror.version, - last_event_id: mirror.last_event_id, - cards: [...cards.values()].sort((a, b) => ( - a.publisher_domain.localeCompare(b.publisher_domain) || a.collection_id.localeCompare(b.collection_id) - )), - }; -} - -function applyEvents(vector) { - const mirror = clone(vector.mirror); - const cards = new Map(mirror.cards.map(c => [keyOf(c), c])); - const seenDelivery = new Set(); - const seenLogical = new Set(); - const stop = (verdict, token) => ({ verdict, repair_token: token, mirror: snapshot(mirror, cards) }); - for (const { webhook, signature_valid: signatureValid = true } of vector.events) { - if (!signatureValid) return stop('repair_conditional', mirror.version); - if (seenDelivery.has(webhook.idempotency_key) || seenLogical.has(webhook.notification_id)) continue; - const previous = webhook.previous_wholesale_feed_version; - if (previous !== undefined && previous !== mirror.version) return stop('repair_conditional', mirror.version); - if (mirror.last_event_id && webhook.event.event_id < mirror.last_event_id) { - return stop('repair_conditional', mirror.version); - } - const { event_type: type, payload } = webhook.event; - if (type === 'collection_catalog.bulk_change') return stop('repair_full', null); - if (type === 'collection_card.updated') cards.set(keyOf(payload.collection), clone(payload.collection)); - else if (type === 'collection_card.removed') cards.delete(keyOf(payload)); - mirror.version = webhook.wholesale_feed_version; - mirror.last_event_id = webhook.event.event_id; - seenDelivery.add(webhook.idempotency_key); - seenLogical.add(webhook.notification_id); - } - return stop('applied', null); -} +const mergedView = input => gradeMergedView(withFixtures(input)); +const gradeResponse = input => gradeSellerResponse(withFixtures(input)); // --------------------------------------------------------------------------- // Replay diff --git a/tests/helpers/collection-copies-grader.cjs b/tests/helpers/collection-copies-grader.cjs new file mode 100644 index 00000000000..57d0b56cd95 --- /dev/null +++ b/tests/helpers/collection-copies-grader.cjs @@ -0,0 +1,307 @@ +'use strict'; + +/** + * Reference grader for experimental collection copies (#8183): the buyer's + * merged view of a seller copy and adagents.json (shared by + * include_collections and list_collections), list_collections seller + * responses, collection card webhook payloads, and card mirror application. + * + * tests/collection-copies-vectors.test.cjs replays the frozen vectors in + * static/compliance/source/test-vectors/collection-copies/ through it, and + * the training agent's collection tests grade the agent's real outputs with + * it. Implementers port it, not the prose. + * + * `mergedView` and `gradeResponse` take a complete vector: a caller that uses + * the shared `fixtures` of vectors.json merges them in first. + */ + +const Ajv = require('ajv'); +const addFormats = require('ajv-formats'); +const { ensureBuiltSchemas, readBuiltSchema, builtSchemaFiles } = require('../../scripts/built-schemas.cjs'); + +ensureBuiltSchemas(); + +function loadSchemas() { + const ajv = new Ajv({ allErrors: true, strict: false }); + addFormats(ajv); + // The published schemas (dist/schemas/latest), with `/schemas/...` ids. + for (const file of builtSchemaFiles()) { + const schema = readBuiltSchema(file); + if (schema.$id && !ajv.getSchema(schema.$id)) ajv.addSchema(schema); + } + return ajv; +} + +const ajv = loadSchemas(); +const validate = { + collection: ajv.getSchema('/schemas/core/collection.json'), + copy: ajv.getSchema('/schemas/core/response-collection.json'), + selector: ajv.getSchema('/schemas/core/collection-selector.json'), + request: ajv.getSchema('/schemas/media-buy/list-collections-request.json'), + response: ajv.getSchema('/schemas/media-buy/list-collections-response.json'), + error: ajv.getSchema('/schemas/core/error.json'), + webhook: ajv.getSchema('/schemas/core/wholesale-feed-webhook.json'), +}; + +// --------------------------------------------------------------------------- +// Reference grader +// --------------------------------------------------------------------------- + +const CATALOG_FEATURE = 'media_buy.collection_catalog'; +const CARDS_FEATURE = 'media_buy.collection_cards'; +const CLOSED_LIST = ['images', 'sample_content', 'url', 'description', 'season', 'cadence']; +const CARD_EVENTS = new Set(['collection_card.updated', 'collection_card.removed', 'collection_catalog.bulk_change']); + +const clone = value => JSON.parse(JSON.stringify(value)); +const keyOf = c => `${c.publisher_domain}\u0000${c.collection_id}`; + +/** Canonical JSON with sorted object keys, for equality and scope comparison. */ +function canonical(value) { + if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`; + if (value && typeof value === 'object') { + return `{${Object.keys(value).sort().map(k => `${JSON.stringify(k)}:${canonical(value[k])}`).join(',')}}`; + } + return JSON.stringify(value); +} + +function belowVersion(version, floor) { + if (!version) return false; + const parse = v => v.split('-')[0].split('.').map(Number); + const [a, b] = [parse(version), parse(floor)]; + return a[0] < b[0] || (a[0] === b[0] && a[1] < b[1]); +} + +function sameIdentifiers(a, b) { + const set = entry => (entry.identifiers || []).map(canonical).sort().join('|'); + return set(a) === set(b); +} + +/** Declared entry overlaid with the closed list from a verified copy. */ +function overlay(publisherDomain, declared, copy) { + const effective = { publisher_domain: publisherDomain, ...clone(declared) }; + for (const field of CLOSED_LIST) if (field in copy) effective[field] = clone(copy[field]); + if (Array.isArray(effective.distribution) && Array.isArray(copy.distribution)) { + for (const entry of copy.distribution) { + if (!entry.followers) continue; + let matches = effective.distribution.filter(d => d.publisher_domain === entry.publisher_domain); + if (matches.length > 1) matches = matches.filter(d => sameIdentifiers(d, entry)); + if (matches.length === 1) matches[0].followers = clone(entry.followers); + } + } + return effective; +} + +function mergedView(vector) { + return vector.keys.map(k => { + const declared = (vector.declarations[k.publisher_domain] || []).find(c => c.collection_id === k.collection_id); + const authorized = vector.seller_authorized.some(sel => ( + sel.publisher_domain === k.publisher_domain && (!sel.collection_ids || sel.collection_ids.includes(k.collection_id)) + )); + const copies = vector.copies.filter(c => keyOf(c) === keyOf(k)); + const fromDeclaration = declared ? { publisher_domain: k.publisher_domain, ...clone(declared) } : null; + const out = (status, effective, display) => ({ + ...k, status, effective, display, carriage_eligible: Boolean(declared), + }); + if (copies.length > 1) return out('unresolvable', fromDeclaration, fromDeclaration); + if (copies.length === 0) return declared ? out('fallback', fromDeclaration, fromDeclaration) : out('unknown', null, null); + if (declared && authorized) { + const effective = overlay(k.publisher_domain, declared, copies[0]); + return out('verified', effective, effective); + } + return out('unverified', null, clone(copies[0])); + }); +} + +function selectorMatches(card, selectors) { + if (!selectors) return true; + return selectors.some(sel => ( + sel.publisher_domain === card.publisher_domain && (!sel.collection_ids || sel.collection_ids.includes(card.collection_id)) + )); +} + +/** RFC 4647 basic filtering, case-insensitive. */ +function languageMatches(range, tag) { + if (!tag) return false; + const [r, t] = [range.toLowerCase(), tag.toLowerCase()]; + return r === '*' || t === r || t.startsWith(`${r}-`); +} + +function filterMatches(card, filters) { + if (!filters) return true; + const any = (values, test) => values.some(test); + if (filters.kinds && !filters.kinds.includes(card.kind || 'series')) return false; + if (filters.statuses && !filters.statuses.includes(card.status)) return false; + if (filters.languages && !any(filters.languages, r => languageMatches(r, card.language))) return false; + const taxonomyOk = !filters.genre_taxonomy || filters.genre_taxonomy === card.genre_taxonomy; + const genres = card.genre || []; + if (filters.genres_include && !(taxonomyOk && any(filters.genres_include, g => genres.includes(g)))) return false; + if (filters.genres_exclude && taxonomyOk && any(filters.genres_exclude, g => genres.includes(g))) return false; + if (filters.talent && !any(filters.talent, t => (card.talent || []).some(p => ( + p.brand_ref && p.brand_ref.domain === t.domain && (!t.brand_id || p.brand_ref.brand_id === t.brand_id) + )))) return false; + if (filters.platforms && !any(filters.platforms, d => (card.distribution || []).some(e => e.publisher_domain === d))) return false; + if (filters.name_contains && !(card.name || '').toLowerCase().includes(filters.name_contains.toLowerCase())) return false; + return true; +} + +/** Selectors and filters as sets; cursor excluded; account only for account-scoped tokens. */ +function scopeOf({ collections, filters, accountId, cacheScope }) { + const selectors = collections + ? [...new Set(collections.map(s => canonical({ + publisher_domain: s.publisher_domain, + ...(s.collection_ids ? { collection_ids: [...new Set(s.collection_ids)].sort() } : {}), + })))].sort() + : null; + const normalizedFilters = filters + ? Object.fromEntries(Object.entries(filters).map(([k, v]) => [ + k, Array.isArray(v) ? [...new Set(v.map(canonical))].sort() : v, + ])) + : null; + return canonical({ + selectors, + filters: normalizedFilters, + account: cacheScope === 'account' ? accountId || null : null, + }); +} + +function idCount(request) { + return (request.collections || []).reduce((sum, s) => sum + (s.collection_ids || []).length, 0); +} + +function gradeResponse(vector) { + const violations = new Set(); + const { seller, seller_catalog: catalog, request, response, task_error: taskError } = vector; + const schemaOk = validate.request(request) + && (taskError ? !response && validate.error(taskError) : validate.response(response)); + if (!schemaOk) return ['schema_invalid']; + + const features = seller.experimental_features || []; + if (response && (!features.includes(CATALOG_FEATURE) || !features.includes(CARDS_FEATURE))) { + violations.add('feature_not_declared'); + } + if (belowVersion(request.adcp_version, '3.3')) { + if (response || taskError.code !== 'UNSUPPORTED_FEATURE') violations.add('version_gate'); + return [...violations]; + } + if (idCount(request) > 100) { + if (response) violations.add('id_limit_not_enforced'); + return [...violations]; + } + if (taskError) { + if (taskError.code === 'REFERENCE_NOT_FOUND') violations.add('per_key_error'); + return [...violations]; + } + + const accountId = request.account && request.account.account_id; + if (!request.account && response.cache_scope === 'account') violations.add('cache_scope_without_account'); + + if (response.outcome === 'unchanged') { + const token = vector.token_issued_for; + const presented = scopeOf({ + collections: token.collections, filters: token.filters, accountId: token.account_id, cacheScope: token.cache_scope, + }); + const current = scopeOf({ + collections: request.collections, filters: request.filters, accountId, cacheScope: token.cache_scope, + }); + if (presented !== current) violations.add('token_scope_mismatch'); + return [...violations]; + } + + const visible = entry => entry.visible_to === 'public' || (accountId && entry.visible_to.includes(accountId)); + const byKey = new Map(catalog.map(entry => [keyOf(entry.card), entry])); + const seen = new Set(); + for (const card of response.collections) { + const key = keyOf(card); + if (seen.has(key)) violations.add('duplicate_key'); + seen.add(key); + const entry = byKey.get(key); + if (!entry || !visible(entry)) violations.add('invisible_entry_returned'); + else if (canonical(card) !== canonical(entry.card)) violations.add('copy_inconsistent'); + if (!selectorMatches(card, request.collections)) violations.add('selector_mismatch'); + if (!filterMatches(card, request.filters)) violations.add('filter_mismatch'); + } + const firstPage = !(request.pagination && request.pagination.cursor); + const lastPage = !(response.pagination && response.pagination.has_more); + if (firstPage && lastPage) { + for (const entry of catalog) { + if (!visible(entry) || seen.has(keyOf(entry.card))) continue; + if (selectorMatches(entry.card, request.collections) && filterMatches(entry.card, request.filters)) { + violations.add('missing_entry'); + } + } + } + return [...violations]; +} + +function gradeWebhook(vector) { + const violations = new Set(); + const { webhook, seller, subscription } = vector; + if (!validate.webhook(webhook)) return ['schema_invalid']; + const type = webhook.notification_type; + if (webhook.notification_id !== webhook.event.event_id) violations.add('notification_id_mismatch'); + if (CARD_EVENTS.has(type) && !(seller.experimental_features || []).includes(CATALOG_FEATURE)) { + violations.add('feature_not_declared'); + } + if (!(seller.wholesale_feed_webhook_event_types || []).includes(type)) violations.add('undeclared_event_type'); + if (!subscription.event_types.includes(type)) violations.add('unsubscribed_event_type'); + if (webhook.account_id !== subscription.account_id) violations.add('account_mismatch'); + if (type === 'collection_card.updated') { + const visible = new Set(vector.subscriber_visible_keys.map(keyOf)); + if (!visible.has(keyOf(webhook.event.payload.collection))) violations.add('visibility_leak'); + } + return [...violations]; +} + +function snapshot(mirror, cards) { + return { + cache_scope: mirror.cache_scope, + version: mirror.version, + last_event_id: mirror.last_event_id, + cards: [...cards.values()].sort((a, b) => ( + a.publisher_domain.localeCompare(b.publisher_domain) || a.collection_id.localeCompare(b.collection_id) + )), + }; +} + +function applyEvents(vector) { + const mirror = clone(vector.mirror); + const cards = new Map(mirror.cards.map(c => [keyOf(c), c])); + const seenDelivery = new Set(); + const seenLogical = new Set(); + const stop = (verdict, token) => ({ verdict, repair_token: token, mirror: snapshot(mirror, cards) }); + for (const { webhook, signature_valid: signatureValid = true } of vector.events) { + if (!signatureValid) return stop('repair_conditional', mirror.version); + if (seenDelivery.has(webhook.idempotency_key) || seenLogical.has(webhook.notification_id)) continue; + const previous = webhook.previous_wholesale_feed_version; + if (previous !== undefined && previous !== mirror.version) return stop('repair_conditional', mirror.version); + if (mirror.last_event_id && webhook.event.event_id < mirror.last_event_id) { + return stop('repair_conditional', mirror.version); + } + const { event_type: type, payload } = webhook.event; + if (type === 'collection_catalog.bulk_change') return stop('repair_full', null); + if (type === 'collection_card.updated') cards.set(keyOf(payload.collection), clone(payload.collection)); + else if (type === 'collection_card.removed') cards.delete(keyOf(payload)); + mirror.version = webhook.wholesale_feed_version; + mirror.last_event_id = webhook.event.event_id; + seenDelivery.add(webhook.idempotency_key); + seenLogical.add(webhook.notification_id); + } + return stop('applied', null); +} + +module.exports = { + CATALOG_FEATURE, + CARDS_FEATURE, + CLOSED_LIST, + CARD_EVENTS, + validate, + canonical, + belowVersion, + languageMatches, + filterMatches, + scopeOf, + mergedView, + gradeResponse, + gradeWebhook, + applyEvents, +};