diff --git a/.github/workflows/training-agent-storyboards.yml b/.github/workflows/training-agent-storyboards.yml index df5742d9181..acc518e06f0 100644 --- a/.github/workflows/training-agent-storyboards.yml +++ b/.github/workflows/training-agent-storyboards.yml @@ -279,10 +279,10 @@ jobs: # Creative-builder retains a large generated-schema graph after its # result is complete on some Node/V8 builds. Run each storyboard in # a bounded child so teardown cannot retain the whole job. Other - # matrix tenants remain monolithic; current /sales has its separate + # matrix tenants except current /creative remain monolithic; current /sales has its separate # isolated aggregate jobs below. isolated_orchestrator_failure=0 - if [ "${{ matrix.tenant }}" = "creative-builder" ]; then + if [ "${{ matrix.tenant }}" = "creative-builder" ] || { [ "${{ matrix.surface }}" = "current" ] && [ "${{ matrix.tenant }}" = "creative" ]; }; then bash scripts/run-storyboards-isolated-shards.sh \ --shard-count 8 --max-parallel 4 2>&1 \ | tee /tmp/storyboards.log || isolated_orchestrator_failure=1 @@ -336,7 +336,7 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" if [ "${isolated_orchestrator_failure}" -ne 0 ]; then - echo "::error::One or more isolated creative-builder orchestrators failed. Counts above are diagnostic only." + echo "::error::One or more isolated ${{ matrix.tenant }} orchestrators failed. Counts above are diagnostic only." exit 1 fi diff --git a/scripts/check-changeset-protocol-scope.cjs b/scripts/check-changeset-protocol-scope.cjs index d0456965341..6ff16a28121 100644 --- a/scripts/check-changeset-protocol-scope.cjs +++ b/scripts/check-changeset-protocol-scope.cjs @@ -103,14 +103,65 @@ function isStoryboardBranchRegistrationOnly(headContent, baseContent) { && headContent === baseContent.replaceAll(oldBranches, newBranches); } +// Only these complete operational substitutions are exempt. Any accompanying +// edit (floors, selection, environment, failure policy, etc.) stays scoped. +const CREATIVE_ISOLATION_SUBSTITUTIONS = new Map(Object.entries({ + "scripts/run-storyboards-matrix.sh": [ + [ + " if { [ \"${FLOOR_SET}\" = \"current\" ] && [ \"${tenant}\" = \"sales\" ]; } \\\n", + " if { [ \"${FLOOR_SET}\" = \"current\" ] && { [ \"${tenant}\" = \"sales\" ] || [ \"${tenant}\" = \"creative\" ]; }; } \\\n" + ] + ], + ".github/workflows/training-agent-storyboards.yml": [ + [ + " # matrix tenants remain monolithic; current /sales has its separate\n", + " # matrix tenants except current /creative remain monolithic; current /sales has its separate\n" + ], + [ + " if [ \"${{ matrix.tenant }}\" = \"creative-builder\" ]; then\n", + " if [ \"${{ matrix.tenant }}\" = \"creative-builder\" ] || { [ \"${{ matrix.surface }}\" = \"current\" ] && [ \"${{ matrix.tenant }}\" = \"creative\" ]; }; then\n" + ], + [ + " echo \"::error::One or more isolated creative-builder orchestrators failed. Counts above are diagnostic only.\"\n", + " echo \"::error::One or more isolated ${{ matrix.tenant }} orchestrators failed. Counts above are diagnostic only.\"\n" + ] + ] +})); + +function isCreativeIsolationOnlyChange(change, readFileAtHead, readFileAtBase) { + if (change.status !== 'M' || change.paths?.length !== 1) return false; + const filePath = normalizePath(change.paths[0]); + const substitutions = CREATIVE_ISOLATION_SUBSTITUTIONS.get(filePath); + if (!substitutions || !readFileAtHead || !readFileAtBase) return false; + try { + let expected = readFileAtBase(filePath); + const head = readFileAtHead(filePath); + if (typeof expected !== 'string' || !expected || typeof head !== 'string') return false; + for (const [before, after] of substitutions) { + if (expected.split(before).length !== 2) return false; + expected = expected.replace(before, after); + } + return expected === head; + } catch { + return false; + } +} + +function isStoryboardBranchRegistrationChange(change, filePath, readFileAtHead, readFileAtBase) { + if (normalizePath(filePath) !== '.github/workflows/training-agent-storyboards.yml' + || change.status !== 'M') return false; + try { + return isStoryboardBranchRegistrationOnly(readFileAtHead(filePath), readFileAtBase(filePath)); + } catch { + return false; + } +} + function isProtocolScopedChange(change, readFileAtHead, readFileAtBase) { + if (isCreativeIsolationOnlyChange(change, readFileAtHead, readFileAtBase)) return false; return (change.paths || []).some(filePath => { if (!isProtocolScopedPath(filePath)) return false; - if (normalizePath(filePath) === '.github/workflows/training-agent-storyboards.yml' - && change.status === 'M' - && isStoryboardBranchRegistrationOnly( - readFileAtHead(filePath), readFileAtBase(filePath) - )) return false; + if (isStoryboardBranchRegistrationChange(change, filePath, readFileAtHead, readFileAtBase)) return false; return true; }); } @@ -306,7 +357,9 @@ function formatViolationMessage(violations) { function run(argv = process.argv.slice(2)) { const baseRef = argv[0] || defaultBaseRef(); - const diffOutput = git(['diff', '--name-status', '--find-renames', `${baseRef}...HEAD`]); + const mergeBase = git(['merge-base', baseRef, 'HEAD']).trim(); + const readFileAtBase = filePath => readFileAtRef(mergeBase, filePath); + const diffOutput = git(['diff', '--name-status', '--find-renames', `${mergeBase}...HEAD`]); const changes = parseNameStatus(diffOutput); if (argv.includes('--is-delete-only-cleanup')) { @@ -333,7 +386,7 @@ function run(argv = process.argv.slice(2)) { const hasProtocol = hasProtocolScopedChanges( changes, readFileAtHead, - filePath => readFileAtRef(baseRef, filePath) + readFileAtBase ); if (hasProtocol) { console.log('Protocol-scoped changes detected.'); @@ -346,7 +399,7 @@ function run(argv = process.argv.slice(2)) { const violations = findChangesetProtocolScopeViolations( changes, readFileAtHead, - filePath => readFileAtRef(baseRef, filePath) + readFileAtBase ); if (violations.length > 0) { diff --git a/scripts/run-storyboards-matrix.sh b/scripts/run-storyboards-matrix.sh index 098491f03ba..866cf7abd2c 100755 --- a/scripts/run-storyboards-matrix.sh +++ b/scripts/run-storyboards-matrix.sh @@ -417,7 +417,7 @@ for entry in "${TENANTS[@]}"; do log=$(mktemp -t "storyboards-${tenant}.XXXXXX.log") orchestrator_failure=0 - if { [ "${FLOOR_SET}" = "current" ] && [ "${tenant}" = "sales" ]; } \ + if { [ "${FLOOR_SET}" = "current" ] && { [ "${tenant}" = "sales" ] || [ "${tenant}" = "creative" ]; }; } \ || [ "${tenant}" = "creative-builder" ]; then TENANT_PATH="${tenant}" \ PUBLIC_TEST_AGENT_TOKEN="${PUBLIC_TEST_AGENT_TOKEN:-storyboard-local-token}" \ diff --git a/tests/changeset-protocol-scope.test.cjs b/tests/changeset-protocol-scope.test.cjs index 191be5c1a4d..27cb0156b96 100644 --- a/tests/changeset-protocol-scope.test.cjs +++ b/tests/changeset-protocol-scope.test.cjs @@ -368,3 +368,113 @@ assert.strictEqual( ); console.log('Changeset protocol scope tests passed.'); + +// Complete before/after contents are required for the exact creative runner +// substitution. Unrelated edits in these operational files remain scoped. +{ + const fs = require('node:fs'); + const path = require('node:path'); + const replacements = { + 'scripts/run-storyboards-matrix.sh': [[ + '&& { [ "${tenant}" = "sales" ] || [ "${tenant}" = "creative" ]; }; }', + '&& [ "${tenant}" = "sales" ]; }', + ]], + '.github/workflows/training-agent-storyboards.yml': [ + ['# matrix tenants except current /creative remain monolithic;', '# matrix tenants remain monolithic;'], + ['if [ "${{ matrix.tenant }}" = "creative-builder" ] || { [ "${{ matrix.surface }}" = "current" ] && [ "${{ matrix.tenant }}" = "creative" ]; }; then', + 'if [ "${{ matrix.tenant }}" = "creative-builder" ]; then'], + ['One or more isolated ${{ matrix.tenant }} orchestrators failed.', 'One or more isolated creative-builder orchestrators failed.'], + ], + }; + for (const [file, substitutions] of Object.entries(replacements)) { + const head = fs.readFileSync(path.join(__dirname, '..', file), 'utf8'); + let base = head; + for (const [after, before] of substitutions) { + assert.ok(base.includes(after)); + base = base.replace(after, before); + } + const changes = [{ status: 'M', paths: [file] }]; + const classify = (candidate = head, prior = base, delta = changes) => + hasProtocolScopedChanges(delta, readFiles({ [file]: candidate }), readFiles({ [file]: prior })); + assert.strictEqual(classify(), false, `${file}: exact isolation substitution is operational`); + assert.strictEqual(hasProtocolScopedChanges(changes), true, 'path-only queries fail closed'); + assert.strictEqual(classify(head, ''), true, 'missing base fails closed'); + assert.strictEqual(classify('', base), true, 'missing head fails closed'); + assert.strictEqual(hasProtocolScopedChanges(changes, () => { throw new Error('unreadable'); }, () => base), true); + assert.strictEqual(classify(head, head), true, 'an absent substitution cannot exempt other edits'); + assert.strictEqual(classify(head + '\n', base), true, 'extra edits fail closed'); + assert.strictEqual(classify(head, base + base), true, 'ambiguous substitution fails closed'); + for (const status of ['A', 'D', 'R100', 'C100']) { + assert.strictEqual(classify(head, base, [{ status, paths: [file] }]), true, status); + } + assert.strictEqual(classify(head, base, [{ status: 'R100', paths: ['previous.sh', file] }]), true); + for (const [before, after] of [['209', '208'], ['49', '48'], ['--shard-count 8', '--shard-count 1'], + ['PUBLIC_TEST_AGENT_TOKEN', 'REPLACED_TOKEN'], ['orchestrator_failure=1', 'orchestrator_failure=0']]) { + assert.ok(head.includes(before)); + assert.strictEqual(classify(head.replace(before, after)), true, `${file}: ${before} must remain scoped`); + } + const mixed = [...changes, { status: 'M', paths: ['static/compliance/source/universal/security.yaml'] }]; + assert.strictEqual(classify(head, base, mixed), true, 'mixed protocol changes still require changesets'); + for (const content of [protocolChangeset, emptyChangeset]) { + const withChangeset = [...changes, { status: 'A', paths: ['.changeset/unnecessary.md'] }]; + assert.strictEqual(findChangesetProtocolScopeViolations(withChangeset, + readFiles({ [file]: head, '.changeset/unnecessary.md': content }), readFiles({ [file]: base })).length, 1, + 'operational substitution rejects gratuitous protocol and empty changesets'); + } + // A separately landed npm-ci substitution may be present identically on + // both sides, but this exemption cannot absorb it into the same delta. + assert.strictEqual(classify(head.replaceAll('run: npm ci', 'run: node .github/scripts/npm-ci.mjs'), + base.replaceAll('run: npm ci', 'run: node .github/scripts/npm-ci.mjs')), false); + if (file.endsWith('.yml')) assert.strictEqual(classify(head.replaceAll('run: npm ci', 'run: node .github/scripts/npm-ci.mjs')), true); + } + // Current main treats release.yml as operational, so use the still-scoped + // storyboard workflow to prove empty readers never exempt an edit. + assert.strictEqual(hasProtocolScopedChanges([{ status: 'M', paths: ['.github/workflows/training-agent-storyboards.yml'] }], () => '', () => ''), true); + assert.strictEqual(hasProtocolScopedChanges([{ status: 'M', paths: ['.github/workflows/release.yml'] }], () => '', () => ''), false); +} + +// Verify the actual CLI reads the diff's merge base, including when main has +// advanced independently. Neither query nor changeset rejection may regress to +// a path-only classification after the unit-level exemption succeeds. +{ + const fs = require('node:fs'); + const os = require('node:os'); + const path = require('node:path'); + const { execFileSync, spawnSync } = require('node:child_process'); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'creative-scope-cli-')); + const git = (...args) => execFileSync('git', args, { cwd: directory, stdio: 'pipe' }).toString().trim(); + const checker = path.resolve(__dirname, '../scripts/check-changeset-protocol-scope.cjs'); + const file = 'scripts/run-storyboards-matrix.sh'; + const head = fs.readFileSync(path.resolve(__dirname, '..', file), 'utf8'); + const base = head.replace('&& { [ "${tenant}" = "sales" ] || [ "${tenant}" = "creative" ]; }; }', + '&& [ "${tenant}" = "sales" ]; }'); + const check = (...args) => spawnSync(process.execPath, [checker, 'main', ...args], { cwd: directory, encoding: 'utf8' }); + try { + git('init', '-b', 'main'); + git('config', 'user.name', 'Scope fixture'); + git('config', 'user.email', 'scope@example.test'); + git('config', 'core.hooksPath', '/dev/null'); + fs.mkdirSync(path.join(directory, 'scripts')); + fs.writeFileSync(path.join(directory, file), base); + git('add', '.'); git('commit', '-m', 'base'); + git('checkout', '-b', 'candidate'); + fs.writeFileSync(path.join(directory, file), head); + git('add', '.'); git('commit', '-m', 'isolate creative'); + assert.strictEqual(check('--has-protocol-scoped-changes').status, 1); + assert.strictEqual(check().status, 0); + git('checkout', 'main'); + fs.appendFileSync(path.join(directory, file), '\n# independent main edit\n'); + git('add', '.'); git('commit', '-m', 'advance main'); + git('checkout', 'candidate'); + assert.match(check('--has-protocol-scoped-changes').stdout, /No protocol-scoped changes detected/); + fs.mkdirSync(path.join(directory, '.changeset')); + fs.writeFileSync(path.join(directory, '.changeset/unnecessary.md'), protocolChangeset); + git('add', '.'); git('commit', '-m', 'gratuitous bump'); + assert.strictEqual(check().status, 1); + fs.writeFileSync(path.join(directory, file), head.replace('creative:49:209', 'creative:49:208')); + git('add', '.'); git('commit', '-m', 'floor change'); + assert.strictEqual(check('--has-protocol-scoped-changes').status, 0); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } +} diff --git a/tests/fixtures/storyboard-isolation-runner.cjs b/tests/fixtures/storyboard-isolation-runner.cjs index f98cd3447c8..922c5a7cfa1 100644 --- a/tests/fixtures/storyboard-isolation-runner.cjs +++ b/tests/fixtures/storyboard-isolation-runner.cjs @@ -48,6 +48,17 @@ const result = { }, }; +if (storyboardId === 'creative_ad_server') { + result.summaries[0].passed = result.totals.passed = 6; + if (process.env.FIXTURE_CREATIVE_ERROR === '1') { + result.summaries[0].passed = result.totals.passed = 4; + result.summaries[0].failed = result.totals.failed = 1; + result.summaries[0].skipped = result.totals.skipped = 1; + result.summaries[0].has_error = true; + result.totals.clean = 0; + } +} + function spawnHangingDescendant() { const descendant = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore', @@ -119,7 +130,7 @@ if (storyboardId === 'hang') { console.log(`ADCP_STORYBOARD_RESULT ${JSON.stringify(result)}`); process.exit(0); } else { - console.log(` ${storyboardId.padEnd(40)} ✓ 1P / 0S / 0N/A`); + console.log(` ${storyboardId.padEnd(40)} ${result.totals.clean ? '✓' : '✗'} ${result.totals.passed}P / ${result.totals.failed}F / ${result.totals.skipped}S / 0N/A`); console.log(`ADCP_STORYBOARD_RESULT ${JSON.stringify(result)}`); process.exit(0); } diff --git a/tests/run-storyboards-sharding.test.cjs b/tests/run-storyboards-sharding.test.cjs index 6784d6b5fa4..94d6257990e 100644 --- a/tests/run-storyboards-sharding.test.cjs +++ b/tests/run-storyboards-sharding.test.cjs @@ -285,7 +285,7 @@ test('creative-builder uses bounded isolated children in local and CI matrices', assert.match( workflow, - /if \[ "\$\{\{ matrix\.tenant \}\}" = "creative-builder" \]; then\n\s+bash scripts\/run-storyboards-isolated-shards\.sh/, + /if \[ "\$\{\{ matrix\.tenant \}\}" = "creative-builder" \] \|\| \{[^\n]+; then\n\s+bash scripts\/run-storyboards-isolated-shards\.sh/, ); assert.match(workflow, /--shard-count 8 --max-parallel 4/); assert.match(workflow, /isolated_orchestrator_failure=0/); @@ -352,3 +352,207 @@ test('3.0 compatibility floors are capability-resolved and mirrored locally', () ); } }); + +// Exercise both unchanged graders with the real routing shell. The fixture +// emits exact boundary counts; no network or training-agent state is involved. +function makeCreativeGradingFixture(t) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'adcp-creative-grading-')); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + fs.mkdirSync(path.join(directory, 'scripts')); + fs.mkdirSync(path.join(directory, 'bin')); + const matrix = fs.readFileSync(MATRIX_RUNNER, 'utf8'); + fs.writeFileSync(path.join(directory, 'scripts/run-storyboards-matrix.sh'), matrix); + for (const kind of ['schemas', 'compliance']) { + fs.mkdirSync(path.join(directory, `dist/${kind}/latest`), { recursive: true }); + fs.writeFileSync(path.join(directory, `dist/${kind}/latest/index.json`), '{"adcp_version":"3.2.0-rc.3"}'); + fs.writeFileSync(path.join(directory, `scripts/build-${kind}.cjs`), ''); + } + const required = [...new Map([...matrix.matchAll(/^ "([^"\n]+)"$/gm)] + .map(match => [match[1].split(':')[0], match[1]])).values()]; + const fixture = `#!/usr/bin/env node +const fs = require('node:fs'); +const isolated = process.argv.includes('isolated'); +const creative = process.env.TENANT_PATH === 'creative'; +fs.appendFileSync(process.env.INVOCATIONS, JSON.stringify({ + tenant: process.env.TENANT_PATH, isolated, args: process.argv.slice(2), + token: process.env.PUBLIC_TEST_AGENT_TOKEN, compliance: process.env.ADCP_COMPLIANCE_DIR, + schema: process.env.ADCP_SCHEMA_ROOT, candidate: process.env.ADCP_STORYBOARD_CANDIDATE_VERSION_MODE, +}) + '\\n'); +for (const requirement of ${JSON.stringify(required)}) { + const [id, passed = '1', skipped = '0'] = requirement.split(':'); + console.log(' ' + id + ' ✓ ' + passed + 'P / ' + skipped + 'S / 0N/A'); +} +if (creative && process.env.OMIT_TOTALS === '1') process.exit(1); +console.log(' storyboards: ' + (creative ? process.env.CREATIVE_CLEAN : '1000') + '/1000 clean'); +console.log(' selection: 59 applicable | 143 not applicable | 2 quarantined | 204 corpus'); +console.log(' steps: ' + (creative ? process.env.CREATIVE_PASSED : '1000') + ' passed | 1 failed | 96 skipped | 0 not applicable'); +process.exit(creative ? Number(process.env.CREATIVE_EXIT) : 0); +`; + const runner = path.join(directory, 'fixture.cjs'); + fs.writeFileSync(runner, fixture); + fs.writeFileSync(path.join(directory, 'scripts/run-storyboards-isolated-shards.sh'), 'exec node ./fixture.cjs isolated "$@"\n'); + const npx = path.join(directory, 'bin/npx'); + fs.writeFileSync(npx, '#!/usr/bin/env bash\nexec node ./fixture.cjs monolithic "$@"\n', { mode: 0o755 }); + const env = { + ...process.env, PATH: `${directory}/bin:${process.env.PATH}`, + INVOCATIONS: path.join(directory, 'invocations.jsonl'), + GITHUB_OUTPUT: path.join(directory, 'outputs'), GITHUB_STEP_SUMMARY: path.join(directory, 'summary'), + PUBLIC_TEST_AGENT_TOKEN: 'fixture-token', ADCP_STORYBOARD_CANDIDATE_VERSION_MODE: '1', + ADCP_SCHEMA_ROOT: path.join(directory, 'dist/schemas/latest'), + ADCP_COMPLIANCE_DIR: path.join(directory, 'dist/compliance/latest'), + CREATIVE_CLEAN: '49', CREATIVE_PASSED: '209', CREATIVE_EXIT: '0', + // The workflow always provides STORYBOARD_ID (empty for whole-tenant rows) + // and its run block uses `set -u`, so the mock environment must too. + STORYBOARD_ID: '', + }; + return { directory, env }; +} + +const workflowDefinition = require('yaml').parse(fs.readFileSync(STORYBOARD_WORKFLOW, 'utf8')); +const workflowSteps = workflowDefinition.jobs.storyboards.steps; +function workflowScript(step, directory, tenant = 'creative', surface = 'current') { + return step.run.replaceAll('${{ matrix.tenant }}', tenant) + .replaceAll('${{ matrix.surface }}', surface) + .replaceAll('${{ steps.compat-bundle.outputs.version }}', '3.0.26') + .replaceAll('/tmp/storyboards.log', path.join(directory, 'storyboards.log')); +} + +for (const target of ['matrix', 'workflow']) { + test(`${target} isolates current /creative and preserves exact floors, counts, environment and failures`, (t) => { + const { directory, env } = makeCreativeGradingFixture(t); + const run = overrides => spawnSync('bash', target === 'matrix' + ? ['scripts/run-storyboards-matrix.sh'] + : ['-c', workflowScript(workflowSteps.find(step => step.id === 'run'), directory)], { + cwd: directory, encoding: 'utf8', timeout: 10000, + env: { ...env, TENANT_PATH: 'creative', ...overrides }, + }); + const success = run({}); + assert.equal(success.status, 0, success.stderr + success.stdout); + const invocations = fs.readFileSync(env.INVOCATIONS, 'utf8').trim().split('\n').map(JSON.parse); + const creative = invocations.find(entry => entry.tenant === 'creative'); + assert.equal(creative.isolated, true); + assert.deepEqual(creative.args, ['isolated', '--shard-count', '8', '--max-parallel', target === 'matrix' ? '2' : '4', + ...(target === 'matrix' ? ['--timeout-ms', '180000'] : [])]); + assert.equal(creative.token, 'fixture-token'); + assert.equal(creative.schema, env.ADCP_SCHEMA_ROOT); + assert.equal(creative.compliance, env.ADCP_COMPLIANCE_DIR); + assert.equal(creative.candidate, '1'); + if (target === 'matrix') { + assert.match(success.stdout, /\/creative: ✓ 49 clean, 209 steps/); + assert.equal(invocations.length, 7); + assert.deepEqual(invocations.filter(entry => entry.isolated).map(entry => entry.tenant), ['sales', 'creative', 'creative-builder']); + } else { + assert.equal(fs.readFileSync(env.GITHUB_OUTPUT, 'utf8'), + 'clean=49\ntotal=1000\napplicable=59\nscope_not_applicable=143\nquarantined=2\ncorpus=204\npassed=209\nfailed=1\nskipped=96\nnot_applicable=0\n'); + } + for (const overrides of [{ CREATIVE_EXIT: '1' }, { OMIT_TOTALS: '1' }]) { + const failed = run(overrides); + assert.equal(failed.status, 1, failed.stderr + failed.stdout); + if (target === 'matrix') assert.match(failed.stdout, /\/si: ✓/); + } + for (const [clean, passed, expected] of [['49', '209', 0], ['48', '209', 1], ['49', '208', 1]]) { + const graded = target === 'matrix' ? run({ CREATIVE_CLEAN: clean, CREATIVE_PASSED: passed }) + : spawnSync('bash', ['-c', workflowScript(workflowSteps.find(step => step.name?.startsWith('Enforce non-regression')), directory)], { + cwd: directory, encoding: 'utf8', env: { ...env, CLEAN: clean, PASSED: passed, MIN_CLEAN: '49', MIN_PASSED: '209' }, + }); + assert.equal(graded.status, expected, graded.stderr + graded.stdout); + } + }); +} + +test('workflow preserves creative job selection, floors, required-clean checks and failure artifact', () => { + const job = workflowDefinition.jobs.storyboards; + assert.equal(job.strategy['fail-fast'], false); + assert.deepEqual(job.strategy.matrix.include.find(row => row.surface === 'current' && row.tenant === 'creative'), + { surface: 'current', tenant: 'creative', min_clean_storyboards: 49, min_passing_steps: 209 }); + assert.equal(job.strategy.matrix.exclude.some(row => row.tenant === 'creative'), false); + const run = workflowSteps.find(step => step.id === 'run'); + assert.equal(run.env.TENANT_PATH, '${{ matrix.tenant_path || matrix.tenant }}'); + assert.equal(run.env.STORYBOARD_ID, '${{ matrix.storyboard_id }}'); + assert.equal(run.env.PUBLIC_TEST_AGENT_TOKEN, 'storyboard-ci-token'); + assert.match(run.env.ADCP_COMPLIANCE_DIR, /dist\/compliance\/latest/); + assert.match(run.env.ADCP_SCHEMA_ROOT, /dist\/schemas\/latest/); + assert.match(run.env.ADCP_STORYBOARD_CANDIDATE_VERSION_MODE, /matrix.surface == 'current'/); + const artifact = workflowSteps.find(step => step.name === 'Upload storyboards log on failure'); + assert.equal(artifact.if, 'failure()'); + assert.deepEqual(artifact.with, { name: 'storyboards-log-${{ matrix.surface }}-${{ matrix.tenant }}', + path: '/tmp/storyboards.log', 'retention-days': 14, 'if-no-files-found': 'ignore' }); + const required = workflowSteps.find(step => step.name === 'Enforce creative required-clean storyboards'); + assert.match(required.run, /"canonical_format_validate_input"\n\s+"creative\/billing_out_of_band"/); + assert.match(required.run, /exit 1/); +}); + +test('workflow leaves 3.0 creative monolithic and preserves creative-builder isolation on both surfaces', (t) => { + const { directory, env } = makeCreativeGradingFixture(t); + for (const [tenant, surface, isolated] of [['creative', '3.0-compat', false], ['creative-builder', 'current', true], ['creative-builder', '3.0-compat', true]]) { + const result = spawnSync('bash', ['-c', workflowScript(workflowSteps.find(step => step.id === 'run'), directory, tenant, surface)], { + cwd: directory, encoding: 'utf8', env: { ...env, TENANT_PATH: tenant }, + }); + assert.equal(result.status, 0, result.stderr + result.stdout); + const invocation = JSON.parse(fs.readFileSync(env.INVOCATIONS, 'utf8').trim().split('\n').at(-1)); + assert.equal(invocation.isolated, isolated); + } +}); + +test('workflow profile rows keep their selected storyboard and are never captured by creative isolation', (t) => { + const rows = workflowDefinition.jobs.storyboards.strategy.matrix.include; + const profileRows = rows.filter(row => row.storyboard_id); + assert.ok(profileRows.length >= 2, 'profile rows are present in the matrix'); + // Isolation keys on the tenant name, so no profile row may be named creative. + for (const row of rows.filter(entry => entry.tenant === 'creative')) { + assert.equal(row.storyboard_id, undefined); + assert.equal(row.tenant_path, undefined); + } + const { directory, env } = makeCreativeGradingFixture(t); + for (const row of profileRows) { + assert.notEqual(row.tenant_path, row.tenant); + fs.rmSync(env.INVOCATIONS, { force: true }); + const result = spawnSync('bash', ['-c', workflowScript(workflowSteps.find(step => step.id === 'run'), directory, row.tenant, row.surface)], { + cwd: directory, encoding: 'utf8', + env: { ...env, TENANT_PATH: row.tenant_path, STORYBOARD_ID: row.storyboard_id }, + }); + assert.equal(result.status, 0, result.stderr + result.stdout); + const invocation = JSON.parse(fs.readFileSync(env.INVOCATIONS, 'utf8').trim().split('\n').at(-1)); + assert.equal(invocation.isolated, false, `${row.tenant} must stay monolithic`); + assert.equal(invocation.tenant, row.tenant_path); + assert.deepEqual(invocation.args.slice(-2), ['--storyboard-id', row.storyboard_id]); + } +}); + +for (const failure of ['', 'crash', 'hang', 'malformed_result', 'inconsistent_result']) { + test(`creative isolated shards preserve exact child results and siblings: ${failure || 'success'}`, () => { + const ids = ['healthy_before', 'creative_ad_server', ...(failure ? [failure] : []), 'healthy_after']; + const result = spawnSync('bash', [ISOLATED_SHARDED_RUNNER, '--shard-count', '2', '--max-parallel', '2', '--timeout-ms', '1500'], { + cwd: REPO_ROOT, encoding: 'utf8', timeout: 15000, + env: { ...process.env, TENANT_PATH: 'creative', STORYBOARD_RUNNER_BIN: path.join(REPO_ROOT, 'tests/fixtures/storyboard-isolation-runner.cjs'), + FIXTURE_STORYBOARDS: ids.join(',') }, + }); + assert.equal(result.status, failure ? 1 : 0, result.stderr + result.stdout); + assert.match(result.stdout, new RegExp(`^ storyboards: 3/${ids.length} clean$`, 'm')); + assert.match(result.stdout, /^ steps: 8 passed \| 0 failed \| 0 skipped \| 0 not applicable$/m); + assert.match(result.stdout, /^ healthy_before\s+✓/m); + assert.match(result.stdout, /^ healthy_after\s+✓/m); + if (failure) assert.match(result.stderr, /Isolated shard .* exited 1/); + }); +} + +test('creative child task errors propagate exact failed and skipped counts through shards', () => { + const result = spawnSync('bash', [ISOLATED_SHARDED_RUNNER, '--shard-count', '2', '--max-parallel', '2'], { + cwd: REPO_ROOT, encoding: 'utf8', timeout: 15000, + env: { ...process.env, TENANT_PATH: 'creative', STORYBOARD_RUNNER_BIN: path.join(REPO_ROOT, 'tests/fixtures/storyboard-isolation-runner.cjs'), + FIXTURE_STORYBOARDS: 'healthy_before,creative_ad_server,healthy_after', FIXTURE_CREATIVE_ERROR: '1' }, + }); + // Task results are graded by the unchanged floors; infrastructure failures + // additionally fail the orchestrator even when aggregate floors are met. + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /^ storyboards: 2\/3 clean$/m); + assert.match(result.stdout, /^ steps: 6 passed \| 1 failed \| 1 skipped \| 0 not applicable$/m); + assert.match(result.stdout, /^ healthy_after\s+✓/m); +}); + +test('registered storyboard suite runs the fail-closed operational changeset scope regressions', () => { + const result = spawnSync(process.execPath, [path.join(REPO_ROOT, 'tests/changeset-protocol-scope.test.cjs')], { + cwd: REPO_ROOT, encoding: 'utf8', timeout: 10000, + }); + assert.equal(result.status, 0, result.stderr + result.stdout); +});