Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/training-agent-storyboards.yml
Original file line number Diff line number Diff line change
Expand Up @@ -279,10 +279,10 @@ jobs:
# Creative-builder retains a large generated-schema graph after its
# result is complete on some Node/V8 builds. Run each storyboard in
# a bounded child so teardown cannot retain the whole job. Other
# matrix tenants remain monolithic; current /sales has its separate
# matrix tenants except current /creative remain monolithic; current /sales has its separate
# isolated aggregate jobs below.
isolated_orchestrator_failure=0
if [ "${{ matrix.tenant }}" = "creative-builder" ]; then
if [ "${{ matrix.tenant }}" = "creative-builder" ] || { [ "${{ matrix.surface }}" = "current" ] && [ "${{ matrix.tenant }}" = "creative" ]; }; then
bash scripts/run-storyboards-isolated-shards.sh \
--shard-count 8 --max-parallel 4 2>&1 \
| tee /tmp/storyboards.log || isolated_orchestrator_failure=1
Expand Down Expand Up @@ -336,7 +336,7 @@ jobs:
} >> "$GITHUB_STEP_SUMMARY"

if [ "${isolated_orchestrator_failure}" -ne 0 ]; then
echo "::error::One or more isolated creative-builder orchestrators failed. Counts above are diagnostic only."
echo "::error::One or more isolated ${{ matrix.tenant }} orchestrators failed. Counts above are diagnostic only."
exit 1
fi

Expand Down
69 changes: 61 additions & 8 deletions scripts/check-changeset-protocol-scope.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -103,14 +103,65 @@ function isStoryboardBranchRegistrationOnly(headContent, baseContent) {
&& headContent === baseContent.replaceAll(oldBranches, newBranches);
}

// Only these complete operational substitutions are exempt. Any accompanying
// edit (floors, selection, environment, failure policy, etc.) stays scoped.
const CREATIVE_ISOLATION_SUBSTITUTIONS = new Map(Object.entries({
"scripts/run-storyboards-matrix.sh": [
[
" if { [ \"${FLOOR_SET}\" = \"current\" ] && [ \"${tenant}\" = \"sales\" ]; } \\\n",
" if { [ \"${FLOOR_SET}\" = \"current\" ] && { [ \"${tenant}\" = \"sales\" ] || [ \"${tenant}\" = \"creative\" ]; }; } \\\n"
]
],
".github/workflows/training-agent-storyboards.yml": [
[
" # matrix tenants remain monolithic; current /sales has its separate\n",
" # matrix tenants except current /creative remain monolithic; current /sales has its separate\n"
],
[
" if [ \"${{ matrix.tenant }}\" = \"creative-builder\" ]; then\n",
" if [ \"${{ matrix.tenant }}\" = \"creative-builder\" ] || { [ \"${{ matrix.surface }}\" = \"current\" ] && [ \"${{ matrix.tenant }}\" = \"creative\" ]; }; then\n"
],
[
" echo \"::error::One or more isolated creative-builder orchestrators failed. Counts above are diagnostic only.\"\n",
" echo \"::error::One or more isolated ${{ matrix.tenant }} orchestrators failed. Counts above are diagnostic only.\"\n"
]
]
}));

function isCreativeIsolationOnlyChange(change, readFileAtHead, readFileAtBase) {
if (change.status !== 'M' || change.paths?.length !== 1) return false;
const filePath = normalizePath(change.paths[0]);
const substitutions = CREATIVE_ISOLATION_SUBSTITUTIONS.get(filePath);
if (!substitutions || !readFileAtHead || !readFileAtBase) return false;
try {
let expected = readFileAtBase(filePath);
const head = readFileAtHead(filePath);
if (typeof expected !== 'string' || !expected || typeof head !== 'string') return false;
for (const [before, after] of substitutions) {
if (expected.split(before).length !== 2) return false;
expected = expected.replace(before, after);
}
return expected === head;
} catch {
return false;
}
}

function isStoryboardBranchRegistrationChange(change, filePath, readFileAtHead, readFileAtBase) {
if (normalizePath(filePath) !== '.github/workflows/training-agent-storyboards.yml'
|| change.status !== 'M') return false;
try {
return isStoryboardBranchRegistrationOnly(readFileAtHead(filePath), readFileAtBase(filePath));
} catch {
return false;
}
}

function isProtocolScopedChange(change, readFileAtHead, readFileAtBase) {
if (isCreativeIsolationOnlyChange(change, readFileAtHead, readFileAtBase)) return false;
return (change.paths || []).some(filePath => {
if (!isProtocolScopedPath(filePath)) return false;
if (normalizePath(filePath) === '.github/workflows/training-agent-storyboards.yml'
&& change.status === 'M'
&& isStoryboardBranchRegistrationOnly(
readFileAtHead(filePath), readFileAtBase(filePath)
)) return false;
if (isStoryboardBranchRegistrationChange(change, filePath, readFileAtHead, readFileAtBase)) return false;
return true;
});
}
Expand Down Expand Up @@ -306,7 +357,9 @@ function formatViolationMessage(violations) {

function run(argv = process.argv.slice(2)) {
const baseRef = argv[0] || defaultBaseRef();
const diffOutput = git(['diff', '--name-status', '--find-renames', `${baseRef}...HEAD`]);
const mergeBase = git(['merge-base', baseRef, 'HEAD']).trim();
const readFileAtBase = filePath => readFileAtRef(mergeBase, filePath);
const diffOutput = git(['diff', '--name-status', '--find-renames', `${mergeBase}...HEAD`]);
const changes = parseNameStatus(diffOutput);

if (argv.includes('--is-delete-only-cleanup')) {
Expand All @@ -333,7 +386,7 @@ function run(argv = process.argv.slice(2)) {
const hasProtocol = hasProtocolScopedChanges(
changes,
readFileAtHead,
filePath => readFileAtRef(baseRef, filePath)
readFileAtBase
);
if (hasProtocol) {
console.log('Protocol-scoped changes detected.');
Expand All @@ -346,7 +399,7 @@ function run(argv = process.argv.slice(2)) {
const violations = findChangesetProtocolScopeViolations(
changes,
readFileAtHead,
filePath => readFileAtRef(baseRef, filePath)
readFileAtBase
);

if (violations.length > 0) {
Expand Down
2 changes: 1 addition & 1 deletion scripts/run-storyboards-matrix.sh
Original file line number Diff line number Diff line change
Expand Up @@ -417,7 +417,7 @@ for entry in "${TENANTS[@]}"; do
log=$(mktemp -t "storyboards-${tenant}.XXXXXX.log")
orchestrator_failure=0

if { [ "${FLOOR_SET}" = "current" ] && [ "${tenant}" = "sales" ]; } \
if { [ "${FLOOR_SET}" = "current" ] && { [ "${tenant}" = "sales" ] || [ "${tenant}" = "creative" ]; }; } \
|| [ "${tenant}" = "creative-builder" ]; then
TENANT_PATH="${tenant}" \
PUBLIC_TEST_AGENT_TOKEN="${PUBLIC_TEST_AGENT_TOKEN:-storyboard-local-token}" \
Expand Down
110 changes: 110 additions & 0 deletions tests/changeset-protocol-scope.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -368,3 +368,113 @@ assert.strictEqual(
);

console.log('Changeset protocol scope tests passed.');

// Complete before/after contents are required for the exact creative runner
// substitution. Unrelated edits in these operational files remain scoped.
{
const fs = require('node:fs');
const path = require('node:path');
const replacements = {
'scripts/run-storyboards-matrix.sh': [[
'&& { [ "${tenant}" = "sales" ] || [ "${tenant}" = "creative" ]; }; }',
'&& [ "${tenant}" = "sales" ]; }',
]],
'.github/workflows/training-agent-storyboards.yml': [
['# matrix tenants except current /creative remain monolithic;', '# matrix tenants remain monolithic;'],
['if [ "${{ matrix.tenant }}" = "creative-builder" ] || { [ "${{ matrix.surface }}" = "current" ] && [ "${{ matrix.tenant }}" = "creative" ]; }; then',
'if [ "${{ matrix.tenant }}" = "creative-builder" ]; then'],
['One or more isolated ${{ matrix.tenant }} orchestrators failed.', 'One or more isolated creative-builder orchestrators failed.'],
],
};
for (const [file, substitutions] of Object.entries(replacements)) {
const head = fs.readFileSync(path.join(__dirname, '..', file), 'utf8');
let base = head;
for (const [after, before] of substitutions) {
assert.ok(base.includes(after));
base = base.replace(after, before);
}
const changes = [{ status: 'M', paths: [file] }];
const classify = (candidate = head, prior = base, delta = changes) =>
hasProtocolScopedChanges(delta, readFiles({ [file]: candidate }), readFiles({ [file]: prior }));
assert.strictEqual(classify(), false, `${file}: exact isolation substitution is operational`);
assert.strictEqual(hasProtocolScopedChanges(changes), true, 'path-only queries fail closed');
assert.strictEqual(classify(head, ''), true, 'missing base fails closed');
assert.strictEqual(classify('', base), true, 'missing head fails closed');
assert.strictEqual(hasProtocolScopedChanges(changes, () => { throw new Error('unreadable'); }, () => base), true);
assert.strictEqual(classify(head, head), true, 'an absent substitution cannot exempt other edits');
assert.strictEqual(classify(head + '\n', base), true, 'extra edits fail closed');
assert.strictEqual(classify(head, base + base), true, 'ambiguous substitution fails closed');
for (const status of ['A', 'D', 'R100', 'C100']) {
assert.strictEqual(classify(head, base, [{ status, paths: [file] }]), true, status);
}
assert.strictEqual(classify(head, base, [{ status: 'R100', paths: ['previous.sh', file] }]), true);
for (const [before, after] of [['209', '208'], ['49', '48'], ['--shard-count 8', '--shard-count 1'],
['PUBLIC_TEST_AGENT_TOKEN', 'REPLACED_TOKEN'], ['orchestrator_failure=1', 'orchestrator_failure=0']]) {
assert.ok(head.includes(before));
assert.strictEqual(classify(head.replace(before, after)), true, `${file}: ${before} must remain scoped`);
}
const mixed = [...changes, { status: 'M', paths: ['static/compliance/source/universal/security.yaml'] }];
assert.strictEqual(classify(head, base, mixed), true, 'mixed protocol changes still require changesets');
for (const content of [protocolChangeset, emptyChangeset]) {
const withChangeset = [...changes, { status: 'A', paths: ['.changeset/unnecessary.md'] }];
assert.strictEqual(findChangesetProtocolScopeViolations(withChangeset,
readFiles({ [file]: head, '.changeset/unnecessary.md': content }), readFiles({ [file]: base })).length, 1,
'operational substitution rejects gratuitous protocol and empty changesets');
}
// A separately landed npm-ci substitution may be present identically on
// both sides, but this exemption cannot absorb it into the same delta.
assert.strictEqual(classify(head.replaceAll('run: npm ci', 'run: node .github/scripts/npm-ci.mjs'),
base.replaceAll('run: npm ci', 'run: node .github/scripts/npm-ci.mjs')), false);
if (file.endsWith('.yml')) assert.strictEqual(classify(head.replaceAll('run: npm ci', 'run: node .github/scripts/npm-ci.mjs')), true);
}
// Current main treats release.yml as operational, so use the still-scoped
// storyboard workflow to prove empty readers never exempt an edit.
assert.strictEqual(hasProtocolScopedChanges([{ status: 'M', paths: ['.github/workflows/training-agent-storyboards.yml'] }], () => '', () => ''), true);
assert.strictEqual(hasProtocolScopedChanges([{ status: 'M', paths: ['.github/workflows/release.yml'] }], () => '', () => ''), false);
}

// Verify the actual CLI reads the diff's merge base, including when main has
// advanced independently. Neither query nor changeset rejection may regress to
// a path-only classification after the unit-level exemption succeeds.
{
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { execFileSync, spawnSync } = require('node:child_process');
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'creative-scope-cli-'));
const git = (...args) => execFileSync('git', args, { cwd: directory, stdio: 'pipe' }).toString().trim();
const checker = path.resolve(__dirname, '../scripts/check-changeset-protocol-scope.cjs');
const file = 'scripts/run-storyboards-matrix.sh';
const head = fs.readFileSync(path.resolve(__dirname, '..', file), 'utf8');
const base = head.replace('&& { [ "${tenant}" = "sales" ] || [ "${tenant}" = "creative" ]; }; }',
'&& [ "${tenant}" = "sales" ]; }');
const check = (...args) => spawnSync(process.execPath, [checker, 'main', ...args], { cwd: directory, encoding: 'utf8' });
try {
git('init', '-b', 'main');
git('config', 'user.name', 'Scope fixture');
git('config', 'user.email', 'scope@example.test');
git('config', 'core.hooksPath', '/dev/null');
fs.mkdirSync(path.join(directory, 'scripts'));
fs.writeFileSync(path.join(directory, file), base);
git('add', '.'); git('commit', '-m', 'base');
git('checkout', '-b', 'candidate');
fs.writeFileSync(path.join(directory, file), head);
git('add', '.'); git('commit', '-m', 'isolate creative');
assert.strictEqual(check('--has-protocol-scoped-changes').status, 1);
assert.strictEqual(check().status, 0);
git('checkout', 'main');
fs.appendFileSync(path.join(directory, file), '\n# independent main edit\n');
git('add', '.'); git('commit', '-m', 'advance main');
git('checkout', 'candidate');
assert.match(check('--has-protocol-scoped-changes').stdout, /No protocol-scoped changes detected/);
fs.mkdirSync(path.join(directory, '.changeset'));
fs.writeFileSync(path.join(directory, '.changeset/unnecessary.md'), protocolChangeset);
git('add', '.'); git('commit', '-m', 'gratuitous bump');
assert.strictEqual(check().status, 1);
fs.writeFileSync(path.join(directory, file), head.replace('creative:49:209', 'creative:49:208'));
git('add', '.'); git('commit', '-m', 'floor change');
assert.strictEqual(check('--has-protocol-scoped-changes').status, 0);
} finally {
fs.rmSync(directory, { recursive: true, force: true });
}
}
13 changes: 12 additions & 1 deletion tests/fixtures/storyboard-isolation-runner.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,17 @@ const result = {
},
};

if (storyboardId === 'creative_ad_server') {
result.summaries[0].passed = result.totals.passed = 6;
if (process.env.FIXTURE_CREATIVE_ERROR === '1') {
result.summaries[0].passed = result.totals.passed = 4;
result.summaries[0].failed = result.totals.failed = 1;
result.summaries[0].skipped = result.totals.skipped = 1;
result.summaries[0].has_error = true;
result.totals.clean = 0;
}
}

function spawnHangingDescendant() {
const descendant = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], {
stdio: 'ignore',
Expand Down Expand Up @@ -119,7 +130,7 @@ if (storyboardId === 'hang') {
console.log(`ADCP_STORYBOARD_RESULT ${JSON.stringify(result)}`);
process.exit(0);
} else {
console.log(` ${storyboardId.padEnd(40)} ✓ 1P / 0S / 0N/A`);
console.log(` ${storyboardId.padEnd(40)} ${result.totals.clean ? '✓' : '✗'} ${result.totals.passed}P / ${result.totals.failed}F / ${result.totals.skipped}S / 0N/A`);
console.log(`ADCP_STORYBOARD_RESULT ${JSON.stringify(result)}`);
process.exit(0);
}
Loading
Loading