From d4116b6492c081e1152b38215380bf1ba964db73 Mon Sep 17 00:00:00 2001 From: JUN Date: Wed, 11 Nov 2026 11:48:59 +0900 Subject: [PATCH] ci: run the unsharded macOS control only on its own lane lane=all, the release evidence run, also started macos control: the whole suite again, unsharded, on one macOS worker, after the four macOS shards had already passed it on the same commit. It is the longest job in the workflow at 50 to 60 minutes, so every release waited on a duplicate, and it took a sixth macOS job against five runner slots. macos control now runs only on an explicit lane=macos-control dispatch (and the retained schedule branch). The aggregate gate expects it only there, release evidence no longer requires it, and a new approved policy entry pins the changed ci.yml and release-evidence.cjs blobs. The shards, Windows, Linux and every other release gate are unchanged. --- .github/scripts/release-evidence-policy.json | 32 +++++++++++++++++++ .github/scripts/release-evidence.cjs | 2 +- .github/workflows/ci.yml | 22 +++++++------ AGENTS.md | 2 +- structure/ops/cross-platform-ci.md | 8 +++-- structure/ops/docs-and-release.md | 2 +- tests/ci-workflows/ci-review-lanes.test.ts | 12 ++++--- tests/ci-workflows/ci-scope-reduction.test.ts | 2 +- tests/ci-workflows/ci-workflows.test.ts | 2 +- 9 files changed, 64 insertions(+), 20 deletions(-) diff --git a/.github/scripts/release-evidence-policy.json b/.github/scripts/release-evidence-policy.json index 835a8e15a4c..f542e337f03 100644 --- a/.github/scripts/release-evidence-policy.json +++ b/.github/scripts/release-evidence-policy.json @@ -302,5 +302,37 @@ "desktop/scripts/appimage-patchelf.py": "63e78ef74160bc48fbcdd882065f1d416db6d13c", "desktop/scripts/linux-packaged-e2e.ts": "9ca92352668ff881c85caea44a9b4747bedc7e2d" } + }, + { + "kind": "ci", + "version": 1, + "files": { + ".github/workflows/ci.yml": "42ee548c7b1d968d2435945846c69017b5bd5163", + ".github/actions/setup-project-bun/action.yml": "cf28f6f9d4385d69784f4e3610914f6952ce789d", + ".github/scripts/release-evidence.cjs": "e89d944b5f0f6208fc8f74a750c65e013ba4b4b3", + ".github/scripts/release-evidence.test.cjs": "de30ad37105cf0e7dd8c538efb5ee1cfafe02990", + "scripts/ci/select-test-lane.sh": "b25836cb55f962d1ffd9e68ed00f45f8b45becc8", + "scripts/ci/windows-sensitive-diff.sh": "03495f256c671aff57e750febffe19ca7face6a1", + "scripts/ci/run-bun-test-batches.sh": "f8aa18e80aff1c95c584cd2c879dfbb6806c11fb", + "scripts/ci/bun-crash-signatures.sh": "d34662ddd07864d97bed909135095db33fe42c9e", + "scripts/ci/bun-install-frozen.sh": "e6d7bc3aec044e12297c7c4c04015a33df0eb22b", + "scripts/ci/docker-smoke.ts": "a8424f93986a31625a0a236cdab5667db67b3405", + "scripts/ci/test-durations.tsv": "6ff82545debbd95286928a7a99abfdeb10020a7a", + "scripts/ci/test-durations.windows.tsv": "18c65823d855bb592cdd59bd102399bd9ead3967", + "scripts/test.ts": "363bf072c4b689478fdbb76a61bf4eadbc054548", + "scripts/lib/test-runner-bun.ts": "8763806104a851dbdefeea8cffe8d7dd8d1c8da1", + "scripts/test-temp-lock.ts": "ee83fb254b728de119028f52b9894bf52d7aea15", + "scripts/ci/run-changed-tests.sh": "9a54d3a5db9849bcbe0dd766954a840185419fa5", + "scripts/keyring-smoke.ts": "cd3be07ad57796172696dfa053ea1fb129f29365", + "scripts/privacy-scan.ts": "86314e891b45e49739b94c76cddd9e4cc2d4340f", + "scripts/structure-ssot.ts": "40ba68401df9b77156afed66d6a11c61df8e9a41", + "scripts/generate-ocx-skill-surface.ts": "155e866656ba2d71e8fd48adda857451caf16322", + "desktop/scripts/prepare-sidecar.ts": "20f7eec61f73635ea681e25f13dd850b62bb41e2", + "desktop/scripts/build-widget.sh": "38e0eb9d713804b603aa75c32383aa18ef01d06c", + "desktop/scripts/verify-macos-runtime.sh": "43f1a31375c6f6fe7dcafb9d96c231b98d1f6ba4", + "desktop/scripts/verify-linux-sidecar.sh": "c502bb5c8ba8346c722a8f5274f2274bb325a934", + "desktop/scripts/appimage-patchelf.py": "63e78ef74160bc48fbcdd882065f1d416db6d13c", + "desktop/scripts/linux-packaged-e2e.ts": "9ca92352668ff881c85caea44a9b4747bedc7e2d" + } } ] diff --git a/.github/scripts/release-evidence.cjs b/.github/scripts/release-evidence.cjs index c4cd8c762ab..e89d944b5f0 100644 --- a/.github/scripts/release-evidence.cjs +++ b/.github/scripts/release-evidence.cjs @@ -21,7 +21,7 @@ const CI_JOBS = Object.freeze([ 'changes', 'select windows runner', 'storage policy', 'api usage', 'gates', ...Array.from({length: 5}, (_, i) => `test ${i + 1}/5`), ...Array.from({length: 4}, (_, i) => `macos ${i + 1}/4`), - 'macos control', ...Array.from({length: 11}, (_, i) => `windows ${i + 1}/11`), + ...Array.from({length: 11}, (_, i) => `windows ${i + 1}/11`), 'keyring ubuntu', 'keyring windows', 'keyring macos', 'docker smoke', 'docs site build', 'structure gate', 'macos widget + bundle', 'desktop shell', ...['ubuntu-latest', 'windows-latest', 'macos-latest'].flatMap(os => [ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc5496ad7e6..42ee548c7b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -871,10 +871,10 @@ jobs: # (123090589363) batch 93/93 before this cap cancelled them, with no failing # tests. The cap stays 20: the suite needs another shard, not a longer wait, # and a wedged shard (#4956) is still cut off as early as before. Four suite - # shards + control + widget/bundle request six macOS jobs for five runner - # slots: one queues, and each run pays one extra shard's setup overhead. - # The full-membership control runs the same bounded batches without sharding - # on explicit dispatch. + # shards + widget/bundle request five macOS jobs for five runner slots; each + # run pays one extra shard's setup overhead. The full-membership control runs + # the same bounded batches without sharding, only on a lane=macos-control + # dispatch. timeout-minutes: 20 strategy: fail-fast: false @@ -940,10 +940,13 @@ jobs: macos-control: name: macos control needs: changes - # Diagnostic-only lanes stay off the ordinary release gates: list every lane - # that should run this job, so a new lane defaults to skipping the diagnostic. + # Diagnostic-only: it runs on an explicit lane=macos-control dispatch and nowhere + # else, so a new lane defaults to skipping it. It used to ride lane=all as well, + # where it reran the whole suite that the four macOS shards had already passed + # on the same commit, on one worker, for 50-60 minutes, and every release waited + # for it. Dispatch it on its own when a sharding-sensitive defect is suspected. if: >- - github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && (github.event.inputs.lane == '' || github.event.inputs.lane == 'all' || github.event.inputs.lane == 'macos-control') + github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && github.event.inputs.lane == 'macos-control' runs-on: macos-latest # Unsharded full-membership control, one worker in sequential fresh batches. # Long-lived Bun isolate pools repeatedly wedged while synchronously reaping @@ -1852,8 +1855,9 @@ jobs: dispatch=requested windows=requested elif [ "$EVENT_NAME" = "workflow_dispatch" ]; then - # Mirror the diagnostic job allowlist; release-gates requests neither suite. - if [ -z "$LANE" ] || [ "$LANE" = "all" ] || [ "$LANE" = "macos-control" ]; then + # Mirror the diagnostic job allowlist: only lane=macos-control requests the + # unsharded control; release-gates requests neither suite. + if [ "$LANE" = "macos-control" ]; then dispatch=requested fi # Only the default/all lane requests the Windows suite shards. diff --git a/AGENTS.md b/AGENTS.md index ea56c79a958..285ca030e6b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -377,7 +377,7 @@ than nudged. `workflow_dispatch` on `dev`. A successful complete `lane=all` run establishes evidence for its exact dev commit; a PR run or narrower manual lane does not. - A manual `lane=all` dispatch is a **batch check on `dev`**, never a per-PR gate. - One run is about 32 jobs (11 Windows, at least 6 macOS, 15 Linux) and six runner-hours, and the + One run is about 31 jobs (11 Windows, at least 5 macOS, 15 Linux) and five runner-hours, and the account has only five macOS slots (a macOS job can queue), so per-PR dispatches starve every other check. On 2026-10-27, parallel agent lanes dispatched 48 of them in twelve hours (about 13,000 runner-minutes). The queue passed 200 runs, and ordinary PR checks waited hours to start. diff --git a/structure/ops/cross-platform-ci.md b/structure/ops/cross-platform-ci.md index 8791d2646d5..7332e0a217a 100644 --- a/structure/ops/cross-platform-ci.md +++ b/structure/ops/cross-platform-ci.md @@ -30,8 +30,12 @@ so every pull request scans exactly once and the coverage does not depend on an list. The aggregate derives the same expectation from its `scoped` result. The macOS suite has four duration-weighted shards (`macos N/4`, `TEST_SHARD=N/4`), each capped -at 20 minutes. With the control and widget/bundle job, `lane=all` requests at least six macOS jobs -for five runner slots; a `ci.yml` change also selects the setup-action macOS leg. +at 20 minutes. With the widget/bundle job, `lane=all` requests at least five macOS jobs for five +runner slots; a `ci.yml` change also selects the setup-action macOS leg. The unsharded `macos +control` runs only on a `lane=macos-control` dispatch: on `lane=all` it repeated the suite the +four shards had already passed on the same commit, on one worker, for 50 to 60 minutes, and +every release waited for it. Release evidence (`.github/scripts/release-evidence.cjs`) therefore +does not require it. Widget/bundle waits for `gates`, so the requests need not overlap and a queue is possible rather than guaranteed. The fourth shard adds setup overhead while dividing the same suite. diff --git a/structure/ops/docs-and-release.md b/structure/ops/docs-and-release.md index 9d12056fa8c..5d5f6a0ccb3 100644 --- a/structure/ops/docs-and-release.md +++ b/structure/ops/docs-and-release.md @@ -242,7 +242,7 @@ full-suite runs, both macOS paths, and `scripts/ci/run-bun-test-batches.sh` exec alone with fresh process homes, including launchd repair and standalone home/lease cases. Hosted batches assign shard membership by the per-file durations in `scripts/ci/test-durations.tsv` (Windows: `scripts/ci/test-durations.windows.tsv`; sorted round-robin when nothing is recorded), run each shard's files in sorted order and split only process boundaries; every selected file still runs once. -Ordinary macOS shards select 1/4, 2/4, 3/4 and 4/4 from the full file list; macOS control selects 1/1. Both +Ordinary macOS shards select 1/4, 2/4, 3/4 and 4/4 from the full file list; macOS control (dispatched only as `lane=macos-control`) selects 1/1. Both execute sequential batches of at most 12 files with one worker. Storage-policy and API-usage families run as singletons, as do manifest-declared files. This preserves full test membership but does not claim cross-batch shared-process coverage. diff --git a/tests/ci-workflows/ci-review-lanes.test.ts b/tests/ci-workflows/ci-review-lanes.test.ts index 84c7fdcdddf..012f6985e8f 100644 --- a/tests/ci-workflows/ci-review-lanes.test.ts +++ b/tests/ci-workflows/ci-review-lanes.test.ts @@ -213,7 +213,7 @@ describe("CI review lanes", () => { } const macosControlIf = ci.jobs?.["macos-control"] as { needs?: string; if?: string } | undefined; expect(macosControlIf?.needs).toBe("changes"); - expect(macosControlIf?.if).toBe("github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && (github.event.inputs.lane == '' || github.event.inputs.lane == 'all' || github.event.inputs.lane == 'macos-control')"); + expect(macosControlIf?.if).toBe("github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && github.event.inputs.lane == 'macos-control'"); }); test("manual release-gates keeps ordinary jobs and skips only diagnostic suites", async () => { @@ -235,8 +235,10 @@ describe("CI review lanes", () => { ["pull_request", "", false, false], ["push", "release-gates", false, false], ["pull_request", "release-gates", false, false], - ["workflow_dispatch", "", true, true], - ["workflow_dispatch", "all", true, true], + // lane=all is the release evidence run; the unsharded control would only + // repeat the macOS shards on one worker, so it waits for its own lane. + ["workflow_dispatch", "", true, false], + ["workflow_dispatch", "all", true, false], ["workflow_dispatch", "macos-control", false, true], ["workflow_dispatch", "release-gates", false, false], // Diagnostic-only lanes stay off ordinary release-gates; unknown lanes @@ -446,7 +448,9 @@ describe("Windows and nightly CI contracts", () => { ["pull_request", "", "false", "not-requested", "not-requested"], ["pull_request", "", undefined, "not-requested", "not-requested"], ["push", "", "true", "not-requested", "not-requested"], - ["workflow_dispatch", "all", "false", "requested", "requested"], + ["workflow_dispatch", "all", "false", "requested", "not-requested"], + ["workflow_dispatch", "", "false", "requested", "not-requested"], + ["workflow_dispatch", "macos-control", "false", "not-requested", "requested"], ["workflow_dispatch", "release-gates", "true", "not-requested", "not-requested"], ] as const) { const env: Record = { diff --git a/tests/ci-workflows/ci-scope-reduction.test.ts b/tests/ci-workflows/ci-scope-reduction.test.ts index 99b2d3bc471..f2656e9b621 100644 --- a/tests/ci-workflows/ci-scope-reduction.test.ts +++ b/tests/ci-workflows/ci-scope-reduction.test.ts @@ -490,7 +490,7 @@ describe("the packaged desktop selection", () => { describe("the smoke matrices", () => { test.skipIf(cannotRunEmitters)("release evidence names match the full-dispatch workflow graph", () => { const { CI_JOBS } = require(repoPath(".github", "scripts", "release-evidence.cjs")) as { CI_JOBS: string[] }; - const scalar = ["changes", "select-windows-runner", "storage-policy", "api-usage", "gates", "macos-control", "docker-smoke", "docs-site-build", "structure-gate", "widget", "desktop-shell", "ci"]; + const scalar = ["changes", "select-windows-runner", "storage-policy", "api-usage", "gates", "docker-smoke", "docs-site-build", "structure-gate", "widget", "desktop-shell", "ci"]; const expected = scalar.map(id => { const name = jobs[id]?.name; if (!name) throw new Error(`missing job name: ${id}`); diff --git a/tests/ci-workflows/ci-workflows.test.ts b/tests/ci-workflows/ci-workflows.test.ts index 7ea5dee018b..1bb182c09ee 100644 --- a/tests/ci-workflows/ci-workflows.test.ts +++ b/tests/ci-workflows/ci-workflows.test.ts @@ -312,7 +312,7 @@ describe("GitHub Actions hardening", () => { } | undefined; expect(macosControlJob?.name).toBe("macos control"); expect(macosControlJob?.needs).toBe("changes"); - expect(macosControlJob?.if).toBe("github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && (github.event.inputs.lane == '' || github.event.inputs.lane == 'all' || github.event.inputs.lane == 'macos-control')"); + expect(macosControlJob?.if).toBe("github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && github.event.inputs.lane == 'macos-control'"); expect(macosControlJob?.["runs-on"]).toBe("macos-latest"); expect(macosControlJob?.strategy).toBeUndefined(); const macosControlSteps = macosControlJob?.steps ?? [];