diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6d7e7ad467..7045e960766 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -696,10 +696,7 @@ jobs: # gui/package.json. Without this the affected shards die on # `Cannot find module 'react/jsx-dev-runtime'` while the other shards pass. - name: Install dependencies - run: | - bun install --frozen-lockfile - cd gui - bun install --frozen-lockfile + run: bash scripts/ci/bun-install-frozen.sh . gui # Nor is the build. Tests that fetch the served dashboard read the session # bootstrap meta tags out of `gui/dist/index.html`, so without a build the @@ -734,7 +731,7 @@ jobs: with: role: test-runner # See the Bun roles contract above test. - name: Install dependencies and build the GUI the tests serve - run: bun install --frozen-lockfile && cd gui && bun install --frozen-lockfile && bun run build + run: bash scripts/ci/bun-install-frozen.sh . gui && cd gui && bun run build - name: Test the files this pull request reaches run: bash scripts/ci/run-changed-tests.sh @@ -760,10 +757,7 @@ jobs: role: test-runner # See the Bun roles contract above test. - name: Install dependencies - run: | - bun install --frozen-lockfile - cd gui - bun install --frozen-lockfile + run: bash scripts/ci/bun-install-frozen.sh . gui - name: Build GUI run: | @@ -802,10 +796,7 @@ jobs: role: test-runner # See the Bun roles contract above test. - name: Install dependencies - run: | - bun install --frozen-lockfile - cd gui - bun install --frozen-lockfile + run: bash scripts/ci/bun-install-frozen.sh . gui - name: Build GUI run: | @@ -840,10 +831,7 @@ jobs: role: test-runner # See the Bun roles contract above test. - name: Install dependencies - run: | - bun install --frozen-lockfile - cd gui - bun install --frozen-lockfile + run: bash scripts/ci/bun-install-frozen.sh . gui - name: GUI lint if: needs.changes.outputs.gui == 'true' diff --git a/scripts/ci/bun-install-frozen.sh b/scripts/ci/bun-install-frozen.sh new file mode 100755 index 00000000000..402fcab7251 --- /dev/null +++ b/scripts/ci/bun-install-frozen.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Frozen-lockfile install with a per-attempt deadline and one total budget. +# +# A registry or runner stall can leave `bun install` printing only its banner until the job's +# timeout cancels the run, which reads as a failed check unrelated to the change under test. +# Each attempt therefore runs under a process deadline, and only an attempt that the deadline +# killed is retried. Any other failure, such as a lockfile that no longer matches, fails at once. +# All directories share one total budget, so retries end well inside the shortest job timeout +# that calls this (5 minutes) and leave time for setup, the GUI build and the tests. +# +# Usage: bash scripts/ci/bun-install-frozen.sh [dir ...] (default: the current directory) +set -euo pipefail + +readonly ATTEMPT_SECONDS="${BUN_INSTALL_ATTEMPT_SECONDS:-45}" +readonly TOTAL_SECONDS="${BUN_INSTALL_TOTAL_SECONDS:-150}" +readonly MIN_ATTEMPT_SECONDS="${BUN_INSTALL_MIN_ATTEMPT_SECONDS:-5}" +readonly KILL_GRACE_SECONDS="${BUN_INSTALL_KILL_GRACE_SECONDS:-5}" +readonly MAX_ATTEMPTS="${BUN_INSTALL_MAX_ATTEMPTS:-3}" +readonly BUN_INSTALL_BIN="${BUN_INSTALL_BIN:-bun}" + +if ! command -v timeout >/dev/null 2>&1 || ! timeout --signal=TERM --kill-after=1s 1s true >/dev/null 2>&1; then + echo "::error::GNU timeout is required to bound bun install." >&2 + exit 69 +fi + +install_dir() { + local dir="$1" attempt status remaining deadline + for ((attempt = 1; attempt <= MAX_ATTEMPTS; attempt++)); do + # The kill grace counts against the budget too, so the worst case stays at TOTAL_SECONDS. + remaining=$(( TOTAL_SECONDS - SECONDS - KILL_GRACE_SECONDS )) + if (( remaining < MIN_ATTEMPT_SECONDS )); then + echo "::error::bun install --frozen-lockfile in ${dir} has no time left in its ${TOTAL_SECONDS}s budget." + return 124 + fi + deadline=$(( remaining < ATTEMPT_SECONDS ? remaining : ATTEMPT_SECONDS )) + status=0 + (cd "$dir" && timeout --signal=TERM --kill-after="${KILL_GRACE_SECONDS}s" "${deadline}s" \ + "$BUN_INSTALL_BIN" install --frozen-lockfile) || status=$? + if [[ $status -eq 0 ]]; then return 0; fi + # 124: the deadline fired and the install exited on TERM; 137: it needed the KILL. + if [[ $status -ne 124 && $status -ne 137 ]]; then + echo "::error::bun install --frozen-lockfile failed in ${dir} (exit ${status}); not retrying." + return "$status" + fi + echo "::warning::bun install --frozen-lockfile in ${dir} exceeded ${deadline}s (attempt ${attempt}/${MAX_ATTEMPTS})." + done + echo "::error::bun install --frozen-lockfile in ${dir} exceeded its deadline ${MAX_ATTEMPTS} times." + return 124 +} + +if [[ $# -eq 0 ]]; then set -- .; fi +for dir in "$@"; do install_dir "$dir"; done diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 4398c8d7639..5f78d0ff358 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -378,7 +378,7 @@ "chutes-provider.test.ts": "providers", "ci-bun-crash-classifier.test.ts": "ci-workflows", "ci-crash-disposition.test.ts": "ci-workflows", - "ci-gui-typecheck-gate.test.ts": "ci-workflows", + "ci-gui-typecheck-gate.test.ts": "ci-workflows", "ci-bun-install-frozen.test.ts": "ci-workflows", "ci-privacy-gate.test.ts": "ci-workflows", "ci-review-lanes.test.ts": "ci-workflows", "ci-scope-reduction.test.ts": "ci-workflows", diff --git a/tests/ci-workflows/ci-bun-install-frozen.test.ts b/tests/ci-workflows/ci-bun-install-frozen.test.ts new file mode 100644 index 00000000000..f7e6f49d0a2 --- /dev/null +++ b/tests/ci-workflows/ci-bun-install-frozen.test.ts @@ -0,0 +1,107 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { repoPath } from "../helpers/repo-root"; + +const script = repoPath("scripts", "ci", "bun-install-frozen.sh"); +const UBUNTU_INSTALL_JOBS = ["test", "test-changed", "storage-policy", "api-usage", "gates"]; + +/** Map each job id in ci.yml to its runs-on value and the run lines it contains. */ +function ciJobs(): Map { + const jobs = new Map(); + let current: { runsOn: string | null; text: string } | null = null; + for (const line of readFileSync(repoPath(".github", "workflows", "ci.yml"), "utf8").split(/\r?\n/)) { + const job = /^ ([a-z0-9-]+):\s*$/.exec(line); + if (job) { current = { runsOn: null, text: "" }; jobs.set(job[1]!, current); continue; } + if (!current) continue; + const runsOn = /^ runs-on:\s*(.+)$/.exec(line); + if (runsOn) current.runsOn = runsOn[1]!.trim(); + current.text += line + "\n"; + } + return jobs; +} + +describe("CI frozen installs are bounded", () => { + test("every Ubuntu job that installs the GUI goes through the bounded install script", () => { + const jobs = ciJobs(); + for (const name of UBUNTU_INSTALL_JOBS) { + const job = jobs.get(name); + expect(job?.runsOn, name).toBe("ubuntu-latest"); + expect(job!.text, name).toContain("bash scripts/ci/bun-install-frozen.sh . gui"); + } + for (const [name, job] of jobs) { + if (job.runsOn !== "ubuntu-latest") continue; + // A bare two-step root+gui install, in block or one-line form, is unbounded again. + expect(/bun install --frozen-lockfile(?:\s*&&\s*|\s*\n\s*)cd gui(?:\s*&&\s*|\s*\n\s*)bun install/.test(job.text), name).toBe(false); + } + }); +}); + +// The script targets the Linux CI runners and needs GNU timeout. +describe.skipIf(process.platform === "win32")("bun-install-frozen.sh retry policy", () => { + const roots: string[] = []; + afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + + /** A fake bun whose Nth invocation behaves as listed: "hang", "ok" or an exit code. */ + function run(plan: string[], options: { dirs?: number; env?: Record } = {}) { + const root = mkdtempSync(join(tmpdir(), "ocx-bun-install-")); + roots.push(root); + const count = join(root, "count"), fake = join(root, "fake-bun"); + writeFileSync(join(root, "plan"), plan.join("\n") + "\n"); + writeFileSync(fake, [ + "#!/usr/bin/env bash", + `n=$(( $(cat "${count}" 2>/dev/null || echo 0) + 1 )); echo "$n" > "${count}"`, + `step=$(sed -n "${"$"}{n}p" "${join(root, "plan")}")`, + 'echo "fake bun $* in $PWD"', + 'case "$step" in hang) exec sleep 30 ;; ok) exit 0 ;; *) exit "$step" ;; esac', + ].join("\n")); + chmodSync(fake, 0o755); + const dirs = Array.from({ length: options.dirs ?? 1 }, () => root); + const started = Date.now(); + const result = spawnSync("bash", [script, ...dirs], { + encoding: "utf8", timeout: 30_000, + env: { ...process.env, BUN_INSTALL_BIN: fake, BUN_INSTALL_ATTEMPT_SECONDS: "1", BUN_INSTALL_KILL_GRACE_SECONDS: "1", + BUN_INSTALL_MIN_ATTEMPT_SECONDS: "1", ...options.env }, + }); + return { status: result.status, out: result.stdout + result.stderr, elapsedMs: Date.now() - started, + calls: existsSync(count) ? Number(readFileSync(count, "utf8")) : 0 }; + } + + test("an attempt killed by its deadline is retried and the next success passes", () => { + const result = run(["hang", "ok"]); + expect(result.status, result.out).toBe(0); + expect(result.calls).toBe(2); + expect(result.out).toContain("::warning::bun install --frozen-lockfile"); + }); + + test("an ordinary install failure such as a lockfile mismatch is not retried", () => { + const result = run(["1", "ok"]); + expect(result.status, result.out).toBe(1); + expect(result.calls).toBe(1); + expect(result.out).toContain("not retrying"); + }); + + test("three deadline kills fail with an error annotation", () => { + const result = run(["hang", "hang", "hang", "ok"]); + expect(result.status, result.out).toBe(124); + expect(result.calls).toBe(3); + expect(result.out).toContain("::error::bun install --frozen-lockfile in"); + }); + + test("every attempt runs the frozen-lockfile install", () => { + const result = run(["ok"]); + expect(result.status, result.out).toBe(0); + expect(result.out).toContain("fake bun install --frozen-lockfile"); + }); + + test("root and GUI stalls share one total budget instead of three attempts each", () => { + // Per-directory retries alone would allow 6 attempts of 2 s plus grace; the 4 s total ends after 2. + const result = run(Array(8).fill("hang"), { dirs: 2, env: { BUN_INSTALL_ATTEMPT_SECONDS: "2", BUN_INSTALL_TOTAL_SECONDS: "4" } }); + expect(result.status, result.out).toBe(124); + expect(result.calls).toBe(2); + expect(result.elapsedMs).toBeLessThan(7_000); + expect(result.out).toContain("budget"); + }); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 8635f25bed1..cdbcc7921e4 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -355,7 +355,7 @@ "bridge-raw-reasoning-hidden.test.ts": "adapters", "bridge-reasoning-replay-batch.test.ts": "adapters", "bridge-stall.test.ts": "adapters", "bridge-terminal-singleness.test.ts": "adapters", "bridge.test.ts": "adapters", "buffered-response-shape-guards.test.ts": "adapters", - "build-desktop-icon-set.test.ts": "ci-workflows", "build-release-changelog.test.ts": "ci-workflows", "ci-gui-typecheck-gate.test.ts": "ci-workflows", + "build-desktop-icon-set.test.ts": "ci-workflows", "build-release-changelog.test.ts": "ci-workflows", "ci-gui-typecheck-gate.test.ts": "ci-workflows", "ci-bun-install-frozen.test.ts": "ci-workflows", "bump-dev-version.test.ts": "ci-workflows", "bun-runtime.test.ts": "ci-workflows", "bun-transpiler-cache-scope.test.ts": "cli", "bun-stream-caps.test.ts": "lib", "cache-diagnostic.test.ts": "usage",