From 66760b167d873b56f5f2f0cec8ca33ede9c0f08a Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Wed, 18 Nov 2026 18:13:17 +0000 Subject: [PATCH] fix(runtime): retry stalled CDN fetches when verifying protocol offloads The image build verifies ~120 protocol archives against the CDN, each with a 60 s timeout and no retry, so one slow response fails the whole deploy (run 44353722716: "The operation was aborted due to timeout" at 64 s). Retry discovery and each archive up to twice (2 s, then 10 s) on a thrown fetch, an aborted or interrupted body, or a 5xx/429. A definitive answer, such as a 4xx, a Fly fallback, a missing R2 ETag or different bytes, still fails at once, and nothing is pruned until every body verifies. Co-Authored-By: Claude Opus 5.5 --- scripts/prune-runtime-artifacts.cjs | 78 ++++++++++++++++++------- tests/runtime-artifact-pruning.test.cjs | 42 ++++++++++++- 2 files changed, 97 insertions(+), 23 deletions(-) diff --git a/scripts/prune-runtime-artifacts.cjs b/scripts/prune-runtime-artifacts.cjs index 6f1949864d7..7069a1cc525 100644 --- a/scripts/prune-runtime-artifacts.cjs +++ b/scripts/prune-runtime-artifacts.cjs @@ -13,6 +13,34 @@ const semver = require("semver"); const PROTOCOL_OFFLOAD_INDEX = ".runtime-offloaded-tarballs.json"; const PROTOCOL_VERIFY_CONCURRENCY = 6; +// Waits before the second and third attempt at one CDN request. Each attempt +// keeps its own 60 s timeout. +const PROTOCOL_FETCH_RETRY_DELAYS_MS = [2_000, 10_000]; + +// The CDN answered, and its answer rules the offload out. Retrying cannot help. +class OffloadRejected extends Error {} + +// A stalled, dropped or 5xx response from the image builder's network is +// retried; a definitive answer (OffloadRejected) fails at once. +async function withFetchRetries(label, attempt, retryDelaysMs) { + for (let tries = 0; ; tries += 1) { + try { + return await attempt(); + } catch (error) { + if (error instanceof OffloadRejected || tries >= retryDelaysMs.length) throw error; + console.warn(`${label}: attempt ${tries + 1} failed (${error.message}); retrying in ${retryDelaysMs[tries]} ms`); + await new Promise((done) => setTimeout(done, retryDelaysMs[tries])); + } + } +} + +function assertCdnResponse(response, unavailable) { + if (response.ok) return; + if (response.status >= 500 || response.status === 429) { + throw new Error(`${unavailable} (HTTP ${response.status})`); + } + throw new OffloadRejected(unavailable); +} function readOffloadedTarballs(repoRoot) { const file = join(repoRoot, "dist", "protocol", PROTOCOL_OFFLOAD_INDEX); @@ -42,19 +70,22 @@ function protocolOffloadCandidates(repoRoot) { // discovery/checksum metadata, before removing any local archive. The Worker // emits R2's strong object ETag; its optional Fly fallback preserves Fly's weak // static-file ETag and request ID. Reject that fallback even if bytes match. -async function verifyProtocolOffloads(repoRoot, fetchArtifact = fetch) { +async function verifyProtocolOffloads( + repoRoot, fetchArtifact = fetch, retryDelaysMs = PROTOCOL_FETCH_RETRY_DELAYS_MS, +) { const candidates = protocolOffloadCandidates(repoRoot); if (!candidates.length) return candidates; const { schemaVersions } = runtimeArtifactManifest(repoRoot); // Worker protocol discovery lists R2 objects directly and has no Fly miss // fallback. A matching origin body alone cannot establish off-app storage. - const discovery = await fetchArtifact("https://adcontextprotocol.org/protocol/", { - redirect: "error", signal: AbortSignal.timeout(60_000), - }); - if (!discovery.ok || discovery.headers.has("fly-request-id")) { - throw new Error("R2 protocol discovery unavailable"); - } - const index = await discovery.json(); + const index = await withFetchRetries("R2 protocol discovery", async () => { + const discovery = await fetchArtifact("https://adcontextprotocol.org/protocol/", { + redirect: "error", signal: AbortSignal.timeout(60_000), + }); + assertCdnResponse(discovery, "R2 protocol discovery unavailable"); + if (discovery.headers.has("fly-request-id")) throw new OffloadRejected("R2 protocol discovery unavailable"); + return discovery.json(); + }, retryDelaysMs); if (!Array.isArray(index?.versions)) throw new Error("Invalid R2 protocol discovery"); const published = new Set(index.versions.map((entry) => entry?.tarball)); // A release cut in this tree reaches R2 only after its tag is published, so @@ -68,20 +99,23 @@ async function verifyProtocolOffloads(repoRoot, fetchArtifact = fetch) { const verifyArchive = async (name) => { const bytes = readFileSync(join(repoRoot, "dist", "protocol", name)); const expected = createHash("sha256").update(bytes).digest("hex"); - const response = await fetchArtifact(`https://adcontextprotocol.org/protocol/${name}`, { - redirect: "error", signal: AbortSignal.timeout(60_000), - }); - if (!response.ok || !response.body) throw new Error(`CDN protocol archive unavailable: ${name}`); - if (!/^"[a-f0-9]{32}(?:-[1-9][0-9]*)?"$/.test(response.headers.get("etag") ?? "") - || response.headers.has("fly-request-id")) { - throw new Error(`CDN protocol archive lacks R2 object evidence: ${name}`); - } - const hash = createHash("sha256"); - let length = 0; - for await (const chunk of response.body) { hash.update(chunk); length += chunk.length; } - if (length !== bytes.length || hash.digest("hex") !== expected) { - throw new Error(`CDN protocol archive differs from committed bytes: ${name}`); - } + await withFetchRetries(`CDN protocol archive ${name}`, async () => { + const response = await fetchArtifact(`https://adcontextprotocol.org/protocol/${name}`, { + redirect: "error", signal: AbortSignal.timeout(60_000), + }); + assertCdnResponse(response, `CDN protocol archive unavailable: ${name}`); + if (!response.body) throw new Error(`CDN protocol archive unavailable: ${name}`); + if (!/^"[a-f0-9]{32}(?:-[1-9][0-9]*)?"$/.test(response.headers.get("etag") ?? "") + || response.headers.has("fly-request-id")) { + throw new OffloadRejected(`CDN protocol archive lacks R2 object evidence: ${name}`); + } + const hash = createHash("sha256"); + let length = 0; + for await (const chunk of response.body) { hash.update(chunk); length += chunk.length; } + if (length !== bytes.length || hash.digest("hex") !== expected) { + throw new OffloadRejected(`CDN protocol archive differs from committed bytes: ${name}`); + } + }, retryDelaysMs); }; // Every image build downloads each offloaded body, so verify a few at a time. for (let start = 0; start < files.length; start += PROTOCOL_VERIFY_CONCURRENCY) { diff --git a/tests/runtime-artifact-pruning.test.cjs b/tests/runtime-artifact-pruning.test.cjs index 7f957da251d..85f2bc3f146 100644 --- a/tests/runtime-artifact-pruning.test.cjs +++ b/tests/runtime-artifact-pruning.test.cjs @@ -303,11 +303,12 @@ test("CDN archive verification rejects unavailable, different and interrupted bo async () => new Response(readFileSync(join(root, "dist", "protocol", "3.0.0.tgz")), { headers: { etag: '"0123456789abcdef0123456789abcdef"', "fly-request-id": "origin-fallback" } }), async () => { throw new Error("network interrupted"); }, + async () => new Response("unavailable", { status: 503 }), ]) { await assert.rejects(() => verifyProtocolOffloads(root, (url, options) => url === "https://adcontextprotocol.org/protocol/" ? Promise.resolve(Response.json({ versions: [{ tarball: "/protocol/3.0.0.tgz" }] })) - : fetchArtifact(url, options))); + : fetchArtifact(url, options), [0, 0])); assert.ok(existsSync(join(root, "dist", "protocol", "3.0.0.tgz"))); assert.ok(existsSync(join(root, "dist", "schemas", "3.1.0-beta.2"))); assert.equal(existsSync(join(root, "dist", "protocol", ".runtime-offloaded-tarballs.json")), false); @@ -318,6 +319,45 @@ test("CDN archive verification rejects unavailable, different and interrupted bo } finally { rmSync(root, { recursive: true, force: true }); } }); +test("a stalled or 5xx CDN request is retried, but a definitive rejection is not", async () => { + const root = fixture(); + try { + const serving = cdnServing(root, ["3.0.0.tgz", "3.1.1.tgz"]); + const calls = new Map(); + // Discovery times out once; one archive answers 503 once, then succeeds. + const flaky = (url, options) => { + const n = (calls.get(url) ?? 0) + 1; + calls.set(url, n); + if (n === 1 && url === "https://adcontextprotocol.org/protocol/") { + return Promise.reject(new DOMException("The operation was aborted due to timeout", "TimeoutError")); + } + if (n === 1 && url.endsWith("/3.1.1.tgz")) return Promise.resolve(new Response("busy", { status: 503 })); + return serving(url, options); + }; + assert.deepEqual(await verifyProtocolOffloads(root, flaky, [0, 0]), ["3.0.0.tgz", "3.1.1.tgz"]); + assert.equal(calls.get("https://adcontextprotocol.org/protocol/"), 2); + assert.equal(calls.get("https://adcontextprotocol.org/protocol/3.1.1.tgz"), 2); + + // Different bytes are an answer, not a stall: one request, then fail. + let archiveCalls = 0; + await assert.rejects(() => verifyProtocolOffloads(root, (url, options) => { + if (!url.endsWith("/3.0.0.tgz")) return serving(url, options); + archiveCalls += 1; + return Promise.resolve(new Response("different bytes", { headers: { etag: '"0123456789abcdef0123456789abcdef"' } })); + }, [0, 0]), /differs from committed bytes/); + assert.equal(archiveCalls, 1); + + // A stall that never clears gives up after one try per delay plus the first. + let stalls = 0; + await assert.rejects(() => verifyProtocolOffloads(root, () => { + stalls += 1; + return Promise.reject(new DOMException("The operation was aborted due to timeout", "TimeoutError")); + }, [0, 0]), /aborted due to timeout/); + assert.equal(stalls, 3); + assert.ok(existsSync(join(root, "dist", "protocol", "3.0.0.tgz"))); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + test("R2 discovery must list every archive outside the required closure and cannot be a Fly fallback", async () => { const root = fixture(); try {