Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 56 additions & 22 deletions scripts/prune-runtime-artifacts.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,34 @@ const semver = require("semver");

const PROTOCOL_OFFLOAD_INDEX = ".runtime-offloaded-tarballs.json";
const PROTOCOL_VERIFY_CONCURRENCY = 6;
// Waits before the second and third attempt at one CDN request. Each attempt
// keeps its own 60 s timeout.
const PROTOCOL_FETCH_RETRY_DELAYS_MS = [2_000, 10_000];

// The CDN answered, and its answer rules the offload out. Retrying cannot help.
class OffloadRejected extends Error {}

// A stalled, dropped or 5xx response from the image builder's network is
// retried; a definitive answer (OffloadRejected) fails at once.
async function withFetchRetries(label, attempt, retryDelaysMs) {
for (let tries = 0; ; tries += 1) {
try {
return await attempt();
} catch (error) {
if (error instanceof OffloadRejected || tries >= retryDelaysMs.length) throw error;
console.warn(`${label}: attempt ${tries + 1} failed (${error.message}); retrying in ${retryDelaysMs[tries]} ms`);
await new Promise((done) => setTimeout(done, retryDelaysMs[tries]));
}
}
}

function assertCdnResponse(response, unavailable) {
if (response.ok) return;
if (response.status >= 500 || response.status === 429) {
throw new Error(`${unavailable} (HTTP ${response.status})`);
}
throw new OffloadRejected(unavailable);
}

function readOffloadedTarballs(repoRoot) {
const file = join(repoRoot, "dist", "protocol", PROTOCOL_OFFLOAD_INDEX);
Expand Down Expand Up @@ -42,19 +70,22 @@ function protocolOffloadCandidates(repoRoot) {
// discovery/checksum metadata, before removing any local archive. The Worker
// emits R2's strong object ETag; its optional Fly fallback preserves Fly's weak
// static-file ETag and request ID. Reject that fallback even if bytes match.
async function verifyProtocolOffloads(repoRoot, fetchArtifact = fetch) {
async function verifyProtocolOffloads(
repoRoot, fetchArtifact = fetch, retryDelaysMs = PROTOCOL_FETCH_RETRY_DELAYS_MS,
) {
const candidates = protocolOffloadCandidates(repoRoot);
if (!candidates.length) return candidates;
const { schemaVersions } = runtimeArtifactManifest(repoRoot);
// Worker protocol discovery lists R2 objects directly and has no Fly miss
// fallback. A matching origin body alone cannot establish off-app storage.
const discovery = await fetchArtifact("https://adcontextprotocol.org/protocol/", {
redirect: "error", signal: AbortSignal.timeout(60_000),
});
if (!discovery.ok || discovery.headers.has("fly-request-id")) {
throw new Error("R2 protocol discovery unavailable");
}
const index = await discovery.json();
const index = await withFetchRetries("R2 protocol discovery", async () => {
const discovery = await fetchArtifact("https://adcontextprotocol.org/protocol/", {
redirect: "error", signal: AbortSignal.timeout(60_000),
});
assertCdnResponse(discovery, "R2 protocol discovery unavailable");
if (discovery.headers.has("fly-request-id")) throw new OffloadRejected("R2 protocol discovery unavailable");
return discovery.json();
}, retryDelaysMs);
if (!Array.isArray(index?.versions)) throw new Error("Invalid R2 protocol discovery");
const published = new Set(index.versions.map((entry) => entry?.tarball));
// A release cut in this tree reaches R2 only after its tag is published, so
Expand All @@ -68,20 +99,23 @@ async function verifyProtocolOffloads(repoRoot, fetchArtifact = fetch) {
const verifyArchive = async (name) => {
const bytes = readFileSync(join(repoRoot, "dist", "protocol", name));
const expected = createHash("sha256").update(bytes).digest("hex");
const response = await fetchArtifact(`https://adcontextprotocol.org/protocol/${name}`, {
redirect: "error", signal: AbortSignal.timeout(60_000),
});
if (!response.ok || !response.body) throw new Error(`CDN protocol archive unavailable: ${name}`);
if (!/^"[a-f0-9]{32}(?:-[1-9][0-9]*)?"$/.test(response.headers.get("etag") ?? "")
|| response.headers.has("fly-request-id")) {
throw new Error(`CDN protocol archive lacks R2 object evidence: ${name}`);
}
const hash = createHash("sha256");
let length = 0;
for await (const chunk of response.body) { hash.update(chunk); length += chunk.length; }
if (length !== bytes.length || hash.digest("hex") !== expected) {
throw new Error(`CDN protocol archive differs from committed bytes: ${name}`);
}
await withFetchRetries(`CDN protocol archive ${name}`, async () => {
const response = await fetchArtifact(`https://adcontextprotocol.org/protocol/${name}`, {
redirect: "error", signal: AbortSignal.timeout(60_000),
});
assertCdnResponse(response, `CDN protocol archive unavailable: ${name}`);
if (!response.body) throw new Error(`CDN protocol archive unavailable: ${name}`);
if (!/^"[a-f0-9]{32}(?:-[1-9][0-9]*)?"$/.test(response.headers.get("etag") ?? "")
|| response.headers.has("fly-request-id")) {
throw new OffloadRejected(`CDN protocol archive lacks R2 object evidence: ${name}`);
}
const hash = createHash("sha256");
let length = 0;
for await (const chunk of response.body) { hash.update(chunk); length += chunk.length; }
if (length !== bytes.length || hash.digest("hex") !== expected) {
throw new OffloadRejected(`CDN protocol archive differs from committed bytes: ${name}`);
}
}, retryDelaysMs);
};
// Every image build downloads each offloaded body, so verify a few at a time.
for (let start = 0; start < files.length; start += PROTOCOL_VERIFY_CONCURRENCY) {
Expand Down
42 changes: 41 additions & 1 deletion tests/runtime-artifact-pruning.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -303,11 +303,12 @@ test("CDN archive verification rejects unavailable, different and interrupted bo
async () => new Response(readFileSync(join(root, "dist", "protocol", "3.0.0.tgz")),
{ headers: { etag: '"0123456789abcdef0123456789abcdef"', "fly-request-id": "origin-fallback" } }),
async () => { throw new Error("network interrupted"); },
async () => new Response("unavailable", { status: 503 }),
]) {
await assert.rejects(() => verifyProtocolOffloads(root, (url, options) =>
url === "https://adcontextprotocol.org/protocol/"
? Promise.resolve(Response.json({ versions: [{ tarball: "/protocol/3.0.0.tgz" }] }))
: fetchArtifact(url, options)));
: fetchArtifact(url, options), [0, 0]));
assert.ok(existsSync(join(root, "dist", "protocol", "3.0.0.tgz")));
assert.ok(existsSync(join(root, "dist", "schemas", "3.1.0-beta.2")));
assert.equal(existsSync(join(root, "dist", "protocol", ".runtime-offloaded-tarballs.json")), false);
Expand All @@ -318,6 +319,45 @@ test("CDN archive verification rejects unavailable, different and interrupted bo
} finally { rmSync(root, { recursive: true, force: true }); }
});

test("a stalled or 5xx CDN request is retried, but a definitive rejection is not", async () => {
const root = fixture();
try {
const serving = cdnServing(root, ["3.0.0.tgz", "3.1.1.tgz"]);
const calls = new Map();
// Discovery times out once; one archive answers 503 once, then succeeds.
const flaky = (url, options) => {
const n = (calls.get(url) ?? 0) + 1;
calls.set(url, n);
if (n === 1 && url === "https://adcontextprotocol.org/protocol/") {
return Promise.reject(new DOMException("The operation was aborted due to timeout", "TimeoutError"));
}
if (n === 1 && url.endsWith("/3.1.1.tgz")) return Promise.resolve(new Response("busy", { status: 503 }));
return serving(url, options);
};
assert.deepEqual(await verifyProtocolOffloads(root, flaky, [0, 0]), ["3.0.0.tgz", "3.1.1.tgz"]);
assert.equal(calls.get("https://adcontextprotocol.org/protocol/"), 2);
assert.equal(calls.get("https://adcontextprotocol.org/protocol/3.1.1.tgz"), 2);

// Different bytes are an answer, not a stall: one request, then fail.
let archiveCalls = 0;
await assert.rejects(() => verifyProtocolOffloads(root, (url, options) => {
if (!url.endsWith("/3.0.0.tgz")) return serving(url, options);
archiveCalls += 1;
return Promise.resolve(new Response("different bytes", { headers: { etag: '"0123456789abcdef0123456789abcdef"' } }));
}, [0, 0]), /differs from committed bytes/);
assert.equal(archiveCalls, 1);

// A stall that never clears gives up after one try per delay plus the first.
let stalls = 0;
await assert.rejects(() => verifyProtocolOffloads(root, () => {
stalls += 1;
return Promise.reject(new DOMException("The operation was aborted due to timeout", "TimeoutError"));
}, [0, 0]), /aborted due to timeout/);
assert.equal(stalls, 3);
assert.ok(existsSync(join(root, "dist", "protocol", "3.0.0.tgz")));
} finally { rmSync(root, { recursive: true, force: true }); }
});

test("R2 discovery must list every archive outside the required closure and cannot be a Fly fallback", async () => {
const root = fixture();
try {
Expand Down
Loading