diff --git a/packages/@emulators/cloudflare/package.json b/packages/@emulators/cloudflare/package.json index a5ea70ecb..7488d0c45 100644 --- a/packages/@emulators/cloudflare/package.json +++ b/packages/@emulators/cloudflare/package.json @@ -52,6 +52,8 @@ "@emulators/planetscale": "workspace:*" }, "devDependencies": { + "esbuild": "0.27.4", + "miniflare": "4.20260702.0", "tsup": "^8", "typescript": "^5.7" } diff --git a/packages/@emulators/cloudflare/src/__tests__/runtime.test.ts b/packages/@emulators/cloudflare/src/__tests__/runtime.test.ts new file mode 100644 index 000000000..b34e8a89d --- /dev/null +++ b/packages/@emulators/cloudflare/src/__tests__/runtime.test.ts @@ -0,0 +1,261 @@ +import { fileURLToPath } from "node:url"; +import { isBuiltin } from "node:module"; +import type { Readable } from "node:stream"; +import { build } from "esbuild"; +import { Log, LogLevel, Miniflare } from "miniflare"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +// Runs the real Worker and Durable Object in workerd and records everything +// Cloudflare could: every tail event (the source of Workers Logs and Issues, +// including uncaught exception events), workerd's own stdout and stderr, and +// the Analytics Engine data points the code writes. Faults are injected at the +// storage and stub boundaries with synthetic secrets in the message, the error +// name and the instance URL; none may reach any of it. +const SECRET = "SYNTHETICtoken482913"; +const SUFFIX = "0123456789abcdef01234567"; +const INSTANCE = `synthetic-${SUFFIX}`; +const SECRETS = [SECRET, SUFFIX]; +const POINT = "probe:analytics-engine"; + +// The probe module wraps the shipped exports. Its Durable Object hands the real +// one a storage proxy that fails the way `x-probe-fault` asks, and its Worker +// can swap in a namespace whose addressing or stub fails. Both get a FAILURES +// dataset that reports each data point as a tagged log line, so the tail sees +// exactly what Analytics Engine would store. +const PROBE = ` +import worker, { EmulatorDurableObject } from "./worker.ts"; +const FAILURES = { writeDataPoint: (point) => console.log("${POINT}", JSON.stringify(point)) }; +const secretError = (flags) => + Object.assign(new Error("uncaught ${SECRET} https://resend.${INSTANCE}.emulators.dev/emails"), { name: "${SECRET}" }, flags); +export class ProbeObject extends EmulatorDurableObject { + constructor(state, env) { + let fault = null; + const storage = new Proxy(state.storage, { + get(target, key) { + if (fault === "do-flagged" && key === "get") return async () => { throw secretError({ retryable: true }); }; + if (fault === "do-plain" && key === "get") return async () => { throw secretError({}); }; + if (fault === "do-put" && key === "put") return async () => { throw secretError({}); }; + const value = target[key]; + return typeof value === "function" ? value.bind(target) : value; + }, + }); + super({ storage, blockConcurrencyWhile: state.blockConcurrencyWhile.bind(state) }, { ...env, FAILURES }); + this.setFault = (next) => { fault = next; }; + } + async fetch(request) { + this.setFault(request.headers.get("x-probe-fault")); + return super.fetch(request); + } +} +export default { + fetch(request, env) { + env = { ...env, FAILURES }; + const fault = request.headers.get("x-probe-fault"); + if (fault === "worker-id") + env = { ...env, EMULATOR: { idFromName() { throw secretError({}); }, get: () => env.EMULATOR.get() } }; + if (fault === "worker-stub") + env = { ...env, EMULATOR: { idFromName: (n) => n, get: () => ({ fetch: async () => { throw secretError({ retryable: true, overloaded: true }); } }) } }; + if (fault === "worker-env") + env = { FAILURES, get EMULATE_HOST_SUFFIX() { throw secretError({}); } }; + return worker.fetch(request, env); + }, +}; +`; + +interface TailEvent { + outcome: string; + event?: { request?: { url: string; headers: Record } }; + entrypoint?: string; + exceptions: Array<{ name: string; message: string; stack?: string }>; + logs: Array<{ level: string; message: unknown[] }>; +} + +let mf: Miniflare; +const tailed: TailEvent[] = []; +const runtimeOutput: string[] = []; + +beforeAll(async () => { + const bundle = await build({ + stdin: { contents: PROBE, resolveDir: fileURLToPath(new URL("..", import.meta.url)), loader: "ts" }, + bundle: true, + write: false, + format: "esm", + platform: "neutral", + mainFields: ["module", "main"], + conditions: ["workerd", "worker", "import"], + logLevel: "silent", + plugins: [ + { + name: "node-builtins", + setup(b) { + b.onResolve({ filter: /.*/ }, (args) => + isBuiltin(args.path) ? { path: `node:${args.path.replace(/^node:/, "")}`, external: true } : undefined, + ); + }, + }, + ], + }); + mf = new Miniflare({ + log: new Log(LogLevel.NONE), + handleRuntimeStdio(stdout: Readable, stderr: Readable) { + stdout.on("data", (chunk: Buffer) => runtimeOutput.push(String(chunk))); + stderr.on("data", (chunk: Buffer) => runtimeOutput.push(String(chunk))); + }, + workers: [ + { + name: "emulate-hosts", + modules: [{ type: "ESModule", path: "/probe/worker.mjs", contents: bundle.outputFiles[0].text }], + modulesRoot: "/probe", + compatibilityDate: "2026-06-08", + compatibilityFlags: ["nodejs_compat"], + durableObjects: { EMULATOR: "ProbeObject" }, + bindings: { EMULATE_HOST_SUFFIX: "emulators.dev" }, + tails: ["sink"], + }, + { + name: "sink", + modules: true, + script: `export default { async tail(events, env) { await env.CAPTURE.fetch("https://capture.invalid", { method: "POST", body: JSON.stringify(events) }); } };`, + compatibilityDate: "2026-06-08", + serviceBindings: { + CAPTURE: async (request: Request) => { + tailed.push(...((await request.json()) as TailEvent[])); + return new Response("ok"); + }, + }, + }, + ], + }); + await mf.ready; +}, 60_000); + +afterAll(async () => { + await mf?.dispose(); +}); + +async function waitForTail(count: number): Promise { + const deadline = Date.now() + 5_000; + while (tailed.length < count && Date.now() < deadline) await new Promise((r) => setTimeout(r, 25)); +} + +// A fault injected in the object's storage reaches it, so the request is one +// Worker invocation plus one object invocation; a Worker fault stops at the +// Worker. Each event is named by its entrypoint, the object's class or none. +const invocations = (fault: string) => (fault.startsWith("do-") ? ["ProbeObject", "worker"] : ["worker"]); +const invocation = (event: TailEvent) => ({ + fault: event.event?.request?.headers["x-probe-fault"], + entrypoint: event.entrypoint ?? "worker", +}); + +const CASES: Array<{ fault: string; method?: string; path: string; status: number; report: Record }> = + [ + { + fault: "do-flagged", + path: "/emails", + status: 503, + report: { error: "emulator_unavailable", route: "/emails", retryable: true, errorClass: "other" }, + }, + { + fault: "do-plain", + path: "/emails", + status: 500, + report: { error: "emulator_error", route: "/emails", retryable: false, errorClass: "other" }, + }, + { + fault: "do-put", + method: "POST", + path: "/_emulate/seed", + status: 500, + report: { error: "emulator_error", route: "/_emulate/seed", errorClass: "other" }, + }, + { + fault: "do-put", + method: "POST", + path: "/_emulate/credentials", + status: 500, + report: { error: "emulator_error", route: "/_emulate/credentials", errorClass: "other" }, + }, + { + fault: "worker-id", + method: "POST", + path: "/emails", + status: 500, + report: { error: "emulator_unavailable", route: "/emails", errorClass: "other" }, + }, + { + fault: "worker-stub", + path: "/emails", + status: 503, + report: { error: "emulator_unavailable", overloaded: true, errorClass: "other" }, + }, + { + fault: "worker-env", + path: "/emails", + status: 500, + report: { error: "worker_error", service: "unknown", instanceId: null }, + }, + ]; + +describe("emulate-hosts in workerd", () => { + it("records no exception and no secret for any injected failure", async () => { + const responses: string[] = []; + for (const c of CASES) { + const before = tailed.length; + const response = await mf.dispatchFetch(`https://emulators.dev/resend/${INSTANCE}${c.path}`, { + method: c.method ?? "GET", + headers: { "x-probe-fault": c.fault, "content-type": "application/json" }, + body: c.method === "POST" ? JSON.stringify({ type: "api-key", login: "synthetic-user" }) : undefined, + }); + const text = await response.text(); + responses.push(text); + expect({ fault: c.fault, status: response.status }).toEqual({ fault: c.fault, status: c.status }); + expect(JSON.parse(text)).toMatchObject(c.report); + // Exactly this request's invocations, and nothing from an earlier one. + const expected = invocations(c.fault); + await waitForTail(before + expected.length); + expect( + tailed + .slice(before) + .map(invocation) + .sort((a, b) => a.entrypoint.localeCompare(b.entrypoint)), + ).toEqual(expected.map((entrypoint) => ({ fault: c.fault, entrypoint }))); + } + + expect(tailed).toHaveLength(CASES.flatMap((c) => invocations(c.fault)).length); + expect(tailed.map((event) => event.outcome).filter((outcome) => outcome !== "ok")).toEqual([]); + expect(tailed.flatMap((event) => event.exceptions)).toEqual([]); + // The code writes nothing to the console: every log line is a data point + // from the probe's dataset, one per failure. + const logged = tailed.flatMap((event) => event.logs.map((log) => log.message.map(String))); + expect(logged.filter(([tag]) => tag !== POINT)).toEqual([]); + const points = logged.map(([, point]) => JSON.parse(point) as { blobs: string[] }); + expect(points).toHaveLength(CASES.length); + expect(points.map((point) => point.blobs[0]).sort()).toEqual(CASES.map((c) => c.report.error).sort()); + + // The secret is nowhere: not in the complete tail events, the runtime's + // output, the data points or the responses. + const everything = [JSON.stringify(tailed), runtimeOutput.join(""), ...responses].join("\n"); + expect(everything).not.toContain(SECRET); + // The instance name is only in each invocation's request: the client's URL + // to the Worker, and the x-emulator-* headers the Worker sets on its request + // to the object. Workers Logs and Issues can store an invocation's request + // with each record, which is why observability is off (see the telemetry + // settings test in worker.test.ts). + const carriers = new Set(); + const visit = (value: unknown, path: string) => { + if (typeof value === "string") { + if (value.includes(SUFFIX)) carriers.add(path.replace(/^\d+\./, "")); + } else if (value && typeof value === "object") { + for (const [key, child] of Object.entries(value)) visit(child, `${path}.${key}`); + } + }; + tailed.forEach((event, i) => visit(event, String(i))); + expect([...carriers].sort()).toEqual([ + "event.request.headers.x-emulator-base-url", + "event.request.headers.x-emulator-instance", + "event.request.url", + ]); + expect(SECRETS.filter((secret) => [runtimeOutput.join(""), ...responses].join("\n").includes(secret))).toEqual([]); + expect(SECRETS.filter((secret) => JSON.stringify(points).includes(secret))).toEqual([]); + }, 60_000); +}); diff --git a/packages/@emulators/cloudflare/src/__tests__/worker.test.ts b/packages/@emulators/cloudflare/src/__tests__/worker.test.ts index 008741538..30af121c2 100644 --- a/packages/@emulators/cloudflare/src/__tests__/worker.test.ts +++ b/packages/@emulators/cloudflare/src/__tests__/worker.test.ts @@ -1,5 +1,7 @@ +import { readFileSync } from "node:fs"; import { describe, expect, it, vi } from "vitest"; import { EmulatorDurableObject } from "../durable-object.js"; +import { instanceId } from "../diagnostics.js"; import worker, { parseHostRoute, type Env } from "../worker.js"; describe("cloudflare worker routing", () => { @@ -252,6 +254,336 @@ describe("cloudflare worker routing", () => { }); }); +// Synthetic secrets: an instance URL is the only access control for its +// emulator, and errors and paths can quote tokens, codes and addresses. None of +// these may reach a failure response or a log line. +const SECRET_INSTANCE = "d040-probe-0123456789abcdef01234567"; +const SECRETS = [ + SECRET_INSTANCE, + "0123456789abcdef01234567", + "emu_resend_SYNTHETICtoken0001", + "SYNTH-CODE-482913", + "pat.synthetic@example.test", + "SYNTHETICtoken482913", +]; +const leakedSecrets = (text: string) => SECRETS.filter((secret) => text.includes(secret)); +const REPORT_KEYS = [ + "error", + "errorClass", + "instanceId", + "method", + "overloaded", + "ray", + "remote", + "retryable", + "route", + "service", +]; +const RAY = "8f1d2c3b4a5e6f70-PHX"; + +// What a failure leaves on Cloudflare: Analytics Engine data points, and +// anything written to the console, which Workers Issues keeps with the +// invocation's URL. The console must stay silent. +type DataPoint = { indexes?: string[]; blobs?: string[]; doubles?: number[] }; +const recordFailures = () => { + const points: DataPoint[] = []; + const consoles = (["log", "info", "warn", "error", "debug"] as const).map((method) => + vi.spyOn(console, method).mockImplementation(() => {}), + ); + return { + sink: { writeDataPoint: (point: DataPoint) => void points.push(point) }, + points, + stored: () => JSON.stringify(points), + consoleCalls: () => consoles.flatMap((spy) => spy.mock.calls), + restore: () => consoles.forEach((spy) => spy.mockRestore()), + }; +}; + +describe("emulate-hosts telemetry settings", () => { + // Workers Issues keeps every 5xx and error log with its invocation's URL, and + // these settings are not part of a Worker version, so the config must turn + // each one off explicitly: a deploy then also undoes a dashboard change. + const config = JSON.parse( + readFileSync(new URL("../../wrangler.jsonc", import.meta.url), "utf8").replace(/^\s*\/\/.*$/gm, ""), + ) as Record; + + it("turns every part of observability off", () => { + expect(config.observability).toEqual({ + enabled: false, + logs: { enabled: false, invocation_logs: false }, + traces: { enabled: false }, + issues: { enabled: false }, + }); + expect(config).not.toHaveProperty("tail_consumers"); + expect(config).not.toHaveProperty("logpush"); + }); + + it("records failures in the Analytics Engine dataset", () => { + expect(config.analytics_engine_datasets).toEqual([{ binding: "FAILURES", dataset: "emulate_hosts_failures" }]); + }); +}); + +describe("cloudflare worker durable object failures", () => { + // Cloudflare raises Durable Object stub failures as exceptions carrying + // `.retryable`, `.overloaded` and `.remote` flags. + const doError = (message: string, flags: { retryable?: boolean; overloaded?: boolean; remote?: boolean }) => + Object.assign(new Error(message), flags); + + const failingEnv = (failures: Error[], sink?: Env["FAILURES"]) => { + const calls: string[] = []; + const env: Env = { + FAILURES: sink, + EMULATOR: { + idFromName: (n) => n, + get: () => ({ + async fetch(request) { + calls.push(`${request.method} ${new URL(request.url).pathname}`); + const failure = failures.shift(); + if (failure) throw failure; + return Response.json({ ok: true }); + }, + }), + }, + }; + return { env, calls }; + }; + + it("answers a retryable reset once, with a report and no replay", async () => { + const failures = recordFailures(); + try { + const { env, calls } = failingEnv([doError("Network connection lost.", { retryable: true })], failures.sink); + const response = await worker.fetch( + new Request(`https://emulators.dev/resend/${SECRET_INSTANCE}/_emulate/reset`, { + method: "POST", + headers: { "cf-ray": RAY }, + body: "{}", + }), + env, + ); + // A retryable flag does not prove the object never ran the reset. + expect(calls).toEqual(["POST /_emulate/reset"]); + expect(response.status).toBe(503); + const report = await response.json(); + expect(report).toEqual({ + error: "emulator_unavailable", + service: "resend", + instanceId: await instanceId("resend", SECRET_INSTANCE), + method: "POST", + route: "/_emulate/reset", + errorClass: "Error", + retryable: true, + overloaded: false, + remote: false, + ray: RAY, + }); + // Recorded once, without the instance hash; nothing goes to the console. + expect(failures.points).toEqual([ + { + indexes: ["resend"], + blobs: ["emulator_unavailable", "resend", "POST", "/_emulate/reset", "Error", RAY], + doubles: [503, 1, 0, 0], + }, + ]); + expect(failures.consoleCalls()).toEqual([]); + } finally { + failures.restore(); + } + }); + + it("does not replay a WorkOS authorize, which issues a code on every call", async () => { + const { env, calls } = failingEnv([doError("Network connection lost.", { retryable: true })]); + const response = await worker.fetch( + new Request( + `https://emulators.dev/workos/${SECRET_INSTANCE}/oauth2/authorize?client_id=c&redirect_uri=https%3A%2F%2Fapp.example.test%2Fcb`, + ), + env, + ); + expect(calls).toEqual(["GET /oauth2/authorize"]); + expect(response.status).toBe(503); + expect(await response.json()).toMatchObject({ route: "/oauth2/authorize", retryable: true }); + }); + + it("keeps tokens, codes, addresses and the instance out of the response and the record", async () => { + const failures = recordFailures(); + try { + const failure = doError( + `lost while serving ${SECRET_INSTANCE}: token emu_resend_SYNTHETICtoken0001 code SYNTH-CODE-482913 for pat.synthetic@example.test`, + { retryable: true }, + ); + const { env } = failingEnv([failure], failures.sink); + const response = await worker.fetch( + new Request( + `https://emulators.dev/resend/${SECRET_INSTANCE}/domains/pat.synthetic@example.test?code=SYNTH-CODE-482913`, + { + headers: { authorization: "Bearer emu_resend_SYNTHETICtoken0001", "cf-ray": RAY }, + }, + ), + env, + ); + const text = await response.text(); + const report = JSON.parse(text) as Record; + expect(Object.keys(report).sort()).toEqual(REPORT_KEYS); + expect(report.route).toBe("/domains/:id"); + expect(leakedSecrets(text)).toEqual([]); + expect(failures.points).toHaveLength(1); + expect(leakedSecrets(failures.stored())).toEqual([]); + expect(failures.stored()).not.toContain(report.instanceId as string); + expect(failures.consoleCalls()).toEqual([]); + } finally { + failures.restore(); + } + }); + + it("does not echo header or path values it cannot vouch for", async () => { + const failures = recordFailures(); + try { + const { env } = failingEnv([doError("Network connection lost.", { retryable: true })], failures.sink); + const response = await worker.fetch( + new Request(`https://emulators.dev/pat.synthetic@example.test/${SECRET_INSTANCE}/emails`, { + headers: { "cf-ray": "SYNTH-CODE-482913" }, + }), + env, + ); + const text = await response.text(); + expect(JSON.parse(text)).toMatchObject({ service: "unknown", route: "unmatched", ray: null }); + expect(leakedSecrets(text)).toEqual([]); + expect(failures.points).toHaveLength(1); + expect(leakedSecrets(failures.stored())).toEqual([]); + expect(failures.stored()).not.toContain("SYNTH-CODE-482913"); + expect(failures.consoleCalls()).toEqual([]); + } finally { + failures.restore(); + } + }); + + it("answers an overloaded object with a 503", async () => { + const { env, calls } = failingEnv([ + doError("Durable Object is overloaded. Too many requests queued.", { retryable: true, overloaded: true }), + ]); + const response = await worker.fetch(new Request(`https://emulators.dev/resend/${SECRET_INSTANCE}/emails`), env); + expect(calls).toEqual(["GET /emails"]); + expect(response.status).toBe(503); + expect(await response.json()).toMatchObject({ overloaded: true, route: "/emails" }); + }); + + it("reports an object killed by its own limits as a 500", async () => { + const { env, calls } = failingEnv([ + doError("Durable Object's isolate exceeded its memory limit and was reset.", { remote: true }), + ]); + const response = await worker.fetch(new Request(`https://emulators.dev/resend/${SECRET_INSTANCE}/emails`), env); + expect(calls).toHaveLength(1); + expect(response.status).toBe(500); + expect(await response.json()).toMatchObject({ remote: true, retryable: false, service: "resend" }); + }); + + it("reports failures to read the body or address the object instead of throwing", async () => { + const failures = recordFailures(); + try { + const secretError = () => new Error(`lost ${SECRET_INSTANCE} token emu_resend_SYNTHETICtoken0001`); + const addressing: Env = { + FAILURES: failures.sink, + EMULATOR: { + idFromName: () => { + throw secretError(); + }, + get: () => ({ fetch: async () => Response.json({ ok: true }) }), + }, + }; + const addressed = await worker.fetch( + new Request(`https://emulators.dev/resend/${SECRET_INSTANCE}/emails`), + addressing, + ); + expect(addressed.status).toBe(500); + expect(await addressed.json()).toMatchObject({ error: "emulator_unavailable", route: "/emails" }); + + const { env, calls } = failingEnv([], failures.sink); + const unreadable = new Request(`https://emulators.dev/resend/${SECRET_INSTANCE}/emails`, { + method: "POST", + body: new ReadableStream({ + start(controller) { + controller.error(secretError()); + }, + }), + duplex: "half", + } as RequestInit); + const read = await worker.fetch(unreadable, env); + expect(calls).toEqual([]); + expect(read.status).toBe(500); + expect(await read.json()).toMatchObject({ error: "emulator_unavailable", route: "/emails" }); + expect(failures.points).toHaveLength(2); + expect(leakedSecrets(failures.stored())).toEqual([]); + expect(failures.consoleCalls()).toEqual([]); + } finally { + failures.restore(); + } + }); + + it("reports any other Worker failure instead of throwing", async () => { + const failures = recordFailures(); + try { + const env = { + FAILURES: failures.sink, + get EMULATE_HOST_SUFFIX(): string { + throw new Error(`config read failed for ${SECRET_INSTANCE}`); + }, + } as unknown as Env; + const response = await worker.fetch(new Request(`https://emulators.dev/resend/${SECRET_INSTANCE}/emails`), env); + const text = await response.text(); + expect(response.status).toBe(500); + expect(JSON.parse(text)).toEqual({ + error: "worker_error", + service: "unknown", + instanceId: null, + method: "GET", + route: "unmatched", + errorClass: "Error", + retryable: false, + overloaded: false, + remote: false, + ray: null, + }); + expect(failures.points).toEqual([ + { + indexes: ["unknown"], + blobs: ["worker_error", "unknown", "GET", "unmatched", "Error", ""], + doubles: [500, 0, 0, 0], + }, + ]); + expect(leakedSecrets(text + failures.stored())).toEqual([]); + expect(failures.consoleCalls()).toEqual([]); + } finally { + failures.restore(); + } + }); + + it("reports only allowlisted error classes and methods", async () => { + const failures = recordFailures(); + try { + const { env } = failingEnv( + [ + doError("lost", { retryable: true }), + Object.assign(doError("lost", { retryable: true }), { name: "SYNTHETICtoken482913" }), + Object.assign(new TypeError("lost"), { retryable: true }), + ], + failures.sink, + ); + const send = (method: string) => + worker + .fetch(new Request(`https://emulators.dev/resend/${SECRET_INSTANCE}/emails`, { method }), env) + .then((r) => r.json() as Promise>); + expect(await send("SYNTHETICTOKEN")).toMatchObject({ method: "OTHER", errorClass: "Error" }); + expect(await send("GET")).toMatchObject({ method: "GET", errorClass: "other" }); + expect(await send("GET")).toMatchObject({ errorClass: "TypeError" }); + expect(failures.points).toHaveLength(3); + expect(failures.stored()).not.toContain("SYNTHETICTOKEN"); + expect(leakedSecrets(failures.stored())).toEqual([]); + expect(failures.consoleCalls()).toEqual([]); + } finally { + failures.restore(); + } + }); +}); + describe("cloudflare durable object control plane", () => { function makeState(options: { limit?: number; initial?: Record } = {}) { const storage = new Map(); @@ -351,6 +683,221 @@ describe("cloudflare durable object control plane", () => { ...extra, }); + // Builds the instance, then makes the next storage write throw `failure` once. + // One-shot, so a later write (the persist after every mutating request) + // cannot rethrow it and hide what the router did with the first one. + const failNextWriteAfterWarmup = async (failure: unknown, sink?: Env["FAILURES"]) => { + const { state } = makeState(); + const durableObject = new EmulatorDurableObject(state, { FAILURES: sink }); + const warm = await durableObject.fetch( + new Request("https://github.my-run.emulators.dev/_emulate/manifest", { headers: idHeaders() }), + ); + expect(warm.status).toBe(200); + const put = state.storage.put; + let thrown = false; + state.storage.put = async (key, value) => { + if (thrown) return put(key, value); + thrown = true; + throw failure; + }; + return durableObject; + }; + const control = (path: string, body: unknown, extra: Record = {}) => + new Request(`https://github.my-run.emulators.dev${path}`, { + method: "POST", + headers: { ...idHeaders(extra), "content-type": "application/json" }, + body: JSON.stringify(body), + }); + const moved = () => + Object.assign(new Error("cannot access storage because object has moved to a different machine"), { + retryable: true, + }); + + it("answers an emulator failure with a report instead of throwing", async () => { + // A 1-byte value cap makes every persist fail the way an oversized value does. + const { state } = makeState({ limit: 1 }); + const durableObject = new EmulatorDurableObject(state, {}); + const response = await durableObject.fetch(control("/_emulate/reset", {}, { "cf-ray": RAY })); + expect(response.status).toBe(500); + expect(await response.json()).toEqual({ + error: "emulator_error", + service: "github", + instanceId: await instanceId("github", "my-run"), + method: "POST", + route: "/_emulate/reset", + errorClass: "Error", + retryable: false, + overloaded: false, + remote: false, + ray: RAY, + }); + }); + + it("keeps secrets in an emulator error out of the response and the record", async () => { + const failures = recordFailures(); + try { + const durableObject = await failNextWriteAfterWarmup( + new Error( + `write failed for ${SECRET_INSTANCE}: token emu_resend_SYNTHETICtoken0001 code SYNTH-CODE-482913 for pat.synthetic@example.test`, + ), + failures.sink, + ); + // The error is thrown inside the router (reset runs in the control plane), + // which used to answer it with its raw message. + const response = await durableObject.fetch(control("/_emulate/reset", {})); + expect(response.status).toBe(500); + const text = await response.text(); + expect(Object.keys(JSON.parse(text)).sort()).toEqual(REPORT_KEYS); + expect(JSON.parse(text)).toMatchObject({ error: "emulator_error", route: "/_emulate/reset" }); + expect(leakedSecrets(text)).toEqual([]); + expect(failures.points).toEqual([ + { + indexes: ["github"], + blobs: ["emulator_error", "github", "POST", "/_emulate/reset", "Error", ""], + doubles: [500, 0, 0, 0], + }, + ]); + expect(leakedSecrets(failures.stored())).toEqual([]); + expect(failures.consoleCalls()).toEqual([]); + } finally { + failures.restore(); + } + }); + + // Every failure below must come back as a report: an error thrown out of the + // object is recorded by Cloudflare with its message, stack and URL. + const reportOf = async (response: Promise) => { + const res = await response; + return { status: res.status, report: (await res.json()) as Record }; + }; + + it("reports Cloudflare's retryable storage failures with their flags instead of throwing", async () => { + const { state } = makeState(); + state.storage.get = async () => { + throw moved(); + }; + const durableObject = new EmulatorDurableObject(state, {}); + // Thrown while the object loads, before the service router runs. + expect(await reportOf(durableObject.fetch(control("/_emulate/reset", {}, { "cf-ray": RAY })))).toEqual({ + status: 503, + report: { + error: "emulator_unavailable", + service: "github", + instanceId: await instanceId("github", "my-run"), + method: "POST", + route: "/_emulate/reset", + errorClass: "Error", + retryable: true, + overloaded: false, + remote: false, + ray: RAY, + }, + }); + }); + + it("reports a flagged failure from inside the service router", async () => { + const durableObject = await failNextWriteAfterWarmup(moved()); + // Reset persists from inside the router, whose error handler used to answer + // every error as a plain 500 without the flags. + expect(await reportOf(durableObject.fetch(control("/_emulate/reset", {})))).toMatchObject({ + status: 503, + report: { error: "emulator_unavailable", route: "/_emulate/reset", retryable: true }, + }); + }); + + it("reports flagged and plain storage failures from seed and credential requests", async () => { + const failures = recordFailures(); + try { + const credentials = () => control("/_emulate/credentials", { type: "bearer-token", login: "synthetic-user" }); + const seed = () => control("/_emulate/seed", { users: [{ login: "synthetic-user" }] }); + const plain = () => + new Error( + `storage write failed for ${SECRET_INSTANCE}: token emu_resend_SYNTHETICtoken0001 for pat.synthetic@example.test`, + ); + for (const [request, route] of [ + [credentials, "/_emulate/credentials"], + [seed, "/_emulate/seed"], + ] as const) { + expect(await reportOf((await failNextWriteAfterWarmup(moved(), failures.sink)).fetch(request()))).toMatchObject( + { + status: 503, + report: { error: "emulator_unavailable", route, retryable: true }, + }, + ); + // A host failure is not the caller's mistake: it used to come back as a + // 400 quoting the raw message. + const res = await (await failNextWriteAfterWarmup(plain(), failures.sink)).fetch(request()); + const text = await res.text(); + expect({ status: res.status, report: JSON.parse(text) }).toMatchObject({ + status: 500, + report: { error: "emulator_error", route, errorClass: "Error" }, + }); + expect(leakedSecrets(text)).toEqual([]); + } + expect(failures.points).toHaveLength(4); + expect(leakedSecrets(failures.stored())).toEqual([]); + expect(failures.consoleCalls()).toEqual([]); + } finally { + failures.restore(); + } + }); + + it("still answers a credential type the emulator does not support with a 400", async () => { + const { state } = makeState(); + const durableObject = new EmulatorDurableObject(state, {}); + const res = await durableObject.fetch(control("/_emulate/credentials", { type: "synthetic-unsupported-type" })); + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: "unsupported", + message: "Credential type synthetic-unsupported-type is not supported by github", + }); + }); + + it("reports an error whose name could carry a secret as class other", async () => { + const failures = recordFailures(); + try { + const named = Object.assign(new Error("synthetic"), { name: "SYNTHETICtoken482913" }); + const durableObject = await failNextWriteAfterWarmup(named, failures.sink); + const res = await durableObject.fetch(control("/_emulate/reset", {})); + const text = await res.text(); + expect(JSON.parse(text)).toMatchObject({ errorClass: "other" }); + expect(text).not.toContain("SYNTHETICtoken482913"); + expect(failures.points).toHaveLength(1); + expect(failures.stored()).not.toContain("SYNTHETICtoken482913"); + expect(failures.consoleCalls()).toEqual([]); + } finally { + failures.restore(); + } + }); + + it("passes the object's flagged report through the Worker as a 503", async () => { + const failures = recordFailures(); + try { + const durableObject = await failNextWriteAfterWarmup(moved(), failures.sink); + const env: Env = { FAILURES: failures.sink, EMULATOR: { idFromName: (n) => n, get: () => durableObject } }; + const response = await worker.fetch( + new Request("https://emulators.dev/github/my-run/_emulate/reset", { + method: "POST", + headers: { "cf-ray": RAY }, + body: "{}", + }), + env, + ); + expect(response.status).toBe(503); + expect(await response.json()).toMatchObject({ + error: "emulator_unavailable", + route: "/_emulate/reset", + retryable: true, + ray: RAY, + }); + // Recorded once, by the object. + expect(failures.points).toHaveLength(1); + expect(failures.consoleCalls()).toEqual([]); + } finally { + failures.restore(); + } + }); + // Executor's cloud onboarding e2e provisions this service exactly this way: // mint an api-key, seed a brand, then resolve the company from a work email. // A missing registration only shows up here, as a 404 from the control plane. diff --git a/packages/@emulators/core/src/__tests__/control-plane.test.ts b/packages/@emulators/core/src/__tests__/control-plane.test.ts index a26496974..bb6915f54 100644 --- a/packages/@emulators/core/src/__tests__/control-plane.test.ts +++ b/packages/@emulators/core/src/__tests__/control-plane.test.ts @@ -3,6 +3,8 @@ import { createServer } from "../server.js"; import { randomInstanceName } from "../control-plane.js"; import type { ServicePlugin } from "../plugin.js"; import type { Store } from "../store.js"; +import { ApiError } from "../middleware/error-handler.js"; +import { ControlPlaneRejection } from "../control-plane-rejection.js"; interface Thing { id: number; @@ -307,3 +309,71 @@ describe("randomInstanceName", () => { expect(name).toMatch(/-[0-9a-f]{24}$/); }); }); + +describe("unexpected route errors", () => { + const throwing: ServicePlugin = { + name: "throwing", + register(app) { + app.get("/missing", () => { + throw new ApiError(404, "Thing not found"); + }); + app.get("/broken", () => { + throw new Error("token emu_demo_SYNTHETIC0001 for pat.synthetic@example.test"); + }); + }, + }; + + it("answers them with their message by default", async () => { + const { app } = createServer(throwing); + const res = await app.request("/broken"); + expect(res.status).toBe(500); + expect(await res.json()).toMatchObject({ message: "token emu_demo_SYNTHETIC0001 for pat.synthetic@example.test" }); + }); + + it("rethrows them for a host that reports failures itself, but keeps API errors", async () => { + const { app } = createServer(throwing, { rethrowUnexpectedErrors: true }); + await expect(app.request("/broken")).rejects.toThrow("emu_demo_SYNTHETIC0001"); + const res = await app.request("/missing"); + expect(res.status).toBe(404); + expect(await res.json()).toMatchObject({ message: "Thing not found" }); + }); +}); + +describe("seed and credential failures", () => { + const plugin: ServicePlugin = { name: "seeded", register() {} }; + const failing = (error: unknown, rethrowUnexpectedErrors = false) => + createServer(plugin, { + rethrowUnexpectedErrors, + seed: () => { + throw error; + }, + issueCredential: () => { + throw error; + }, + }).app; + const post = (app: ReturnType, path: string) => + app.request(path, { method: "POST", headers: { "content-type": "application/json" }, body: "{}" }); + + it("answers the emulator's own rejections with a 400 and their message", async () => { + const app = failing(new ControlPlaneRejection("Credential type synthetic is not supported by seeded")); + const seed = await post(app, "/_emulate/seed"); + expect(seed.status).toBe(400); + expect(await seed.json()).toEqual({ + error: "invalid_seed", + message: "Credential type synthetic is not supported by seeded", + }); + const credentials = await post(app, "/_emulate/credentials"); + expect(credentials.status).toBe(400); + expect(await credentials.json()).toMatchObject({ error: "unsupported" }); + }); + + it("sends any other error to the app's error handler, not a 400", async () => { + const error = () => new Error("storage write failed: token emu_demo_SYNTHETIC0001"); + for (const path of ["/_emulate/seed", "/_emulate/credentials"]) { + const res = await post(failing(error()), path); + expect(res.status).toBe(500); + const thrown = error(); + await expect(post(failing(thrown, true), path)).rejects.toBe(thrown); + } + }); +}); diff --git a/packages/@emulators/core/src/__tests__/http.test.ts b/packages/@emulators/core/src/__tests__/http.test.ts index 82421938c..f616f21f8 100644 --- a/packages/@emulators/core/src/__tests__/http.test.ts +++ b/packages/@emulators/core/src/__tests__/http.test.ts @@ -71,4 +71,12 @@ describe("internal http layer", () => { expect(res.headers.get("Access-Control-Allow-Headers")).toBe("x-test"); expect(res.headers.get("Access-Control-Max-Age")).toBe("60"); }); + + it("names the route pattern a request would match", () => { + const app = new Hono(); + app.get("/domains/:id", (c) => c.text("ok")); + expect(app.routePattern("GET", "/domains/pat.synthetic@example.test")).toBe("/domains/:id"); + expect(app.routePattern("HEAD", "/domains/x")).toBe("/domains/:id"); + expect(app.routePattern("POST", "/domains/x")).toBeUndefined(); + }); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 720449c25..f00fba7a9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -563,6 +563,12 @@ importers: specifier: workspace:* version: link:../x devDependencies: + esbuild: + specifier: 0.27.4 + version: 0.27.4 + miniflare: + specifier: 4.20260702.0 + version: 4.20260702.0 tsup: specifier: ^8 version: 8.5.1(jiti@2.6.1)(postcss@8.5.8)(typescript@5.9.3)(yaml@2.9.0)