From 4154137aa16a66ed3b86aa6f2514ff7965f8d2fc Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:35:09 -0700 Subject: [PATCH 1/5] Report Durable Object failures with their cause instead of an opaque 500 --- .../cloudflare/src/durable-object.ts | 20 +++++ packages/@emulators/cloudflare/src/worker.ts | 79 +++++++++++++++++-- packages/@emulators/cloudflare/wrangler.jsonc | 6 ++ 3 files changed, 98 insertions(+), 7 deletions(-) diff --git a/packages/@emulators/cloudflare/src/durable-object.ts b/packages/@emulators/cloudflare/src/durable-object.ts index edf3b449f..784ebdcd5 100644 --- a/packages/@emulators/cloudflare/src/durable-object.ts +++ b/packages/@emulators/cloudflare/src/durable-object.ts @@ -449,6 +449,26 @@ export class EmulatorDurableObject { } async fetch(request: Request): Promise { + try { + return await this.handle(request); + } catch (error) { + // Answer with the cause instead of throwing: an exception crossing the stub + // reaches the client as an opaque Cloudflare 500 page with no message. + const report = { + error: "emulator_error", + message: error instanceof Error ? error.message : String(error), + service: request.headers.get("x-emulator-service"), + instance: request.headers.get("x-emulator-instance"), + method: request.method, + path: new URL(request.url).pathname, + ray: request.headers.get("cf-ray"), + }; + console.error(JSON.stringify({ ...report, stack: error instanceof Error ? error.stack : undefined })); + return Response.json(report, { status: 500 }); + } + } + + private async handle(request: Request): Promise { const service = request.headers.get("x-emulator-service") ?? ""; const instance = request.headers.get("x-emulator-instance") ?? "default"; const baseUrl = request.headers.get("x-emulator-base-url") ?? new URL(request.url).origin; diff --git a/packages/@emulators/cloudflare/src/worker.ts b/packages/@emulators/cloudflare/src/worker.ts index 9bc704688..31715a52a 100644 --- a/packages/@emulators/cloudflare/src/worker.ts +++ b/packages/@emulators/cloudflare/src/worker.ts @@ -78,14 +78,79 @@ async function forwardToDurableObject( // Manual redirect: the emulator's OAuth callbacks return 302s (e.g. → the app's // redirect_uri). Without this, the Worker→DO stub.fetch FOLLOWS the redirect // internally and re-fetches the DO by the Location path, mangling it. - const inner = new Request(`${origin}${opts.innerPath}${opts.search}`, { - method: request.method, - headers, - body, - redirect: "manual", - }); + const inner = () => + new Request(`${origin}${opts.innerPath}${opts.search}`, { + method: request.method, + headers, + body, + redirect: "manual", + }); const id = env.EMULATOR.idFromName(`${opts.service}:${opts.instance}`); - return env.EMULATOR.get(id).fetch(inner); + const idempotent = isIdempotent(request.method, opts.innerPath); + for (let attempt = 1; ; attempt++) { + try { + // A stub that threw is not reused: Cloudflare documents that many + // exceptions leave it broken, so every attempt gets a fresh one. + return await env.EMULATOR.get(id).fetch(inner()); + } catch (error) { + const failure = durableObjectFailure(error); + if (idempotent && failure.retryable && !failure.overloaded && attempt < DO_MAX_ATTEMPTS) { + await sleep(DO_BASE_BACKOFF_MS * Math.random() * 2 ** attempt); + continue; + } + const report = { + error: "emulator_unavailable", + message: failure.message, + service: opts.service, + instance: opts.instance, + method: request.method, + path: opts.innerPath, + attempts: attempt, + retryable: failure.retryable, + overloaded: failure.overloaded, + remote: failure.remote, + ray: request.headers.get("cf-ray"), + }; + console.error(JSON.stringify(report)); + // 503 when Cloudflare says the object was unreachable or overloaded; 500 + // when the emulator itself threw or was killed (remote, not retryable). + return Response.json(report, { status: failure.retryable || failure.overloaded ? 503 : 500 }); + } + } +} + +// Cloudflare's Durable Object error contract: an exception with `.retryable` +// may be retried if the request is idempotent; one with `.overloaded` must not +// be; `.remote` marks an error raised by (or a limit hit in) the object itself. +// https://developers.cloudflare.com/durable-objects/best-practices/error-handling/ +const DO_MAX_ATTEMPTS = 3; +const DO_BASE_BACKOFF_MS = 100; +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +// Only requests whose repetition cannot change the result are retried: GET and +// HEAD (idempotent by RFC 9110) and `POST /_emulate/reset`, which always restores +// the instance's seeded state and answers `{ ok: true }`. Seeds, credentials and +// provider writes are not idempotent and fail precisely instead. +const isIdempotent = (method: string, innerPath: string): boolean => + method === "GET" || method === "HEAD" || (method === "POST" && innerPath === "/_emulate/reset"); + +function durableObjectFailure(error: unknown): { + message: string; + retryable: boolean; + overloaded: boolean; + remote: boolean; +} { + const flags = (typeof error === "object" && error !== null ? error : {}) as { + retryable?: unknown; + overloaded?: unknown; + remote?: unknown; + }; + return { + message: error instanceof Error ? error.message : String(error), + retryable: flags.retryable === true, + overloaded: flags.overloaded === true, + remote: flags.remote === true, + }; } // Router: diff --git a/packages/@emulators/cloudflare/wrangler.jsonc b/packages/@emulators/cloudflare/wrangler.jsonc index f80216292..f87efc93b 100644 --- a/packages/@emulators/cloudflare/wrangler.jsonc +++ b/packages/@emulators/cloudflare/wrangler.jsonc @@ -22,6 +22,12 @@ "zone_name": "emulators.dev" } ], + // Workers Logs: keeps every invocation's logs and uncaught exceptions, + // including the structured `emulator_unavailable` reports from worker.ts, so + // a 5xx seen by a client can be traced by its cf-ray. + "observability": { + "enabled": true + }, "vars": { "EMULATE_HOST_SUFFIX": "emulators.dev" }, From 5e44f3629d44a1b70ba6e25647eefc83b2fa4f81 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:51:23 -0700 Subject: [PATCH 2/5] Let Cloudflare's retryable failures reach the Worker with their flags --- packages/@emulators/cloudflare/src/durable-object.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/@emulators/cloudflare/src/durable-object.ts b/packages/@emulators/cloudflare/src/durable-object.ts index 784ebdcd5..0ccc6edd6 100644 --- a/packages/@emulators/cloudflare/src/durable-object.ts +++ b/packages/@emulators/cloudflare/src/durable-object.ts @@ -105,6 +105,11 @@ const ledgerEntryKey = (id: string): string => `${LEDGER_ENTRY_PREFIX}${encodeKe // instance survives eviction. Auth is FAITHFUL by // default (strict): only seeded or minted tokens work; everything else gets the // real API's 401/403. Mint tokens at runtime via `POST /__token`. +const isCloudflareFailure = (error: unknown): boolean => + typeof error === "object" && + error !== null && + ((error as { retryable?: unknown }).retryable === true || (error as { overloaded?: unknown }).overloaded === true); + export class EmulatorDurableObject { private live?: Live; // The JSON of every snapshot and ledger value last written to storage, by key. @@ -452,6 +457,10 @@ export class EmulatorDurableObject { try { return await this.handle(request); } catch (error) { + // Cloudflare's own failures (e.g. "object has moved to a different machine") + // carry `.retryable`/`.overloaded`; rethrow them so the Worker sees the flags + // and applies Cloudflare's retry contract. + if (isCloudflareFailure(error)) throw error; // Answer with the cause instead of throwing: an exception crossing the stub // reaches the client as an opaque Cloudflare 500 page with no message. const report = { From fc120a49ab171de6298442afa67221d525a3c5a7 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:38:04 -0700 Subject: [PATCH 3/5] Report Durable Object failures without secrets and drop the retry The Worker no longer retries a failed stub call: a retryable flag does not prove the object never ran the request, and replaying a reset or an OAuth authorize changes state twice. Reports carry an instance hash, the route template, the error class and Cloudflare's flags, never a message, path or stack. The core router and control plane pass flagged platform failures through, and the Durable Object rethrows unexpected router errors so they are reported the same way. Invocation logs, which record request URLs, are off. --- .../@emulators/cloudflare/src/diagnostics.ts | 92 ++++++++++++++++++ .../cloudflare/src/durable-object.ts | 32 +++---- packages/@emulators/cloudflare/src/worker.ts | 96 +++++-------------- packages/@emulators/cloudflare/wrangler.jsonc | 13 ++- packages/@emulators/core/src/control-plane.ts | 3 + packages/@emulators/core/src/http.ts | 7 ++ packages/@emulators/core/src/index.ts | 1 + .../@emulators/core/src/platform-failure.ts | 10 ++ packages/@emulators/core/src/server.ts | 20 +++- 9 files changed, 181 insertions(+), 93 deletions(-) create mode 100644 packages/@emulators/cloudflare/src/diagnostics.ts create mode 100644 packages/@emulators/core/src/platform-failure.ts diff --git a/packages/@emulators/cloudflare/src/diagnostics.ts b/packages/@emulators/cloudflare/src/diagnostics.ts new file mode 100644 index 000000000..7f71db472 --- /dev/null +++ b/packages/@emulators/cloudflare/src/diagnostics.ts @@ -0,0 +1,92 @@ +import { createServer } from "@emulators/core"; +import { SERVICES } from "./services.js"; + +// Failure reports go to the client and to Workers Logs, so they carry only +// values that cannot hold a secret or user data. The instance name is the +// sole access control for its emulator, request paths hold ids, emails and +// codes, and error messages and stacks can quote any of them. A report names +// the instance by a short hash, the request by its route template, and the +// error by its class and Cloudflare's flags. +export interface FailureReport { + error: "emulator_unavailable" | "emulator_error"; + service: string; + instanceId: string; + method: string; + route: string; + errorClass: string; + retryable: boolean; + overloaded: boolean; + remote: boolean; + ray: string | null; +} + +export async function failureReport( + error: FailureReport["error"], + cause: unknown, + request: { service: string; instance: string; method: string; path: string; headers: Headers }, +): Promise { + const flags = (typeof cause === "object" && cause !== null ? cause : {}) as { + retryable?: unknown; + overloaded?: unknown; + remote?: unknown; + }; + const known = Object.hasOwn(SERVICES, request.service); + return { + error, + service: known ? request.service : "unknown", + instanceId: await instanceId(request.service, request.instance), + method: METHOD.test(request.method) ? request.method : "OTHER", + route: known ? routeTemplate(request.service, request.method, request.path) : "unmatched", + errorClass: errorClass(cause), + retryable: flags.retryable === true, + overloaded: flags.overloaded === true, + remote: flags.remote === true, + ray: rayOf(request.headers), + }; +} + +const METHOD = /^[A-Z]{1,10}$/; +const RAY = /^[0-9a-f]{16}(-[A-Z]{3})?$/; +const ERROR_CLASS = /^[A-Za-z][A-Za-z0-9]{0,39}$/; + +// The first 12 hex digits of SHA-256(`:`), the Durable Object +// name. Instance suffixes carry 96 random bits, so the hash cannot be reversed, +// and anyone holding the instance URL can compute it to find their reports. +export async function instanceId(service: string, instance: string): Promise { + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(`${service}:${instance}`)); + return Array.from(new Uint8Array(digest).slice(0, 6), (b) => b.toString(16).padStart(2, "0")).join(""); +} + +function errorClass(cause: unknown): string { + const name = cause instanceof Error ? cause.name : typeof cause; + return ERROR_CLASS.test(name) ? name : "Unknown"; +} + +function rayOf(headers: Headers): string | null { + const ray = headers.get("cf-ray"); + return ray && RAY.test(ray) ? ray : null; +} + +// Paths the Durable Object serves itself, outside the service's router. +const OBJECT_ROUTES = new Set(["/__seed", "/__reset", "/__token"]); + +// One route table per service and isolate, built only when a failure needs it: +// the routes are registered statically, so a throwaway server answers which +// pattern a path matches without touching any instance's state. +const routers = new Map["app"]>(); + +function routeTemplate(service: string, method: string, path: string): string { + if (OBJECT_ROUTES.has(path)) return path; + try { + let app = routers.get(service); + if (!app) { + const entry = SERVICES[service]; + app = createServer(entry.plugin, { baseUrl: "https://route.invalid", manifest: entry.manifest }).app; + routers.set(service, app); + } + return app.routePattern(method, path) ?? "unmatched"; + } catch { + // A report must never fail on its way out; the template is a convenience. + return "unknown"; + } +} diff --git a/packages/@emulators/cloudflare/src/durable-object.ts b/packages/@emulators/cloudflare/src/durable-object.ts index 0ccc6edd6..907582ae6 100644 --- a/packages/@emulators/cloudflare/src/durable-object.ts +++ b/packages/@emulators/cloudflare/src/durable-object.ts @@ -1,5 +1,6 @@ import { createServer, + isPlatformFailure, type AppKeyResolver, type LedgerEntry, type LedgerSnapshot, @@ -9,6 +10,7 @@ import { type TokenMap, } from "@emulators/core"; import { SERVICES, issueCloudflareCredential } from "./services.js"; +import { failureReport } from "./diagnostics.js"; // Minimal CF runtime types (avoid a hard dep on @cloudflare/workers-types here). interface DurableObjectStorage { @@ -105,11 +107,6 @@ const ledgerEntryKey = (id: string): string => `${LEDGER_ENTRY_PREFIX}${encodeKe // instance survives eviction. Auth is FAITHFUL by // default (strict): only seeded or minted tokens work; everything else gets the // real API's 401/403. Mint tokens at runtime via `POST /__token`. -const isCloudflareFailure = (error: unknown): boolean => - typeof error === "object" && - error !== null && - ((error as { retryable?: unknown }).retryable === true || (error as { overloaded?: unknown }).overloaded === true); - export class EmulatorDurableObject { private live?: Live; // The JSON of every snapshot and ledger value last written to storage, by key. @@ -336,6 +333,7 @@ export class EmulatorDurableObject { manifest: entry.manifest, instance, ledgerPersistent: true, + rethrowUnexpectedErrors: true, reset: () => resetService(), seed: async (seed) => { if (seed && entry.seedFromConfig) { @@ -458,21 +456,19 @@ export class EmulatorDurableObject { return await this.handle(request); } catch (error) { // Cloudflare's own failures (e.g. "object has moved to a different machine") - // carry `.retryable`/`.overloaded`; rethrow them so the Worker sees the flags - // and applies Cloudflare's retry contract. - if (isCloudflareFailure(error)) throw error; - // Answer with the cause instead of throwing: an exception crossing the stub - // reaches the client as an opaque Cloudflare 500 page with no message. - const report = { - error: "emulator_error", - message: error instanceof Error ? error.message : String(error), - service: request.headers.get("x-emulator-service"), - instance: request.headers.get("x-emulator-instance"), + // carry `.retryable`/`.overloaded`; rethrow them so the Worker reports the flags. + if (isPlatformFailure(error)) throw error; + // Answer with a report instead of throwing: an exception crossing the stub + // reaches the client as an opaque Cloudflare 500 page. The report and the + // log hold no message or stack, which can quote tokens, codes or emails. + const report = await failureReport("emulator_error", error, { + service: request.headers.get("x-emulator-service") ?? "", + instance: request.headers.get("x-emulator-instance") ?? "default", method: request.method, path: new URL(request.url).pathname, - ray: request.headers.get("cf-ray"), - }; - console.error(JSON.stringify({ ...report, stack: error instanceof Error ? error.stack : undefined })); + headers: request.headers, + }); + console.error(JSON.stringify(report)); return Response.json(report, { status: 500 }); } } diff --git a/packages/@emulators/cloudflare/src/worker.ts b/packages/@emulators/cloudflare/src/worker.ts index 31715a52a..2791edc64 100644 --- a/packages/@emulators/cloudflare/src/worker.ts +++ b/packages/@emulators/cloudflare/src/worker.ts @@ -9,6 +9,7 @@ import { EmulatorDurableObject } from "./durable-object.js"; import { SERVICES } from "./services.js"; import { SERVICE_ICONS } from "./icons.js"; import { consoleHtml } from "./console-html.js"; +import { failureReport } from "./diagnostics.js"; export { EmulatorDurableObject }; @@ -78,81 +79,36 @@ async function forwardToDurableObject( // Manual redirect: the emulator's OAuth callbacks return 302s (e.g. → the app's // redirect_uri). Without this, the Worker→DO stub.fetch FOLLOWS the redirect // internally and re-fetches the DO by the Location path, mangling it. - const inner = () => - new Request(`${origin}${opts.innerPath}${opts.search}`, { + const inner = new Request(`${origin}${opts.innerPath}${opts.search}`, { + method: request.method, + headers, + body, + redirect: "manual", + }); + const id = env.EMULATOR.idFromName(`${opts.service}:${opts.instance}`); + try { + return await env.EMULATOR.get(id).fetch(inner); + } catch (error) { + // The stub failed: Cloudflare could not reach the object, the object was + // overloaded, or a platform failure inside it (flagged `.retryable` or + // `.overloaded`) propagated out. Uncaught, this reaches the client as an + // opaque Cloudflare 500 page. Nothing is retried: a flag does not prove the + // object never ran the request, and replaying a reset, an OAuth authorize or + // a write can change state twice. + const report = await failureReport("emulator_unavailable", error, { + service: opts.service, + instance: opts.instance, method: request.method, - headers, - body, - redirect: "manual", + path: opts.innerPath, + headers: request.headers, }); - const id = env.EMULATOR.idFromName(`${opts.service}:${opts.instance}`); - const idempotent = isIdempotent(request.method, opts.innerPath); - for (let attempt = 1; ; attempt++) { - try { - // A stub that threw is not reused: Cloudflare documents that many - // exceptions leave it broken, so every attempt gets a fresh one. - return await env.EMULATOR.get(id).fetch(inner()); - } catch (error) { - const failure = durableObjectFailure(error); - if (idempotent && failure.retryable && !failure.overloaded && attempt < DO_MAX_ATTEMPTS) { - await sleep(DO_BASE_BACKOFF_MS * Math.random() * 2 ** attempt); - continue; - } - const report = { - error: "emulator_unavailable", - message: failure.message, - service: opts.service, - instance: opts.instance, - method: request.method, - path: opts.innerPath, - attempts: attempt, - retryable: failure.retryable, - overloaded: failure.overloaded, - remote: failure.remote, - ray: request.headers.get("cf-ray"), - }; - console.error(JSON.stringify(report)); - // 503 when Cloudflare says the object was unreachable or overloaded; 500 - // when the emulator itself threw or was killed (remote, not retryable). - return Response.json(report, { status: failure.retryable || failure.overloaded ? 503 : 500 }); - } + console.error(JSON.stringify(report)); + // 503 when Cloudflare says the object was unreachable or overloaded; 500 + // when the emulator itself threw or was killed (remote, not retryable). + return Response.json(report, { status: report.retryable || report.overloaded ? 503 : 500 }); } } -// Cloudflare's Durable Object error contract: an exception with `.retryable` -// may be retried if the request is idempotent; one with `.overloaded` must not -// be; `.remote` marks an error raised by (or a limit hit in) the object itself. -// https://developers.cloudflare.com/durable-objects/best-practices/error-handling/ -const DO_MAX_ATTEMPTS = 3; -const DO_BASE_BACKOFF_MS = 100; -const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - -// Only requests whose repetition cannot change the result are retried: GET and -// HEAD (idempotent by RFC 9110) and `POST /_emulate/reset`, which always restores -// the instance's seeded state and answers `{ ok: true }`. Seeds, credentials and -// provider writes are not idempotent and fail precisely instead. -const isIdempotent = (method: string, innerPath: string): boolean => - method === "GET" || method === "HEAD" || (method === "POST" && innerPath === "/_emulate/reset"); - -function durableObjectFailure(error: unknown): { - message: string; - retryable: boolean; - overloaded: boolean; - remote: boolean; -} { - const flags = (typeof error === "object" && error !== null ? error : {}) as { - retryable?: unknown; - overloaded?: unknown; - remote?: unknown; - }; - return { - message: error instanceof Error ? error.message : String(error), - retryable: flags.retryable === true, - overloaded: flags.overloaded === true, - remote: flags.remote === true, - }; -} - // Router: // - `.` -> the service-level control plane (no shared instance). // - `..` -> a named instance host (requires a 2-label cert). diff --git a/packages/@emulators/cloudflare/wrangler.jsonc b/packages/@emulators/cloudflare/wrangler.jsonc index f87efc93b..f0954998e 100644 --- a/packages/@emulators/cloudflare/wrangler.jsonc +++ b/packages/@emulators/cloudflare/wrangler.jsonc @@ -22,11 +22,16 @@ "zone_name": "emulators.dev" } ], - // Workers Logs: keeps every invocation's logs and uncaught exceptions, - // including the structured `emulator_unavailable` reports from worker.ts, so - // a 5xx seen by a client can be traced by its cf-ray. + // Workers Logs keeps the structured failure reports from worker.ts and + // durable-object.ts, so a 5xx seen by a client can be traced by its cf-ray. + // Invocation logs are off: each one records the request URL, and an + // instance URL is the only access control for its emulator. "observability": { - "enabled": true + "enabled": true, + "logs": { + "enabled": true, + "invocation_logs": false + } }, "vars": { "EMULATE_HOST_SUFFIX": "emulators.dev" diff --git a/packages/@emulators/core/src/control-plane.ts b/packages/@emulators/core/src/control-plane.ts index 55c05841a..4aec773c3 100644 --- a/packages/@emulators/core/src/control-plane.ts +++ b/packages/@emulators/core/src/control-plane.ts @@ -8,6 +8,7 @@ import { coverageReport, enrichManifest, resolveConnections } from "./manifest.j import type { TokenMap } from "./middleware/auth.js"; import { escapeHtml, renderCardPage } from "./ui.js"; import type { FaultArmInput, FaultRegistry } from "./faults.js"; +import { isPlatformFailure } from "./platform-failure.js"; export interface CredentialRequest { type?: string; @@ -198,6 +199,7 @@ export function registerControlPlane(app: Hono, options: ControlPlaneOpt try { await options.seed(body); } catch (err) { + if (isPlatformFailure(err)) throw err; return c.json({ error: "invalid_seed", message: err instanceof Error ? err.message : "Seed failed." }, 400); } return c.json({ ok: true }); @@ -215,6 +217,7 @@ export function registerControlPlane(app: Hono, options: ControlPlaneOpt } return c.json({ credential }); } catch (err) { + if (isPlatformFailure(err)) throw err; return c.json( { error: "unsupported", message: err instanceof Error ? err.message : "Credential creation failed." }, 400, diff --git a/packages/@emulators/core/src/http.ts b/packages/@emulators/core/src/http.ts index 153c4f478..78952afc9 100644 --- a/packages/@emulators/core/src/http.ts +++ b/packages/@emulators/core/src/http.ts @@ -1,4 +1,5 @@ import { createServer as createNodeServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import { isPlatformFailure } from "./platform-failure.js"; type BodyInit = ConstructorParameters[0]; type HeadersInit = ConstructorParameters[0]; @@ -285,10 +286,16 @@ export class Hono { const response = await this.dispatch(context, matched.handlers); return context.finalize(response ?? (await this.notFoundHandler(context))); } catch (err) { + if (isPlatformFailure(err)) throw err; return context.finalize(await this.errorHandler(err, context)); } }; + /** The registered pattern (e.g. `/repos/:owner/:repo`) that would serve this request, if any. */ + routePattern(method: string, path: string): string | undefined { + return this.match(method.toUpperCase(), path).routePattern; + } + private match( method: string, path: string, diff --git a/packages/@emulators/core/src/index.ts b/packages/@emulators/core/src/index.ts index c784244ee..e8daa8eab 100644 --- a/packages/@emulators/core/src/index.ts +++ b/packages/@emulators/core/src/index.ts @@ -156,3 +156,4 @@ export { type ManifestResponse, type SpecsResponse, } from "./client.js"; +export { isPlatformFailure } from "./platform-failure.js"; diff --git a/packages/@emulators/core/src/platform-failure.ts b/packages/@emulators/core/src/platform-failure.ts new file mode 100644 index 000000000..8cc743731 --- /dev/null +++ b/packages/@emulators/core/src/platform-failure.ts @@ -0,0 +1,10 @@ +// Cloudflare raises its own runtime failures (a Durable Object that moved to a +// different machine, a lost connection, an overloaded object) as exceptions +// flagged `.retryable` or `.overloaded`. Emulator code must not answer these as +// its own errors: they propagate to the host, which reports them with the flags. +// https://developers.cloudflare.com/durable-objects/best-practices/error-handling/ +export function isPlatformFailure(err: unknown): boolean { + if (typeof err !== "object" || err === null) return false; + const flags = err as { retryable?: unknown; overloaded?: unknown }; + return flags.retryable === true || flags.overloaded === true; +} diff --git a/packages/@emulators/core/src/server.ts b/packages/@emulators/core/src/server.ts index 3c38d8216..7797fda2b 100644 --- a/packages/@emulators/core/src/server.ts +++ b/packages/@emulators/core/src/server.ts @@ -33,6 +33,13 @@ export interface ServerOptions { reset?: () => void | Promise; seed?: (seed: unknown) => void | Promise; issueCredential?: (request: CredentialRequest) => IssuedCredential | Promise; + /** + * Rethrow errors that carry no HTTP `status` instead of answering them with + * their message. A host that reports failures itself (the Cloudflare Durable + * Object) sets this so raw messages, which can quote tokens, codes or emails, + * never reach a response. + */ + rethrowUnexpectedErrors?: boolean; } export function createServer(plugin: ServicePlugin, options: ServerOptions = {}) { @@ -60,7 +67,15 @@ export function createServer(plugin: ServicePlugin, options: ServerOptions = {}) registerFontRoutes(app); - app.onError(createApiErrorHandler(docsUrl)); + const apiErrorHandler = createApiErrorHandler(docsUrl); + app.onError( + options.rethrowUnexpectedErrors + ? (err, c) => { + if (!hasHttpStatus(err)) throw err; + return apiErrorHandler(err, c); + } + : apiErrorHandler, + ); app.use("*", cors()); app.use("*", createErrorHandler(docsUrl)); app.use("*", authMiddleware(tokenMap, options.appKeyResolver, options.fallbackUser)); @@ -146,3 +161,6 @@ export function createServer(plugin: ServicePlugin, options: ServerOptions = {}) return { app, store, webhooks, ledger, faults, port, baseUrl, tokenMap }; } + +const hasHttpStatus = (err: unknown): boolean => + typeof err === "object" && err !== null && typeof (err as { status?: unknown }).status === "number"; From 71dce9e7b87a267b42420bd38e9774c04901784b Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Thu, 8 Oct 2026 00:26:05 -0700 Subject: [PATCH 4/5] Answer every Worker and Durable Object failure with an allowlisted report Neither boundary lets an error escape to Cloudflare's exception logging. The Durable Object reports Cloudflare's flagged failures itself instead of rethrowing them, so core no longer passes them through its router. Report fields come from fixed lists: error class names, HTTP methods, registered services and declared route templates. Seed and credential requests answer 400 only for typed rejections; other errors go to the error handler. --- .../@emulators/cloudflare/src/diagnostics.ts | 96 ++++- .../cloudflare/src/durable-object.ts | 19 +- .../@emulators/cloudflare/src/services.ts | 10 +- packages/@emulators/cloudflare/src/worker.ts | 350 +++++++++--------- packages/@emulators/cloudflare/wrangler.jsonc | 5 +- .../core/src/control-plane-rejection.ts | 22 ++ packages/@emulators/core/src/control-plane.ts | 13 +- packages/@emulators/core/src/http.ts | 2 - packages/@emulators/core/src/index.ts | 2 +- .../@emulators/core/src/platform-failure.ts | 10 - packages/@emulators/planetscale/src/index.ts | 8 +- packages/emulate/src/registry.ts | 8 +- 12 files changed, 318 insertions(+), 227 deletions(-) create mode 100644 packages/@emulators/core/src/control-plane-rejection.ts delete mode 100644 packages/@emulators/core/src/platform-failure.ts diff --git a/packages/@emulators/cloudflare/src/diagnostics.ts b/packages/@emulators/cloudflare/src/diagnostics.ts index 7f71db472..a0aef08a1 100644 --- a/packages/@emulators/cloudflare/src/diagnostics.ts +++ b/packages/@emulators/cloudflare/src/diagnostics.ts @@ -1,16 +1,18 @@ import { createServer } from "@emulators/core"; import { SERVICES } from "./services.js"; -// Failure reports go to the client and to Workers Logs, so they carry only -// values that cannot hold a secret or user data. The instance name is the -// sole access control for its emulator, request paths hold ids, emails and -// codes, and error messages and stacks can quote any of them. A report names -// the instance by a short hash, the request by its route template, and the -// error by its class and Cloudflare's flags. +// Failure reports go to the client and to Workers Logs, so every field is +// either a fixed value or chosen from a fixed list: no field copies text from +// the request or the error. The instance name is the sole access control for +// its emulator, request paths hold ids, emails and codes, and error messages, +// names and stacks can quote any of them. A report names the service from the +// registry, the instance by a short hash, the request by a route template the +// service's router declares, and the error by an allowlisted class name and +// Cloudflare's flags. export interface FailureReport { - error: "emulator_unavailable" | "emulator_error"; + error: "emulator_unavailable" | "emulator_error" | "worker_error"; service: string; - instanceId: string; + instanceId: string | null; method: string; route: string; errorClass: string; @@ -34,8 +36,8 @@ export async function failureReport( return { error, service: known ? request.service : "unknown", - instanceId: await instanceId(request.service, request.instance), - method: METHOD.test(request.method) ? request.method : "OTHER", + instanceId: known ? await instanceId(request.service, request.instance) : null, + method: METHODS.has(request.method) ? request.method : "OTHER", route: known ? routeTemplate(request.service, request.method, request.path) : "unmatched", errorClass: errorClass(cause), retryable: flags.retryable === true, @@ -45,21 +47,83 @@ export async function failureReport( }; } -const METHOD = /^[A-Z]{1,10}$/; +// The response for a failure at the Worker or Durable Object boundary: the +// report, logged once and returned as JSON. 503 when Cloudflare flags the +// failure retryable or overloaded, otherwise 500. Building it never throws, so +// no raw error escapes the boundary to Cloudflare's exception logging. +export async function failureResponse( + error: FailureReport["error"], + cause: unknown, + request: Parameters[2], +): Promise { + let report: FailureReport; + try { + report = await failureReport(error, cause, request); + } catch { + report = { + error, + service: "unknown", + instanceId: null, + method: "OTHER", + route: "unknown", + errorClass: "other", + retryable: false, + overloaded: false, + remote: false, + ray: null, + }; + } + console.error(JSON.stringify(report)); + return Response.json(report, { status: report.retryable || report.overloaded ? 503 : 500 }); +} + +// Cloudflare raises its own runtime failures (a Durable Object that moved to a +// different machine, a lost connection, an overloaded object) as errors +// flagged `.retryable` or `.overloaded`. +// https://developers.cloudflare.com/durable-objects/best-practices/error-handling/ +export function isPlatformFailure(err: unknown): boolean { + if (typeof err !== "object" || err === null) return false; + const flags = err as { retryable?: unknown; overloaded?: unknown }; + return flags.retryable === true || flags.overloaded === true; +} + +const METHODS = new Set(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"]); const RAY = /^[0-9a-f]{16}(-[A-Z]{3})?$/; -const ERROR_CLASS = /^[A-Za-z][A-Za-z0-9]{0,39}$/; +// JavaScript's built-in error classes and the DOMException names the Workers +// runtime raises. Cloudflare's Durable Object failures are plain `Error`s told +// apart by their flags. Any other name, which code can set to anything, is +// reported as "other". +const ERROR_CLASSES = new Set([ + "Error", + "TypeError", + "RangeError", + "SyntaxError", + "ReferenceError", + "EvalError", + "URIError", + "AggregateError", + "AbortError", + "TimeoutError", + "DataCloneError", + "QuotaExceededError", + "InvalidStateError", + "NetworkError", + "OperationError", +]); // The first 12 hex digits of SHA-256(`:`), the Durable Object -// name. Instance suffixes carry 96 random bits, so the hash cannot be reversed, -// and anyone holding the instance URL can compute it to find their reports. +// name; anyone holding the instance URL can compute it to find their reports. +// Generated instance names end in 96 random bits, so their hashes cannot be +// enumerated. The hash does not hide a predictable or low-entropy name (one a +// caller chose, or a legacy fixed name): hashing guesses confirms it. export async function instanceId(service: string, instance: string): Promise { const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(`${service}:${instance}`)); return Array.from(new Uint8Array(digest).slice(0, 6), (b) => b.toString(16).padStart(2, "0")).join(""); } function errorClass(cause: unknown): string { - const name = cause instanceof Error ? cause.name : typeof cause; - return ERROR_CLASS.test(name) ? name : "Unknown"; + const name = typeof cause === "object" && cause !== null ? (cause as { name?: unknown }).name : undefined; + return typeof name === "string" && ERROR_CLASSES.has(name) ? name : "other"; } function rayOf(headers: Headers): string | null { diff --git a/packages/@emulators/cloudflare/src/durable-object.ts b/packages/@emulators/cloudflare/src/durable-object.ts index 907582ae6..f70129a68 100644 --- a/packages/@emulators/cloudflare/src/durable-object.ts +++ b/packages/@emulators/cloudflare/src/durable-object.ts @@ -1,6 +1,5 @@ import { createServer, - isPlatformFailure, type AppKeyResolver, type LedgerEntry, type LedgerSnapshot, @@ -10,7 +9,7 @@ import { type TokenMap, } from "@emulators/core"; import { SERVICES, issueCloudflareCredential } from "./services.js"; -import { failureReport } from "./diagnostics.js"; +import { failureResponse, isPlatformFailure } from "./diagnostics.js"; // Minimal CF runtime types (avoid a hard dep on @cloudflare/workers-types here). interface DurableObjectStorage { @@ -455,21 +454,19 @@ export class EmulatorDurableObject { try { return await this.handle(request); } catch (error) { - // Cloudflare's own failures (e.g. "object has moved to a different machine") - // carry `.retryable`/`.overloaded`; rethrow them so the Worker reports the flags. - if (isPlatformFailure(error)) throw error; - // Answer with a report instead of throwing: an exception crossing the stub - // reaches the client as an opaque Cloudflare 500 page. The report and the - // log hold no message or stack, which can quote tokens, codes or emails. - const report = await failureReport("emulator_error", error, { + // Nothing may escape this boundary. A thrown error reaches the client as + // an opaque Cloudflare 500, and Cloudflare records it as an exception + // event with its message, stack and the request URL, any of which can + // quote tokens, codes, emails or the instance name. Cloudflare's own + // failures (e.g. "object has moved to a different machine") keep their + // `.retryable`/`.overloaded` flags in the report and its 503 status. + return failureResponse(isPlatformFailure(error) ? "emulator_unavailable" : "emulator_error", error, { service: request.headers.get("x-emulator-service") ?? "", instance: request.headers.get("x-emulator-instance") ?? "default", method: request.method, path: new URL(request.url).pathname, headers: request.headers, }); - console.error(JSON.stringify(report)); - return Response.json(report, { status: 500 }); } } diff --git a/packages/@emulators/cloudflare/src/services.ts b/packages/@emulators/cloudflare/src/services.ts index 1c56b0005..84259cf1e 100644 --- a/packages/@emulators/cloudflare/src/services.ts +++ b/packages/@emulators/cloudflare/src/services.ts @@ -10,6 +10,7 @@ import type { TokenMap, WebhookDispatcher, } from "@emulators/core"; +import { ControlPlaneRejection } from "@emulators/core"; import { getGitHubStore, githubPlugin, @@ -400,7 +401,7 @@ export function issueCloudflareCredential( } const type = request.type ?? entry.manifest.auth[0]?.type ?? "bearer-token"; if (type === "bearer-token" || type === "api-key") { - if (!tokenMap) throw new Error(`Credential type ${type} is not supported by ${service}`); + if (!tokenMap) throw new ControlPlaneRejection(`Credential type ${type} is not supported by ${service}`); const login = request.login ?? "admin"; const scopes = Array.isArray(request.scopes) ? request.scopes.filter((s): s is string => typeof s === "string") @@ -419,13 +420,14 @@ export function issueCloudflareCredential( type === "oauth-client-credentials" || type === "dynamic-client-registration" ) { - if (!entry.seedFromConfig) throw new Error(`Credential type ${type} is not supported by ${service}`); + if (!entry.seedFromConfig) + throw new ControlPlaneRejection(`Credential type ${type} is not supported by ${service}`); const clientId = request.client_id ?? defaultClientId(service); const clientSecret = request.client_secret ?? defaultClientSecret(service); const redirectUris = normalizeRedirectUris(request.redirect_uris); const name = request.name ?? `${entry.manifest.name} Client`; const seed = credentialSeed(service, { clientId, clientSecret, redirectUris, name, request }); - if (!seed) throw new Error(`Credential type ${type} is not supported by ${service}`); + if (!seed) throw new ControlPlaneRejection(`Credential type ${type} is not supported by ${service}`); entry.seedFromConfig(store, baseUrl, seed); return { type, @@ -437,7 +439,7 @@ export function issueCloudflareCredential( }; } - throw new Error(`Credential type ${type} is not supported by ${service}`); + throw new ControlPlaneRejection(`Credential type ${type} is not supported by ${service}`); } function tokenPrefix(service: string, type: string): string { diff --git a/packages/@emulators/cloudflare/src/worker.ts b/packages/@emulators/cloudflare/src/worker.ts index 2791edc64..235486735 100644 --- a/packages/@emulators/cloudflare/src/worker.ts +++ b/packages/@emulators/cloudflare/src/worker.ts @@ -9,7 +9,7 @@ import { EmulatorDurableObject } from "./durable-object.js"; import { SERVICES } from "./services.js"; import { SERVICE_ICONS } from "./icons.js"; import { consoleHtml } from "./console-html.js"; -import { failureReport } from "./diagnostics.js"; +import { failureResponse } from "./diagnostics.js"; export { EmulatorDurableObject }; @@ -68,44 +68,41 @@ async function forwardToDurableObject( request: Request, opts: { service: string; instance: string; baseUrl: string; innerPath: string; search: string; mcpMode?: string }, ): Promise { - const origin = new URL(request.url).origin; - const headers = new Headers(request.headers); - headers.set("x-emulator-service", opts.service); - headers.set("x-emulator-instance", opts.instance); - headers.set("x-emulator-base-url", opts.baseUrl); - if (opts.mcpMode) headers.set("x-emulator-mcp-mode", opts.mcpMode); - const hasBody = request.method !== "GET" && request.method !== "HEAD"; - const body = hasBody ? await request.arrayBuffer() : undefined; - // Manual redirect: the emulator's OAuth callbacks return 302s (e.g. → the app's - // redirect_uri). Without this, the Worker→DO stub.fetch FOLLOWS the redirect - // internally and re-fetches the DO by the Location path, mangling it. - const inner = new Request(`${origin}${opts.innerPath}${opts.search}`, { - method: request.method, - headers, - body, - redirect: "manual", - }); - const id = env.EMULATOR.idFromName(`${opts.service}:${opts.instance}`); try { + const origin = new URL(request.url).origin; + const headers = new Headers(request.headers); + headers.set("x-emulator-service", opts.service); + headers.set("x-emulator-instance", opts.instance); + headers.set("x-emulator-base-url", opts.baseUrl); + if (opts.mcpMode) headers.set("x-emulator-mcp-mode", opts.mcpMode); + const hasBody = request.method !== "GET" && request.method !== "HEAD"; + const body = hasBody ? await request.arrayBuffer() : undefined; + // Manual redirect: the emulator's OAuth callbacks return 302s (e.g. → the app's + // redirect_uri). Without this, the Worker→DO stub.fetch FOLLOWS the redirect + // internally and re-fetches the DO by the Location path, mangling it. + const inner = new Request(`${origin}${opts.innerPath}${opts.search}`, { + method: request.method, + headers, + body, + redirect: "manual", + }); + const id = env.EMULATOR.idFromName(`${opts.service}:${opts.instance}`); + // The object answers its own failures, Cloudflare's flagged ones included, + // with a report; this passes that report through unchanged. return await env.EMULATOR.get(id).fetch(inner); } catch (error) { - // The stub failed: Cloudflare could not reach the object, the object was - // overloaded, or a platform failure inside it (flagged `.retryable` or - // `.overloaded`) propagated out. Uncaught, this reaches the client as an - // opaque Cloudflare 500 page. Nothing is retried: a flag does not prove the - // object never ran the request, and replaying a reset, an OAuth authorize or - // a write can change state twice. - const report = await failureReport("emulator_unavailable", error, { + // Reading the body, addressing the object or the stub itself failed: + // Cloudflare could not reach the object, the object was overloaded, or it + // was killed by its own limits. Nothing is retried: a flag does not prove + // the object never ran the request, and replaying a reset, an OAuth + // authorize or a write can change state twice. + return failureResponse("emulator_unavailable", error, { service: opts.service, instance: opts.instance, method: request.method, path: opts.innerPath, headers: request.headers, }); - console.error(JSON.stringify(report)); - // 503 when Cloudflare says the object was unreachable or overloaded; 500 - // when the emulator itself threw or was killed (remote, not retryable). - return Response.json(report, { status: report.retryable || report.overloaded ? 503 : 500 }); } } @@ -116,89 +113,94 @@ async function forwardToDurableObject( // - apex / bare `/` -> the catalog (SPA for browsers, server-rendered for agents). export default { async fetch(request: Request, env: Env): Promise { - const url = new URL(request.url); - const suffix = env.EMULATE_HOST_SUFFIX ?? DEFAULT_HOST_SUFFIX; - - // The docs site is a separate worker on the docs. custom domain. - // Routes take precedence over custom domains on the same hostname, so the - // *./* route lands here first; a same-URL fetch passes the request - // through to the custom-domain worker (the documented route -> custom - // domain chaining pattern). - if (url.hostname === `docs.${suffix}`) return fetch(request); - const segments = url.pathname - .replace(/^\/+/, "") - .split("/") - .filter((s) => s.length > 0); - const hostRoute = parseHostRoute(url.hostname, suffix); - const apexOrigin = hostRoute ? `${url.protocol}//${suffix}` : url.origin; - - // Official provider brand icons, served from any host (falls back to a - // monogram client-side if missing). - if (url.pathname.startsWith("/_emulate/icons/") && (request.method === "GET" || request.method === "HEAD")) { - const id = url.pathname.slice("/_emulate/icons/".length).replace(/\.svg$/, ""); - const svg = SERVICE_ICONS[id]; - if (!svg) return new Response("not found", { status: 404 }); - return new Response(svg, { - headers: { "content-type": "image/svg+xml; charset=utf-8", "cache-control": "public, max-age=86400" }, + try { + return await route(request, env); + } catch (error) { + // Nothing may escape the Worker: Cloudflare records an uncaught + // exception with its message, stack and the request URL, and the + // instance URL is the only access control for its emulator. + return failureResponse("worker_error", error, { + service: "", + instance: "", + method: request.method, + path: "", + headers: request.headers, }); } + }, +}; - // Machine-readable catalog of every service this host serves (any host). - if (url.pathname === "/_emulate/services" && (request.method === "GET" || request.method === "HEAD")) { - return servicesCatalog(serviceCatalogEntries(), { - origin: apexOrigin, - protocol: url.protocol, - hostSuffix: suffix, - }); - } +async function route(request: Request, env: Env): Promise { + const url = new URL(request.url); + const suffix = env.EMULATE_HOST_SUFFIX ?? DEFAULT_HOST_SUFFIX; + + // The docs site is a separate worker on the docs. custom domain. + // Routes take precedence over custom domains on the same hostname, so the + // *./* route lands here first; a same-URL fetch passes the request + // through to the custom-domain worker (the documented route -> custom + // domain chaining pattern). + if (url.hostname === `docs.${suffix}`) return fetch(request); + const segments = url.pathname + .replace(/^\/+/, "") + .split("/") + .filter((s) => s.length > 0); + const hostRoute = parseHostRoute(url.hostname, suffix); + const apexOrigin = hostRoute ? `${url.protocol}//${suffix}` : url.origin; - // SERVICE HOST: . with no instance label. Control plane - // only — there is deliberately NO shared default instance behind it. A - // well-known host with world-readable state and a world-writable control - // plane is exactly the polling target the unguessable instance names exist - // to prevent, so provider traffic requires an instance of your own. - if (hostRoute?.service && !hostRoute.instance) { - const service = hostRoute.service; - const entry = SERVICES[service]; + // Official provider brand icons, served from any host (falls back to a + // monogram client-side if missing). + if (url.pathname.startsWith("/_emulate/icons/") && (request.method === "GET" || request.method === "HEAD")) { + const id = url.pathname.slice("/_emulate/icons/".length).replace(/\.svg$/, ""); + const svg = SERVICE_ICONS[id]; + if (!svg) return new Response("not found", { status: 404 }); + return new Response(svg, { + headers: { "content-type": "image/svg+xml; charset=utf-8", "cache-control": "public, max-age=86400" }, + }); + } - // Create a NAMED, isolated instance. Returned in the cert-safe path form - // (a 2-label instance subdomain has no Universal SSL certificate). A - // caller-supplied name only prefixes the generated one: the instance URL - // is the sole access control, so it must never be guessable. - if (url.pathname === "/_emulate/instances" && request.method === "POST") { - const body = (await request.json().catch(() => ({}))) as { instance?: string }; - const instance = randomInstanceName(body.instance); - return Response.json( - buildInstanceCreation({ - service, - instance, - providerBaseUrl: `${apexOrigin}/${service}/${instance}`, - pathOrigin: apexOrigin, - hostSuffix: suffix, - }), - ); - } + // Machine-readable catalog of every service this host serves (any host). + if (url.pathname === "/_emulate/services" && (request.method === "GET" || request.method === "HEAD")) { + return servicesCatalog(serviceCatalogEntries(), { + origin: apexOrigin, + protocol: url.protocol, + hostSuffix: suffix, + }); + } - if (!entry) return html(consoleHtml); + // SERVICE HOST: . with no instance label. Control plane + // only — there is deliberately NO shared default instance behind it. A + // well-known host with world-readable state and a world-writable control + // plane is exactly the polling target the unguessable instance names exist + // to prevent, so provider traffic requires an instance of your own. + if (hostRoute?.service && !hostRoute.instance) { + const service = hostRoute.service; + const entry = SERVICES[service]; - // Root: browsers get the interactive console; agents/raw fetches get the - // server-rendered service landing (no JS) or the service-level manifest. - if (url.pathname === "/" || url.pathname === "") { - if (isBrowserNavigation(request)) return html(consoleHtml); - const landing = serviceHostControlPlane(wantsJson(request) ? "/_emulate/manifest" : "/_emulate", "GET", { - manifest: entry.manifest, + // Create a NAMED, isolated instance. Returned in the cert-safe path form + // (a 2-label instance subdomain has no Universal SSL certificate). A + // caller-supplied name only prefixes the generated one: the instance URL + // is the sole access control, so it must never be guessable. + if (url.pathname === "/_emulate/instances" && request.method === "POST") { + const body = (await request.json().catch(() => ({}))) as { instance?: string }; + const instance = randomInstanceName(body.instance); + return Response.json( + buildInstanceCreation({ service, - origin: apexOrigin, - protocol: url.protocol, + instance, + providerBaseUrl: `${apexOrigin}/${service}/${instance}`, + pathOrigin: apexOrigin, hostSuffix: suffix, - ledgerPersistent: true, - }); - if (landing) return landing; - } + }), + ); + } - // Service-level control plane (manifest, quickstart, specs, coverage, - // connections, openapi) — answerable without any instance. - const controlPlane = serviceHostControlPlane(url.pathname, request.method, { + if (!entry) return html(consoleHtml); + + // Root: browsers get the interactive console; agents/raw fetches get the + // server-rendered service landing (no JS) or the service-level manifest. + if (url.pathname === "/" || url.pathname === "") { + if (isBrowserNavigation(request)) return html(consoleHtml); + const landing = serviceHostControlPlane(wantsJson(request) ? "/_emulate/manifest" : "/_emulate", "GET", { manifest: entry.manifest, service, origin: apexOrigin, @@ -206,79 +208,91 @@ export default { hostSuffix: suffix, ledgerPersistent: true, }); - if (controlPlane) return controlPlane; - - // Browsers exploring other paths still get the console SPA. - if (isBrowserNavigation(request)) return html(consoleHtml); - - // Provider routes have no shared instance to serve: point the caller at - // instance creation instead. - return Response.json( - { - error: "instance_required", - message: `${service}.${suffix} is a service host with no shared instance. Create one with POST ${url.protocol}//${service}.${suffix}/_emulate/instances and use the returned providerBaseUrl.`, - createInstance: `${url.protocol}//${service}.${suffix}/_emulate/instances`, - }, - { status: 404 }, - ); - } - - // NAMED INSTANCE HOST: .. (needs a 2-label cert). - if (hostRoute?.service && hostRoute.instance) { - return forwardToDurableObject(env, request, { - service: hostRoute.service, - instance: hostRoute.instance, - baseUrl: url.origin, - innerPath: url.pathname, - search: url.search, - }); + if (landing) return landing; } - // PATH FORM (apex/local): `///...`. Peel a leading - // `.well-known/` (RFC 8414 / 9728 path-aware metadata discovery: clients - // probe `/.well-known//`) so the resolved resource is - // instance-prefixed and the emulator answers with instance-scoped endpoints. - let wellKnownType: string | undefined; - let resource = segments; - if (segments[0] === ".well-known" && segments.length >= 2) { - wellKnownType = segments[1]; - resource = segments.slice(2); - } + // Service-level control plane (manifest, quickstart, specs, coverage, + // connections, openapi) — answerable without any instance. + const controlPlane = serviceHostControlPlane(url.pathname, request.method, { + manifest: entry.manifest, + service, + origin: apexOrigin, + protocol: url.protocol, + hostSuffix: suffix, + ledgerPersistent: true, + }); + if (controlPlane) return controlPlane; - const service = resource[0]; - const instance = resource[1]; + // Browsers exploring other paths still get the console SPA. + if (isBrowserNavigation(request)) return html(consoleHtml); - // Apex root or bare `/` → the catalog. Browser → SPA; agent → a - // server-rendered list of emulators (readable without running JS). - if (!service || !instance) { - if (isBrowserNavigation(request)) - return serveCatalogConsole({ origin: apexOrigin, protocol: url.protocol, hostSuffix: suffix }); - if (wantsJson(request)) { - return servicesCatalog(serviceCatalogEntries(), { - origin: apexOrigin, - protocol: url.protocol, - hostSuffix: suffix, - }); - } - return html( - renderCatalogPage(serviceCatalogEntries(), { origin: apexOrigin, protocol: url.protocol, hostSuffix: suffix }), - ); - } + // Provider routes have no shared instance to serve: point the caller at + // instance creation instead. + return Response.json( + { + error: "instance_required", + message: `${service}.${suffix} is a service host with no shared instance. Create one with POST ${url.protocol}//${service}.${suffix}/_emulate/instances and use the returned providerBaseUrl.`, + createInstance: `${url.protocol}//${service}.${suffix}/_emulate/instances`, + }, + { status: 404 }, + ); + } - // `/github/oauth/mcp` etc.: the instance segment is the MCP connection type. - const mcpMode = MCP_PRESETS.has(instance) ? instance : undefined; - const afterPrefix = resource.slice(2); - const innerSegs = wellKnownType ? [".well-known", wellKnownType, ...afterPrefix] : afterPrefix; + // NAMED INSTANCE HOST: .. (needs a 2-label cert). + if (hostRoute?.service && hostRoute.instance) { return forwardToDurableObject(env, request, { - service, - instance, - baseUrl: `${url.origin}/${service}/${instance}`, - innerPath: `/${innerSegs.join("/")}`, + service: hostRoute.service, + instance: hostRoute.instance, + baseUrl: url.origin, + innerPath: url.pathname, search: url.search, - mcpMode, }); - }, -}; + } + + // PATH FORM (apex/local): `///...`. Peel a leading + // `.well-known/` (RFC 8414 / 9728 path-aware metadata discovery: clients + // probe `/.well-known//`) so the resolved resource is + // instance-prefixed and the emulator answers with instance-scoped endpoints. + let wellKnownType: string | undefined; + let resource = segments; + if (segments[0] === ".well-known" && segments.length >= 2) { + wellKnownType = segments[1]; + resource = segments.slice(2); + } + + const service = resource[0]; + const instance = resource[1]; + + // Apex root or bare `/` → the catalog. Browser → SPA; agent → a + // server-rendered list of emulators (readable without running JS). + if (!service || !instance) { + if (isBrowserNavigation(request)) + return serveCatalogConsole({ origin: apexOrigin, protocol: url.protocol, hostSuffix: suffix }); + if (wantsJson(request)) { + return servicesCatalog(serviceCatalogEntries(), { + origin: apexOrigin, + protocol: url.protocol, + hostSuffix: suffix, + }); + } + return html( + renderCatalogPage(serviceCatalogEntries(), { origin: apexOrigin, protocol: url.protocol, hostSuffix: suffix }), + ); + } + + // `/github/oauth/mcp` etc.: the instance segment is the MCP connection type. + const mcpMode = MCP_PRESETS.has(instance) ? instance : undefined; + const afterPrefix = resource.slice(2); + const innerSegs = wellKnownType ? [".well-known", wellKnownType, ...afterPrefix] : afterPrefix; + return forwardToDurableObject(env, request, { + service, + instance, + baseUrl: `${url.origin}/${service}/${instance}`, + innerPath: `/${innerSegs.join("/")}`, + search: url.search, + mcpMode, + }); +} export function parseHostRoute( hostname: string, diff --git a/packages/@emulators/cloudflare/wrangler.jsonc b/packages/@emulators/cloudflare/wrangler.jsonc index f0954998e..1cf443ead 100644 --- a/packages/@emulators/cloudflare/wrangler.jsonc +++ b/packages/@emulators/cloudflare/wrangler.jsonc @@ -25,7 +25,10 @@ // Workers Logs keeps the structured failure reports from worker.ts and // durable-object.ts, so a 5xx seen by a client can be traced by its cf-ray. // Invocation logs are off: each one records the request URL, and an - // instance URL is the only access control for its emulator. + // instance URL is the only access control for its emulator. That setting + // does not stop Cloudflare recording uncaught exceptions with their message, + // stack and URL, so neither the Worker nor the Durable Object lets an error + // escape: both answer it with a report instead. "observability": { "enabled": true, "logs": { diff --git a/packages/@emulators/core/src/control-plane-rejection.ts b/packages/@emulators/core/src/control-plane-rejection.ts new file mode 100644 index 000000000..a29b13f66 --- /dev/null +++ b/packages/@emulators/core/src/control-plane-rejection.ts @@ -0,0 +1,22 @@ +const BRAND = Symbol.for("@emulators/core/ControlPlaneRejection"); + +// A seed or credential request the emulator refuses on its own terms (an +// unsupported credential type, an invalid OAuth client). The control plane +// answers these with a 400 and their message, which the emulator wrote. Any +// other error from a seed or credential request is unexpected, such as a host +// storage failure, and goes to the app's error handler instead; on Cloudflare +// that is a redacted failure report, since a raw message can quote anything. +export class ControlPlaneRejection extends Error { + readonly [BRAND] = true; + + constructor(message: string) { + super(message); + this.name = "ControlPlaneRejection"; + } + + // Checked by brand, not `instanceof`: emulator packages can load their own + // copy of core. + static is(err: unknown): err is ControlPlaneRejection { + return typeof err === "object" && err !== null && (err as { [BRAND]?: unknown })[BRAND] === true; + } +} diff --git a/packages/@emulators/core/src/control-plane.ts b/packages/@emulators/core/src/control-plane.ts index 4aec773c3..383347c70 100644 --- a/packages/@emulators/core/src/control-plane.ts +++ b/packages/@emulators/core/src/control-plane.ts @@ -8,7 +8,7 @@ import { coverageReport, enrichManifest, resolveConnections } from "./manifest.j import type { TokenMap } from "./middleware/auth.js"; import { escapeHtml, renderCardPage } from "./ui.js"; import type { FaultArmInput, FaultRegistry } from "./faults.js"; -import { isPlatformFailure } from "./platform-failure.js"; +import { ControlPlaneRejection } from "./control-plane-rejection.js"; export interface CredentialRequest { type?: string; @@ -199,8 +199,8 @@ export function registerControlPlane(app: Hono, options: ControlPlaneOpt try { await options.seed(body); } catch (err) { - if (isPlatformFailure(err)) throw err; - return c.json({ error: "invalid_seed", message: err instanceof Error ? err.message : "Seed failed." }, 400); + if (!ControlPlaneRejection.is(err)) throw err; + return c.json({ error: "invalid_seed", message: err.message }, 400); } return c.json({ ok: true }); }); @@ -217,11 +217,8 @@ export function registerControlPlane(app: Hono, options: ControlPlaneOpt } return c.json({ credential }); } catch (err) { - if (isPlatformFailure(err)) throw err; - return c.json( - { error: "unsupported", message: err instanceof Error ? err.message : "Credential creation failed." }, - 400, - ); + if (!ControlPlaneRejection.is(err)) throw err; + return c.json({ error: "unsupported", message: err.message }, 400); } }); app.post("/_emulate/instances", async (c) => { diff --git a/packages/@emulators/core/src/http.ts b/packages/@emulators/core/src/http.ts index 78952afc9..0095f40b3 100644 --- a/packages/@emulators/core/src/http.ts +++ b/packages/@emulators/core/src/http.ts @@ -1,5 +1,4 @@ import { createServer as createNodeServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; -import { isPlatformFailure } from "./platform-failure.js"; type BodyInit = ConstructorParameters[0]; type HeadersInit = ConstructorParameters[0]; @@ -286,7 +285,6 @@ export class Hono { const response = await this.dispatch(context, matched.handlers); return context.finalize(response ?? (await this.notFoundHandler(context))); } catch (err) { - if (isPlatformFailure(err)) throw err; return context.finalize(await this.errorHandler(err, context)); } }; diff --git a/packages/@emulators/core/src/index.ts b/packages/@emulators/core/src/index.ts index e8daa8eab..595c2433e 100644 --- a/packages/@emulators/core/src/index.ts +++ b/packages/@emulators/core/src/index.ts @@ -156,4 +156,4 @@ export { type ManifestResponse, type SpecsResponse, } from "./client.js"; -export { isPlatformFailure } from "./platform-failure.js"; +export { ControlPlaneRejection } from "./control-plane-rejection.js"; diff --git a/packages/@emulators/core/src/platform-failure.ts b/packages/@emulators/core/src/platform-failure.ts deleted file mode 100644 index 8cc743731..000000000 --- a/packages/@emulators/core/src/platform-failure.ts +++ /dev/null @@ -1,10 +0,0 @@ -// Cloudflare raises its own runtime failures (a Durable Object that moved to a -// different machine, a lost connection, an overloaded object) as exceptions -// flagged `.retryable` or `.overloaded`. Emulator code must not answer these as -// its own errors: they propagate to the host, which reports them with the flags. -// https://developers.cloudflare.com/durable-objects/best-practices/error-handling/ -export function isPlatformFailure(err: unknown): boolean { - if (typeof err !== "object" || err === null) return false; - const flags = err as { retryable?: unknown; overloaded?: unknown }; - return flags.retryable === true || flags.overloaded === true; -} diff --git a/packages/@emulators/planetscale/src/index.ts b/packages/@emulators/planetscale/src/index.ts index 5000d06d7..d7a5a51eb 100644 --- a/packages/@emulators/planetscale/src/index.ts +++ b/packages/@emulators/planetscale/src/index.ts @@ -9,6 +9,7 @@ import type { TokenMap, WebhookDispatcher, } from "@emulators/core"; +import { ControlPlaneRejection } from "@emulators/core"; import { generatePublicId } from "./constants.js"; import { registerMcpRoutes } from "./routes/mcp.js"; import { registerClient, registerOAuthRoutes } from "./routes/oauth.js"; @@ -124,7 +125,8 @@ export function seedFromConfig(store: Store, _baseUrl: string, config: PlanetSca for (const client of config.oauth_clients ?? []) { if (client.client_id && ps.oauthClients.findOneBy("client_id", client.client_id)) continue; const result = registerClient(ps, client); - if (!result.ok) throw new Error(`Invalid PlanetScale OAuth client seed: ${result.error_description}`); + if (!result.ok) + throw new ControlPlaneRejection(`Invalid PlanetScale OAuth client seed: ${result.error_description}`); } } @@ -136,7 +138,7 @@ export function seedFromConfig(store: Store, _baseUrl: string, config: PlanetSca export function issueCredential(store: Store, baseUrl: string, request: CredentialRequest): IssuedCredential { const type = request.type ?? "dynamic-client-registration"; if (type !== "dynamic-client-registration" && type !== "oauth-authorization-code") { - throw new Error(`Credential type ${type} is not supported by planetscale`); + throw new ControlPlaneRejection(`Credential type ${type} is not supported by planetscale`); } const result = registerClient(getPlanetScaleStore(store), { client_name: request.name ?? "PlanetScale Client", @@ -145,7 +147,7 @@ export function issueCredential(store: Store, baseUrl: string, request: Credenti client_id: request.client_id, client_secret: request.client_secret, }); - if (!result.ok) throw new Error(result.error_description); + if (!result.ok) throw new ControlPlaneRejection(result.error_description); return { type, client_id: result.client.client_id, diff --git a/packages/emulate/src/registry.ts b/packages/emulate/src/registry.ts index 5c100e880..e1e442799 100644 --- a/packages/emulate/src/registry.ts +++ b/packages/emulate/src/registry.ts @@ -11,6 +11,7 @@ import type { Hono, AppEnv, } from "@emulators/core"; +import { ControlPlaneRejection } from "@emulators/core"; export interface LoadedService { plugin: ServicePlugin; @@ -96,13 +97,14 @@ export function issueServiceCredential( type === "oauth-client-credentials" || type === "dynamic-client-registration" ) { - if (!loaded.seedFromConfig) throw new Error(`Credential type ${type} is not supported by ${service}`); + if (!loaded.seedFromConfig) + throw new ControlPlaneRejection(`Credential type ${type} is not supported by ${service}`); const clientId = request.client_id ?? defaultClientId(service); const clientSecret = request.client_secret ?? defaultClientSecret(service); const redirectUris = normalizeRedirectUris(request.redirect_uris); const name = request.name ?? `${SERVICE_REGISTRY[service].label.replace(/ emulator$/i, "")} Client`; const seed = credentialSeed(service, { clientId, clientSecret, redirectUris, name, request }); - if (!seed) throw new Error(`Credential type ${type} is not supported by ${service}`); + if (!seed) throw new ControlPlaneRejection(`Credential type ${type} is not supported by ${service}`); loaded.seedFromConfig(store, baseUrl, seed, webhooks); return { type, @@ -114,7 +116,7 @@ export function issueServiceCredential( }; } - throw new Error(`Credential type ${type} is not supported by ${service}`); + throw new ControlPlaneRejection(`Credential type ${type} is not supported by ${service}`); } function defaultToken(service: ServiceName, type: string): string { From 2cedc9b76d4122a3fe2eb5f7f951926cc1b6c84f Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Thu, 8 Oct 2026 02:23:36 -0700 Subject: [PATCH 5/5] Keep failure reports out of Workers Logs and Issues; record them in Analytics Engine --- .../@emulators/cloudflare/src/diagnostics.ts | 54 ++++++++++++++----- .../cloudflare/src/durable-object.ts | 23 ++++---- packages/@emulators/cloudflare/src/worker.ts | 36 +++++++------ packages/@emulators/cloudflare/wrangler.jsonc | 32 +++++++---- 4 files changed, 99 insertions(+), 46 deletions(-) diff --git a/packages/@emulators/cloudflare/src/diagnostics.ts b/packages/@emulators/cloudflare/src/diagnostics.ts index a0aef08a1..90b25ed5d 100644 --- a/packages/@emulators/cloudflare/src/diagnostics.ts +++ b/packages/@emulators/cloudflare/src/diagnostics.ts @@ -1,14 +1,14 @@ import { createServer } from "@emulators/core"; import { SERVICES } from "./services.js"; -// Failure reports go to the client and to Workers Logs, so every field is -// either a fixed value or chosen from a fixed list: no field copies text from -// the request or the error. The instance name is the sole access control for -// its emulator, request paths hold ids, emails and codes, and error messages, -// names and stacks can quote any of them. A report names the service from the -// registry, the instance by a short hash, the request by a route template the -// service's router declares, and the error by an allowlisted class name and -// Cloudflare's flags. +// Failure reports go to the client, so every field is either a fixed value or +// chosen from a fixed list: no field copies text from the request or the +// error. The instance name is the sole access control for its emulator, +// request paths hold ids, emails and codes, and error messages, names and +// stacks can quote any of them. A report names the service from the registry, +// the instance by a short hash, the request by a route template the service's +// router declares, and the error by an allowlisted class name and Cloudflare's +// flags. export interface FailureReport { error: "emulator_unavailable" | "emulator_error" | "worker_error"; service: string; @@ -47,14 +47,26 @@ export async function failureReport( }; } +// The subset of a Workers Analytics Engine dataset binding the reports use. +export interface FailureSink { + writeDataPoint(point: { indexes?: string[]; blobs?: string[]; doubles?: number[] }): void; +} + // The response for a failure at the Worker or Durable Object boundary: the -// report, logged once and returned as JSON. 503 when Cloudflare flags the -// failure retryable or overloaded, otherwise 500. Building it never throws, so -// no raw error escapes the boundary to Cloudflare's exception logging. +// report as JSON, 503 when Cloudflare flags the failure retryable or +// overloaded, otherwise 500. Building it never throws, so no raw error escapes +// the boundary. +// +// Nothing is written to the console. Workers Issues keeps each error log and +// each 5xx with its invocation's URL, which for an instance holds the instance +// name, so observability stays off (wrangler.jsonc) and the report is recorded +// once in Analytics Engine instead. A data point holds only what is written to +// it. export async function failureResponse( error: FailureReport["error"], cause: unknown, request: Parameters[2], + env?: { FAILURES?: FailureSink }, ): Promise { let report: FailureReport; try { @@ -73,8 +85,24 @@ export async function failureResponse( ray: null, }; } - console.error(JSON.stringify(report)); - return Response.json(report, { status: report.retryable || report.overloaded ? 503 : 500 }); + const status = report.retryable || report.overloaded ? 503 : 500; + try { + env?.FAILURES?.writeDataPoint(failurePoint(report, status)); + } catch { + // Recording is best effort; the client still gets its report. + } + return Response.json(report, { status }); +} + +// A report as an Analytics Engine data point, without the instance hash: the +// stored record names the service, route and failure but no instance. The ray +// joins it to the client's copy of the report, which has the hash. +export function failurePoint(report: FailureReport, status: number) { + return { + indexes: [report.service], + blobs: [report.error, report.service, report.method, report.route, report.errorClass, report.ray ?? ""], + doubles: [status, Number(report.retryable), Number(report.overloaded), Number(report.remote)], + }; } // Cloudflare raises its own runtime failures (a Durable Object that moved to a diff --git a/packages/@emulators/cloudflare/src/durable-object.ts b/packages/@emulators/cloudflare/src/durable-object.ts index f70129a68..9ccea854b 100644 --- a/packages/@emulators/cloudflare/src/durable-object.ts +++ b/packages/@emulators/cloudflare/src/durable-object.ts @@ -9,7 +9,7 @@ import { type TokenMap, } from "@emulators/core"; import { SERVICES, issueCloudflareCredential } from "./services.js"; -import { failureResponse, isPlatformFailure } from "./diagnostics.js"; +import { failureResponse, isPlatformFailure, type FailureSink } from "./diagnostics.js"; // Minimal CF runtime types (avoid a hard dep on @cloudflare/workers-types here). interface DurableObjectStorage { @@ -117,7 +117,7 @@ export class EmulatorDurableObject { constructor( private readonly state: DurableObjectState, - _env: unknown, + private readonly env: { FAILURES?: FailureSink }, ) {} private async readPersistedState(): Promise { @@ -460,13 +460,18 @@ export class EmulatorDurableObject { // quote tokens, codes, emails or the instance name. Cloudflare's own // failures (e.g. "object has moved to a different machine") keep their // `.retryable`/`.overloaded` flags in the report and its 503 status. - return failureResponse(isPlatformFailure(error) ? "emulator_unavailable" : "emulator_error", error, { - service: request.headers.get("x-emulator-service") ?? "", - instance: request.headers.get("x-emulator-instance") ?? "default", - method: request.method, - path: new URL(request.url).pathname, - headers: request.headers, - }); + return failureResponse( + isPlatformFailure(error) ? "emulator_unavailable" : "emulator_error", + error, + { + service: request.headers.get("x-emulator-service") ?? "", + instance: request.headers.get("x-emulator-instance") ?? "default", + method: request.method, + path: new URL(request.url).pathname, + headers: request.headers, + }, + this.env, + ); } } diff --git a/packages/@emulators/cloudflare/src/worker.ts b/packages/@emulators/cloudflare/src/worker.ts index 235486735..0ae1026e9 100644 --- a/packages/@emulators/cloudflare/src/worker.ts +++ b/packages/@emulators/cloudflare/src/worker.ts @@ -9,7 +9,7 @@ import { EmulatorDurableObject } from "./durable-object.js"; import { SERVICES } from "./services.js"; import { SERVICE_ICONS } from "./icons.js"; import { consoleHtml } from "./console-html.js"; -import { failureResponse } from "./diagnostics.js"; +import { failureResponse, type FailureSink } from "./diagnostics.js"; export { EmulatorDurableObject }; @@ -41,6 +41,8 @@ interface EmulatorNamespace { export interface Env { EMULATOR: EmulatorNamespace; EMULATE_HOST_SUFFIX?: string; + // Analytics Engine dataset for failure reports (wrangler.jsonc). + FAILURES?: FailureSink; } const DEFAULT_HOST_SUFFIX = "emulators.dev"; @@ -96,13 +98,18 @@ async function forwardToDurableObject( // was killed by its own limits. Nothing is retried: a flag does not prove // the object never ran the request, and replaying a reset, an OAuth // authorize or a write can change state twice. - return failureResponse("emulator_unavailable", error, { - service: opts.service, - instance: opts.instance, - method: request.method, - path: opts.innerPath, - headers: request.headers, - }); + return failureResponse( + "emulator_unavailable", + error, + { + service: opts.service, + instance: opts.instance, + method: request.method, + path: opts.innerPath, + headers: request.headers, + }, + env, + ); } } @@ -119,13 +126,12 @@ export default { // Nothing may escape the Worker: Cloudflare records an uncaught // exception with its message, stack and the request URL, and the // instance URL is the only access control for its emulator. - return failureResponse("worker_error", error, { - service: "", - instance: "", - method: request.method, - path: "", - headers: request.headers, - }); + return failureResponse( + "worker_error", + error, + { service: "", instance: "", method: request.method, path: "", headers: request.headers }, + env, + ); } }, }; diff --git a/packages/@emulators/cloudflare/wrangler.jsonc b/packages/@emulators/cloudflare/wrangler.jsonc index 1cf443ead..a38972def 100644 --- a/packages/@emulators/cloudflare/wrangler.jsonc +++ b/packages/@emulators/cloudflare/wrangler.jsonc @@ -22,20 +22,34 @@ "zone_name": "emulators.dev" } ], - // Workers Logs keeps the structured failure reports from worker.ts and - // durable-object.ts, so a 5xx seen by a client can be traced by its cf-ray. - // Invocation logs are off: each one records the request URL, and an - // instance URL is the only access control for its emulator. That setting - // does not stop Cloudflare recording uncaught exceptions with their message, - // stack and URL, so neither the Worker nor the Durable Object lets an error - // escape: both answer it with a report instead. + // Observability is off, every part of it, and set explicitly so each deploy + // also turns off anything enabled from the dashboard. Workers Issues keeps + // every 5xx response and every console.error with its invocation's URL, + // whether or not Workers Logs is on; logs and traces are invocation + // telemetry too. An instance URL is the only access control for its + // emulator. Failures are recorded instead as allowlisted reports in the + // FAILURES Analytics Engine dataset (diagnostics.ts), which stores only the + // fields written to it. These settings are not part of a Worker version: + // `wrangler rollback` leaves them as they are. "observability": { - "enabled": true, + "enabled": false, "logs": { - "enabled": true, + "enabled": false, "invocation_logs": false + }, + "traces": { + "enabled": false + }, + "issues": { + "enabled": false } }, + "analytics_engine_datasets": [ + { + "binding": "FAILURES", + "dataset": "emulate_hosts_failures" + } + ], "vars": { "EMULATE_HOST_SUFFIX": "emulators.dev" },