Report Durable Object failures without secrets instead of an opaque 500

packages/@emulators/cloudflare/src/diagnostics.ts

@@ -0,0 +1,184 @@
+import { createServer } from "@emulators/core";
+import { SERVICES } from "./services.js";
+
+// Failure reports go to the client, so every field is either a fixed value or
+// chosen from a fixed list: no field copies text from the request or the
+// error. The instance name is the sole access control for its emulator,
+// request paths hold ids, emails and codes, and error messages, names and
+// stacks can quote any of them. A report names the service from the registry,
+// the instance by a short hash, the request by a route template the service's
+// router declares, and the error by an allowlisted class name and Cloudflare's
+// flags.
+export interface FailureReport {
+  error: "emulator_unavailable" | "emulator_error" | "worker_error";
+  service: string;
+  instanceId: string | null;
+  method: string;
+  route: string;
+  errorClass: string;
+  retryable: boolean;
+  overloaded: boolean;
+  remote: boolean;
+  ray: string | null;
+}
+
+export async function failureReport(
+  error: FailureReport["error"],
+  cause: unknown,
+  request: { service: string; instance: string; method: string; path: string; headers: Headers },
+): Promise<FailureReport> {
+  const flags = (typeof cause === "object" && cause !== null ? cause : {}) as {
+    retryable?: unknown;
+    overloaded?: unknown;
+    remote?: unknown;
+  };
+  const known = Object.hasOwn(SERVICES, request.service);
+  return {
+    error,
+    service: known ? request.service : "unknown",
+    instanceId: known ? await instanceId(request.service, request.instance) : null,
+    method: METHODS.has(request.method) ? request.method : "OTHER",
+    route: known ? routeTemplate(request.service, request.method, request.path) : "unmatched",
+    errorClass: errorClass(cause),
+    retryable: flags.retryable === true,
+    overloaded: flags.overloaded === true,
+    remote: flags.remote === true,
+    ray: rayOf(request.headers),
+  };
+}
+
+// The subset of a Workers Analytics Engine dataset binding the reports use.
+export interface FailureSink {
+  writeDataPoint(point: { indexes?: string[]; blobs?: string[]; doubles?: number[] }): void;
+}
+
+// The response for a failure at the Worker or Durable Object boundary: the
+// report as JSON, 503 when Cloudflare flags the failure retryable or
+// overloaded, otherwise 500. Building it never throws, so no raw error escapes
+// the boundary.
+//
+// Nothing is written to the console. Workers Issues keeps each error log and
+// each 5xx with its invocation's URL, which for an instance holds the instance
+// name, so observability stays off (wrangler.jsonc) and the report is recorded
+// once in Analytics Engine instead. A data point holds only what is written to
+// it.
+export async function failureResponse(
+  error: FailureReport["error"],
+  cause: unknown,
+  request: Parameters<typeof failureReport>[2],
+  env?: { FAILURES?: FailureSink },
+): Promise<Response> {
+  let report: FailureReport;
+  try {
+    report = await failureReport(error, cause, request);
+  } catch {
+    report = {
+      error,
+      service: "unknown",
+      instanceId: null,
+      method: "OTHER",
+      route: "unknown",
+      errorClass: "other",
+      retryable: false,
+      overloaded: false,
+      remote: false,
+      ray: null,
+    };
+  }
+  const status = report.retryable || report.overloaded ? 503 : 500;
+  try {
+    env?.FAILURES?.writeDataPoint(failurePoint(report, status));
+  } catch {
+    // Recording is best effort; the client still gets its report.
+  }
+  return Response.json(report, { status });
+}
+
+// A report as an Analytics Engine data point, without the instance hash: the
+// stored record names the service, route and failure but no instance. The ray
+// joins it to the client's copy of the report, which has the hash.
+export function failurePoint(report: FailureReport, status: number) {
+  return {
+    indexes: [report.service],
+    blobs: [report.error, report.service, report.method, report.route, report.errorClass, report.ray ?? ""],
+    doubles: [status, Number(report.retryable), Number(report.overloaded), Number(report.remote)],
+  };
+}
+
+// Cloudflare raises its own runtime failures (a Durable Object that moved to a
+// different machine, a lost connection, an overloaded object) as errors
+// flagged `.retryable` or `.overloaded`.
+// https://developers.cloudflare.com/durable-objects/best-practices/error-handling/
+export function isPlatformFailure(err: unknown): boolean {
+  if (typeof err !== "object" || err === null) return false;
+  const flags = err as { retryable?: unknown; overloaded?: unknown };
+  return flags.retryable === true || flags.overloaded === true;
+}
+
+const METHODS = new Set(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"]);
+const RAY = /^[0-9a-f]{16}(-[A-Z]{3})?$/;
+// JavaScript's built-in error classes and the DOMException names the Workers
+// runtime raises. Cloudflare's Durable Object failures are plain `Error`s told
+// apart by their flags. Any other name, which code can set to anything, is
+// reported as "other".
+const ERROR_CLASSES = new Set([
+  "Error",
+  "TypeError",
+  "RangeError",
+  "SyntaxError",
+  "ReferenceError",
+  "EvalError",
+  "URIError",
+  "AggregateError",
+  "AbortError",
+  "TimeoutError",
+  "DataCloneError",
+  "QuotaExceededError",
+  "InvalidStateError",
+  "NetworkError",
+  "OperationError",
+]);
+
+// The first 12 hex digits of SHA-256(`<service>:<instance>`), the Durable Object
+// name; anyone holding the instance URL can compute it to find their reports.
+// Generated instance names end in 96 random bits, so their hashes cannot be
+// enumerated. The hash does not hide a predictable or low-entropy name (one a
+// caller chose, or a legacy fixed name): hashing guesses confirms it.
+export async function instanceId(service: string, instance: string): Promise<string> {
+  const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(`${service}:${instance}`));
+  return Array.from(new Uint8Array(digest).slice(0, 6), (b) => b.toString(16).padStart(2, "0")).join("");
+}
+
+function errorClass(cause: unknown): string {
+  const name = typeof cause === "object" && cause !== null ? (cause as { name?: unknown }).name : undefined;
+  return typeof name === "string" && ERROR_CLASSES.has(name) ? name : "other";
+}
+
+function rayOf(headers: Headers): string | null {
+  const ray = headers.get("cf-ray");
+  return ray && RAY.test(ray) ? ray : null;
+}
+
+// Paths the Durable Object serves itself, outside the service's router.
+const OBJECT_ROUTES = new Set(["/__seed", "/__reset", "/__token"]);
+
+// One route table per service and isolate, built only when a failure needs it:
+// the routes are registered statically, so a throwaway server answers which
+// pattern a path matches without touching any instance's state.
+const routers = new Map<string, ReturnType<typeof createServer>["app"]>();
+
+function routeTemplate(service: string, method: string, path: string): string {
+  if (OBJECT_ROUTES.has(path)) return path;
+  try {
+    let app = routers.get(service);
+    if (!app) {
+      const entry = SERVICES[service];
+      app = createServer(entry.plugin, { baseUrl: "https://route.invalid", manifest: entry.manifest }).app;
+      routers.set(service, app);
+    }
+    return app.routePattern(method, path) ?? "unmatched";
+  } catch {
+    // A report must never fail on its way out; the template is a convenience.
+    return "unknown";
+  }
+}

packages/@emulators/cloudflare/src/durable-object.ts

@@ -9,6 +9,7 @@ import {
   type TokenMap,
 } from "@emulators/core";
 import { SERVICES, issueCloudflareCredential } from "./services.js";
+import { failureResponse, isPlatformFailure, type FailureSink } from "./diagnostics.js";
 
 // Minimal CF runtime types (avoid a hard dep on @cloudflare/workers-types here).
 interface DurableObjectStorage {
@@ -116,7 +117,7 @@ export class EmulatorDurableObject {
 
   constructor(
     private readonly state: DurableObjectState,
-    _env: unknown,
+    private readonly env: { FAILURES?: FailureSink },
   ) {}
 
   private async readPersistedState(): Promise<PersistedState> {
@@ -331,6 +332,7 @@ export class EmulatorDurableObject {
       manifest: entry.manifest,
       instance,
       ledgerPersistent: true,
+      rethrowUnexpectedErrors: true,
       reset: () => resetService(),
       seed: async (seed) => {
         if (seed && entry.seedFromConfig) {
@@ -449,6 +451,31 @@ export class EmulatorDurableObject {
   }
 
   async fetch(request: Request): Promise<Response> {
+    try {
+      return await this.handle(request);
+    } catch (error) {
+      // Nothing may escape this boundary. A thrown error reaches the client as
+      // an opaque Cloudflare 500, and Cloudflare records it as an exception
+      // event with its message, stack and the request URL, any of which can
+      // quote tokens, codes, emails or the instance name. Cloudflare's own
+      // failures (e.g. "object has moved to a different machine") keep their
+      // `.retryable`/`.overloaded` flags in the report and its 503 status.
+      return failureResponse(
+        isPlatformFailure(error) ? "emulator_unavailable" : "emulator_error",
+        error,
+        {
+          service: request.headers.get("x-emulator-service") ?? "",
+          instance: request.headers.get("x-emulator-instance") ?? "default",
+          method: request.method,
+          path: new URL(request.url).pathname,
+          headers: request.headers,
+        },
+        this.env,
+      );
+    }
+  }
+
+  private async handle(request: Request): Promise<Response> {
     const service = request.headers.get("x-emulator-service") ?? "";
     const instance = request.headers.get("x-emulator-instance") ?? "default";
     const baseUrl = request.headers.get("x-emulator-base-url") ?? new URL(request.url).origin;

packages/@emulators/cloudflare/src/services.ts

@@ -10,6 +10,7 @@ import type {
   TokenMap,
   WebhookDispatcher,
 } from "@emulators/core";
+import { ControlPlaneRejection } from "@emulators/core";
 import {
   getGitHubStore,
   githubPlugin,
@@ -400,7 +401,7 @@ export function issueCloudflareCredential(
   }
   const type = request.type ?? entry.manifest.auth[0]?.type ?? "bearer-token";
   if (type === "bearer-token" || type === "api-key") {
-    if (!tokenMap) throw new Error(`Credential type ${type} is not supported by ${service}`);
+    if (!tokenMap) throw new ControlPlaneRejection(`Credential type ${type} is not supported by ${service}`);
     const login = request.login ?? "admin";
     const scopes = Array.isArray(request.scopes)
       ? request.scopes.filter((s): s is string => typeof s === "string")
@@ -419,13 +420,14 @@ export function issueCloudflareCredential(
     type === "oauth-client-credentials" ||
     type === "dynamic-client-registration"
   ) {
-    if (!entry.seedFromConfig) throw new Error(`Credential type ${type} is not supported by ${service}`);
+    if (!entry.seedFromConfig)
+      throw new ControlPlaneRejection(`Credential type ${type} is not supported by ${service}`);
     const clientId = request.client_id ?? defaultClientId(service);
     const clientSecret = request.client_secret ?? defaultClientSecret(service);
     const redirectUris = normalizeRedirectUris(request.redirect_uris);
     const name = request.name ?? `${entry.manifest.name} Client`;
     const seed = credentialSeed(service, { clientId, clientSecret, redirectUris, name, request });
-    if (!seed) throw new Error(`Credential type ${type} is not supported by ${service}`);
+    if (!seed) throw new ControlPlaneRejection(`Credential type ${type} is not supported by ${service}`);
     entry.seedFromConfig(store, baseUrl, seed);
     return {
       type,
@@ -437,7 +439,7 @@ export function issueCloudflareCredential(
     };
   }
 
-  throw new Error(`Credential type ${type} is not supported by ${service}`);
+  throw new ControlPlaneRejection(`Credential type ${type} is not supported by ${service}`);
 }
 
 function tokenPrefix(service: string, type: string): string {