Retain correlated search failure diagnostics

docs/api.md

@@ -50,6 +50,26 @@ arrive before the `provider` event.
 Chat uses the shared OpenAI-compatible gateway with Claude Haiku 5.5 at low reasoning effort. An unavailable, rate-limited,
 or failed gateway request terminates the stream with an `error` event; no alternate provider exists.
 
+API responses include `X-Request-ID`, including security rejections and HTTP-200 SSE streams.
+Application logs distinguish HTTP completion (`api_request_completed`) from a terminal SSE
+error (`search_stream_failed`); HTTP 200 alone does not establish that the answer succeeded.
+Embedding rate-limit records retain the upstream status, workload, model, gateway request ID,
+and `Retry-After` when supplied. Missing retry timing does not replace the original provider cause.
+An in-band Responses failure retains its finite `terminalReason` on the request-correlated log;
+that protocol outcome is not reported as a transport HTTP 429.
+
+### Browser failure diagnostics
+
+POST `/api/diagnostics/search-failures` accepts bounded search-transport observations and returns
+202. It requires the same CSRF protection as other browser POSTs. Reports identify the API path,
+failure kind (`HTTP`, `SSE`, `NETWORK`, or `PROTOCOL`), optional HTTP status/request ID/error code/
+retry timing, page path, loaded module asset, and elapsed milliseconds. URL queries, prompts,
+cookies, authorization headers, and raw response bodies are not report fields and are not emitted into logs.
+
+Browser reports use `event=browser_search_failed` and `provenance=client_reported`; they are not
+verified server failures. A search failure sends one report without replaying the search. A failed
+diagnostic submission remains console-visible and never recursively reports itself.
+
 Example:
 
 ```bash

frontend/e2e/search-diagnostics.spec.ts

@@ -0,0 +1,47 @@
+import { expect, test } from "@playwright/test";
+import { BrowserSearchFailureSchema } from "../src/lib/validation/schemas";
+
+for (const failureKind of ["HTTP", "SSE"] as const) {
+  test(`${failureKind} search failure reaches diagnostics without replaying the question`, async ({
+    page,
+  }) => {
+    const searchRequests: string[] = [];
+    await page.route("**/api/security/csrf", async (route) => route.fulfill({ status: 204 }));
+    await page.route("**/api/diagnostics/search-failures", async (route) =>
+      route.fulfill({ status: 202 }),
+    );
+    await page.route("**/api/chat/stream", async (route) => {
+      searchRequests.push(route.request().url());
+      await route.fulfill({
+        status: failureKind === "HTTP" ? 429 : 200,
+        headers: { "X-Request-ID": "browser-refusal-test", "Retry-After": "12" },
+        contentType: failureKind === "HTTP" ? "application/json" : "text/event-stream",
+        body:
+          failureKind === "HTTP"
+            ? JSON.stringify({ status: "error", message: "Search is temporarily rate limited" })
+            : 'event:error\ndata:{"message":"Search is temporarily rate limited","code":"test.rate_limited","retryable":true}\n\n',
+      });
+    });
+    await page.goto("/");
+    const diagnosticSubmission = page.waitForRequest("**/api/diagnostics/search-failures");
+    await page.getByLabel("Message input").fill("Explain Java records");
+    await page.getByLabel("Message input").press("Enter");
+    await expect(
+      page
+        .locator(".message.assistant.error .bubble")
+        .getByText("Search is temporarily rate limited", { exact: true }),
+    ).toBeVisible();
+    const diagnosticRequest = await diagnosticSubmission;
+    const parsedDiagnostic: unknown = diagnosticRequest.postDataJSON();
+    const diagnostic = BrowserSearchFailureSchema.safeParse(parsedDiagnostic);
+    if (!diagnostic.success) throw diagnostic.error;
+    expect(diagnostic.data).toMatchObject({
+      path: "/api/chat/stream",
+      kind: failureKind,
+      httpStatus: failureKind === "HTTP" ? 429 : 200,
+      requestId: "browser-refusal-test",
+    });
+    expect(diagnosticRequest.postData()).not.toContain("Explain Java records");
+    expect(searchRequests).toHaveLength(1);
+  });
+}

frontend/src/lib/services/csrf.ts

@@ -52,7 +52,7 @@ export function csrfHeader(): Record<string, string> {
   return { [CSRF_HEADER_NAME]: tokenText };
 }
 
-async function readCsrfError(
+export async function readApiError(
   httpResponse: Response,
   source: string,
 ): Promise<ApiErrorResponse | null> {
@@ -84,7 +84,7 @@ export async function extractApiErrorMessage(
   httpResponse: Response,
   source: string,
 ): Promise<string | null> {
-  const apiError = await readCsrfError(httpResponse, source);
+  const apiError = await readApiError(httpResponse, source);
   if (!apiError) {
     return null;
   }
@@ -154,7 +154,7 @@ export async function fetchWithCsrfRetry(
     return initialHttpResponse;
   }
 
-  const initialCsrfError = await readCsrfError(initialHttpResponse.clone(), source);
+  const initialCsrfError = await readApiError(initialHttpResponse.clone(), source);
   if (!initialCsrfError || !isRecoverableCsrfErrorMessage(initialCsrfError.message)) {
     return initialHttpResponse;
   }
@@ -174,7 +174,7 @@ export async function fetchWithCsrfRetry(
     return retriedHttpResponse;
   }
 
-  const retriedCsrfError = await readCsrfError(retriedHttpResponse.clone(), `${source}:retry`);
+  const retriedCsrfError = await readApiError(retriedHttpResponse.clone(), `${source}:retry`);
   if (retriedCsrfError && isRecoverableCsrfErrorMessage(retriedCsrfError.message)) {
     maybeToastExpired();
   }

frontend/src/lib/services/searchDiagnostics.test.ts

@@ -0,0 +1,174 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { BrowserSearchFailureSchema } from "../validation/schemas";
+import { streamSse, streamSseGet } from "./sse";
+
+describe("search failure diagnostic transport", () => {
+  afterEach(() => {
+    window.history.replaceState({}, "", "/");
+    vi.unstubAllGlobals();
+    vi.restoreAllMocks();
+  });
+
+  it.each([1, 160, 161])("retains HTTP diagnostics at page length %i", async (pagePathLength) => {
+    const pagePath = `/${"a".repeat(pagePathLength - 1)}`;
+    window.history.replaceState({}, "", `${pagePath}?question=private-query#private-fragment`);
+    const submittedReports: string[] = [];
+    const searchRequests: string[] = [];
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async (url: string, options?: RequestInit) => {
+        if (url === "/api/diagnostics/search-failures") {
+          if (typeof options?.body !== "string")
+            throw new Error("Expected a JSON diagnostic request body");
+          submittedReports.push(options.body);
+          return new Response(null, { status: 202 });
+        }
+        searchRequests.push(url);
+        return new Response(null, {
+          status: 429,
+          headers: {
+            "X-Request-ID": "refusal-report-test",
+            "Retry-After": "12",
+          },
+        });
+      }),
+    );
+
+    await expect(
+      streamSse(
+        "/api/chat/stream",
+        { latest: "private search question" },
+        { onText: vi.fn() },
+        "chat.ts",
+      ),
+    ).rejects.toMatchObject({ httpStatus: 429 });
+    await vi.waitFor(() => expect(submittedReports).toHaveLength(1));
+    expect(searchRequests).toEqual(["/api/chat/stream"]);
+    expect(submittedReports[0]).not.toContain("private search question");
+    expect(submittedReports[0]).not.toContain("private-query");
+    expect(submittedReports[0]).not.toContain("private-fragment");
+    const diagnostic = BrowserSearchFailureSchema.safeParse(
+      JSON.parse(submittedReports[0] ?? "null"),
+    );
+    if (!diagnostic.success) throw diagnostic.error;
+    expect(diagnostic.data).toMatchObject({
+      kind: "HTTP",
+      path: "/api/chat/stream",
+      httpStatus: 429,
+      requestId: "refusal-report-test",
+      retryAfterSeconds: 12,
+      pagePath: `/${"a".repeat(Math.min(pagePathLength, 160) - 1)}`,
+    });
+  });
+
+  it.each([
+    { method: "GET", path: "/api/guided/content/stream" },
+    { method: "POST", path: "/api/chat/stream" },
+  ])(
+    "reports a missing $method response body as PROTOCOL without replaying the search",
+    async ({ method, path }) => {
+      const submittedReports: string[] = [];
+      const searchRequests: string[] = [];
+      const searchUrl = `${path}?slug=java-records`;
+      vi.stubGlobal(
+        "fetch",
+        vi.fn(async (url: string, options?: RequestInit) => {
+          if (url === "/api/diagnostics/search-failures") {
+            if (typeof options?.body !== "string")
+              throw new Error("Expected a JSON diagnostic request body");
+            submittedReports.push(options.body);
+            return new Response(null, { status: 202 });
+          }
+          searchRequests.push(url);
+          return new Response(null, {
+            status: 200,
+            headers: { "X-Request-ID": "missing-body-report-test" },
+          });
+        }),
+      );
+
+      const streamingRequest =
+        method === "GET"
+          ? streamSseGet(searchUrl, { onText: vi.fn() }, "guided.lesson-content.ts")
+          : streamSse(
+              searchUrl,
+              { latest: "private search question" },
+              { onText: vi.fn() },
+              "chat.ts",
+            );
+      await expect(streamingRequest).rejects.toMatchObject({
+        name: "StreamFailureError",
+        message: "No response body",
+        retryable: true,
+      });
+      await vi.waitFor(() => expect(submittedReports).toHaveLength(1));
+      expect(searchRequests).toEqual([searchUrl]);
+      expect(submittedReports[0]).not.toContain("private search question");
+      expect(submittedReports[0]).not.toContain("java-records");
+      const diagnostic = BrowserSearchFailureSchema.safeParse(
+        JSON.parse(submittedReports[0] ?? "null"),
+      );
+      if (!diagnostic.success) throw diagnostic.error;
+      expect(diagnostic.data).toMatchObject({
+        kind: "PROTOCOL",
+        path,
+        httpStatus: 200,
+        requestId: "missing-body-report-test",
+        retryable: true,
+      });
+    },
+  );
+
+  it("distinguishes an SSE error inside HTTP 200 from an HTTP refusal", async () => {
+    const submittedReports: string[] = [];
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async (url: string, options?: RequestInit) => {
+        if (url === "/api/diagnostics/search-failures") {
+          if (typeof options?.body !== "string")
+            throw new Error("Expected a JSON diagnostic request body");
+          submittedReports.push(options.body);
+          return new Response(null, { status: 202 });
+        }
+        return new Response(
+          'event:error\ndata:{"message":"Temporarily unavailable","code":"stream.provider_retryable_error","retryable":true}\n\n',
+          { status: 200, headers: { "X-Request-ID": "sse-report-test" } },
+        );
+      }),
+    );
+    await expect(
+      streamSse(
+        "/api/guided/stream",
+        { latest: "private question" },
+        { onText: vi.fn() },
+        "guided.ts",
+      ),
+    ).rejects.toThrow("Temporarily unavailable");
+    await vi.waitFor(() => expect(submittedReports).toHaveLength(1));
+    expect(JSON.parse(submittedReports[0] ?? "null")).toMatchObject({
+      kind: "SSE",
+      httpStatus: 200,
+      requestId: "sse-report-test",
+      code: "stream.provider_retryable_error",
+    });
+  });
+
+  it("does not recursively report an unavailable diagnostic endpoint", async () => {
+    const requestedPaths: string[] = [];
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async (url: string) => {
+        requestedPaths.push(url);
+        throw new TypeError("Connection unavailable");
+      }),
+    );
+    vi.spyOn(console, "error").mockImplementation(() => undefined);
+    vi.spyOn(console, "warn").mockImplementation(() => undefined);
+    await expect(
+      streamSse("/api/chat/stream", { latest: "question" }, { onText: vi.fn() }, "chat.ts"),
+    ).rejects.toThrow("Couldn't reach the server");
+    await vi.waitFor(() =>
+      expect(requestedPaths).toEqual(["/api/chat/stream", "/api/diagnostics/search-failures"]),
+    );
+  });
+});

frontend/src/lib/services/searchDiagnostics.ts

@@ -0,0 +1,33 @@
+import { BrowserSearchFailureSchema, type BrowserSearchFailure } from "../validation/schemas";
+import { validateWithSchema } from "../validation/validate";
+import { fetchWithCsrfRetry } from "./csrf";
+
+/** Sends one browser observation into the existing application log transport, without replaying searches. */
+export async function reportSearchFailure(failure: BrowserSearchFailure): Promise<void> {
+  const diagnosticValidation = validateWithSchema(
+    BrowserSearchFailureSchema,
+    failure,
+    "search-diagnostics",
+  );
+  if (!diagnosticValidation.success) {
+    return;
+  }
+  try {
+    const diagnosticReply = await fetchWithCsrfRetry(
+      "/api/diagnostics/search-failures",
+      {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(diagnosticValidation.validated),
+        keepalive: true,
+      },
+      "search-diagnostics",
+    );
+    if (!diagnosticReply.ok) {
+      console.warn("Search diagnostics were rejected", { status: diagnosticReply.status });
+    }
+  } catch (diagnosticFailure) {
+    // A failed report must neither replace the search's original error nor recursively report itself.
+    console.warn("Search diagnostics could not reach the server", diagnosticFailure);
+  }
+}

frontend/src/lib/services/sse.test.ts

@@ -1,6 +1,10 @@
 import { afterEach, describe, expect, it, vi } from "vitest";
 import { streamSse, streamSseGet } from "./sse";
 
+vi.mock("./searchDiagnostics", () => ({
+  reportSearchFailure: vi.fn().mockResolvedValue(undefined),
+}));
+
 const SSE_STREAM_RESPONSE_STATUS = 200;
 const HTTP_BAD_REQUEST_STATUS = 400;
 const HTTP_TOO_MANY_REQUESTS_STATUS = 429;
@@ -180,6 +184,39 @@ describe("streamSse transport handling", () => {
     });
   });
 
+  it("preserves HTTP refusal diagnostics and authoritative retry timing", async () => {
+    vi.stubGlobal(
+      "fetch",
+      vi.fn().mockResolvedValue(
+        new Response(
+          JSON.stringify({
+            status: "error",
+            message: "Search is temporarily rate limited",
+            details: "The upstream request budget resets shortly",
+          }),
+          {
+            status: HTTP_TOO_MANY_REQUESTS_STATUS,
+            headers: {
+              "Content-Type": "application/json",
+              "X-Request-ID": "search-rejection-test",
+              "Retry-After": "12",
+            },
+          },
+        ),
+      ),
+    );
+
+    await expect(
+      streamSseGet("/api/chat/stream", { onText: vi.fn() }, "sse.test.ts"),
+    ).rejects.toMatchObject({
+      message: "Search is temporarily rate limited",
+      details: "The upstream request budget resets shortly",
+      httpStatus: HTTP_TOO_MANY_REQUESTS_STATUS,
+      requestId: "search-rejection-test",
+      retryAfterSeconds: 12,
+    });
+  });
+
   it("reports a 4xx client error as a non-retryable stream failure", async () => {
     const badRequestMessage = `HTTP ${HTTP_BAD_REQUEST_STATUS}: Bad Request`;
     vi.stubGlobal(