From 15d64e5cfbb5d880cb05327675dd7c907456b832 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 08:01:52 +0000 Subject: [PATCH 1/5] Assistant answers bind mirror admission and private accounting Fix mirror admission before model work so a later source activation cannot widen an ordinary answer. Mark private output first and retain required stream accounting through owned cancellation cleanup; SDKs require terminal completion. Signed-off-by: zlareb1 --- docs/architecture/assistant.md | 32 +++ docs/architecture/document-sources.md | 12 + docs/reference/operations/assistant.md | 1 + sdk/python/loom/assistant.py | 10 +- sdk/python/loom/ops.py | 1 + sdk/typescript/src/assistant.test.ts | 25 ++ sdk/typescript/src/assistant.ts | 8 +- sdk/typescript/src/generated/ops.ts | 2 + server/assistant/api/models.py | 6 + server/assistant/app/ask.py | 1 + server/assistant/app/bridge.py | 128 ++++++++-- server/assistant/app/routes.py | 56 ++++- server/auth/delegation.py | 2 + server/auth/jwt.py | 13 +- server/core/errors.py | 12 +- server/core/request_tail.py | 31 ++- server/core/request_work.py | 17 ++ server/knowledge/app/source_access.py | 2 + server/transport/mcp_outcome.py | 47 ++-- .../test_mirror_checked_search_http.py | 27 +++ tests/unit/test_assistant_bridge.py | 222 ++++++++++++++++++ tests/unit/test_assistant_mirror_boundary.py | 191 +++++++++++++++ tests/unit/test_sdk_assistant_client.py | 26 ++ .../test_session_protection_declaration.py | 7 +- 24 files changed, 818 insertions(+), 61 deletions(-) create mode 100644 tests/unit/test_assistant_mirror_boundary.py diff --git a/docs/architecture/assistant.md b/docs/architecture/assistant.md index 1a114ae279..163ace0e8f 100644 --- a/docs/architecture/assistant.md +++ b/docs/architecture/assistant.md @@ -71,6 +71,38 @@ session and components, and links verified citations into the memory and documen ## The bridge: each caller's own identity +Before checking model spend, the bridge reads the authorized namespace's public +mirrored-source registrations. Any active registration marks the complete answer, +even when its tools admit no document. The chat route sends `data: {"mirrored":true}` +as its first SSE event, or `mirrored: true` as its whole response's first field. +`assistant.ask` carries the same first field. A marked refusal carries it too. This +decision also routes the answer's cost row to private storage and suppresses public +diagnostics before subsequent model, budget or tool work. + +The answer token keeps the caller's `session_protection` promise and a separate +`answer_mirrors` admission bit. Only an already marked answer for a declared caller +receives positive admission. Missing or false admission disables its mirror legs, +including when a source becomes active during an initially ordinary answer. Ordinary +namespace reads retain their authorization. Current ACL checks still decide each +mirrored item; the bit grants no access by itself. Ordinary delegated children omit +both claims and cannot regain mirror admission through a request header. An answer +token or any descendant cannot start another assistant answer. + +Streaming output is provisional until `[DONE]`. The response owns its required +accounting lifetime beyond HTTP headers: it closes and joins model and citation work, +captures final usage once, and confirms private cost delivery before sending `[DONE]` +or a terminal error. Disconnects join producer cleanup and required delivery before +propagating cancellation. A failed cost write cannot publish a completed answer. +Whole answers and checked tool responses retain their accounting-before-publication +boundary. Incremental text precedes final usage, so its completion boundary is the +terminal event. A streaming client must reject EOF without `[DONE]` as incomplete. + +Mirrored-source creation stays unavailable until every serving build and consumer +implements the admission, first-output marker, session and accounting boundaries. +An older verifier accepting an unknown claim does not enforce it; schema equality +does not establish this capability. The fleet and rollback requirements in +[rolling upgrades](rolling-upgrades.md) still apply. + LibreChat's agents API authenticates a LibreChat API key and runs the agent as that key's LibreChat user. The agent's tools are Loom's MCP server, and what they may read is decided by the token on each tool call, never by the LibreChat user. LibreChat v0.8.8 diff --git a/docs/architecture/document-sources.md b/docs/architecture/document-sources.md index 06b7c31086..e954ca35cd 100644 --- a/docs/architecture/document-sources.md +++ b/docs/architecture/document-sources.md @@ -613,6 +613,18 @@ their contributing proofs. A raw download owns the same required marked accounti attempt as a registered operation. Cancellation and disconnects still propagate through owned cleanup. These checks end at the server's response-send boundary. +The assistant's incremental stream has a terminal accounting boundary because final +usage is unavailable before its early text. Its server-owned first event is +`data: {"mirrored":true}` when marked. Partial text remains provisional until the +response closes and joins its producers, captures final usage and confirms its +required private cost row before `[DONE]` or a terminal error. Disconnect cleanup +retains and joins that required delivery. Whole answers and checked source frames +keep the accounting-before-publication boundary above. The answer token's distinct +`answer_mirrors` bit fixes mirror admission before model work, so a source activated +during a negative answer cannot introduce an unmarked tool leg. Missing admission is +false; ordinary delegation drops it. See the +[assistant boundary](assistant.md#the-bridge-each-callers-own-identity). + A multi-entry MCP batch refuses every response-bearing marked entry with the same instruction to send it individually, including denied and empty entries. Notifications remain silent. It discards each marked entry's complete frame before buffering it or diff --git a/docs/reference/operations/assistant.md b/docs/reference/operations/assistant.md index 431a746653..8776e14406 100644 --- a/docs/reference/operations/assistant.md +++ b/docs/reference/operations/assistant.md @@ -23,6 +23,7 @@ Ask the Loom assistant one question about this namespace and get one cited answe | Output field | Type | Meaning | | --- | --- | --- | +| `mirrored` | `bool` | True when this answer is private to the caller's protected session; false by default. | | `answer` | `str` | The assistant's answer, with its citations as the agent wrote them. | | `run_id` | `str` | The answer's id: the run its tool calls are ledgered under (agent_events.run_id). | | `model` | `str` | The model that answered. | diff --git a/sdk/python/loom/assistant.py b/sdk/python/loom/assistant.py index 9283a597ad..13680cd9dc 100644 --- a/sdk/python/loom/assistant.py +++ b/sdk/python/loom/assistant.py @@ -170,12 +170,16 @@ async def stream( the text written so far.""" body = {"stream": True, "messages": _messages(question, history)} reading = _Reading() + complete = False async with self._agent.transport.stream( ROUTE, json=body, headers=self._headers(namespace), read_timeout_s=read_timeout_s ) as resp: request_id = resp.headers.get("X-Request-Id", "") async for line in resp.aiter_lines(): - if not line.startswith("data: ") or line == "data: [DONE]": + if line == "data: [DONE]": + complete = True + continue + if complete or not line.startswith("data: "): continue try: event = json.loads(line[6:]) @@ -196,6 +200,10 @@ async def stream( err = LoomError(f"{ROUTE}: the stream ended before the answer began") err.request_id = request_id raise err + if not complete: + err = LoomError(f"{ROUTE}: the stream ended before completion") + err.request_id = request_id + raise err yield reading.answer() async def ask( diff --git a/sdk/python/loom/ops.py b/sdk/python/loom/ops.py index 11cd07f75b..641018970a 100644 --- a/sdk/python/loom/ops.py +++ b/sdk/python/loom/ops.py @@ -5367,6 +5367,7 @@ class AssistantAskBody(TypedDict, total=False): class AssistantAskOutput(TypedDict, total=False): + mirrored: NotRequired[bool] answer: NotRequired[str] run_id: NotRequired[str] model: NotRequired[str] diff --git a/sdk/typescript/src/assistant.test.ts b/sdk/typescript/src/assistant.test.ts index 3f7dc2c43c..95d361c92e 100644 --- a/sdk/typescript/src/assistant.test.ts +++ b/sdk/typescript/src/assistant.test.ts @@ -338,6 +338,31 @@ describe("loom.assistant.stream", () => { await expect(collect(loom)).rejects.toThrow(/before the answer began/); }); + it("keeps partial text incomplete when EOF arrives without DONE", async () => { + const { loom, sent } = client(() => new Response(ANSWER.replace(/data: \[DONE\]\n\n$/, ""))); + const seen: AssistantEvent[] = []; + const read = async () => { + for await (const event of loom.assistant.stream({ question: "q" })) seen.push(event); + }; + await expect(read()).rejects.toThrow(/before completion/); + expect(seen).toEqual([ + { type: "text", text: "Deploys freeze " }, + { type: "text", text: "on Fridays." }, + ]); + expect(sent).toHaveLength(1); + }); + + it("ignores text received after DONE", async () => { + const { loom } = client(() => new Response(ANSWER + sse(chunk("discarded")))); + const out = await collect(loom); + expect(out.slice(0, 2)).toEqual([ + { type: "text", text: "Deploys freeze " }, + { type: "text", text: "on Fridays." }, + ]); + expect(out).toHaveLength(3); + expect(out[2]?.type === "answer" && out[2].answer.text).toBe("Deploys freeze on Fridays."); + }); + it("throws a connection cut mid-stream as NETWORK", async () => { const { loom } = client(() => new Response(body(sse(chunk("Deploys ")), [5], new Error("reset")))); const err = await collect(loom).then( diff --git a/sdk/typescript/src/assistant.ts b/sdk/typescript/src/assistant.ts index f1c9584196..e44140d01d 100644 --- a/sdk/typescript/src/assistant.ts +++ b/sdk/typescript/src/assistant.ts @@ -151,9 +151,14 @@ export function assistantChat(transport: Transport): AssistantChat { steps: [], stepsCut: false, }; + let complete = false; try { for await (const data of events(res, transport.baseUrl, requestId, cut.signal, touch)) { - if (data === "[DONE]") continue; + if (data === "[DONE]") { + complete = true; + continue; + } + if (complete) continue; const event = parse(data); if (!event) continue; if (isObject(event.error)) { @@ -173,6 +178,7 @@ export function assistantChat(transport: Transport): AssistantChat { } if (!answer.runId) throw new LoomError(`${ASSISTANT_ROUTE}: the stream ended before the answer began`, { requestId }); + if (!complete) throw new LoomError(`${ASSISTANT_ROUTE}: the stream ended before completion`, { requestId }); pause(); yield { type: "answer", answer }; } finally { diff --git a/sdk/typescript/src/generated/ops.ts b/sdk/typescript/src/generated/ops.ts index b1faf0d710..f3bfe4b544 100644 --- a/sdk/typescript/src/generated/ops.ts +++ b/sdk/typescript/src/generated/ops.ts @@ -84,6 +84,8 @@ export interface AssistantCitation { } export interface AssistantAskOutput { + /** True when this answer is private to the caller's protected session; false by default. */ + mirrored?: boolean; answer?: string; run_id?: string; model?: string; diff --git a/server/assistant/api/models.py b/server/assistant/api/models.py index 9bdeae725e..d597cdaee6 100644 --- a/server/assistant/api/models.py +++ b/server/assistant/api/models.py @@ -63,6 +63,12 @@ class AssistantCitation(BaseModel): class AssistantAskOutput(BaseModel): + mirrored: bool = Field( + default=False, + description=( + "True when this answer is private to the caller's protected session; false by default." + ), + ) # The assistant's answer, with its citations as the agent wrote them. answer: str = "" # The answer's id: the run its tool calls are ledgered under (agent_events.run_id). diff --git a/server/assistant/app/ask.py b/server/assistant/app/ask.py index 6ec7d6900e..c3da4ef0a6 100644 --- a/server/assistant/app/ask.py +++ b/server/assistant/app/ask.py @@ -18,6 +18,7 @@ async def handle_assistant_ask(inp: AssistantAskInput) -> AssistantAskOutput: messages = [m.model_dump() for m in inp.history] + [{"role": "user", "content": inp.question}] answer = await bridge.ask(auth, inp.org, inp.namespace, messages) return AssistantAskOutput( + mirrored=answer.mirrored, answer="".join(answer.text), run_id=answer.run_id, citations=answer.citations, diff --git a/server/assistant/app/bridge.py b/server/assistant/app/bridge.py index 0a6830e5bd..dcb3d41342 100644 --- a/server/assistant/app/bridge.py +++ b/server/assistant/app/bridge.py @@ -16,7 +16,7 @@ import json import logging import time -from collections.abc import AsyncGenerator, AsyncIterator, Iterator +from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Iterator from dataclasses import asdict, dataclass, field from typing import Any, NoReturn @@ -34,7 +34,9 @@ from server.auth.directory import directory from server.auth.jwt import AuthInfo, new_jti from server.auth.session_protection import protected_for +from server.core import request_tail, request_work, shielded from server.core.ch import set_query_tag +from server.core.diagnostics import private_work from server.core.errors import ( AssistantSelfCallError, AssistantUnavailableError, @@ -49,6 +51,7 @@ error_kind, ) from server.identity.domain.read_scope import request_read_narrowed +from server.knowledge.store import source_access from server.llm.customer import ModelCallsDisabled, switched_off from server.observability import cost_accumulator from server.observability.budget import operation_over_share @@ -90,6 +93,7 @@ class Ticket: # Loom's line about the caller's credential (`asker_line`), the last line of the # question LibreChat is sent. asker: str + mirrored: bool = False def release(self) -> None: """Give the answer's slot back; a second call does nothing.""" @@ -100,6 +104,7 @@ def release(self) -> None: class Answer: """What one answer produced, filled while it streams.""" + mirrored: bool = False run_id: str = "" text: list[str] = field(default_factory=list) prompt_tokens: int = 0 @@ -279,7 +284,7 @@ async def _afford(org: str, namespace: str) -> tuple[str, str]: return answering -async def answer_token(auth: AuthInfo, org: str, namespace: str) -> str: +async def answer_token(auth: AuthInfo, org: str, namespace: str, *, mirrored: bool = False) -> str: """The token one answer's tool calls read under: delegated from the caller's, `viewer` on the namespace alone, no longer-lived than an answer may run, and ended with the caller's token, subject or sign-in session.""" @@ -301,6 +306,7 @@ async def answer_token(auth: AuthInfo, org: str, namespace: str) -> str: jti=f"{ANSWER_JTI_PREFIX}{new_jti()}", reads_as_creator=bool(reader and reader == delegated.creator), session_protection=protected_for(auth), + answer_mirrors=mirrored and protected_for(auth), ) @@ -322,15 +328,41 @@ async def open_answer(auth: AuthInfo, org: str, namespace: str) -> Ticket: answer's slot, its token and its asker line, run before anything streams, so a refused request is an HTTP error and pays for no line. The stream gives the slot back; so does `Ticket.release` on a refusal after this.""" - agent_id, api_key = await _afford(org, namespace) - slot = _take_slot(org, namespace) + mirrored = await _mirror_admission(org, namespace) + with private_work() if mirrored else contextlib.nullcontext(): + agent_id, api_key = await _afford(org, namespace) + slot = _take_slot(org, namespace) + try: + token = await answer_token(auth, org, namespace, mirrored=mirrored) + asker = await asker_line(auth, org, namespace) + except BaseException: + _give_back(org, namespace, slot) + raise + return Ticket(org, namespace, agent_id, api_key, token, slot, auth.subject, asker, mirrored) + + +async def _mirror_admission(org: str, namespace: str) -> bool: + """Fix the answer's privacy before model work, without reading private evidence.""" + if not settings.auth_actually_required(): + return False # mirror reads require authenticated serving + set_query_tag("assistant.mirror.admission") try: - token = await answer_token(auth, org, namespace) - asker = await asker_line(auth, org, namespace) - except BaseException: - _give_back(org, namespace, slot) - raise - return Ticket(org, namespace, agent_id, api_key, token, slot, auth.subject, asker) + sources = await source_access.active_sources(org, [namespace]) + except Exception: # noqa: BLE001 — unknown admission must never become an ordinary answer + request_work.mark_mirrored(unread=True) + with private_work(): + pass + raise BackendUnavailableError( + "the assistant's source access could not be checked" + ) from None + if not sources: + return False + if not request_work.mark_mirrored(): + raise BackendUnavailableError("the assistant's source access could not be checked") + # Install diagnostic filters before the marked answer's first further await. + with private_work(): + pass + return True def _with_asker(messages: list[dict[str, str]], asker: str) -> list[dict[str, str]]: @@ -477,6 +509,60 @@ async def stream( the whole list so far; a `loom.text_done` event says the text is complete before the checks start; the `loom.citations` event then carries the steps of one read of the run after it ends.""" + answer.mirrored = ticket.mirrored + if ticket.mirrored: + yield b'data: {"mirrored":true}\n\n' + chunks = _stream(ticket, messages, answer, live=live) + try: + while True: + with private_work() if ticket.mirrored else contextlib.nullcontext(): + try: + chunk = await anext(chunks) + except StopAsyncIteration: + break + yield chunk + finally: + with private_work() if ticket.mirrored else contextlib.nullcontext(): + if ticket.mirrored: + await _close_owned(chunks.aclose(), name="assistant.stream.producer") + else: + await chunks.aclose() + + +@contextlib.asynccontextmanager +async def _answer_client( + ticket: Ticket, http_timeout: httpx.Timeout +) -> AsyncIterator[httpx.AsyncClient]: + client = httpx.AsyncClient(base_url=str(settings.assistant_librechat_url), timeout=http_timeout) + try: + yield client + finally: + if ticket.mirrored: + await _close_owned(client.aclose(), name="assistant.stream.client") + else: + await client.aclose() + + +async def _close_upstream(response: httpx.Response, tasks: list[asyncio.Task[None]]) -> None: + for task in tasks: + task.cancel() + try: + await response.aclose() + finally: + await asyncio.gather(*tasks, return_exceptions=True) + + +async def _close_owned(action: Awaitable[None], *, name: str) -> None: + async def bounded() -> None: + async with asyncio.timeout(request_tail.REQUIRED_STEP_TIMEOUT_S): + await action + + await shielded.to_its_end(bounded(), name=name) + + +async def _stream( + ticket: Ticket, messages: list[dict[str, str]], answer: Answer, *, live: bool +) -> AsyncGenerator[bytes]: body = { "model": ticket.agent_id, "messages": _with_asker(messages, ticket.asker), @@ -493,9 +579,7 @@ async def stream( started = time.time() answered = False # LibreChat's stream ended: the slot is held through the checks try: - async with httpx.AsyncClient( - base_url=str(settings.assistant_librechat_url), timeout=timeout - ) as client: + async with _answer_client(ticket, timeout) as client: async with asyncio.timeout_at(deadline): r = await client.send( client.build_request( @@ -518,16 +602,12 @@ async def stream( async for chunk in relayed: yield chunk finally: - # The request closes first (a cancellation delivered again at every await, - # as in an anyio scope, then cannot skip it), then the cancelled reader and - # poll are awaited. Under such a cancellation the await is cut short too: - # neither catches the cancellation, so both end on the loop's next pass. - for t in tasks: - t.cancel() - try: - await r.aclose() - finally: - await asyncio.gather(*tasks, return_exceptions=True) + # A marked answer joins response, reader and poll cleanup despite + # repeated cancellation, before final usage can be captured. + if ticket.mirrored: + await _close_owned(_close_upstream(r, tasks), name="assistant.stream.upstream") + else: + await _close_upstream(r, tasks) answered = True except httpx.HTTPError as e: log.warning("reaching LibreChat for an answer failed: %s", e) @@ -718,7 +798,7 @@ async def ask(auth: AuthInfo, org: str, namespace: str, messages: Any) -> Answer kept = admitted_messages(messages) await authorize(auth, org, namespace) ticket = await open_answer(auth, org, namespace) - answer = Answer() + answer = Answer(mirrored=ticket.mirrored) try: async for _ in stream(ticket, kept, answer): pass diff --git a/server/assistant/app/routes.py b/server/assistant/app/routes.py index fabf8f4eab..9b90662da8 100644 --- a/server/assistant/app/routes.py +++ b/server/assistant/app/routes.py @@ -30,7 +30,7 @@ from server.auth.context import set_current_auth from server.auth.gateway import effective_auth from server.auth.jwt import AuthInfo -from server.core import fail_open, tenant_slots +from server.core import fail_open, request_tail, request_work, shielded, tenant_slots from server.core.errors import ( AssistantUnavailableError, AuthInsufficientError, @@ -90,9 +90,14 @@ def __init__(self, auth: AuthInfo, org: str, namespace: str) -> None: self.request_id = uuid.uuid4().hex self.started = time.perf_counter() self.written = False + self.work = request_work.begin() cost_accumulator.begin_op() # the answer's spend lands on this row def write(self, error: BaseException | None) -> None: + with request_work.bind(self.work): + self._write(error) + + def _write(self, error: BaseException | None) -> None: if self.written: return self.written = True @@ -129,7 +134,10 @@ def write(self, error: BaseException | None) -> None: slo_class=slo_class, budget_ms=budget_ms, ) - except Exception as e: # noqa: BLE001 — telemetry never fails an answer + except Exception as e: # noqa: BLE001 — ordinary telemetry stays best effort + if self.work.mirror_marked: + request_tail.fail_required() + raise request_tail.RequiredTailError() from None log.debug("assistant answer: ledger row not written: %s", e) @@ -162,7 +170,7 @@ class _Held: def __init__(self, ticket: bridge.Ticket, row: _Row, slots: contextlib.ExitStack) -> None: self.ticket, self.row, self.slots = ticket, row, slots - self.answer = bridge.Answer() + self.answer = bridge.Answer(mirrored=ticket.mirrored) self.finished = False # the stream reached its end, or its error event def close(self, error: BaseException | None) -> None: @@ -188,6 +196,15 @@ def __init__(self, body: AsyncIterator[bytes], held: _Held) -> None: self.held = held async def __call__(self, scope: Any, receive: Any, send: Any) -> None: + lifetime = ( + request_tail.streaming_scope() + if self.held.ticket.mirrored + else contextlib.nullcontext() + ) + async with lifetime: + await self._serve(scope, receive, send) + + async def _serve(self, scope: Any, receive: Any, send: Any) -> None: error: BaseException | None = None try: await super().__call__(scope, receive, send) @@ -195,15 +212,25 @@ async def __call__(self, scope: Any, receive: Any, send: Any) -> None: error = e raise finally: - try: - # Ends the body however the response ended, so its charge lands on the row. + if self.held.ticket.mirrored: + await shielded.to_its_end(self._close(error), name="assistant.stream.close") + else: + await self._close(error) + + async def _close(self, error: BaseException | None) -> None: + try: + # Join producers before capturing final usage, including repeated cancellation. + async with asyncio.timeout(request_tail.REQUIRED_STEP_TIMEOUT_S): await self.body_iterator.aclose() # type: ignore[attr-defined] - finally: - if error is None and not self.held.finished: - # The response returned with the stream unfinished: the client left - # (below ASGI 2.4 Starlette absorbs the disconnect). - error = asyncio.CancelledError() + finally: + if error is None and not self.held.finished: + # ASGI 2.3 can absorb a disconnect without raising it to the response. + error = asyncio.CancelledError() + try: self.held.close(error) + except request_tail.RequiredTailError: + if error is None: + raise @router.post("/v1/assistant/chat/completions") @@ -280,10 +307,16 @@ async def chat_completions(request: Request) -> Any: async def _streamed(chunks: AsyncIterator[bytes], held: _Held) -> AsyncIterator[bytes]: try: async for chunk in chunks: + if chunk == b"data: [DONE]\n\n": + await chunks.aclose() # type: ignore[attr-defined] + held.close(None) + await request_tail.settle_current() yield chunk except LoomServerError as e: # The status is sent: LibreChat's failure goes out as an error event, and on the row. - held.row.write(e) + await chunks.aclose() # type: ignore[attr-defined] + held.close(e) + await request_tail.settle_current() held.finished = True yield bridge.error_event(e) return @@ -305,6 +338,7 @@ async def _whole(chunks: AsyncIterator[bytes], held: _Held) -> JSONResponse: held.close(error) return JSONResponse( { + **({"mirrored": True} if answer.mirrored else {}), "id": answer.run_id, "object": "chat.completion", "model": service.current_definition().model, diff --git a/server/auth/delegation.py b/server/auth/delegation.py index bc789ab47d..036413fad9 100644 --- a/server/auth/delegation.py +++ b/server/auth/delegation.py @@ -76,6 +76,7 @@ def mint( consumer: str | None = None, reads_as_creator: bool = False, session_protection: bool = False, + answer_mirrors: bool = False, ) -> str: return jwt_mod.mint_token( subject=subject, @@ -96,6 +97,7 @@ def mint( creator=self.creator, reads_as_creator=reads_as_creator, session_protection=session_protection, + answer_mirrors=answer_mirrors, saves_to=self.saves_to or None, ) diff --git a/server/auth/jwt.py b/server/auth/jwt.py index be8e2bf5f0..e7e91835f8 100644 --- a/server/auth/jwt.py +++ b/server/auth/jwt.py @@ -161,6 +161,8 @@ class AuthInfo: reads_as_creator: bool = False # The bridge fixes the caller's session-protection promise for one answer. session_protection: bool = False + # A marked answer's fixed mirror admission, separate from its caller's promise. + answer_mirrors: bool = False # `saves_to` claim: where a list token's calls that name no namespace save, # chosen at mint and read by `caller_home._pick_default`; '' = none recorded. saves_to: str = "" @@ -266,6 +268,7 @@ def mint_token( creator: str | None = None, reads_as_creator: bool = False, session_protection: bool = False, + answer_mirrors: bool = False, saves_to: str | None = None, issued_at: int | None = None, ) -> str: @@ -350,8 +353,12 @@ def mint_token( payload["creator"] = creator if reads_as_creator: payload["reads_as_creator"] = True - if _bool_claim({"session_protection": session_protection}, "session_protection"): - payload["session_protection"] = True + for name, value in ( + ("session_protection", session_protection), + ("answer_mirrors", answer_mirrors), + ): + if _bool_claim({name: value}, name): + payload[name] = True if saves_to: payload["saves_to"] = saves_to if ttl_seconds is not None: @@ -467,6 +474,7 @@ def verify_token(token: str, *, secret: str | None = None) -> AuthInfo: creator=_creator_claim(decoded), reads_as_creator=_bool_claim(decoded, "reads_as_creator"), session_protection=_bool_claim(decoded, "session_protection"), + answer_mirrors=_bool_claim(decoded, "answer_mirrors"), saves_to=_optional_str_claim(decoded, "saves_to"), session_id=_optional_str_claim(decoded, "sid"), first_party=_bool_claim(decoded, "fps"), @@ -508,6 +516,7 @@ def _end_user_key(token: str) -> tuple[str, EndUserIssuer, Any] | None: # Claims an end-user token may not carry: its identity is the user, nothing more. _NOT_FOR_END_USERS = ( + "answer_mirrors", "namespaces", "ns_write", "ns_roles", diff --git a/server/core/errors.py b/server/core/errors.py index 0c107dcbde..6458c7c8b7 100644 --- a/server/core/errors.py +++ b/server/core/errors.py @@ -686,12 +686,14 @@ def register_exception_handler(app: FastAPI) -> None: from fastapi.responses import JSONResponse, Response from starlette.exceptions import HTTPException as StarletteHTTPException + from server.core.request_work import flag_response, mirror_marked + @app.exception_handler(LoomServerError) async def _handle_loom_error(_: Request, exc: LoomServerError) -> JSONResponse: detail = classified(exc.to_detail(), exc.status_code) return JSONResponse( status_code=exc.status_code, - content={"detail": detail}, + content=flag_response({"detail": detail}), headers=_with_help_link(exc.headers(), detail), ) @@ -701,6 +703,12 @@ async def _handle_http_error(request: Request, exc: StarletteHTTPException) -> R detail = exc.detail if isinstance(detail, dict) and detail.get("code"): detail = classified(detail, exc.status_code) + if mirror_marked() and exc.status_code not in (204, 304): + return JSONResponse( + status_code=exc.status_code, + content=flag_response({"detail": detail}), + headers=headers, + ) # Never write onto exc: a module-level HTTPException is raised on every request. answer = StarletteHTTPException(exc.status_code, detail=detail, headers=headers) return await http_exception_handler(request, answer) @@ -710,4 +718,4 @@ async def _handle_request_validation(_: Request, exc: RequestValidationError) -> # FastAPI's own 422, except that a non-finite input (`1e400`) is echoed as a # string: JSON has no inf, and the response would fail as a 500. errors = loads_keeping_non_finite(json.dumps(jsonable_encoder(exc.errors()))) - return JSONResponse(status_code=422, content={"detail": errors}) + return JSONResponse(status_code=422, content=flag_response({"detail": errors})) diff --git a/server/core/request_tail.py b/server/core/request_tail.py index 3f5502baf1..fee64f68b4 100644 --- a/server/core/request_tail.py +++ b/server/core/request_tail.py @@ -4,8 +4,8 @@ import asyncio import logging -from collections.abc import Awaitable, Callable, Iterator -from contextlib import contextmanager +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator +from contextlib import asynccontextmanager, contextmanager from contextvars import Context, ContextVar, copy_context from typing import Any @@ -356,6 +356,33 @@ async def settle_current() -> None: await settle(tail) +@asynccontextmanager +async def streaming_scope() -> AsyncIterator[None]: + """Own final streamed usage after the HTTP header tail has closed. + + The caller settles before its terminal event. Cancellation still joins the + owned required delivery before this scope returns. + """ + tail = _Tail() + token = _tail.set(tail) + group_token = _publication_group.set(None) + primary: BaseException | None = None + try: + yield + except BaseException as error: + primary = error + raise + finally: + try: + await settle(tail) + except BaseException: + if primary is None: + raise + finally: + _publication_group.reset(group_token) + _tail.reset(token) + + class RequestTailMiddleware: def __init__(self, app: Any) -> None: self.app = app diff --git a/server/core/request_work.py b/server/core/request_work.py index af268f421c..829d0c3c7f 100644 --- a/server/core/request_work.py +++ b/server/core/request_work.py @@ -132,6 +132,16 @@ def begin() -> RequestWork: return bag +@contextmanager +def bind(bag: RequestWork) -> Iterator[None]: + """Restore a logical owner's bag when a stream closes in another context.""" + token = _current.set(bag) + try: + yield + finally: + _current.reset(token) + + def current() -> RequestWork | None: return None if codes_only() else _current.get() @@ -161,6 +171,13 @@ def mirror_marked() -> bool: return bag is not None and bag.mirror_marked +def flag_response(body: dict[str, Any]) -> dict[str, Any]: + """Place the server-owned privacy flag before a marked logical call's fields.""" + if not mirror_marked(): + return body + return {"mirrored": True, **{k: v for k, v in body.items() if k != "mirrored"}} + + def marked_event_pending() -> bool: """A raw logical mark still owes its one required event registration.""" bag = _current.get() diff --git a/server/knowledge/app/source_access.py b/server/knowledge/app/source_access.py index c34fa71e1d..88c31dadf4 100644 --- a/server/knowledge/app/source_access.py +++ b/server/knowledge/app/source_access.py @@ -8,6 +8,7 @@ from typing import Any from server.auth.aliases import canonical_subject +from server.auth.answer_tokens import is_answer_token from server.auth.authz import check_authorized from server.auth.context import current_auth from server.auth.reader_identity import ReaderIdentity, reader_subject, resolve_reader @@ -254,6 +255,7 @@ async def prepare(org: str, namespaces: list[str]) -> ReadPlan: auth is None or not settings.auth_actually_required() or any(is_mirror_name(ns) for ns in namespaces) + or (is_answer_token(auth) and not (auth.session_protection and auth.answer_mirrors)) ): return plan set_query_tag("knowledge.mirror.prepare") diff --git a/server/transport/mcp_outcome.py b/server/transport/mcp_outcome.py index de0970243e..3e192ece02 100644 --- a/server/transport/mcp_outcome.py +++ b/server/transport/mcp_outcome.py @@ -185,6 +185,7 @@ def _handler_failed( HTTP's body and is recorded under its code; any other exception is one that says only "internal error" and is logged with its traceback.""" from server.core.errors import handler_failure_detail + from server.core.request_work import flag_response from server.transport.mcp_refusals import refusal_body, refusal_text, server_failure refused = refusal_body(e) @@ -192,6 +193,7 @@ def _handler_failed( if failure is not None: failure = handler_failure_detail(failure, failure["status"], op.idempotent) if refused is not None: + refused = flag_response(refused) log.info("mcp.refused %s request_id=%s: %s", op.name, request_id, refused.get("code")) elif failure is not None: # the raiser logged its cause; no traceback for a named 5xx log.warning("mcp.unavailable %s request_id=%s: %s", op.name, request_id, failure["code"]) @@ -232,27 +234,40 @@ def _handler_failed( except Exception: # noqa: BLE001 — telemetry must never mask the handler's error pass if refused is not None: - return _ok( - req_id, - { - "content": [{"type": "text", "text": refusal_text(refused)}], - "isError": True, - "structuredContent": refused, - }, + return flag_response( + _ok( + req_id, + { + "content": [ + { + "type": "text", + "text": json.dumps(refused) + if refused.get("mirrored") is True + else refusal_text(refused), + } + ], + "isError": True, + "structuredContent": refused, + }, + ) ) if failure is not None: # HTTP's 503/504 body, as the error's message and data - return _err( + return flag_response( + _err( + req_id, + TOOL_EXEC_FAILED, + f"tool execution failed: {refusal_text(failure)}; request_id={request_id}", + data=flag_response({**failure, "tool": name, "request_id": request_id}), + ) + ) + # The exception's text (ClickHouse / SQL) stays in the log line above. + return flag_response( + _err( req_id, TOOL_EXEC_FAILED, - f"tool execution failed: {refusal_text(failure)}; request_id={request_id}", - data={**failure, "tool": name, "request_id": request_id}, + f"tool execution failed: internal error; request_id={request_id}", + data=flag_response({"code": "INTERNAL_ERROR", "tool": name, "request_id": request_id}), ) - # The exception's text (ClickHouse / SQL) stays in the log line above. - return _err( - req_id, - TOOL_EXEC_FAILED, - f"tool execution failed: internal error; request_id={request_id}", - data={"code": "INTERNAL_ERROR", "tool": name, "request_id": request_id}, ) diff --git a/tests/integration/test_mirror_checked_search_http.py b/tests/integration/test_mirror_checked_search_http.py index 08661e21a4..0b24c5cdab 100644 --- a/tests/integration/test_mirror_checked_search_http.py +++ b/tests/integration/test_mirror_checked_search_http.py @@ -412,6 +412,33 @@ async def download(token): assert len((await listing(owner_bot))["rows"]) == 2 assert len((await inspect(owner_bot))["listed"]) == 2 assert len((await source_inspect(owner_bot))["documents"]) == 1 + # An anyone-readable ACL cannot widen an answer's pre-output admission. + for promise, admission in [(True, False), (False, True), (True, True)]: + answer = mint_token( + "source-owner", + org, + [namespace], + ["viewer"], + ttl_seconds=600, + is_service_account=False, + session_id=str(uuid4()), + consumer="loom-assistant", + jti="assistant-answer-" + str(uuid4()), + session_protection=promise, + answer_mirrors=admission, + ) + found = await search(answer) + addresses = [r["source_uri"] for r in found["results"]] + assert "ordinary:visible" in addresses + document = await get(answer) + if promise and admission: + assert public_uri in addresses and document["content"] == private_content + assert found["mirrored"] is True and document["mirrored"] is True + else: + assert public_uri not in addresses and not document["found"] + assert private_content not in json.dumps(found) + assert found.get("mirrored") is not True + assert document.get("mirrored") is not True await sources.insert_source(replace(registration, active=False)) assert await listing(owner) == ordinary_list assert (await inspect(owner))["listed"] == ordinary_inspect["listed"] diff --git a/tests/unit/test_assistant_bridge.py b/tests/unit/test_assistant_bridge.py index 2964efa31b..08437997c9 100644 --- a/tests/unit/test_assistant_bridge.py +++ b/tests/unit/test_assistant_bridge.py @@ -80,6 +80,8 @@ def __init__(self) -> None: self.denied = False # Another answer for LibreChat to give, in place of the stream above. self.override: Any = None + self.mirror_sources: list[dict[str, str]] = [] + self.source_queries = 0 def handle(self, request: httpx.Request) -> httpx.Response: if self.override is not None: @@ -93,6 +95,7 @@ def handle(self, request: httpx.Request) -> httpx.Response: def world(monkeypatch: pytest.MonkeyPatch) -> _World: w = _World() real_client = httpx.AsyncClient + w.client_factory = real_client async def check_authorized(auth: Any, role: Role, org: str, ns: str) -> None: if w.denied: @@ -107,6 +110,11 @@ async def over_cap(org: str, ns: str, **kw: Any) -> bool: raise ConnectionError("ClickHouse is down") return w.over_cap + async def active_sources(org: str, namespaces: list[str]) -> list[dict[str, str]]: + assert org == ORG and len(namespaces) == 1 + w.source_queries += 1 + return list(w.mirror_sources) + class _Delegated: creator = "alice" @@ -118,8 +126,10 @@ def mint( jti: str | None = None, reads_as_creator: bool = False, session_protection: bool = False, + answer_mirrors: bool = False, ) -> str: w.delegated["minted"] = (subject, consumer) + w.delegated["answer_mirrors"] = answer_mirrors return "answer-token" async def delegate(parent: AuthInfo, **kw: Any) -> _Delegated: @@ -141,6 +151,7 @@ async def admits(self, ns: str, role: Role) -> bool: monkeypatch.setattr(bridge, "_place", lambda org, ns: "namespace") monkeypatch.setattr(bridge, "switched_off", switched_off) monkeypatch.setattr(bridge, "operation_over_share", over_cap) + monkeypatch.setattr(bridge.source_access, "active_sources", active_sources) # The answer's tool calls found nothing (a test may say otherwise). monkeypatch.setattr(bridge.support.ledger, "run_rows", _found_nothing) monkeypatch.setattr(bridge.support, "_wait", _no_wait) @@ -234,6 +245,7 @@ def test_the_answer_token_is_viewer_on_the_namespace_for_one_answer(world: _Worl "is_service_account": False, "ends_with_session": True, "minted": ("alice", "loom-assistant"), + "answer_mirrors": False, } @@ -1274,6 +1286,7 @@ def mint( jti: str | None = None, reads_as_creator: bool = False, session_protection: bool = False, + answer_mirrors: bool = False, ) -> str: minted.update(subject=subject, consumer=consumer, jti=jti) return "answer-token" @@ -1797,3 +1810,212 @@ async def delegate(parent: AuthInfo, **kwargs: Any) -> Delegated: assert response.status_code == 403 assert response.json()["detail"]["code"] == "ASSISTANT_SELF_CALL" assert len(rows) == 2 and world.librechat_body == {} + + +def _mirrored_namespace(monkeypatch: pytest.MonkeyPatch) -> None: + async def active(org: str, namespaces: list[str]) -> list[dict[str, str]]: + assert org == ORG and namespaces == [NS] + return [{"namespace": NS, "source_id": "source"}] + + monkeypatch.setattr(settings, "auth_required", True) + monkeypatch.setattr(bridge.source_access, "active_sources", active) + + +@pytest.mark.parametrize("streamed", [False, True]) +def test_marked_chat_has_a_server_owned_first_flag( + client: Any, world: _World, monkeypatch: pytest.MonkeyPatch, streamed: bool +) -> None: + _mirrored_namespace(monkeypatch) + http, rows = client + response = http.post(ROUTE, headers=HEADERS, json={"messages": QUESTION, "stream": streamed}) + assert response.status_code == 200 + if streamed: + assert response.content.startswith(b'data: {"mirrored":true}\n\n') + assert b"Seven " in response.content and b"data: [DONE]" in response.content + else: + assert response.content.startswith(b'{"mirrored":true,') + assert response.json()["choices"][0]["message"]["content"] == "Seven memories." + assert len(rows) == 1 and bridge._running == {} + # No declaration on this request: the model's tools cannot read mirrors. + assert world.delegated["answer_mirrors"] is False + + +def test_a_marked_pre_stream_refusal_keeps_its_first_flag( + client: Any, world: _World, monkeypatch: pytest.MonkeyPatch +) -> None: + _mirrored_namespace(monkeypatch) + world.over_cap = True + http, rows = client + response = http.post(ROUTE, headers=HEADERS, json={"messages": QUESTION, "stream": True}) + assert response.status_code == 429 + assert response.content.startswith(b'{"mirrored":true,') + assert world.librechat_body == {} and len(rows) == 1 + + +@pytest.mark.parametrize("failure", [False, True]) +def test_marked_stream_commits_real_ledger_delivery_before_done( + world: _World, monkeypatch: pytest.MonkeyPatch, failure: bool +) -> None: + import contextlib + + from server.assistant.app import routes + from server.core import request_work + from server.core.request_tail import RequiredTailError + from server.observability import events + + _mirrored_namespace(monkeypatch) + committed: list[Any] = [] + sent: list[bytes] = [] + + async def commit(event: Any) -> None: + await asyncio.sleep(0) + assert event.mirror_marked and event.input_tokens == 1000 and event.output_tokens == 100 + if failure: + raise ConnectionError("private cost store unavailable") + committed.append(event) + + monkeypatch.setattr(events, "write_marked_request_event", commit) + + async def run() -> None: + scope = {"type": "http", "asgi": {"spec_version": "2.4"}} + with request_work.http_owner(scope): + row = routes._Row(_caller(), ORG, NS) + ticket = await bridge.open_answer(_caller(), ORG, NS) + held = routes._Held(ticket, row, contextlib.ExitStack()) + response = routes._AnswerStream( + routes._streamed(bridge.stream(ticket, QUESTION, held.answer), held), held + ) + + async def send(message: dict[str, Any]) -> None: + body = message.get("body", b"") + if body == b"data: [DONE]\n\n": + assert len(committed) == 1 + if body: + sent.append(body) + + async def receive() -> dict[str, Any]: + pytest.fail("ASGI 2.4 unexpectedly waited for disconnect") + + if failure: + with pytest.raises(RequiredTailError): + await response(scope, receive, send) + else: + await response(scope, receive, send) + + asyncio.run(run()) + assert sent[0] == b'data: {"mirrored":true}\n\n' + assert (b"data: [DONE]\n\n" in sent) is (not failure) + assert len(committed) == (0 if failure else 1) + assert bridge._running == {} + + +def test_a_source_activated_after_an_ordinary_answer_cannot_add_a_mirror_leg( + world: _World, +) -> None: + from server.auth.context import current_auth_var + from server.auth.session_protection import request_declaration + from server.core import request_work + from server.knowledge.app import source_access + + async def run() -> None: + bag = request_work.begin() + with request_declaration({"headers": [(b"x-loom-session-protection", b"v1")]}): + ticket = await bridge.open_answer(_caller(), ORG, NS) + try: + assert not ticket.mirrored and not bag.mirror_marked + world.mirror_sources = [{"namespace": NS, "source_id": "activated"}] + auth = _caller( + token_id=f"{bridge.ANSWER_JTI_PREFIX}ordinary", + session_protection=True, + answer_mirrors=world.delegated["answer_mirrors"], + ) + token = current_auth_var.set(auth) + try: + plan = await source_access.prepare(ORG, [NS]) + finally: + current_auth_var.reset(token) + assert not plan.marked and not plan.sources and not bag.mirror_marked + assert world.source_queries == 1 # preflight only, no later mirror discovery + finally: + ticket.release() + + asyncio.run(run()) + + +def test_marked_repeated_cancellation_joins_upstream_before_its_private_row( + world: _World, monkeypatch: pytest.MonkeyPatch +) -> None: + import contextlib + + from server.assistant.app import routes + from server.core import request_work + from server.observability import events + + _mirrored_namespace(monkeypatch) + + async def run() -> None: + text_seen, closing, release = asyncio.Event(), asyncio.Event(), asyncio.Event() + closed: list[str] = [] + committed: list[Any] = [] + + class Body(httpx.AsyncByteStream): + async def __aiter__(self) -> Any: + try: + yield b'data: {"id":"partial","choices":[{"delta":{"content":"text"}}]}\n\n' + await asyncio.Event().wait() + finally: + closed.append("reader") + + async def aclose(self) -> None: + closing.set() + await release.wait() + closed.append("response") + + class Upstream(httpx.MockTransport): + async def aclose(self) -> None: + closed.append("client") + + world.override = lambda request: httpx.Response(200, stream=Body()) + monkeypatch.setattr( + bridge.httpx, + "AsyncClient", + lambda **kw: world.client_factory(transport=Upstream(world.handle), **kw), + ) + + async def commit(event: Any) -> None: + assert "reader" in closed and "response" in closed and "client" in closed + assert event.mirror_marked and event.error_code == "HTTP_499" + assert event.upstream_cost_usd == pytest.approx(settings.assistant_answer_ceiling_usd) + committed.append(event) + + monkeypatch.setattr(events, "write_marked_request_event", commit) + scope = {"type": "http", "asgi": {"spec_version": "2.4"}} + with request_work.http_owner(scope): + row = routes._Row(_caller(), ORG, NS) + ticket = await bridge.open_answer(_caller(), ORG, NS) + held = routes._Held(ticket, row, contextlib.ExitStack()) + response = routes._AnswerStream( + routes._streamed(bridge.stream(ticket, QUESTION, held.answer), held), held + ) + + async def send(message: dict[str, Any]) -> None: + if b'"partial"' in message.get("body", b""): + text_seen.set() + + async def receive() -> dict[str, Any]: + pytest.fail("ASGI 2.4 unexpectedly waited for disconnect") + + task = asyncio.create_task(response(scope, receive, send)) + await text_seen.wait() + task.cancel() + await closing.wait() + task.cancel() + await asyncio.sleep(0) + assert not task.done() and committed == [] + release.set() + with pytest.raises(asyncio.CancelledError): + await task + assert len(committed) == 1 and bridge._running == {} + assert not (asyncio.all_tasks() - {asyncio.current_task()}) + + asyncio.run(run()) diff --git a/tests/unit/test_assistant_mirror_boundary.py b/tests/unit/test_assistant_mirror_boundary.py new file mode 100644 index 0000000000..5c74dfb393 --- /dev/null +++ b/tests/unit/test_assistant_mirror_boundary.py @@ -0,0 +1,191 @@ +"""An answer cannot discover private tools outside its pre-output admission.""" + +from __future__ import annotations + +from dataclasses import replace +from types import SimpleNamespace +from typing import Any + +import jwt +import pytest + +from server.auth.answer_tokens import ANSWER_JTI_PREFIX +from server.auth.context import current_auth_var +from server.auth.jwt import AuthInfo, TokenError, mint_token, verify_token +from server.core import request_work +from server.knowledge.app import source_access +from server.operations.base import Role +from server.settings import settings + +_SECRET = "assistant-mirror-admission-unit-key" + + +def _caller(**kwargs: Any) -> AuthInfo: + return AuthInfo( + **{ + "subject": "reader", + "org": "example", + "namespaces": ("notes",), + "roles": (Role.VIEWER,), + "is_service_account": False, + "token_id": f"{ANSWER_JTI_PREFIX}answer", + **kwargs, + } + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "identity", + [ + {"consumer": "loom-assistant", "token_id": "legacy"}, + {}, + {"token_id": "child", "parents": (f"{ANSWER_JTI_PREFIX}answer",)}, + ], +) +@pytest.mark.parametrize("protected", [False, True]) +async def test_an_answer_without_admission_never_discovers_a_mirror( + monkeypatch: pytest.MonkeyPatch, identity: dict[str, Any], protected: bool +) -> None: + seen: list[str] = [] + + async def authorized(*args: Any) -> None: + seen.append("authorized") + + async def active(*args: Any) -> list[dict[str, Any]]: + seen.append("discovered") + return [{"namespace": "notes", "source_id": "source"}] + + async def prepare(*args: Any) -> None: + seen.append("private") + + monkeypatch.setattr(settings, "auth_required", True) + monkeypatch.setattr(source_access, "check_authorized", authorized) + monkeypatch.setattr(source_access.store, "active_sources", active) + monkeypatch.setattr(source_access, "_prepare", prepare) + auth = _caller(**identity, session_protection=protected) + token = current_auth_var.set(auth) + try: + request_work.begin() + plan = await source_access.prepare("example", ["notes"]) + finally: + current_auth_var.reset(token) + assert not plan.marked + assert not plan.sources + assert seen == [] + + +@pytest.mark.parametrize("admitted", [False, True]) +def test_answer_admission_survives_signature_verification(admitted: bool) -> None: + token = mint_token( + "reader", + "example", + ["notes"], + ["viewer"], + secret=_SECRET, + answer_mirrors=admitted, + ) + assert verify_token(token, secret=_SECRET).answer_mirrors is admitted + + +@pytest.mark.asyncio +async def test_positive_admission_still_requires_the_fixed_caller_promise( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def unexpected(*args: Any) -> Any: + pytest.fail("an undeclared answer queried mirrored registrations") + + monkeypatch.setattr(settings, "auth_required", True) + monkeypatch.setattr(source_access, "check_authorized", unexpected) + monkeypatch.setattr(source_access.store, "active_sources", unexpected) + token = current_auth_var.set(replace(_caller(), answer_mirrors=True)) + try: + request_work.begin() + assert not (await source_access.prepare("example", ["notes"])).marked + finally: + current_auth_var.reset(token) + + +@pytest.mark.parametrize("claim", ["true", "false", 0, 1, None, [], {}]) +def test_non_boolean_answer_admission_is_refused(claim: Any) -> None: + with pytest.raises(TokenError, match="must be a boolean"): + mint_token("reader", "example", ["notes"], ["viewer"], secret=_SECRET, answer_mirrors=claim) + signed = mint_token("reader", "example", ["notes"], ["viewer"], secret=_SECRET) + payload = jwt.decode(signed, options={"verify_signature": False}) + payload["answer_mirrors"] = claim + signed = jwt.encode(payload, _SECRET, algorithm="HS256", headers={"kid": "override"}) + with pytest.raises(TokenError, match="must be a boolean"): + verify_token(signed, secret=_SECRET) + + +@pytest.mark.asyncio +async def test_an_admitted_protected_answer_keeps_the_checked_mirror_path( + monkeypatch: pytest.MonkeyPatch, +) -> None: + seen: list[str] = [] + + async def authorized(*args: Any) -> None: + seen.append("authorized") + + async def active(*args: Any) -> list[dict[str, str]]: + seen.append("discovered") + return [{"namespace": "notes", "source_id": "source"}] + + async def prepare(*args: Any) -> None: + assert request_work.mirror_marked() + seen.append("checked") + + monkeypatch.setattr(settings, "auth_required", True) + monkeypatch.setattr(source_access, "check_authorized", authorized) + monkeypatch.setattr(source_access.store, "active_sources", active) + monkeypatch.setattr(source_access, "_prepare", prepare) + token = current_auth_var.set(_caller(session_protection=True, answer_mirrors=True)) + try: + bag = request_work.begin() + plan = await source_access.prepare("example", ["notes"]) + finally: + current_auth_var.reset(token) + assert plan.marked and bag.mirror_marked + assert seen == ["authorized", "discovered", "checked"] + + +@pytest.mark.parametrize("status", [403, 503, 500]) +def test_mcp_failures_keep_the_mark_after_protocol_unwrapping( + monkeypatch: pytest.MonkeyPatch, status: int +) -> None: + import json + + from server.core.diagnostics import private_work + from server.core.errors import AuthInsufficientError, BackendUnavailableError + from server.transport import mcp_outcome + + error = { + 403: AuthInsufficientError("refused"), + 503: BackendUnavailableError("unavailable"), + 500: RuntimeError("private detail"), + }[status] + monkeypatch.setattr(mcp_outcome, "_emit_mcp_event", lambda *a, **kw: None) + request_work.begin() + request_work.mark_mirrored() + with private_work(): + result = mcp_outcome._handler_failed( + error, + 1, + "loom_assistant_ask", + SimpleNamespace(name="assistant.ask", idempotent=False), + {}, + None, + _caller(), + "request", + "session", + rl_ns="notes", + rl_org="example", + elapsed_ms=1, + ) + assert next(iter(result)) == "mirrored" and result["mirrored"] is True + if status == 403: + body = result["result"] + assert next(iter(body["structuredContent"])) == "mirrored" + assert json.loads(body["content"][0]["text"])["mirrored"] is True + else: + assert next(iter(result["error"]["data"])) == "mirrored" diff --git a/tests/unit/test_sdk_assistant_client.py b/tests/unit/test_sdk_assistant_client.py index a795c4decb..d43f7e66e1 100644 --- a/tests/unit/test_sdk_assistant_client.py +++ b/tests/unit/test_sdk_assistant_client.py @@ -108,6 +108,32 @@ def handler(r: httpx.Request) -> httpx.Response: } +def test_eof_after_partial_text_does_not_yield_a_completed_answer() -> None: + seen: list[Any] = [] + sent: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + sent.append(request) + return httpx.Response(200, content=ANSWER.removesuffix(b"data: [DONE]\n\n")) + + async def run() -> None: + async with _agent(handler) as agent: + async for event in agent.assistant.stream("q"): + seen.append(event) + + with pytest.raises(LoomError, match="before completion"): + asyncio.run(run()) + assert seen == [AssistantText("Deploys freeze "), AssistantText("on Fridays.")] + assert len(sent) == 1 + + +def test_text_after_done_cannot_change_the_completed_answer() -> None: + body = ANSWER + _sse(_chunk("discarded")) + events = _collect(_agent(lambda request: httpx.Response(200, content=body))) + assert events[:2] == [AssistantText("Deploys freeze "), AssistantText("on Fridays.")] + assert len(events) == 3 and events[-1].text == "Deploys freeze on Fridays." + + @pytest.mark.parametrize("support", ["a-later-state", None]) def test_an_unknown_or_missing_support_reads_as_unchecked(support: Any) -> None: event: dict[str, Any] = {"object": "loom.citations", "id": "chatcmpl-1", "citations": []} diff --git a/tests/unit/test_session_protection_declaration.py b/tests/unit/test_session_protection_declaration.py index d4c4cae7d0..c2cfa51b57 100644 --- a/tests/unit/test_session_protection_declaration.py +++ b/tests/unit/test_session_protection_declaration.py @@ -164,9 +164,10 @@ def test_a_signed_non_boolean_promise_is_refused(claim: Any) -> None: @pytest.mark.parametrize("caller_protected", [False, True]) +@pytest.mark.parametrize("mirrored", [False, True]) @pytest.mark.asyncio async def test_the_bridge_fixes_the_callers_declaration_in_the_answer( - monkeypatch: pytest.MonkeyPatch, caller_protected: bool + monkeypatch: pytest.MonkeyPatch, caller_protected: bool, mirrored: bool ) -> None: parent = _auth() @@ -189,13 +190,15 @@ async def delegate(auth: AuthInfo, **kwargs: Any) -> Delegated: monkeypatch.setattr(bridge, "delegate", delegate) with request_declaration({"headers": [_HEADER] if caller_protected else []}): - token = await bridge.answer_token(parent, "example", "notes") + token = await bridge.answer_token(parent, "example", "notes", mirrored=mirrored) answer = verify_token(token) assert answer.session_protection is caller_protected + assert answer.answer_mirrors is (caller_protected and mirrored) with request_declaration({"headers": [] if caller_protected else [_HEADER]}): assert protected_for(answer) is caller_protected child = verify_token((await delegate(answer)).mint("sa:child")) assert not child.session_protection + assert not child.answer_mirrors assert not protected_for(child) # A forged local field is not inherited by ordinary delegation either. assert not verify_token( From 53eff65f4536ad18c9c4fd8264a297e5f8d54def Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 08:12:23 +0000 Subject: [PATCH 2/5] Assistant reads mirror registration through the knowledge app Keep the registration-only admission lookup within the package contract while preserving the conservative privacy boundary. Signed-off-by: zlareb1 --- server/assistant/app/bridge.py | 4 ++-- server/knowledge/app/source_access.py | 5 +++++ tests/unit/test_assistant_bridge.py | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/server/assistant/app/bridge.py b/server/assistant/app/bridge.py index dcb3d41342..aa145440d0 100644 --- a/server/assistant/app/bridge.py +++ b/server/assistant/app/bridge.py @@ -51,7 +51,7 @@ error_kind, ) from server.identity.domain.read_scope import request_read_narrowed -from server.knowledge.store import source_access +from server.knowledge.app import source_access from server.llm.customer import ModelCallsDisabled, switched_off from server.observability import cost_accumulator from server.observability.budget import operation_over_share @@ -347,7 +347,7 @@ async def _mirror_admission(org: str, namespace: str) -> bool: return False # mirror reads require authenticated serving set_query_tag("assistant.mirror.admission") try: - sources = await source_access.active_sources(org, [namespace]) + sources = await source_access.has_active_mirrored_sources(org, namespace) except Exception: # noqa: BLE001 — unknown admission must never become an ordinary answer request_work.mark_mirrored(unread=True) with private_work(): diff --git a/server/knowledge/app/source_access.py b/server/knowledge/app/source_access.py index 88c31dadf4..ecdff4e46e 100644 --- a/server/knowledge/app/source_access.py +++ b/server/knowledge/app/source_access.py @@ -247,6 +247,11 @@ def _head_serves(head: store.ReadHead, candidate: dict[str, Any], chunks: bool) ) +async def has_active_mirrored_sources(org: str, namespace: str) -> bool: + """Read public registration presence after the caller authorizes the namespace.""" + return bool(await store.active_sources(org, [namespace])) + + async def prepare(org: str, namespaces: list[str]) -> ReadPlan: """Read public registrations first, then bind one marked, fail-closed mirror plan.""" plan = ReadPlan(org) diff --git a/tests/unit/test_assistant_bridge.py b/tests/unit/test_assistant_bridge.py index 08437997c9..e839623a3d 100644 --- a/tests/unit/test_assistant_bridge.py +++ b/tests/unit/test_assistant_bridge.py @@ -151,7 +151,7 @@ async def admits(self, ns: str, role: Role) -> bool: monkeypatch.setattr(bridge, "_place", lambda org, ns: "namespace") monkeypatch.setattr(bridge, "switched_off", switched_off) monkeypatch.setattr(bridge, "operation_over_share", over_cap) - monkeypatch.setattr(bridge.source_access, "active_sources", active_sources) + monkeypatch.setattr(bridge.source_access.store, "active_sources", active_sources) # The answer's tool calls found nothing (a test may say otherwise). monkeypatch.setattr(bridge.support.ledger, "run_rows", _found_nothing) monkeypatch.setattr(bridge.support, "_wait", _no_wait) @@ -1818,7 +1818,7 @@ async def active(org: str, namespaces: list[str]) -> list[dict[str, str]]: return [{"namespace": NS, "source_id": "source"}] monkeypatch.setattr(settings, "auth_required", True) - monkeypatch.setattr(bridge.source_access, "active_sources", active) + monkeypatch.setattr(bridge.source_access.store, "active_sources", active) @pytest.mark.parametrize("streamed", [False, True]) From bff10f343a7e4022c57f940a9e2ea7a5733fc0a8 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 08:33:59 +0000 Subject: [PATCH 3/5] Marked operation successes retain their privacy flag Attach the server-owned mark after typed output serialization and keep it through structured and text protocol unwrapping. Signed-off-by: zlareb1 --- docs/architecture/assistant.md | 7 ++ server/transport/arg_check.py | 4 +- server/transport/mcp_outcome.py | 37 ++++++---- tests/unit/test_assistant_mirror_boundary.py | 75 ++++++++++++++++++++ 4 files changed, 110 insertions(+), 13 deletions(-) diff --git a/docs/architecture/assistant.md b/docs/architecture/assistant.md index 163ace0e8f..42e9c04111 100644 --- a/docs/architecture/assistant.md +++ b/docs/architecture/assistant.md @@ -79,6 +79,13 @@ as its first SSE event, or `mirrored: true` as its whole response's first field. decision also routes the answer's cost row to private storage and suppresses public diagnostics before subsequent model, budget or tool work. +Marked operation results carry the same server-owned flag after their typed output is +serialized. HTTP success responses retain it before post-handler rows. MCP success +responses retain it in the outer envelope, tool result, structured content and canonical +JSON text. The search tool's prose summary follows a separate first JSON marker block, +so consuming its text blocks in order observes protection before the summary. Ordinary +unmarked results keep their existing shapes. + The answer token keeps the caller's `session_protection` promise and a separate `answer_mirrors` admission bit. Only an already marked answer for a declared caller receives positive admission. Missing or false admission disables its mirror legs, diff --git a/server/transport/arg_check.py b/server/transport/arg_check.py index 6e78b9c0f1..0532e64239 100644 --- a/server/transport/arg_check.py +++ b/server/transport/arg_check.py @@ -230,6 +230,7 @@ def _note(op_name: str, keys: list[str]) -> None: def response_body(op: Any, args: Any, result: Any) -> Any: """The dumped result a transport returns, with `ignored_args` / `did_you_mean` when the call carried keys the op does not read.""" + from server.core.request_work import flag_response from server.transport.epoch_nulls import scrub body = result.model_dump(mode="json") if hasattr(result, "model_dump") else result @@ -237,7 +238,8 @@ def response_body(op: Any, args: Any, result: Any) -> Any: block = ignored_block(op, args) if block: _note(op.name, block["ignored_args"]) - return attach(body, block) + body = attach(body, block) + return flag_response(body) if isinstance(body, dict) else body def header_value(op: Any, args: Any, result: Any) -> str: diff --git a/server/transport/mcp_outcome.py b/server/transport/mcp_outcome.py index 3e192ece02..e50dab7068 100644 --- a/server/transport/mcp_outcome.py +++ b/server/transport/mcp_outcome.py @@ -10,7 +10,7 @@ from collections.abc import Callable from typing import TYPE_CHECKING, Any -from server.core import llm_free +from server.core import llm_free, request_work from server.core.naming import present from server.transport._common import ledger_identity from server.transport.lean_results import mcp_op_result @@ -95,8 +95,15 @@ def _search_answer(call: _AdmittedCall, result: Any) -> dict[str, Any]: from server.transport.mcp_tools import search_answer answer = search_answer(result.model_dump(), call.arguments, call.rl_ns, call.mcp_session_id) - answer["structuredContent"] = _coerce_bigints(answer["structuredContent"]) - return answer + answer["structuredContent"] = request_work.flag_response( + _coerce_bigints(answer["structuredContent"]) + ) + if request_work.mirror_marked(): + answer["content"] = [ + {"type": "text", "text": json.dumps({"mirrored": True})}, + *answer["content"], + ] + return request_work.flag_response(answer) async def _success_response( @@ -113,6 +120,8 @@ async def _success_response( auth, request_id, rl_ns, rl_org = call.auth, call.request_id, call.rl_ns, call.rl_org result_dict = result.model_dump(mode="json") if hasattr(result, "model_dump") else result result_dict = _coerce_bigints(result_dict) + if isinstance(result_dict, dict): + result_dict = request_work.flag_response(result_dict) # Budget violation, audit and idempotency rows in one concurrent round, # not three awaited writes in a row. @@ -137,7 +146,7 @@ async def _success_response( ) if op.name == "search.query": - return _ok(req_id, _search_answer(call, result)) + return request_work.flag_response(_ok(req_id, _search_answer(call, result))) # content[0].text is raw JSON (no prose prefix) — some clients parse it # for display and a prefix breaks the parse. structuredContent mirrors it @@ -151,14 +160,18 @@ async def _success_response( if resolved and isinstance(lean_dict, dict): lean_dict = {**lean_dict, "namespace_resolved": resolved} if isinstance(lean_dict, dict): # a non-home default says so - lean_dict = {**lean_dict, **default_namespace_note()} - return _ok( - req_id, - { - "content": [{"type": "text", "text": json.dumps(lean_dict)}], - "isError": False, - "structuredContent": lean_dict, - }, + lean_dict = request_work.flag_response({**lean_dict, **default_namespace_note()}) + return request_work.flag_response( + _ok( + req_id, + request_work.flag_response( + { + "content": [{"type": "text", "text": json.dumps(lean_dict)}], + "isError": False, + "structuredContent": lean_dict, + } + ), + ) ) diff --git a/tests/unit/test_assistant_mirror_boundary.py b/tests/unit/test_assistant_mirror_boundary.py index 5c74dfb393..bd51b0fbf3 100644 --- a/tests/unit/test_assistant_mirror_boundary.py +++ b/tests/unit/test_assistant_mirror_boundary.py @@ -189,3 +189,78 @@ def test_mcp_failures_keep_the_mark_after_protocol_unwrapping( assert json.loads(body["content"][0]["text"])["mirrored"] is True else: assert next(iter(result["error"]["data"])) == "mirrored" + + +@pytest.mark.parametrize("name", ["knowledge.get", "knowledge.search"]) +@pytest.mark.parametrize("marked", [False, True]) +def test_http_success_serialization_retains_the_server_owned_mark(name, marked): + from server.operations.registry import OPERATIONS_BY_NAME + from server.transport import arg_check + + op = OPERATIONS_BY_NAME[name] + result = ( + op.output_model(found=True, source_uri="example", content="private") + if name == "knowledge.get" + else op.output_model(results=[], embedding_model="fake") + ) + with request_work.bind(request_work.RequestWork()): + if marked: + request_work.mark_mirrored() + body = arg_check.response_body(op, {}, result) + if marked: + assert next(iter(body)) == "mirrored" and body["mirrored"] is True + assert {k: v for k, v in body.items() if k != "mirrored"} == result.model_dump(mode="json") + else: + assert body == result.model_dump(mode="json") + + +@pytest.mark.parametrize("name", ["knowledge.get", "search.query"]) +@pytest.mark.parametrize("marked", [False, True]) +async def test_mcp_success_unwrapping_keeps_the_mark(monkeypatch, name, marked): + import json + + from server.operations.registry import OPERATIONS_BY_NAME + from server.transport import mcp_outcome, post_rows + + op = OPERATIONS_BY_NAME[name] + result = ( + op.output_model(found=True, source_uri="example", content="private") + if name == "knowledge.get" + else op.output_model() + ) + rows = [] + + async def record_rows(op, **kwargs): + rows.append(kwargs["response_body"]) + + monkeypatch.setattr(post_rows, "post_handler_rows", record_rows) + call = SimpleNamespace( + req_id=1, + op=op, + input_obj=SimpleNamespace(agent="example"), + arguments={}, + auth=None, + request_id="request", + rl_ns="notes", + rl_org="example", + mcp_session_id="session", + ) + with request_work.bind(request_work.RequestWork()): + if marked: + request_work.mark_mirrored() + reply = await mcp_outcome._success_response(call, result, 1, (None, None), None) + body = reply["result"] + if marked: + for envelope in [reply, body, body["structuredContent"], rows[0]]: + assert next(iter(envelope)) == "mirrored" and envelope["mirrored"] is True + assert json.loads(body["content"][0]["text"])["mirrored"] is True + else: + assert all( + "mirrored" not in envelope + for envelope in [reply, body, body["structuredContent"], rows[0]] + ) + if name == "knowledge.get": + assert body["structuredContent"]["content"] == "private" + assert json.loads(body["content"][0]["text"])["content"] == "private" + else: + assert "loom_search" in body["content"][-1]["text"] From 333daf87fd49b816ce2143dc15beaf8583ad5589 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 08:48:50 +0000 Subject: [PATCH 4/5] Inactive mirror fixtures require an unmarked ordinary response Active registrations mark responses before ACL filtering. Compare inactive responses with the same content and no flag; keep active and stale-evidence expectations intact. Signed-off-by: zlareb1 --- .../test_mirror_checked_search_http.py | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_mirror_checked_search_http.py b/tests/integration/test_mirror_checked_search_http.py index 0b24c5cdab..23fdc43e5e 100644 --- a/tests/integration/test_mirror_checked_search_http.py +++ b/tests/integration/test_mirror_checked_search_http.py @@ -440,12 +440,22 @@ async def download(token): assert found.get("mirrored") is not True assert document.get("mirrored") is not True await sources.insert_source(replace(registration, active=False)) - assert await listing(owner) == ordinary_list - assert (await inspect(owner))["listed"] == ordinary_inspect["listed"] - assert await search(owner) == baseline - assert await get(owner) == missing_get - assert await synthesize(owner) == missing_synthesis - assert await outline(owner) == missing_outline + # Active registration baselines are marked even when no private row is visible. + for inactive, active in ( + (await listing(owner), ordinary_list), + (await search(owner), baseline), + (await get(owner), missing_get), + (await synthesize(owner), missing_synthesis), + (await outline(owner), missing_outline), + ): + assert active["mirrored"] is True + assert "mirrored" not in inactive + assert inactive == { + key: value for key, value in active.items() if key != "mirrored" + } + inactive_inspect = await inspect(owner) + assert "mirrored" not in inactive_inspect + assert inactive_inspect["listed"] == ordinary_inspect["listed"] assert await download(owner) == missing_download await sources.insert_source(registration) assert (await get(owner))["found"] From 041f82bdf22506b83a828f409981190d52847f6a Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 08:50:01 +0000 Subject: [PATCH 5/5] Inactive mirror reads match pre-registration responses Capture ordinary responses before registration so inactive-source assertions retain complete JSON, download bytes and header comparisons without removing privacy metadata from expected values. Signed-off-by: zlareb1 --- .../test_mirror_checked_search_http.py | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/tests/integration/test_mirror_checked_search_http.py b/tests/integration/test_mirror_checked_search_http.py index 23fdc43e5e..7a1e9f86d2 100644 --- a/tests/integration/test_mirror_checked_search_http.py +++ b/tests/integration/test_mirror_checked_search_http.py @@ -226,10 +226,21 @@ async def download(token): access="mirrored", active=True, ) + await write_document(namespace, "ordinary:visible", query + " ordinary") + inactive_baselines = ( + await listing(outsider), + await search(outsider), + await get(outsider), + await synthesize(outsider), + await outline(outsider), + ) + inactive_inspect_baseline = await inspect(outsider) + inactive_download_baseline = await download(outsider) + for ordinary in (*inactive_baselines, inactive_inspect_baseline): + assert "mirrored" not in ordinary await sources.insert_source(registration) await mirrors.add_mirror(org, namespace, mirror) await mirrors.record_listing(org, mirror, source, "checked", now) - await write_document(namespace, "ordinary:visible", query + " ordinary") baseline = await search(outsider) assert [r["source_uri"] for r in baseline["results"]] == ["ordinary:visible"] ordinary_list = await listing(outsider) @@ -440,23 +451,16 @@ async def download(token): assert found.get("mirrored") is not True assert document.get("mirrored") is not True await sources.insert_source(replace(registration, active=False)) - # Active registration baselines are marked even when no private row is visible. - for inactive, active in ( - (await listing(owner), ordinary_list), - (await search(owner), baseline), - (await get(owner), missing_get), - (await synthesize(owner), missing_synthesis), - (await outline(owner), missing_outline), - ): - assert active["mirrored"] is True - assert "mirrored" not in inactive - assert inactive == { - key: value for key, value in active.items() if key != "mirrored" - } - inactive_inspect = await inspect(owner) - assert "mirrored" not in inactive_inspect - assert inactive_inspect["listed"] == ordinary_inspect["listed"] - assert await download(owner) == missing_download + inactive_responses = ( + await listing(owner), + await search(owner), + await get(owner), + await synthesize(owner), + await outline(owner), + ) + assert inactive_responses == inactive_baselines + assert await inspect(owner) == inactive_inspect_baseline + assert await download(owner) == inactive_download_baseline await sources.insert_source(registration) assert (await get(owner))["found"] stale = now - timedelta(seconds=settings.knowledge_source_acl_stale_seconds + 1)