From 6d816d925d4f5a9cff1a78522b703603ed65682e Mon Sep 17 00:00:00 2001 From: Dizhan Xue Date: Fri, 20 Nov 2026 11:20:57 +0000 Subject: [PATCH] fix(agent): archive inline images so replayed turns keep the cache A saved turn replaced its inline image parts with [image], so the next turn replayed a placeholder where the picture had been. The provider's cached prefix broke at that message and every later turn repaid the prompt behind it. The save now archives the exact data URI in _image_refs[].url on the message. The display content keeps the [image] placeholder, and the replay projection expands the archived string before the provider call. A missing or malformed URL leaves the placeholder; remote URLs are untouched; token estimates include the archived picture. This keeps the session self-contained: deleting the session deletes its images, and an exported session carries them. The cost is a larger session JSONL and a linear session load for image-heavy transcripts; the trajectory index is unaffected because it reads the span log and bounded artifact previews, not the session file. --- CONTEXT.md | 8 + raven/agent/loop/_shared.py | 9 +- raven/agent/loop/turn_path.py | 38 ++-- raven/agent/window/images.py | 8 +- raven/context_engine/runtime_context.py | 3 +- raven/context_engine/segments/render.py | 7 +- raven/utils/image_store.py | 153 ++++++++++++++ raven/utils/tokens.py | 7 +- tests/test_read_file_image.py | 263 +++++++++++++++++++++++- 9 files changed, 459 insertions(+), 37 deletions(-) create mode 100644 raven/utils/image_store.py diff --git a/CONTEXT.md b/CONTEXT.md index 2337aa8cb8..6c3b2120c4 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -162,6 +162,14 @@ shell calls. And saying Action owns the loop: the shell keeps iteration accounti phases, tool execution and approval, the three in-turn recoveries, persistence and event order. A replacement loop may name its own roles, which is why the paper is factory-loop tier. +**Image Ref** (`utils/image_store.py`, field `_image_refs`): +The index-and-URL record filed beside an inline picture's `[image]` placeholder when a +message is saved. The exact data URI is archived in that private field, so the next +turn's replay sends the same bytes, character for character, while the display content +keeps the placeholder. A malformed reference falls back to the placeholder. +_Avoid_: calling the schema's `content` a copy of the model's picture — the display form +keeps `[image]`, and the private refs hold the bytes. + **Worker Table** (`agent/subagent/delegate.py`, generated by `playbook/agent_generator.py`): The workers one Turn may dispatch to, written for that Turn before it starts when `playbooks.agentHarness` is `generate`. Each row is a label, the roster agent behind it, and diff --git a/raven/agent/loop/_shared.py b/raven/agent/loop/_shared.py index e20420e5ae..b1742eb1e7 100644 --- a/raven/agent/loop/_shared.py +++ b/raven/agent/loop/_shared.py @@ -652,11 +652,12 @@ def turn_question(messages: list[dict[str, Any]] | None) -> str: def _strip_inline_images(content: list[Any]) -> list[Any]: - """Replace inline base64 images with a text placeholder, for persistence. + """Replace inline base64 images with a text placeholder. - Images live for exactly the turn that produced them. Keeping the bytes would - bloat the session JSONL by megabytes per picture, and every later turn would - replay them to the model — paying for an image nobody asked about again. + The placeholder-only projection, for callers that keep no archived refs. The + save path uses :func:`~raven.utils.image_store.capture_images`, which archives + the exact data URI in the message's private refs so a later turn can send the + same picture again. A *new* list is returned: the input is the live message the model is still working from this turn, and `_save_turn` only shallow-copies the entry, so diff --git a/raven/agent/loop/turn_path.py b/raven/agent/loop/turn_path.py index 3d8d3e7fb0..00a13a5c9d 100644 --- a/raven/agent/loop/turn_path.py +++ b/raven/agent/loop/turn_path.py @@ -61,7 +61,6 @@ _stamp_context_used, _stamp_reasoning_ms, _stamp_replied_via_tool, - _strip_inline_images, append_hook_note, asyncio, autofill_resolver, @@ -4250,6 +4249,7 @@ def _save_turn( carries_runtime_context, ) from raven.context_engine.scratch import TURN_SCRATCH_KEY, cut_turn_scratch + from raven.utils.image_store import IMAGE_REFS_KEY, capture_images # Both exits that persist a turn come through here, so a question noted # on the turn is filed once whichever exit runs. The early inbound write @@ -4341,7 +4341,7 @@ def _save_turn( entry["file_written"] = tool_written if tool_metadata := entry.pop(_TOOL_METADATA_KEY, None): entry["metadata"] = tool_metadata - # Provenance of pictures that lived for this turn only; nothing to file. + # Provenance of pictures is turn-only; the pictures are filed below. entry.pop(IMAGE_SOURCES_KEY, None) # A withdrawn picture's note is filed without this turn's reasons: on # the tool-result transport the message outlives the turn, and a @@ -4367,13 +4367,14 @@ def _save_turn( if (took := entry.pop(private_key, None)) is not None: entry[stored_key] = took if role == "tool" and isinstance(content, list): - # A multimodal tool result. Images must never reach the JSONL: - # a single one adds megabytes that are then replayed on every - # resume and re-fed to the model, and unlike a code bug that is - # not revertible once written. The char cap below cannot catch - # it either — it guards `str` content only. - content = _strip_inline_images(content) + # A multimodal tool result. Its bytes are archived in the + # message's private refs; the display content keeps a + # placeholder. The char cap below guards `str` content only, so + # pictures are the part of a list it cannot see. + content, image_refs = capture_images(content) entry["content"] = content + if image_refs: + entry[IMAGE_REFS_KEY] = image_refs cap = self._TOOL_RESULT_MAX_CHARS if role == "tool" and isinstance(content, str) and cap and len(content) > cap: # The marker is what retrieval reads to say a span was cut on the @@ -4398,19 +4399,20 @@ def _save_turn( entry["content"] = inbound_original entry.pop(ATTACHMENT_NOTES_KEY, None) if isinstance(content, list): - filtered = [ - c - for c in _strip_inline_images(content) - if not ( - isinstance(c, dict) - and c.get("type") == "text" - and isinstance(c.get("text"), str) - and c["text"].startswith(ContextBuilder._RUNTIME_CONTEXT_TAG) - ) - ] + filtered, image_refs = capture_images( + content, + drop=lambda part: ( + isinstance(part, dict) + and part.get("type") == "text" + and isinstance(part.get("text"), str) + and part["text"].startswith(ContextBuilder._RUNTIME_CONTEXT_TAG) + ), + ) if not filtered: continue entry["content"] = filtered + if image_refs: + entry[IMAGE_REFS_KEY] = image_refs if role == "user" and runtime_lines and not carries_runtime_context(entry.get("content"), runtime_lines): # Filed beside the user's words, which ``content`` keeps alone, so # the history projection replays the block this turn sent. Lines diff --git a/raven/agent/window/images.py b/raven/agent/window/images.py index e2f24d06b1..ab8b8bf0a6 100644 --- a/raven/agent/window/images.py +++ b/raven/agent/window/images.py @@ -14,6 +14,7 @@ import re from typing import Any +from raven.utils.image_store import IMAGE_REFS_KEY from raven.utils.images import is_image_part, is_inline_image # Marks the synthetic user message that carries images a transport cannot put in @@ -29,8 +30,10 @@ #: transcript and read when it leaves it, so the note that stands in for a withdrawn #: picture can say which tool showed which page in which round. Not sent (the #: provider allow-list drops it) and not filed in the session (``_save_turn`` pops -#: it): the picture lives for the turn only, and so does its provenance. The -#: tracing ``llm.input`` artifact records the request as handed to the provider, +#: it): the provenance lives for the turn only, while the picture itself is +#: archived in the message's private refs and replayed from them. The tracing +#: ``llm.input`` artifact +#: records the request as handed to the provider, #: before the allow-list, so on a stock install the entries do appear there, #: beside the pictures they describe. IMAGE_SOURCES_KEY = "_image_sources" @@ -160,4 +163,5 @@ def without_pictures(message: dict[str, Any], *, reason: str = "superseded", kee parts.append({"type": "text", "text": note}) clean = dict(message) clean["content"] = parts + clean.pop(IMAGE_REFS_KEY, None) return clean diff --git a/raven/context_engine/runtime_context.py b/raven/context_engine/runtime_context.py index 247329f5b9..ea44c6e78f 100644 --- a/raven/context_engine/runtime_context.py +++ b/raven/context_engine/runtime_context.py @@ -19,6 +19,7 @@ from typing import Any from raven.context_engine.segments.render import RUNTIME_CONTEXT_TAG +from raven.utils.image_store import expand_images RUNTIME_CONTEXT_KEY = "_runtime_context" """Set on the envelope the assembler built, holding the lines it opened with. @@ -64,7 +65,7 @@ def replayed_content(message: dict[str, Any]) -> Any: A user entry filed before the field existed, and every other entry, comes back as it is. """ - content = message.get("content") + content = expand_images(message.get("content"), message) lines = message.get(RUNTIME_CONTEXT_FIELD) if message.get("role") != "user" or not isinstance(lines, str) or not lines: return content diff --git a/raven/context_engine/segments/render.py b/raven/context_engine/segments/render.py index 5c6d980459..de074d421d 100644 --- a/raven/context_engine/segments/render.py +++ b/raven/context_engine/segments/render.py @@ -977,10 +977,9 @@ def build_user_content( so their paths are surfaced as a text note for the model to read on demand. Each image also gets its path named in the text, the same way non-image - attachments already do. The base64 lives for exactly this turn — it is - replaced by a placeholder on the way into the session — so without the path - the model loses any way to look at the picture again, and a follow-up - question about it has nothing to work from. + attachments already do. The base64 is archived in the session's private + image refs and replayed from them; the path is what lets the model read the + file again after the message is compacted away. ``can_see_images=False`` (the model has no vision) turns a picture into the same kind of note the other attachments get. Said out loud rather than diff --git a/raven/utils/image_store.py b/raven/utils/image_store.py new file mode 100644 index 0000000000..ce90d660d9 --- /dev/null +++ b/raven/utils/image_store.py @@ -0,0 +1,153 @@ +"""The archived image refs a session replays. + +A message's display content keeps ``[image]`` in place of an inline picture: the +bytes would otherwise bloat every transcript view and the session's text. But +the next turn has to send the same picture, character for character, or the +provider's cached prefix breaks at it and everything after it is prefilled +again. + +The save pass therefore archives the exact data URI in a private +``_image_refs`` field beside the placeholder. The replay pass reads it back, so +what reaches the provider is the picture that was sent; ``content`` stays the +display form. The provider allow-list drops the refs, and the session view +never sees them. + +Archiving trades session-load time for exact replay: the session loader parses +every line, so an image-heavy session opens more slowly than one holding only a +reference. The bytes stay in this private field, never in the display content, +so transcript views stay small. + +A malformed reference leaves the placeholder: the turn still runs, and only the +cache entry behind the picture is lost. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from raven.utils.images import estimate_content_part_tokens, image_block, is_inline_image + +IMAGE_REFS_KEY = "_image_refs" +"""References filed beside a saved message's image placeholders. + +Each entry carries ``at`` (the block index), ``url`` (the exact data URI), +``tokens`` and the part's non-URL fields. Dropped by the provider allow-lists; +read only by the replay projection and the token estimator. +""" + +IMAGE_PLACEHOLDER = "[image]" +"""What the display content holds in place of an inline picture.""" + + +def capture_images( + content: Any, + *, + drop: Callable[[Any], bool] | None = None, +) -> tuple[Any, list[dict[str, Any]]]: + """Replace inline pictures with placeholders, archiving their exact data URIs. + + A non-list comes back unchanged. ``drop`` removes blocks before the index a + reference records, so a caller that also strips a runtime block gets indices + into the list it actually stores. + """ + if not isinstance(content, list): + return content, [] + kept: list[Any] = [] + refs: list[dict[str, Any]] = [] + for part in content: + if drop is not None and drop(part): + continue + if not is_inline_image(part): + kept.append(part) + continue + ref = _ref_for_part(part) + kept.append({"type": "text", "text": IMAGE_PLACEHOLDER}) + if ref is not None: + ref["at"] = len(kept) - 1 + refs.append(ref) + return kept, refs + + +def expand_images(content: Any, message: dict[str, Any]) -> Any: + """``content`` with each recorded placeholder replaced by the picture it stood for. + + A message without references comes back as it is. A reference whose URL is + malformed leaves the placeholder. + """ + refs = message.get(IMAGE_REFS_KEY) + if not isinstance(content, list) or not isinstance(refs, list): + return content + expanded = list(content) + for ref in refs: + if not isinstance(ref, dict): + continue + at = ref.get("at") + if not isinstance(at, int) or not 0 <= at < len(expanded): + continue + placeholder = expanded[at] + if not ( + isinstance(placeholder, dict) + and placeholder.get("type") == "text" + and placeholder.get("text") == IMAGE_PLACEHOLDER + ): + continue + rebuilt = _rebuild_part(ref) + if rebuilt is not None: + expanded[at] = rebuilt + return expanded + + +def image_tokens(message: dict[str, Any]) -> int: + """The image weight a saved message's references still cost the prompt.""" + refs = message.get(IMAGE_REFS_KEY) + if not isinstance(refs, list): + return 0 + total = 0 + for ref in refs: + if isinstance(ref, dict): + tokens = ref.get("tokens") + if isinstance(tokens, int) and not isinstance(tokens, bool) and tokens > 0: + total += tokens + return total + + +def _ref_for_part(part: dict[str, Any]) -> dict[str, Any] | None: + image_url = part.get("image_url") + url = image_url.get("url") if isinstance(image_url, dict) else None + if not isinstance(url, str) or not url.startswith("data:image/"): + return None + ref: dict[str, Any] = {"url": url, "tokens": estimate_content_part_tokens(part) or 0} + top = {key: value for key, value in part.items() if key not in ("type", "image_url")} + url_extra = {key: value for key, value in image_url.items() if key != "url"} + if top: + ref["part"] = top + if url_extra: + ref["image_url"] = url_extra + return ref + + +def _rebuild_part(ref: dict[str, Any]) -> dict[str, Any] | None: + url = ref.get("url") + if not isinstance(url, str) or not url.startswith("data:image/"): + return None + rebuilt: dict[str, Any] = dict(image_block(url)) + extra = ref.get("part") + if isinstance(extra, dict): + rebuilt = {**rebuilt, **{key: value for key, value in extra.items() if key not in ("type", "image_url")}} + extra_url = ref.get("image_url") + if isinstance(extra_url, dict): + rebuilt["image_url"] = { + **rebuilt["image_url"], + **{key: value for key, value in extra_url.items() if key != "url"}, + } + return rebuilt + + +__all__ = [ + "IMAGE_PLACEHOLDER", + "IMAGE_REFS_KEY", + "capture_images", + "expand_images", + "image_tokens", +] diff --git a/raven/utils/tokens.py b/raven/utils/tokens.py index 723f6cfb1a..3aa140d65d 100644 --- a/raven/utils/tokens.py +++ b/raven/utils/tokens.py @@ -15,6 +15,7 @@ import tiktoken +from raven.utils.image_store import image_tokens from raven.utils.images import estimate_content_part_tokens # A cl100k regex piece cannot cross a newline followed by non-whitespace. @@ -180,7 +181,7 @@ def estimate_message_tokens(message: dict[str, Any]) -> int: """Estimate prompt tokens contributed by one persisted message.""" content = message.get("content") parts: list[str] = [] - extra_tokens = 0 + extra_tokens = image_tokens(message) if isinstance(content, str): parts.append(content) elif isinstance(content, list): @@ -189,8 +190,8 @@ def estimate_message_tokens(message: dict[str, Any]) -> int: text = part.get("text", "") if text: parts.append(text) - elif (image_tokens := estimate_content_part_tokens(part)) is not None: - extra_tokens += image_tokens + elif (part_tokens := estimate_content_part_tokens(part)) is not None: + extra_tokens += part_tokens else: parts.append(json.dumps(part, ensure_ascii=False)) elif content is not None: diff --git a/tests/test_read_file_image.py b/tests/test_read_file_image.py index ff0ac2ea28..37329e7704 100644 --- a/tests/test_read_file_image.py +++ b/tests/test_read_file_image.py @@ -1,7 +1,8 @@ """read_file's image branch: preprocessing, capability routing, persistence. -Covers the invariant that makes the feature safe to ship — base64 reaches the -model for exactly one turn and never reaches disk. +Covers the invariant that makes the feature safe to ship: base64 reaches the +model and is archived in the session's private image refs, while the display +content keeps the placeholder. """ from __future__ import annotations @@ -374,9 +375,10 @@ def test_strip_inline_images_leaves_remote_urls_alone() -> None: @pytest.mark.parametrize("role", ["tool", "user"]) -def test_save_turn_never_writes_base64_to_the_session(role: str, tmp_path: Path) -> None: - """Regression: a list-shaped tool result skipped the char-cap guard entirely, - so a multi-megabyte data URI landed in the JSONL — unrecoverably.""" +def test_save_turn_keeps_base64_out_of_the_display_content(role: str, tmp_path: Path) -> None: + """The display content keeps a placeholder; the exact data URI is archived in + the message's private refs. A list-shaped result skipped the char cap, so the + split is what keeps a multi-megabyte URI out of transcript views.""" from unittest.mock import MagicMock from raven.agent.loop.main import AgentLoop @@ -406,10 +408,261 @@ def test_save_turn_never_writes_base64_to_the_session(role: str, tmp_path: Path) assert {"type": "text", "text": "[image]"} in stored # The path survives, so the model can ask to see the image again. assert any("/tmp/x.png" in str(part) for part in stored) + assert session.messages[0]["_image_refs"][0]["url"] == big # And the in-flight message still holds the picture. assert messages[0]["content"][1]["type"] == "image_url" +# -------------------------------------------------------------------------- +# exact replay of inline images +# -------------------------------------------------------------------------- + + +def _png_header(width: int = 1000, height: int = 1000) -> bytes: + return b"\x89PNG\r\n\x1a\n" + b"\x00\x00\x00\x0dIHDR" + width.to_bytes(4, "big") + height.to_bytes(4, "big") + + +def _data_uri(raw: bytes, mime: str = "image/png") -> str: + return f"data:{mime};base64,{base64.b64encode(raw).decode()}" + + +def _saving_loop(tmp_path: Path): + from datetime import datetime + from unittest.mock import MagicMock + + from raven.agent.loop.main import AgentLoop + + loop = object.__new__(AgentLoop) + loop._now_fn = MagicMock(return_value=datetime(2026, 11, 20, 12, 0)) + loop.workspace = tmp_path + return loop + + +def test_tool_image_is_saved_as_a_ref_and_replayed_byte_identical(tmp_path: Path) -> None: + from raven.agent.loop.main import AgentLoop + from raven.context_engine.history_trimmer import HistoryTrimmer + from raven.session.manager import Session + + uri = _data_uri(_png_header()) + live = [ + {"type": "text", "text": "a chart"}, + {"type": "image_url", "image_url": {"url": uri, "detail": "high"}}, + ] + session = Session(key="cli:u") + + AgentLoop._save_turn( + _saving_loop(tmp_path), + session, + [{"role": "tool", "tool_call_id": "c1", "name": "read_file", "content": live}], + 0, + ) + + saved = session.messages[0] + assert saved["content"] == [{"type": "text", "text": "a chart"}, {"type": "text", "text": "[image]"}] + assert "base64" not in json.dumps(saved["content"]) + ref = saved["_image_refs"][0] + assert ref["at"] == 1 + assert ref["url"] == uri + + replayed = HistoryTrimmer.history_from_ids(session.messages, [0])[0] + assert replayed["content"] == live + assert "_image_refs" not in replayed + + +def test_the_session_archives_the_exact_image_wire_form(tmp_path: Path) -> None: + from raven.agent.loop.main import AgentLoop + from raven.context_engine.history_trimmer import HistoryTrimmer + from raven.session.manager import Session + from raven.utils.images import image_block + + uri = _data_uri(_png_header()) + session = Session(key="cli:u") + AgentLoop._save_turn( + _saving_loop(tmp_path), + session, + [{"role": "tool", "tool_call_id": "c1", "content": [image_block(uri)]}], + 0, + ) + + saved = session.messages[0] + assert saved["content"] == [{"type": "text", "text": "[image]"}] + assert saved["_image_refs"][0]["url"] == uri + assert uri in json.dumps(saved) + assert "_image_store" not in saved + media = tmp_path / "media" + assert not media.exists() or not any(media.rglob("*")) + + replayed = HistoryTrimmer.history_from_ids(session.messages, [0])[0] + assert replayed["content"] == [image_block(uri)] + + +def test_user_image_with_runtime_context_replays_the_sent_message(tmp_path: Path) -> None: + from raven.agent.context import ContextBuilder + from raven.agent.loop.main import AgentLoop + from raven.context_engine.history_trimmer import HistoryTrimmer + from raven.context_engine.runtime_context import RUNTIME_CONTEXT_KEY, with_runtime_context + from raven.session.manager import Session + from raven.utils.images import image_block + + tag = ContextBuilder._RUNTIME_CONTEXT_TAG + lines = f"{tag}\nCurrent Time: 2026-11-20 12:00 (Friday) (UTC)\nChat ID: u" + live = with_runtime_context( + lines, + [image_block(_data_uri(_png_header())), {"type": "text", "text": "what is this?"}], + ) + session = Session(key="cli:u") + + AgentLoop._save_turn( + _saving_loop(tmp_path), + session, + [{"role": "user", "content": live, RUNTIME_CONTEXT_KEY: lines}], + 0, + ) + + saved = session.messages[0] + assert "base64" not in json.dumps(saved["content"]) + assert saved["_image_refs"][0]["url"] == live[1]["image_url"]["url"] + assert saved["runtime_context"] == lines + assert [part.get("text") for part in saved["content"]] == ["[image]", "what is this?"] + + replayed = HistoryTrimmer.history_from_ids(session.messages, [0])[0] + assert replayed["content"] == live + + +def test_missing_image_url_falls_back_to_the_placeholder(tmp_path: Path) -> None: + from raven.agent.loop.main import AgentLoop + from raven.context_engine.history_trimmer import HistoryTrimmer + from raven.session.manager import Session + from raven.utils.images import image_block + + session = Session(key="cli:u") + AgentLoop._save_turn( + _saving_loop(tmp_path), + session, + [{"role": "tool", "tool_call_id": "c1", "content": [image_block(_data_uri(_png_header()))]}], + 0, + ) + + saved = session.messages[0] + saved["_image_refs"][0].pop("url") + + replayed = HistoryTrimmer.history_from_ids(session.messages, [0])[0] + assert replayed["content"] == [{"type": "text", "text": "[image]"}] + + +def test_saved_image_message_keeps_its_token_weight(tmp_path: Path) -> None: + from raven.agent.loop.main import AgentLoop + from raven.session.manager import Session + from raven.utils.images import image_block + from raven.utils.tokens import estimate_message_tokens + + live = {"role": "tool", "tool_call_id": "c1", "content": [image_block(_data_uri(_png_header()))]} + session = Session(key="cli:u") + AgentLoop._save_turn(_saving_loop(tmp_path), session, [live], 0) + + saved_tokens = estimate_message_tokens(session.messages[0]) + live_tokens = estimate_message_tokens(live) + assert saved_tokens >= live_tokens + assert saved_tokens - live_tokens <= 3 + + +def test_malformed_image_url_falls_back_to_the_placeholder(tmp_path: Path) -> None: + from raven.agent.loop.main import AgentLoop + from raven.context_engine.history_trimmer import HistoryTrimmer + from raven.session.manager import Session + from raven.utils.images import image_block + + session = Session(key="cli:u") + AgentLoop._save_turn( + _saving_loop(tmp_path), + session, + [{"role": "tool", "tool_call_id": "c1", "content": [image_block(_data_uri(_png_header()))]}], + 0, + ) + + saved = session.messages[0] + saved["_image_refs"][0]["url"] = "not a data uri" + + replayed = HistoryTrimmer.history_from_ids(session.messages, [0])[0] + assert replayed["content"] == [{"type": "text", "text": "[image]"}] + + +def test_malformed_image_ref_extras_do_not_crash_replay(tmp_path: Path) -> None: + from raven.agent.loop.main import AgentLoop + from raven.context_engine.history_trimmer import HistoryTrimmer + from raven.session.manager import Session + from raven.utils.images import image_block + + uri = _data_uri(_png_header()) + session = Session(key="cli:u") + AgentLoop._save_turn( + _saving_loop(tmp_path), + session, + [{"role": "tool", "tool_call_id": "c1", "content": [image_block(uri)]}], + 0, + ) + + saved = session.messages[0] + saved["_image_refs"][0]["part"] = "not a mapping" + saved["_image_refs"][0]["image_url"] = ["not", "a", "mapping"] + assert HistoryTrimmer.history_from_ids(session.messages, [0])[0]["content"] == [image_block(uri)] + + saved["_image_refs"][0]["part"] = {"image_url": "bad", "type": "text"} + saved["_image_refs"][0]["image_url"] = {"url": "data:image/png;base64,bad", "detail": "high"} + replayed = HistoryTrimmer.history_from_ids(session.messages, [0])[0] + assert replayed["content"] == [{"type": "image_url", "image_url": {"url": uri, "detail": "high"}}] + + +def test_session_jsonl_archives_the_image_and_reloads_the_picture(tmp_path: Path) -> None: + from raven.agent.loop.main import AgentLoop + from raven.context_engine.history_trimmer import HistoryTrimmer + from raven.session.manager import SessionManager + + workspace = tmp_path / "ws" + manager = SessionManager(workspace) + session = manager.get_or_create("cli:u") + uri = _data_uri(_png_header()) + live = [{"type": "text", "text": "a chart"}, {"type": "image_url", "image_url": {"url": uri}}] + AgentLoop._save_turn( + _saving_loop(workspace), + session, + [{"role": "tool", "tool_call_id": "c1", "content": live}], + 0, + ) + manager.save(session) + + text = manager.session_path("cli:u").read_text(encoding="utf-8") + assert uri in text + + reloaded = SessionManager(workspace).get_or_create("cli:u") + ids = [index for index, message in enumerate(reloaded.messages) if message.get("role") == "tool"] + assert "base64" not in json.dumps(reloaded.messages[ids[0]]["content"]) + assert HistoryTrimmer.history_from_ids(reloaded.messages, ids)[0]["content"] == live + + +def test_remote_image_url_is_not_copied_or_rewritten(tmp_path: Path) -> None: + from raven.agent.loop.main import AgentLoop + from raven.context_engine.history_trimmer import HistoryTrimmer + from raven.session.manager import Session + + live = [ + {"type": "text", "text": "a chart"}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, + ] + session = Session(key="cli:u") + AgentLoop._save_turn( + _saving_loop(tmp_path), + session, + [{"role": "tool", "tool_call_id": "c1", "content": live}], + 0, + ) + + saved = session.messages[0] + assert saved["content"] == live + assert "_image_refs" not in saved + assert HistoryTrimmer.history_from_ids(session.messages, [0])[0]["content"] == live + + # -------------------------------------------------------------------------- # emergency shrink # --------------------------------------------------------------------------