Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CONTEXT.md
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,14 @@ shell calls. And saying Action owns the loop: the shell keeps iteration accounti
phases, tool execution and approval, the three in-turn recoveries, persistence and event
order. A replacement loop may name its own roles, which is why the paper is factory-loop tier.

**Image Ref** (`utils/image_store.py`, field `_image_refs`):
The index-and-URL record filed beside an inline picture's `[image]` placeholder when a
message is saved. The exact data URI is archived in that private field, so the next
turn's replay sends the same bytes, character for character, while the display content
keeps the placeholder. A malformed reference falls back to the placeholder.
_Avoid_: calling the schema's `content` a copy of the model's picture — the display form
keeps `[image]`, and the private refs hold the bytes.

**Worker Table** (`agent/subagent/delegate.py`, generated by `playbook/agent_generator.py`):
The workers one Turn may dispatch to, written for that Turn before it starts when
`playbooks.agentHarness` is `generate`. Each row is a label, the roster agent behind it, and
Expand Down
9 changes: 5 additions & 4 deletions raven/agent/loop/_shared.py
Original file line number Diff line number Diff line change
Expand Up @@ -652,11 +652,12 @@ def turn_question(messages: list[dict[str, Any]] | None) -> str:


def _strip_inline_images(content: list[Any]) -> list[Any]:
"""Replace inline base64 images with a text placeholder, for persistence.
"""Replace inline base64 images with a text placeholder.

Images live for exactly the turn that produced them. Keeping the bytes would
bloat the session JSONL by megabytes per picture, and every later turn would
replay them to the model — paying for an image nobody asked about again.
The placeholder-only projection, for callers that keep no archived refs. The
save path uses :func:`~raven.utils.image_store.capture_images`, which archives
the exact data URI in the message's private refs so a later turn can send the
same picture again.

A *new* list is returned: the input is the live message the model is still
working from this turn, and `_save_turn` only shallow-copies the entry, so
Expand Down
38 changes: 20 additions & 18 deletions raven/agent/loop/turn_path.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@
_stamp_context_used,
_stamp_reasoning_ms,
_stamp_replied_via_tool,
_strip_inline_images,
append_hook_note,
asyncio,
autofill_resolver,
Expand Down Expand Up @@ -4250,6 +4249,7 @@ def _save_turn(
carries_runtime_context,
)
from raven.context_engine.scratch import TURN_SCRATCH_KEY, cut_turn_scratch
from raven.utils.image_store import IMAGE_REFS_KEY, capture_images

# Both exits that persist a turn come through here, so a question noted
# on the turn is filed once whichever exit runs. The early inbound write
Expand Down Expand Up @@ -4341,7 +4341,7 @@ def _save_turn(
entry["file_written"] = tool_written
if tool_metadata := entry.pop(_TOOL_METADATA_KEY, None):
entry["metadata"] = tool_metadata
# Provenance of pictures that lived for this turn only; nothing to file.
# Provenance of pictures is turn-only; the pictures are filed below.
entry.pop(IMAGE_SOURCES_KEY, None)
# A withdrawn picture's note is filed without this turn's reasons: on
# the tool-result transport the message outlives the turn, and a
Expand All @@ -4367,13 +4367,14 @@ def _save_turn(
if (took := entry.pop(private_key, None)) is not None:
entry[stored_key] = took
if role == "tool" and isinstance(content, list):
# A multimodal tool result. Images must never reach the JSONL:
# a single one adds megabytes that are then replayed on every
# resume and re-fed to the model, and unlike a code bug that is
# not revertible once written. The char cap below cannot catch
# it either — it guards `str` content only.
content = _strip_inline_images(content)
# A multimodal tool result. Its bytes are archived in the
# message's private refs; the display content keeps a
# placeholder. The char cap below guards `str` content only, so
# pictures are the part of a list it cannot see.
content, image_refs = capture_images(content)
entry["content"] = content
if image_refs:
entry[IMAGE_REFS_KEY] = image_refs
cap = self._TOOL_RESULT_MAX_CHARS
if role == "tool" and isinstance(content, str) and cap and len(content) > cap:
# The marker is what retrieval reads to say a span was cut on the
Expand All @@ -4398,19 +4399,20 @@ def _save_turn(
entry["content"] = inbound_original
entry.pop(ATTACHMENT_NOTES_KEY, None)
if isinstance(content, list):
filtered = [
c
for c in _strip_inline_images(content)
if not (
isinstance(c, dict)
and c.get("type") == "text"
and isinstance(c.get("text"), str)
and c["text"].startswith(ContextBuilder._RUNTIME_CONTEXT_TAG)
)
]
filtered, image_refs = capture_images(
content,
drop=lambda part: (
isinstance(part, dict)
and part.get("type") == "text"
and isinstance(part.get("text"), str)
and part["text"].startswith(ContextBuilder._RUNTIME_CONTEXT_TAG)
),
)
if not filtered:
continue
entry["content"] = filtered
if image_refs:
entry[IMAGE_REFS_KEY] = image_refs
if role == "user" and runtime_lines and not carries_runtime_context(entry.get("content"), runtime_lines):
# Filed beside the user's words, which ``content`` keeps alone, so
# the history projection replays the block this turn sent. Lines
Expand Down
8 changes: 6 additions & 2 deletions raven/agent/window/images.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import re
from typing import Any

from raven.utils.image_store import IMAGE_REFS_KEY
from raven.utils.images import is_image_part, is_inline_image

# Marks the synthetic user message that carries images a transport cannot put in
Expand All @@ -29,8 +30,10 @@
#: transcript and read when it leaves it, so the note that stands in for a withdrawn
#: picture can say which tool showed which page in which round. Not sent (the
#: provider allow-list drops it) and not filed in the session (``_save_turn`` pops
#: it): the picture lives for the turn only, and so does its provenance. The
#: tracing ``llm.input`` artifact records the request as handed to the provider,
#: it): the provenance lives for the turn only, while the picture itself is
#: archived in the message's private refs and replayed from them. The tracing
#: ``llm.input`` artifact
#: records the request as handed to the provider,
#: before the allow-list, so on a stock install the entries do appear there,
#: beside the pictures they describe.
IMAGE_SOURCES_KEY = "_image_sources"
Expand Down Expand Up @@ -160,4 +163,5 @@ def without_pictures(message: dict[str, Any], *, reason: str = "superseded", kee
parts.append({"type": "text", "text": note})
clean = dict(message)
clean["content"] = parts
clean.pop(IMAGE_REFS_KEY, None)
return clean
3 changes: 2 additions & 1 deletion raven/context_engine/runtime_context.py

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Blocking: replayed pictures skip the gates that bound and route live pictures, so user photos accumulate in every request until the endpoint refuses, and a model or transport that cannot take an image is sent one on every later turn.

replayed_content now runs expand_images on every history message (runtime_context.py:68), with no input about the request it feeds. The two things that keep pictures in check were built on base's premise that a picture lives for one turn, and neither sees a replayed one the way it should:

  1. The standing image window never counts a user picture. window_images only collects role == "tool" or _attached_image messages (shrink.py:250-255); its docstring says why: "A picture the user sent is the subject of the turn rather than a result, and stays". That held while a user picture was filed as [image]. With _image_refs on the user branch (turn_path.py:4402-4415), every photo from every earlier turn is sent again and imageWindowBudgetBytes (12,000,000 by default) never applies to them. When the endpoint then refuses for size, the refusal ladder closes the window with any_role (harness/memory.py:425-433) and withdraws every picture, including the one attached to the question being asked. without_pictures pops the refs only from the live copy, so the session file still has them and the next turn repeats the refusal.

  2. Capability gates apply only to the current message. _supports_vision gates the user's new attachment (render.py:1040) and _supports_image_tool_result gates new tool results (organ_glue.py:92-103). An archived picture is replayed after a session switches to a model without vision, or to a transport that cannot carry images in role=tool, so each later turn first sends a request base never sent, gets refused, and retries. With a refusal worded outside the patterns in providers/base.py, the turn fails instead.

Measured on head 6d816d9 merged onto main fc0a30f against base fc0a30f, real AgentLoop._process_message, stub providers, pyt.sh with scratch homes:

case                                         base                        head
14 turns, each attaching one ~1 MB photo     1 picture / request (1.0 MB) turn N carries N pictures (14.0 MB at turn 14)
  session file after 14 turns                25,259 bytes                14,053,015 bytes
same, endpoint refuses requests over 12 MB   0 refusals, picture kept     turns 12-14: 1 refused call each, then
                                                                          answered WITHOUT the picture just sent
tool picture on turn 1, then switch to a     turns 2-3: 0 images          turns 2-3: image sent at [3,'tool'],
  model without vision                                                    refused, retried without it
same, switch to a transport without          turns 2-3: 0 images          turns 2-3: image in role=tool, refused,
  tool-result images                                                      then demoted to a user message

The cache goal does not need either behaviour. A shape that keeps it: let the window count replayed user pictures from earlier turns (exempting only the current turn's question), and make the replay projection honour the same two capability verdicts the live path uses, leaving the placeholder when the receiving model or transport cannot take the picture. That keeps the bytes identical for the requests that can carry them, which is where the cached prefix matters.

Not blocking, for the same revision: raven/templates/TOOLS.md:23 still tells the model "An image is visible for the turn that read it. Later turns see a placeholder".

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Blocking: archived replay must honor the existing image-window and capability gates before merge.

This holds. I reproduced the first path directly: fourteen archived user messages with 1,000,000-byte base64 payloads project to 14,000,000 bytes and 14 images, then window_images(..., budget=12_000_000) returns (0, 0) and leaves all 14 images because ordinary role="user" messages are outside its candidate set. The second path follows from the same unconditional projection: HistoryTrimmer.history_from_ids calls replayed_content without the selected model or transport capability, so an old image is reconstructed after the current-message capability decisions have already been made.

The unchanged image suite still passes (116 tests), which confirms the current coverage does not exercise these replay gates. I am updating my own stance to blocking; I am not resolving this thread because it is yours and remains open.

Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from typing import Any

from raven.context_engine.segments.render import RUNTIME_CONTEXT_TAG
from raven.utils.image_store import expand_images

RUNTIME_CONTEXT_KEY = "_runtime_context"
"""Set on the envelope the assembler built, holding the lines it opened with.
Expand Down Expand Up @@ -64,7 +65,7 @@ def replayed_content(message: dict[str, Any]) -> Any:
A user entry filed before the field existed, and every other entry, comes
back as it is.
"""
content = message.get("content")
content = expand_images(message.get("content"), message)
lines = message.get(RUNTIME_CONTEXT_FIELD)
if message.get("role") != "user" or not isinstance(lines, str) or not lines:
return content
Expand Down
7 changes: 3 additions & 4 deletions raven/context_engine/segments/render.py
Original file line number Diff line number Diff line change
Expand Up @@ -977,10 +977,9 @@ def build_user_content(
so their paths are surfaced as a text note for the model to read on demand.

Each image also gets its path named in the text, the same way non-image
attachments already do. The base64 lives for exactly this turn — it is
replaced by a placeholder on the way into the session — so without the path
the model loses any way to look at the picture again, and a follow-up
question about it has nothing to work from.
attachments already do. The base64 is archived in the session's private
image refs and replayed from them; the path is what lets the model read the
file again after the message is compacted away.

``can_see_images=False`` (the model has no vision) turns a picture into the
same kind of note the other attachments get. Said out loud rather than
Expand Down
153 changes: 153 additions & 0 deletions raven/utils/image_store.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
"""The archived image refs a session replays.

A message's display content keeps ``[image]`` in place of an inline picture: the
bytes would otherwise bloat every transcript view and the session's text. But
the next turn has to send the same picture, character for character, or the
provider's cached prefix breaks at it and everything after it is prefilled
again.

The save pass therefore archives the exact data URI in a private
``_image_refs`` field beside the placeholder. The replay pass reads it back, so
what reaches the provider is the picture that was sent; ``content`` stays the
display form. The provider allow-list drops the refs, and the session view
never sees them.

Archiving trades session-load time for exact replay: the session loader parses
every line, so an image-heavy session opens more slowly than one holding only a
reference. The bytes stay in this private field, never in the display content,
so transcript views stay small.

A malformed reference leaves the placeholder: the turn still runs, and only the
cache entry behind the picture is lost.
"""

from __future__ import annotations

from collections.abc import Callable
from typing import Any

from raven.utils.images import estimate_content_part_tokens, image_block, is_inline_image

IMAGE_REFS_KEY = "_image_refs"
"""References filed beside a saved message's image placeholders.

Each entry carries ``at`` (the block index), ``url`` (the exact data URI),
``tokens`` and the part's non-URL fields. Dropped by the provider allow-lists;
read only by the replay projection and the token estimator.
"""

IMAGE_PLACEHOLDER = "[image]"
"""What the display content holds in place of an inline picture."""


def capture_images(
content: Any,
*,
drop: Callable[[Any], bool] | None = None,
) -> tuple[Any, list[dict[str, Any]]]:
"""Replace inline pictures with placeholders, archiving their exact data URIs.

A non-list comes back unchanged. ``drop`` removes blocks before the index a
reference records, so a caller that also strips a runtime block gets indices
into the list it actually stores.
"""
if not isinstance(content, list):
return content, []
kept: list[Any] = []
refs: list[dict[str, Any]] = []
for part in content:
if drop is not None and drop(part):
continue
if not is_inline_image(part):
kept.append(part)
continue
ref = _ref_for_part(part)
kept.append({"type": "text", "text": IMAGE_PLACEHOLDER})
if ref is not None:
ref["at"] = len(kept) - 1
refs.append(ref)
return kept, refs


def expand_images(content: Any, message: dict[str, Any]) -> Any:
"""``content`` with each recorded placeholder replaced by the picture it stood for.

A message without references comes back as it is. A reference whose URL is
malformed leaves the placeholder.
"""
refs = message.get(IMAGE_REFS_KEY)
if not isinstance(content, list) or not isinstance(refs, list):
return content
expanded = list(content)
for ref in refs:
if not isinstance(ref, dict):
continue
at = ref.get("at")
if not isinstance(at, int) or not 0 <= at < len(expanded):
continue
placeholder = expanded[at]
if not (
isinstance(placeholder, dict)
and placeholder.get("type") == "text"
and placeholder.get("text") == IMAGE_PLACEHOLDER
):
continue
rebuilt = _rebuild_part(ref)
if rebuilt is not None:
expanded[at] = rebuilt
return expanded


def image_tokens(message: dict[str, Any]) -> int:
"""The image weight a saved message's references still cost the prompt."""
refs = message.get(IMAGE_REFS_KEY)
if not isinstance(refs, list):
return 0
total = 0
for ref in refs:
if isinstance(ref, dict):
tokens = ref.get("tokens")
if isinstance(tokens, int) and not isinstance(tokens, bool) and tokens > 0:
total += tokens
return total


def _ref_for_part(part: dict[str, Any]) -> dict[str, Any] | None:
image_url = part.get("image_url")
url = image_url.get("url") if isinstance(image_url, dict) else None
if not isinstance(url, str) or not url.startswith("data:image/"):
return None
ref: dict[str, Any] = {"url": url, "tokens": estimate_content_part_tokens(part) or 0}
top = {key: value for key, value in part.items() if key not in ("type", "image_url")}
url_extra = {key: value for key, value in image_url.items() if key != "url"}
if top:
ref["part"] = top
if url_extra:
ref["image_url"] = url_extra
return ref


def _rebuild_part(ref: dict[str, Any]) -> dict[str, Any] | None:
url = ref.get("url")
if not isinstance(url, str) or not url.startswith("data:image/"):
return None
rebuilt: dict[str, Any] = dict(image_block(url))
extra = ref.get("part")
if isinstance(extra, dict):
rebuilt = {**rebuilt, **{key: value for key, value in extra.items() if key not in ("type", "image_url")}}
extra_url = ref.get("image_url")
if isinstance(extra_url, dict):
rebuilt["image_url"] = {
**rebuilt["image_url"],
**{key: value for key, value in extra_url.items() if key != "url"},
}
return rebuilt


__all__ = [
"IMAGE_PLACEHOLDER",
"IMAGE_REFS_KEY",
"capture_images",
"expand_images",
"image_tokens",
]
7 changes: 4 additions & 3 deletions raven/utils/tokens.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

import tiktoken

from raven.utils.image_store import image_tokens
from raven.utils.images import estimate_content_part_tokens

# A cl100k regex piece cannot cross a newline followed by non-whitespace.
Expand Down Expand Up @@ -180,7 +181,7 @@ def estimate_message_tokens(message: dict[str, Any]) -> int:
"""Estimate prompt tokens contributed by one persisted message."""
content = message.get("content")
parts: list[str] = []
extra_tokens = 0
extra_tokens = image_tokens(message)
if isinstance(content, str):
parts.append(content)
elif isinstance(content, list):
Expand All @@ -189,8 +190,8 @@ def estimate_message_tokens(message: dict[str, Any]) -> int:
text = part.get("text", "")
if text:
parts.append(text)
elif (image_tokens := estimate_content_part_tokens(part)) is not None:
extra_tokens += image_tokens
elif (part_tokens := estimate_content_part_tokens(part)) is not None:
extra_tokens += part_tokens
else:
parts.append(json.dumps(part, ensure_ascii=False))
elif content is not None:
Expand Down
Loading
Loading