From ed578796c9e6cb96b3ba1099e64f3179752dedac Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 15 Nov 2026 16:21:02 -0800 Subject: [PATCH 1/3] fix(computer): a frame an agent gets carries no capture time, which was the driver's real time The driver keeps real time, so every screenshot result's captured_at gave an agent on a computer following the universe clock the real date and a UTC clock beside the universe's (2026-11-15T23:18Z, in a Saturday 6 February 2027 run). No agent needs it and nothing read it: FrameData drops it, and Frame, which only copied it to the result, too. Recordings keep stamping their own frames. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/computer/frames.py | 4 ---- .../src/agentenv_protocol/computer/results.py | 3 ++- packages/agentenv-protocol/tests/test_computer_frames.py | 8 ++------ 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py index b611fe0c9..0ca3834f1 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py @@ -6,7 +6,6 @@ import math from collections import OrderedDict from dataclasses import dataclass, field -from datetime import datetime, timezone from PIL import Image as PILImage @@ -32,7 +31,6 @@ class Frame: origin: tuple[int, int] root_size: tuple[int, int] parent_id: str | None - captured_at: datetime inputs: tuple = field(repr=False) #: The page it shows, on a computer whose screen is a website browser's active tab. page: PageState | None = None @@ -72,7 +70,6 @@ def result(self) -> FrameData: height=self.height, display=DisplayData(width=self.display.width, height=self.display.height, scale=self.display.scale), parent_frame_id=self.parent_id, - captured_at=self.captured_at.isoformat().replace("+00:00", "Z"), page=_page_data(self.page), ) @@ -116,7 +113,6 @@ def add(self, image: PILImage.Image, display: Display, inputs: tuple, *, identit origin=(parent.origin[0] + origin[0], parent.origin[1] + origin[1]) if parent is not None else (0, 0), root_size=parent.root_size if parent is not None else image.size, parent_id=parent.id if parent is not None else None, - captured_at=parent.captured_at if parent is not None else datetime.now(timezone.utc), inputs=parent.inputs if parent is not None else inputs, page=parent.page if parent is not None else page, ) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/results.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/results.py index dc1a72a83..8b6270fa9 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/results.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/results.py @@ -36,12 +36,13 @@ class PageData(BaseModel): class FrameData(BaseModel): + """A frame as an agent sees it. When it was captured isn't part of it: the driver keeps real time, which in a + computer on the universe's clock would tell the agent the real date.""" frame_id: str width: int height: int display: DisplayData parent_frame_id: Optional[str] = None - captured_at: str page: Optional[PageData] = None diff --git a/packages/agentenv-protocol/tests/test_computer_frames.py b/packages/agentenv-protocol/tests/test_computer_frames.py index 6c5eb9b79..d350014c9 100644 --- a/packages/agentenv-protocol/tests/test_computer_frames.py +++ b/packages/agentenv-protocol/tests/test_computer_frames.py @@ -29,7 +29,8 @@ async def test_a_screenshot_is_a_native_frame_with_its_geometry(): data = first.structuredContent assert data["frame_id"] == "f1" and (data["width"], data["height"]) == (1280, 800) assert data["display"] == {"width": 1280, "height": 800, "scale": 1.0} and "parent_frame_id" not in data - assert data["captured_at"].endswith("Z") and data["summary"] == "Captured f1: 1280×800 of the 1280×800 display" + assert data["summary"] == "Captured f1: 1280×800 of the 1280×800 display" + assert "captured_at" not in data # the driver's real time, which would tell an agent the real date assert first.content[0].text == data["summary"] and first.content[1].mimeType == "image/png" assert _image(first).size == (1280, 800) assert second.structuredContent["frame_id"] == "f2" and second.content[1].mimeType == "image/jpeg" @@ -56,16 +57,11 @@ async def test_a_region_is_a_native_crop_whose_points_map_back_exactly(): assert crop.structuredContent["parent_frame_id"] == "f1" and (crop.structuredContent["width"], crop.structuredContent["height"]) == (300, 100) assert crop.structuredContent["summary"] == "Cropped f2: 300×100 of f1 at (600, 400)" - assert crop.structuredContent["captured_at"] == (await _captured_at(driver, "f1")) assert _image(crop).getpixel((100, 50))[:3] == (1, 2, 3) assert nested.structuredContent["parent_frame_id"] == "f2" and _image(nested).getpixel((0, 0))[:3] == (1, 2, 3) assert not clicked.isError and driver.screen.log[-1][1:3] == (701, 451) -async def _captured_at(driver: FakeDriver, frame_id: str) -> str: - return driver._computer.frames.get(frame_id, "default").result().captured_at - - @pytest.mark.asyncio @pytest.mark.parametrize("args, code", [ ({"frame_id": "f1"}, "invalid_argument"), From c0e806e2480db1156046c18ad20f4ed817bba0ea Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 15 Nov 2026 16:31:26 -0800 Subject: [PATCH 2/3] fix(computer): a frame's capture time goes in the answer's _meta, which recordings line up with and agents aren't shown The recording test lines each screenshot up with the video by when the driver captured it, so that time stays, as `agentenv.io/capturedAt` in the tool answer's _meta (MCP clients don't show _meta to their model), not in the structured content an agent reads. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/computer/frames.py | 3 +++ .../src/agentenv_protocol/computer/results.py | 17 +++++++++++++---- .../src/agentenv_protocol/computer/runtime.py | 2 +- .../src/agentenv_protocol/computer/tools.py | 4 ++-- .../tests/test_computer_frames.py | 5 ++++- .../recording/artifacts/drive_desk/verify.py | 3 ++- 6 files changed, 25 insertions(+), 9 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py index 0ca3834f1..4e937140a 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py @@ -6,6 +6,7 @@ import math from collections import OrderedDict from dataclasses import dataclass, field +from datetime import datetime, timezone from PIL import Image as PILImage @@ -31,6 +32,7 @@ class Frame: origin: tuple[int, int] root_size: tuple[int, int] parent_id: str | None + captured_at: datetime inputs: tuple = field(repr=False) #: The page it shows, on a computer whose screen is a website browser's active tab. page: PageState | None = None @@ -113,6 +115,7 @@ def add(self, image: PILImage.Image, display: Display, inputs: tuple, *, identit origin=(parent.origin[0] + origin[0], parent.origin[1] + origin[1]) if parent is not None else (0, 0), root_size=parent.root_size if parent is not None else image.size, parent_id=parent.id if parent is not None else None, + captured_at=parent.captured_at if parent is not None else datetime.now(timezone.utc), inputs=parent.inputs if parent is not None else inputs, page=parent.page if parent is not None else page, ) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/results.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/results.py index 8b6270fa9..359b03578 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/results.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/results.py @@ -2,6 +2,7 @@ answers is what its schema says.""" from __future__ import annotations +from datetime import datetime from typing import Literal, Optional from mcp.types import CallToolResult, ImageContent, TextContent @@ -9,6 +10,9 @@ from .operations import Effect +#: The ``_meta`` key of a tool answer that carries a frame: when the driver captured it, in real time (RFC 3339, UTC). +CAPTURED_AT_META = "agentenv.io/capturedAt" + class DisplayData(BaseModel): width: int @@ -37,7 +41,8 @@ class PageData(BaseModel): class FrameData(BaseModel): """A frame as an agent sees it. When it was captured isn't part of it: the driver keeps real time, which in a - computer on the universe's clock would tell the agent the real date.""" + computer on the universe's clock would tell the agent the real date; the tool's answer carries it in its ``_meta`` + (``CAPTURED_AT_META``) for the callers that line frames up with a recording.""" frame_id: str width: int height: int @@ -119,10 +124,14 @@ class PointerListResult(BaseModel): pointers: list[PointerData] -def send(result: BaseModel, image: tuple[str, str] | None = None) -> CallToolResult: +def send(result: BaseModel, image: tuple[str, str] | None = None, + captured_at: datetime | None = None) -> CallToolResult: """A result as a tool answers: the summary as the text block, then the image (base64, MIME type) when there is - one, and the result as structured content with its unset fields left out.""" + one, and the result as structured content with its unset fields left out; when its frame was ``captured_at``, + that real time in its ``_meta``, which a client doesn't show its model.""" content: list = [TextContent(type="text", text=result.summary)] if image is not None: content.append(ImageContent(type="image", data=image[0], mimeType=image[1])) - return CallToolResult(content=content, structuredContent=result.model_dump(mode="json", exclude_none=True)) + meta = {CAPTURED_AT_META: captured_at.isoformat().replace("+00:00", "Z")} if captured_at is not None else None + return CallToolResult(content=content, structuredContent=result.model_dump(mode="json", exclude_none=True), + _meta=meta) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py index 8723045fb..2cbf236cb 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py @@ -154,7 +154,7 @@ async def act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Fra result.summary = f"{result.summary}; the screenshot after it failed ({e.code}: {e.message})" return send(result) result.summary, result.frame = f"{result.summary}, then captured {new.id}{new.note()}", new.result() - return send(result, new.encode()) + return send(result, new.encode(), new.captured_at) async def _hold_for(self, ms: int, keys: list[str]) -> None: """Keep ``keys`` down for ``ms``, then release them, also when the call is cancelled meanwhile.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py index 4f36213a4..d85558e94 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py @@ -99,7 +99,7 @@ async def computer_screenshot( identity=identity, parent=parent, origin=(box[0], box[1])) summary = f"Cropped {frame.id}: {frame.width}×{frame.height} of {parent.id} at ({box[0]}, {box[1]})" result = ScreenshotResult(summary=summary, **frame.result().model_dump()) - return send(result, frame.encode(format, quality)) + return send(result, frame.encode(format, quality), frame.captured_at) @tool(annotations=_INPUT) async def computer_click( @@ -242,7 +242,7 @@ async def computer_wait( frame = await self._computer.capture(self.caller_identity(ctx)) return send(WaitResult(summary=f"{summary}, then captured {frame.id}{frame.note()}", waited_ms=ms, frame=frame.result()), - frame.encode()) + frame.encode(), frame.captured_at) @tool(annotations=_INPUT) async def computer_pointer_move( diff --git a/packages/agentenv-protocol/tests/test_computer_frames.py b/packages/agentenv-protocol/tests/test_computer_frames.py index d350014c9..3bfd4c117 100644 --- a/packages/agentenv-protocol/tests/test_computer_frames.py +++ b/packages/agentenv-protocol/tests/test_computer_frames.py @@ -15,6 +15,7 @@ from agentenv_protocol import ROLE_META_KEY # noqa: E402 from agentenv_protocol.computer.frames import KEEP # noqa: E402 +from agentenv_protocol.computer.results import CAPTURED_AT_META # noqa: E402 def _image(result) -> PILImage.Image: @@ -31,6 +32,7 @@ async def test_a_screenshot_is_a_native_frame_with_its_geometry(): assert data["display"] == {"width": 1280, "height": 800, "scale": 1.0} and "parent_frame_id" not in data assert data["summary"] == "Captured f1: 1280×800 of the 1280×800 display" assert "captured_at" not in data # the driver's real time, which would tell an agent the real date + assert first.meta[CAPTURED_AT_META].endswith("Z") # kept for whoever lines frames up with a recording assert first.content[0].text == data["summary"] and first.content[1].mimeType == "image/png" assert _image(first).size == (1280, 800) assert second.structuredContent["frame_id"] == "f2" and second.content[1].mimeType == "image/jpeg" @@ -50,13 +52,14 @@ async def test_a_region_is_a_native_crop_whose_points_map_back_exactly(): driver = FakeDriver(FakeScreen(1280, 800)) driver.screen.paint(700, 450, (1, 2, 3)) async with mcp_session(driver) as session: - await session.call_tool("computer_screenshot", {}) + first_at = (await session.call_tool("computer_screenshot", {})).meta[CAPTURED_AT_META] crop = await session.call_tool("computer_screenshot", {"region": [600, 400, 900, 500]}) nested = await session.call_tool("computer_screenshot", {"region": [100, 50, 200, 100], "frame_id": "f2"}) clicked = await session.call_tool("computer_click", {"x": 1, "y": 1, "frame_id": "f3"}) assert crop.structuredContent["parent_frame_id"] == "f1" and (crop.structuredContent["width"], crop.structuredContent["height"]) == (300, 100) assert crop.structuredContent["summary"] == "Cropped f2: 300×100 of f1 at (600, 400)" + assert crop.meta[CAPTURED_AT_META] == first_at # a crop was captured when its frame was assert _image(crop).getpixel((100, 50))[:3] == (1, 2, 3) assert nested.structuredContent["parent_frame_id"] == "f2" and _image(nested).getpixel((0, 0))[:3] == (1, 2, 3) assert not clicked.isError and driver.screen.log[-1][1:3] == (701, 451) diff --git a/tst/integration/computer/recording/artifacts/drive_desk/verify.py b/tst/integration/computer/recording/artifacts/drive_desk/verify.py index 77ebea583..1ab4771e6 100644 --- a/tst/integration/computer/recording/artifacts/drive_desk/verify.py +++ b/tst/integration/computer/recording/artifacts/drive_desk/verify.py @@ -11,6 +11,7 @@ from PIL import Image from agentenv_protocol.computer.conformance.client import Driver +from agentenv_protocol.computer.results import CAPTURED_AT_META THUMB = (64, 40) DESK = {name: resources.files("agentenv_protocol.computer.conformance").joinpath(name).read_bytes() @@ -58,7 +59,7 @@ def centre(box): if result.isError: return [{"id": "drove", "result": False, "description": f"{tool}: {result.content[0].text}"}] shots.append({"step": tool, "args": args, "sent_at": sent_at, - "captured_at": result.structuredContent["frame"]["captured_at"], "thumb": _thumb(result), + "captured_at": result.meta[CAPTURED_AT_META], "thumb": _thumb(result), "cell": list(_image(result).convert("RGB").getpixel((clicked[0] + 8, clicked[1] + 8)))}) await asyncio.sleep(0.7) return [{"id": "drove", "result": True, "description": json.dumps({"shots": shots, "clicked_cell": clicked})}] From 1a8b95b97a64f8b823acbfd233fa2451c02f8393 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 15 Nov 2026 16:38:24 -0800 Subject: [PATCH 3/3] fix(protocol): a driver on standalone FastMCP keeps its answer's _meta, where a frame's capture time is Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/agent_env_environment.py | 5 +++-- packages/agentenv-protocol/tests/test_computer_spaces.py | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py index 360005a78..c871c8146 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py @@ -990,7 +990,7 @@ async def on_call_tool(self, context: Any, call_next: Callable) -> Any: def _fastmcp_tool_result(answer: Any) -> Any: """A hook's ``mcp.types.CallToolResult`` as the ``ToolResult`` a fastmcp middleware chain returns, the error - flag kept, under the field spellings of mcp 1 and 2.""" + flag and ``_meta`` kept, under the field spellings of mcp 1 and 2.""" from fastmcp.tools import ToolResult # mcp 2's spellings first: on mcp 1 they are plain misses, on mcp 2 the camel-case ones are deprecation shims. structured = getattr(answer, "structured_content", None) @@ -999,7 +999,8 @@ def _fastmcp_tool_result(answer: Any) -> Any: is_error = getattr(answer, "is_error", None) if is_error is None: is_error = getattr(answer, "isError", False) - return ToolResult(content=list(answer.content), structured_content=structured, is_error=bool(is_error)) + return ToolResult(content=list(answer.content), structured_content=structured, meta=getattr(answer, "meta", None), + is_error=bool(is_error)) def _mcp_tool_annotations(annotations: Any) -> Any: diff --git a/packages/agentenv-protocol/tests/test_computer_spaces.py b/packages/agentenv-protocol/tests/test_computer_spaces.py index 9352dbf1b..ceddb4765 100644 --- a/packages/agentenv-protocol/tests/test_computer_spaces.py +++ b/packages/agentenv-protocol/tests/test_computer_spaces.py @@ -16,6 +16,7 @@ from agentenv_protocol.computer import ComputerSettings, InvalidArgument, OutOfBounds # noqa: E402 from agentenv_protocol.computer.frames import FrameStore # noqa: E402 from agentenv_protocol.computer.operations import Display # noqa: E402 +from agentenv_protocol.computer.results import CAPTURED_AT_META # noqa: E402 from agentenv_protocol.computer.spaces import check_space, frame_point # noqa: E402 @@ -97,12 +98,13 @@ async def test_setting_the_default_space_rewrites_the_descriptions_on_fastmcp(): card = (await http.get("/.well-known/agent-env.json")).json() async with fastmcp.Client(app) as client: listed = {t.name: t for t in await client.list_tools()} - await client.call_tool("computer_screenshot", {}) + shot = await client.call_tool("computer_screenshot", {}) clicked = await client.call_tool("computer_click", {"x": 500, "y": 500}) card_click = next(t for t in card["capabilities"]["tools"] if t["name"] == "computer_click") assert "norm space by default: 0 to 1000" in listed["computer_click"].description assert card_click["description"] == listed["computer_click"].description assert clicked.structured_content["summary"].startswith("Clicked left at (500, 500) (norm 1000) in f1") + assert shot.meta[CAPTURED_AT_META].endswith("Z") and "captured_at" not in shot.structured_content @pytest.mark.asyncio