Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,12 @@
NUDGE_S = 0.5
#: How long a frame may take to capture, longer than a screenshot an agent waits on, since a frame is taken in the
#: background: a page too busy to paint, or between documents mid-navigation, gets this long to show, a capture the
#: browser refused for being between documents asked again every ``FRAME_RETRY_S``.
#: browser refused for the moment asked again every ``FRAME_RETRY_S``.
FRAME_CAPTURE_S = 10.0
FRAME_RETRY_S = 0.25
#: What the browser says of a capture refused for the moment, as while a navigation swaps the tab's document.
BETWEEN_DOCUMENTS = ("Not attached to an active page", "Unable to capture screenshot")
#: What the browser says of a capture refused for the moment: while a navigation swaps the tab's document, or once the
#: session it was asked on is gone, as when the tab closes or the browser attaches it anew.
FOR_THE_MOMENT = ("Not attached to an active page", "Unable to capture screenshot", "Session with given id not found")
#: How often a wait for a tab to load looks again.
LOAD_POLL_S = 0.1
#: Where an agent's files to upload are, and where downloads land, under the session user's home.
Expand Down Expand Up @@ -658,14 +659,20 @@ async def frame(self, target: Optional[str], load_wait_s: float) -> tuple[str, s
except asyncio.TimeoutError:
raise DeviceUnavailable(f"the tab painted nothing within {FRAME_CAPTURE_S:g} s") from None
except DevToolsError as e:
if not any(why in str(e) for why in BETWEEN_DOCUMENTS) or loop.time() + FRAME_RETRY_S >= deadline:
if not any(why in str(e) for why in FOR_THE_MOMENT) or loop.time() + FRAME_RETRY_S >= deadline:
raise DeviceUnavailable(f"the tab couldn't be captured: {e}") from None
except ConnectionError as e:
raise DeviceUnavailable(f"the tab couldn't be captured: {e}") from None
await asyncio.sleep(FRAME_RETRY_S)
if session not in self._sessions:
session = await self.active_session()
target = self._sessions[session]
while session not in self._sessions:
try:
session = next((s for s, t in self._sessions.items() if t == target), None) or (
await self.active_session())
except DeviceUnavailable:
if loop.time() + FRAME_RETRY_S >= deadline:
raise
await asyncio.sleep(FRAME_RETRY_S)
target = self._sessions[session]
self._frames[target] = shot["data"], "jpg"
return shot["data"], "jpg", target, self.url_of(target)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -400,24 +400,27 @@ def note(self, kind: str, body: dict) -> int:
self.write(seq, {kind: body})
return seq

def read(self, after: int = 0, limit: int = READ_LIMIT) -> dict:
def read(self, after: int = 0, limit: int = READ_LIMIT, settle_through: Optional[int] = None) -> dict:
"""The lines after sequence number ``after``, in order, up to ``limit`` of them and to the last whose every
earlier line is written or stalled: ``lines``, ``last``, the sequence number of that last one, and ``more``
when the limit left some."""
complete, picked, more = self._picked(after, limit)
when the limit left some. An action begun at or before ``settle_through`` and still going counts as stalled
now, as a recording that ends while an action is in flight needs."""
complete, picked, more = self._picked(after, limit, settle_through)
return self._loaded(after, complete, picked, more)

async def read_async(self, after: int = 0, limit: int = READ_LIMIT) -> dict:
async def read_async(self, after: int = 0, limit: int = READ_LIMIT, settle_through: Optional[int] = None) -> dict:
"""``read``, its file read in a thread."""
complete, picked, more = self._picked(after, limit)
complete, picked, more = self._picked(after, limit, settle_through)
return await asyncio.to_thread(self._loaded, after, complete, picked, more)

def _picked(self, after: int, limit: int) -> tuple[int, list[tuple[int, int, int]], bool]:
def _picked(self, after: int, limit: int, settle_through: Optional[int] = None
) -> tuple[int, list[tuple[int, int, int]], bool]:
"""Where the lines a read returns are: the last sequence number it may return, and each line's number, offset
and length, in order."""
now = time.monotonic()
for seq, began in list(self._pending.items()):
if now - began >= STALLED_S and (begun := self._begun.get(seq)) is not None:
settled = settle_through is not None and seq <= settle_through
if (settled or now - began >= STALLED_S) and (begun := self._begun.get(seq)) is not None:
self._stood_in.add(seq)
self.write(seq, {"action": {**begun["action"], "unfinished": True}}, at=begun["at"])
complete = min(self._pending) - 1 if self._pending else self._seq
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -244,15 +244,18 @@ async def end(self, grant: Optional[WriteNamespaceGrant], reason: str) -> dict:
return wire

async def _log_lines(self) -> list[dict]:
"""The log's lines from the recording's start to its end, once every one of them is written, or what's
written of them after ``LOG_WAIT_S``."""
"""The log's lines from the recording's start to its end, once every one of them is written; after
``LOG_WAIT_S``, an action still going is written as unfinished, as it was at the end, and what's written of
them is what there is."""
manifest, log = self.manifest, self.recordings.log
deadline = asyncio.get_running_loop().time() + LOG_WAIT_S
wanted = manifest.log_last - manifest.log_after
while True:
lines = [line for line in (await log.read_async(manifest.log_after, limit=wanted))["lines"]
if line.get("seq", 0) <= manifest.log_last]
if len(lines) >= wanted or asyncio.get_running_loop().time() >= deadline:
late = asyncio.get_running_loop().time() >= deadline
read = await log.read_async(manifest.log_after, limit=wanted,
settle_through=manifest.log_last if late else None)
lines = [line for line in read["lines"] if line.get("seq", 0) <= manifest.log_last]
if len(lines) >= wanted or late:
return lines
await asyncio.sleep(LOG_POLL_S)

Expand Down
13 changes: 13 additions & 0 deletions packages/agentenv-protocol/tests/test_browser_log.py
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,19 @@ def test_an_action_still_going_after_the_limit_is_written_unfinished_and_its_out
assert [line["seq"] for line in log.read()["lines"]] == [1, 2, 3]


def test_a_read_settling_through_an_entry_writes_the_actions_still_going_up_to_it_unfinished_now(tmp_path):
log = BrowserLog(tmp_path / "browser-chromium.jsonl")
early = log.begin({"via": "mcp", "tool": "browser_navigate", "acts": True, "tab": "T1"})
late = log.begin({"via": "cdp", "method": "Runtime.evaluate", "acts": True, "tab": "T1"})

read = log.read(settle_through=early["seq"])

assert [(line["seq"], line["action"]["unfinished"]) for line in read["lines"]] == [(early["seq"], True)]
assert read["last"] == early["seq"] and log.read(after=early["seq"])["lines"] == []
log.finish(late, ok=True)
assert [line["seq"] for line in log.read(after=early["seq"])["lines"]] == [late["seq"]]


def test_a_restarted_server_carries_on_the_sequence_on_a_new_line_after_a_partial_one(tmp_path):
path = tmp_path / "browser-chromium.jsonl"
path.write_text('{"seq": 1, "at": "t", "reset": {}}\n{"seq": 2, "at": "t", "tab": {}}\n{"seq": 3, "at"')
Expand Down
43 changes: 39 additions & 4 deletions packages/agentenv-protocol/tests/test_browser_recording.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,19 +112,25 @@ async def test_a_burst_still_settling_at_the_end_gets_its_frame_before_the_last(


@pytest.mark.asyncio
async def test_an_end_waits_for_entries_in_flight_and_lists_any_still_unwritten_as_missing(recorded, monkeypatch):
async def test_an_end_waits_for_entries_in_flight_writes_an_action_still_going_as_unfinished_and_lists_the_rest(
recorded, monkeypatch):
log, control, uploads, recordings = recorded
monkeypatch.setattr(browser_recording, "LOG_WAIT_S", 0.3)
started = await recordings.start("run-1:record", _grant(), frames=False)
finishing = log.begin({"via": "cdp", "who": "ada", "method": "Page.navigate", "acts": True, "tab": "T1"})
asyncio.get_running_loop().call_later(0.1, lambda: log.finish(finishing, ok=True))
complete = await recordings.end(started["recording_id"])
started = await recordings.start("run-2:record", _grant(), frames=False)
stuck = log.begin({"via": "cdp", "who": "ada", "method": "Page.navigate", "acts": True, "tab": "T1"})
incomplete = await recordings.end(started["recording_id"])
going = log.begin({"via": "mcp", "who": "default", "tool": "browser_navigate", "acts": True, "tab": "T1"})
after = _acted(log, "cdp", "Input.insertText", "T1")
held = log.reserve()
ended = await recordings.end(started["recording_id"])
lines = [json.loads(line) for line in uploads.by_path[
f"agentenv-browser-recordings/r1/{ended['recording_id']}/log.jsonl"].decode().splitlines()]

assert complete["log_missing"] == [] and complete["log_last"] == finishing["seq"]
assert incomplete["log_missing"] == [stuck["seq"]]
assert ended["log_missing"] == [held] and [line["seq"] for line in lines] == [going["seq"], after]
assert lines[0]["action"]["unfinished"] is True


@pytest.mark.asyncio
Expand Down Expand Up @@ -324,6 +330,35 @@ async def test_a_tab_that_closes_while_its_frame_is_asked_again_is_framed_as_the
assert framed[:3] == held[:3] and held[2] == "T1" and [s for s, _, _ in pipe.captured] == ["s-T1", "s-T2"]


@pytest.mark.asyncio
async def test_a_frame_whose_session_is_gone_is_taken_on_the_session_its_tab_was_attached_on_anew():
pipe = RefusingPipe(refusals=1, why="Page.captureScreenshot: Session with given id not found")
control = _one_tab(pipe)

def reattach() -> None:
pipe.emit("Target.detachedFromTarget", {"sessionId": "s-T1"})
pipe.emit("Target.attachedToTarget", {"sessionId": "s-T1b", "targetInfo": {"targetId": "T1", "type": "page"}})
asyncio.get_running_loop().call_later(0.05, reattach)

_, _, tab, _ = await control.frame("T1", load_wait_s=0)

assert tab == "T1" and [s for s, _, _ in pipe.captured] == ["s-T1", "s-T1b"]


@pytest.mark.asyncio
async def test_a_frame_whose_only_tab_is_attached_anew_only_after_a_retry_waits_for_it_within_its_budget():
pipe = RefusingPipe(refusals=1, why="Page.captureScreenshot: Session with given id not found")
control = _one_tab(pipe)
loop = asyncio.get_running_loop()
loop.call_later(0.05, pipe.emit, "Target.detachedFromTarget", {"sessionId": "s-T1"})
loop.call_later(0.6, pipe.emit, "Target.attachedToTarget",
{"sessionId": "s-T1b", "targetInfo": {"targetId": "T1", "type": "page"}})

_, _, tab, _ = await control.frame("T1", load_wait_s=0)

assert tab == "T1" and [s for s, _, _ in pipe.captured] == ["s-T1", "s-T1b"]


@pytest.mark.asyncio
async def test_a_frame_the_browser_refuses_for_another_reason_is_missing_at_once():
pipe = RefusingPipe(refusals=5, why="Page.captureScreenshot: Internal error")
Expand Down
Loading