Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,10 @@
PASSWORD_CHECK_TIMEOUT_S = 2.0


class _TabGone(Exception):
"""The tab a capture was asked of went away while the capture was waiting: the browser answers it never."""


class BrowserControl:
"""agentenv's control of a browser over its pipe: at each launch it attaches to every page with flattened sessions
and keeps a record of the browser since this server started, its open tabs in opened order, those closed and who
Expand Down Expand Up @@ -750,6 +754,10 @@ async def frame(self, target: Optional[str], load_wait_s: float) -> tuple[str, s
except DevToolsError as e:
if not any(why in str(e) for why in FOR_THE_MOMENT) or loop.time() + FRAME_RETRY_S >= deadline:
raise DeviceUnavailable(f"the tab couldn't be captured: {e}") from None
except _TabGone:
if loop.time() + FRAME_RETRY_S >= deadline:
raise DeviceUnavailable(f"the tab went away while its frame was taken, {FRAME_CAPTURE_S:g} s in all"
) from None
except ConnectionError as e:
raise DeviceUnavailable(f"the tab couldn't be captured: {e}") from None
await asyncio.sleep(FRAME_RETRY_S)
Expand All @@ -776,7 +784,8 @@ def _held_frame(self, target: str) -> tuple[str, str, Optional[str], Optional[st

async def _nudged_capture(self, session: str, timeout_s: float) -> dict:
"""The tab's JPEG from the first of its captures to answer: a second follows one unanswered after ``NUDGE_S``,
since a tab behind the active one answers only then, all within ``timeout_s``."""
since a tab behind the active one answers only then, all within ``timeout_s``; _TabGone once the tab's session
is gone with none answered, as a closed tab's captures never are."""
loop = asyncio.get_running_loop()
deadline = loop.time() + timeout_s

Expand All @@ -791,11 +800,13 @@ def capture() -> asyncio.Future:
if not done:
captures.add(capture())
while True:
done, captures = await asyncio.wait(captures, return_when=asyncio.FIRST_COMPLETED)
done, captures = await asyncio.wait(captures, timeout=FRAME_RETRY_S, return_when=asyncio.FIRST_COMPLETED)
if answers := [c.result() for c in done if c.exception() is None]:
return answers[0]
if not captures:
raise next(iter(done)).exception()
if session not in self._sessions:
raise _TabGone()
finally:
for pending in captures:
pending.cancel()
Expand Down
31 changes: 31 additions & 0 deletions packages/agentenv-protocol/tests/test_browser_recording.py
Original file line number Diff line number Diff line change
Expand Up @@ -359,6 +359,37 @@ async def test_a_frame_whose_only_tab_is_attached_anew_only_after_a_retry_waits_
assert tab == "T1" and [s for s, _, _ in pipe.captured] == ["s-T1", "s-T1b"]


class HangingPipe(Pipe):
"""A pipe whose captures of the sessions in ``hanging`` never answer, as a tab's that closes mid-capture."""

def __init__(self, *hanging: str) -> None:
super().__init__()
self.hanging = set(hanging)

async def call(self, method, params=None, *, session_id=None, timeout_s=None):
if method == "Page.captureScreenshot" and session_id in self.hanging:
self.captured.append((session_id, params, asyncio.get_running_loop().time()))
await asyncio.sleep(timeout_s)
raise asyncio.TimeoutError()
return await super().call(method, params, session_id=session_id, timeout_s=timeout_s)


@pytest.mark.asyncio
async def test_a_frame_whose_tab_closes_while_its_capture_waits_is_taken_of_the_tab_left_at_once():
pipe = HangingPipe("s-T2")
control = BrowserControl(pipe, "agent", (1280, 800))
for tab in ("T1", "T2"):
pipe.emit("Target.targetCreated", {"targetInfo": {"targetId": tab, "type": "page", "url": "about:blank"}})
pipe.emit("Target.attachedToTarget", {"sessionId": f"s-{tab}", "targetInfo": {"targetId": tab, "type": "page"}})
loop = asyncio.get_running_loop()
loop.call_later(0.1, pipe.emit, "Target.detachedFromTarget", {"sessionId": "s-T2"})
started = loop.time()

_, _, tab, _ = await control.frame("T2", load_wait_s=0)

assert tab == "T1" and pipe.captured[-1][0] == "s-T1" and loop.time() - started < 2


@pytest.mark.asyncio
async def test_a_frame_the_browser_refuses_for_another_reason_is_missing_at_once():
pipe = RefusingPipe(refusals=5, why="Page.captureScreenshot: Internal error")
Expand Down
82 changes: 82 additions & 0 deletions tst/integration/computer/browser_computer_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -1375,6 +1375,88 @@ async def _cdp_agent_acts_behind(ws_url: str) -> str:
return target


@pytest.mark.asyncio
async def test_a_browser_recording_that_ends_while_an_agents_command_is_in_flight_logs_it_unfinished(browser_computer):
with namespace_routing():
deployed = await Env.get(browser_computer.id).deploy(
env_params={"app_settings": {"chromium": {"start_urls": [PAGE_URL]}}})
context = TaskStepContext(instance_id=f"run-{uuid.uuid4().hex[:8]}", deployed_envs=[deployed])
try:
context = await StartBrowserRecordingTaskStep(id="start", version=None, env_id=browser_computer.id,
name="flight").execute(context)
ws_url = (await _cdp(deployed, "pi"))["ws_url"]
async with _cdp_calls(ws_url, reply_s=300) as waiting, _cdp_calls(ws_url) as call:
target = await _page_target(call)
held = (await waiting("Target.attachToTarget", {"targetId": target, "flatten": True}))["result"]["sessionId"]
hanging = asyncio.ensure_future(waiting("Runtime.evaluate", {
"expression": "new Promise((done) => setTimeout(done, 120000))", "awaitPromise": True}, held))
await asyncio.sleep(1)
session = (await call("Target.attachToTarget", {"targetId": target, "flatten": True}))["result"]["sessionId"]
for n in range(3):
await call("Runtime.evaluate", {"expression": f"document.body.append('after {n}')"}, session)
context = await EndBrowserRecordingTaskStep(id="end", version=None, name="flight",
fail_task_on_error=False).execute(context)
hanging.cancel()
finally:
report = await teardown_run(context)
grant_server(None, None).close()
entry = context.metadata[BROWSER_RECORDINGS]["flight"]
lines = _recorded_log(entry)

evaluated = [line["action"] for line in lines if (line.get("action") or {}).get("method") == "Runtime.evaluate"
and line["action"].get("who") == "pi"]
assert not report.failed and (entry["state"], entry["log_missing"]) == ("ended", 0), entry
assert [bool(action.get("unfinished")) for action in evaluated] == [True, False, False, False], evaluated


@pytest.mark.asyncio
async def test_a_browser_recording_frames_the_active_tab_when_the_agents_tab_closes_mid_capture(browser_computer):
with namespace_routing():
deployed = await Env.get(browser_computer.id).deploy(
env_params={"app_settings": {"chromium": {"start_urls": [PAGE_URL]}}})
context = TaskStepContext(instance_id=f"run-{uuid.uuid4().hex[:8]}", deployed_envs=[deployed])
try:
context = await StartBrowserRecordingTaskStep(id="start", version=None, env_id=browser_computer.id,
name="closing").execute(context)
async with _cdp_calls((await _cdp(deployed, "pi"))["ws_url"]) as call:
tab = (await call("Target.createTarget", {"url": BEHIND[0]}))["result"]["targetId"]
session = (await call("Target.attachToTarget", {"targetId": tab, "flatten": True}))["result"]["sessionId"]
await asyncio.sleep(2)
await call("Runtime.evaluate", {"expression":
"setTimeout(() => { const t = Date.now(); while (Date.now() - t < 8000) {} }, 200)"}, session)
await asyncio.sleep(1.5)
await call("Target.closeTarget", {"targetId": tab})
await asyncio.sleep(3)
context = await EndBrowserRecordingTaskStep(id="end", version=None, name="closing",
fail_task_on_error=False).execute(context)
finally:
report = await teardown_run(context)
grant_server(None, None).close()
entry = context.metadata[BROWSER_RECORDINGS]["closing"]
manifest = _recorded_manifest(entry)

frames = manifest["frames"]
missing = [(frame["index"], frame.get("error")) for frame in frames if not frame["uploaded"]]
assert not report.failed and (entry["state"], missing) == ("ended", []), missing


async def _page_target(call) -> str:
"""The first page the browser has open, as a CDP agent finds it."""
targets = (await call("Target.getTargets", {}))["result"]["targetInfos"]
return next(t["targetId"] for t in targets if t["type"] == "page")


def _recorded_manifest(entry: dict) -> dict:
universe = Artifact.get(entry["artifact"]["id"], version=entry["artifact"]["version"])
return json.loads(get_config().get_object_store().get(Artifact.get(universe.file_artifact_ids["manifest.json"]).object_url))


def _recorded_log(entry: dict) -> list[dict]:
universe = Artifact.get(entry["artifact"]["id"], version=entry["artifact"]["version"])
text = get_config().get_object_store().get(Artifact.get(universe.file_artifact_ids["log.jsonl"]).object_url).decode()
return [json.loads(line) for line in text.splitlines() if line.strip()]


async def _mcp(deployed, work):
"""What ``work`` makes of one MCP session on the computer, as a role-less agent opens one."""
async with streamablehttp_client(deployed.mcp_url) as (read, write, _):
Expand Down
Loading