diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py index 4f0c360b7..05a480013 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py @@ -85,6 +85,10 @@ PASSWORD_CHECK_TIMEOUT_S = 2.0 +class _TabGone(Exception): + """The tab a capture was asked of went away while the capture was waiting: the browser answers it never.""" + + class BrowserControl: """agentenv's control of a browser over its pipe: at each launch it attaches to every page with flattened sessions and keeps a record of the browser since this server started, its open tabs in opened order, those closed and who @@ -750,6 +754,10 @@ async def frame(self, target: Optional[str], load_wait_s: float) -> tuple[str, s except DevToolsError as e: if not any(why in str(e) for why in FOR_THE_MOMENT) or loop.time() + FRAME_RETRY_S >= deadline: raise DeviceUnavailable(f"the tab couldn't be captured: {e}") from None + except _TabGone: + if loop.time() + FRAME_RETRY_S >= deadline: + raise DeviceUnavailable(f"the tab went away while its frame was taken, {FRAME_CAPTURE_S:g} s in all" + ) from None except ConnectionError as e: raise DeviceUnavailable(f"the tab couldn't be captured: {e}") from None await asyncio.sleep(FRAME_RETRY_S) @@ -776,7 +784,8 @@ def _held_frame(self, target: str) -> tuple[str, str, Optional[str], Optional[st async def _nudged_capture(self, session: str, timeout_s: float) -> dict: """The tab's JPEG from the first of its captures to answer: a second follows one unanswered after ``NUDGE_S``, - since a tab behind the active one answers only then, all within ``timeout_s``.""" + since a tab behind the active one answers only then, all within ``timeout_s``; _TabGone once the tab's session + is gone with none answered, as a closed tab's captures never are.""" loop = asyncio.get_running_loop() deadline = loop.time() + timeout_s @@ -791,11 +800,13 @@ def capture() -> asyncio.Future: if not done: captures.add(capture()) while True: - done, captures = await asyncio.wait(captures, return_when=asyncio.FIRST_COMPLETED) + done, captures = await asyncio.wait(captures, timeout=FRAME_RETRY_S, return_when=asyncio.FIRST_COMPLETED) if answers := [c.result() for c in done if c.exception() is None]: return answers[0] if not captures: raise next(iter(done)).exception() + if session not in self._sessions: + raise _TabGone() finally: for pending in captures: pending.cancel() diff --git a/packages/agentenv-protocol/tests/test_browser_recording.py b/packages/agentenv-protocol/tests/test_browser_recording.py index 3a23d94f8..e3376c88f 100644 --- a/packages/agentenv-protocol/tests/test_browser_recording.py +++ b/packages/agentenv-protocol/tests/test_browser_recording.py @@ -359,6 +359,37 @@ async def test_a_frame_whose_only_tab_is_attached_anew_only_after_a_retry_waits_ assert tab == "T1" and [s for s, _, _ in pipe.captured] == ["s-T1", "s-T1b"] +class HangingPipe(Pipe): + """A pipe whose captures of the sessions in ``hanging`` never answer, as a tab's that closes mid-capture.""" + + def __init__(self, *hanging: str) -> None: + super().__init__() + self.hanging = set(hanging) + + async def call(self, method, params=None, *, session_id=None, timeout_s=None): + if method == "Page.captureScreenshot" and session_id in self.hanging: + self.captured.append((session_id, params, asyncio.get_running_loop().time())) + await asyncio.sleep(timeout_s) + raise asyncio.TimeoutError() + return await super().call(method, params, session_id=session_id, timeout_s=timeout_s) + + +@pytest.mark.asyncio +async def test_a_frame_whose_tab_closes_while_its_capture_waits_is_taken_of_the_tab_left_at_once(): + pipe = HangingPipe("s-T2") + control = BrowserControl(pipe, "agent", (1280, 800)) + for tab in ("T1", "T2"): + pipe.emit("Target.targetCreated", {"targetInfo": {"targetId": tab, "type": "page", "url": "about:blank"}}) + pipe.emit("Target.attachedToTarget", {"sessionId": f"s-{tab}", "targetInfo": {"targetId": tab, "type": "page"}}) + loop = asyncio.get_running_loop() + loop.call_later(0.1, pipe.emit, "Target.detachedFromTarget", {"sessionId": "s-T2"}) + started = loop.time() + + _, _, tab, _ = await control.frame("T2", load_wait_s=0) + + assert tab == "T1" and pipe.captured[-1][0] == "s-T1" and loop.time() - started < 2 + + @pytest.mark.asyncio async def test_a_frame_the_browser_refuses_for_another_reason_is_missing_at_once(): pipe = RefusingPipe(refusals=5, why="Page.captureScreenshot: Internal error") diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index bee44425c..4ad96acb0 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -1375,6 +1375,88 @@ async def _cdp_agent_acts_behind(ws_url: str) -> str: return target +@pytest.mark.asyncio +async def test_a_browser_recording_that_ends_while_an_agents_command_is_in_flight_logs_it_unfinished(browser_computer): + with namespace_routing(): + deployed = await Env.get(browser_computer.id).deploy( + env_params={"app_settings": {"chromium": {"start_urls": [PAGE_URL]}}}) + context = TaskStepContext(instance_id=f"run-{uuid.uuid4().hex[:8]}", deployed_envs=[deployed]) + try: + context = await StartBrowserRecordingTaskStep(id="start", version=None, env_id=browser_computer.id, + name="flight").execute(context) + ws_url = (await _cdp(deployed, "pi"))["ws_url"] + async with _cdp_calls(ws_url, reply_s=300) as waiting, _cdp_calls(ws_url) as call: + target = await _page_target(call) + held = (await waiting("Target.attachToTarget", {"targetId": target, "flatten": True}))["result"]["sessionId"] + hanging = asyncio.ensure_future(waiting("Runtime.evaluate", { + "expression": "new Promise((done) => setTimeout(done, 120000))", "awaitPromise": True}, held)) + await asyncio.sleep(1) + session = (await call("Target.attachToTarget", {"targetId": target, "flatten": True}))["result"]["sessionId"] + for n in range(3): + await call("Runtime.evaluate", {"expression": f"document.body.append('after {n}')"}, session) + context = await EndBrowserRecordingTaskStep(id="end", version=None, name="flight", + fail_task_on_error=False).execute(context) + hanging.cancel() + finally: + report = await teardown_run(context) + grant_server(None, None).close() + entry = context.metadata[BROWSER_RECORDINGS]["flight"] + lines = _recorded_log(entry) + + evaluated = [line["action"] for line in lines if (line.get("action") or {}).get("method") == "Runtime.evaluate" + and line["action"].get("who") == "pi"] + assert not report.failed and (entry["state"], entry["log_missing"]) == ("ended", 0), entry + assert [bool(action.get("unfinished")) for action in evaluated] == [True, False, False, False], evaluated + + +@pytest.mark.asyncio +async def test_a_browser_recording_frames_the_active_tab_when_the_agents_tab_closes_mid_capture(browser_computer): + with namespace_routing(): + deployed = await Env.get(browser_computer.id).deploy( + env_params={"app_settings": {"chromium": {"start_urls": [PAGE_URL]}}}) + context = TaskStepContext(instance_id=f"run-{uuid.uuid4().hex[:8]}", deployed_envs=[deployed]) + try: + context = await StartBrowserRecordingTaskStep(id="start", version=None, env_id=browser_computer.id, + name="closing").execute(context) + async with _cdp_calls((await _cdp(deployed, "pi"))["ws_url"]) as call: + tab = (await call("Target.createTarget", {"url": BEHIND[0]}))["result"]["targetId"] + session = (await call("Target.attachToTarget", {"targetId": tab, "flatten": True}))["result"]["sessionId"] + await asyncio.sleep(2) + await call("Runtime.evaluate", {"expression": + "setTimeout(() => { const t = Date.now(); while (Date.now() - t < 8000) {} }, 200)"}, session) + await asyncio.sleep(1.5) + await call("Target.closeTarget", {"targetId": tab}) + await asyncio.sleep(3) + context = await EndBrowserRecordingTaskStep(id="end", version=None, name="closing", + fail_task_on_error=False).execute(context) + finally: + report = await teardown_run(context) + grant_server(None, None).close() + entry = context.metadata[BROWSER_RECORDINGS]["closing"] + manifest = _recorded_manifest(entry) + + frames = manifest["frames"] + missing = [(frame["index"], frame.get("error")) for frame in frames if not frame["uploaded"]] + assert not report.failed and (entry["state"], missing) == ("ended", []), missing + + +async def _page_target(call) -> str: + """The first page the browser has open, as a CDP agent finds it.""" + targets = (await call("Target.getTargets", {}))["result"]["targetInfos"] + return next(t["targetId"] for t in targets if t["type"] == "page") + + +def _recorded_manifest(entry: dict) -> dict: + universe = Artifact.get(entry["artifact"]["id"], version=entry["artifact"]["version"]) + return json.loads(get_config().get_object_store().get(Artifact.get(universe.file_artifact_ids["manifest.json"]).object_url)) + + +def _recorded_log(entry: dict) -> list[dict]: + universe = Artifact.get(entry["artifact"]["id"], version=entry["artifact"]["version"]) + text = get_config().get_object_store().get(Artifact.get(universe.file_artifact_ids["log.jsonl"]).object_url).decode() + return [json.loads(line) for line in text.splitlines() if line.strip()] + + async def _mcp(deployed, work): """What ``work`` makes of one MCP session on the computer, as a role-less agent opens one.""" async with streamablehttp_client(deployed.mcp_url) as (read, write, _):