Skip to content

Commit 6f8dc81

Browse files
fix(browser): a start tab already on its URL's page is reloaded there, so a load of cookies alone reaches it (#685)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent cabad0b commit 6f8dc81

6 files changed

Lines changed: 247 additions & 41 deletions

File tree

‎packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py‎

Lines changed: 53 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@
3333
ARRIVAL_POLL_S = 0.05
3434
#: How long a start tab on a site the env doesn't serve may take to load before it's left loading.
3535
LIVE_START_TAB_TIMEOUT_S = 30.0
36+
#: How long a start tab's move within its document may take to land after the browser answers it, before its reload.
37+
MOVE_WITHIN_S = 5.0
3638
#: How a start tab settled.
3739
LOADED, FAILED, TIMED_OUT = "loaded", "failed", "timed_out"
3840
#: How much of a download each read hashes.
@@ -326,7 +328,8 @@ async def load_start_tabs(self, urls: list[str], site_of: Callable[[str], Option
326328
as ``site_of`` names them, an error status is too, or once it has been waited on ``own_timeout_s`` on such a
327329
site, else ``live_timeout_s``. Each tab's outcome, in order, kept though the browser has gone and the first
328330
can't come to the front. A page already loaded doesn't read cookies set after it, so a tab a load signed in
329-
shows it only once it loads again."""
331+
shows it only once it loads again: a tab already on its URL's document, as a hash-routed page, is reloaded
332+
there, since going to the URL alone only moves within the document (``_start_navigation``)."""
330333
if not urls:
331334
return []
332335
targets = [target for target, tab in self.tabs.items()
@@ -357,7 +360,8 @@ async def _first_tab(self) -> str:
357360
async def _start_load(self, target: str, url: str, site: Optional[str]) -> _StartLoad:
358361
"""The tab ``target`` sent to ``url``, its navigation followed by what the browser says of it. The browser
359362
answers a navigation once it commits, which a page that never answers never does, so its answer is awaited
360-
apart, for as long as the tab is."""
363+
apart, for as long as the tab is, and a navigation that stays in the tab's document is followed by a reload
364+
(``_start_navigation``)."""
361365
session = next((s for s, t in self._sessions.items() if t == target), None)
362366
if session is None:
363367
raise DeviceUnavailable(f"the browser's tab {target} has no session to load a start tab in")
@@ -368,14 +372,43 @@ async def _start_load(self, target: str, url: str, site: Optional[str]) -> _Star
368372
load.error = f"{type(e).__name__}: {e}"
369373
load.check()
370374
return load
371-
landed = asyncio.get_running_loop().create_future()
372-
self.causes.navigating(target, landed, _cause())
373-
load.navigating = asyncio.ensure_future(self.devtools.request("Page.navigate", {"url": url},
374-
session_id=session))
375+
load.navigating = asyncio.ensure_future(self._start_navigation(target, load, session))
375376
load.navigating.add_done_callback(load.answered)
376-
load.navigating.add_done_callback(lambda done: _landed_at(landed, done))
377377
return load
378378

379+
async def _start_navigation(self, target: str, load: _StartLoad, session: str) -> dict:
380+
"""``load``'s tab sent to its URL, which the log says who asked for: the browser's answer. A tab already on the
381+
URL's document, as a hash-routed page is, only moves within it, which loads nothing and names no loader, so the
382+
page wouldn't read what a load just set: it's reloaded there once the move has landed, which the browser
383+
answers before it does (else the reload loads where the tab was), the load taking its main frame's first
384+
document after for its own, and the log that navigation for the reload."""
385+
cause, landed = _cause(), asyncio.get_running_loop().create_future()
386+
self.causes.navigating(target, landed, cause)
387+
try:
388+
answer = await self.devtools.request("Page.navigate", {"url": load.url}, session_id=session)
389+
except BaseException:
390+
landed.cancel()
391+
raise
392+
result = answer.get("result") or {}
393+
if "error" in answer or result.get("errorText"):
394+
landed.cancel()
395+
return answer
396+
landed.set_result(result.get("loaderId"))
397+
if result.get("loaderId"):
398+
return answer
399+
with contextlib.suppress(asyncio.TimeoutError):
400+
await asyncio.wait_for(load.moved.wait(), MOVE_WITHIN_S)
401+
load.reloading = True
402+
self.causes.navigates_next(target, cause)
403+
try:
404+
reloaded = await self.devtools.request("Page.reload", {}, session_id=session)
405+
except BaseException:
406+
self.causes.navigated_nowhere(target, cause)
407+
raise
408+
if "error" in reloaded:
409+
self.causes.navigated_nowhere(target, cause)
410+
return reloaded
411+
379412
async def _settled(self, target: str, load: _StartLoad, timeout_s: float) -> dict:
380413
"""``load``'s outcome once it settles, or once it has been waited on ``timeout_s``."""
381414
try:
@@ -1039,6 +1072,8 @@ def _on_Page_frameNavigated(self, params: dict, session: Optional[str]) -> None:
10391072
self._spawn(self._log_navigated(self.log.reserve(), log_time(), target, frame.get("url", ""),
10401073
frame.get("loaderId")))
10411074
if (load := self._loading.get(session)) is not None and not frame.get("parentId") and frame.get("loaderId"):
1075+
if load.reloading and load.loader is None:
1076+
load.loader = frame["loaderId"]
10421077
load.commits.append(frame["loaderId"])
10431078
load.check()
10441079

@@ -1050,14 +1085,18 @@ def _on_Page_frameAttached(self, params: dict, session: Optional[str]) -> None:
10501085

10511086
def _on_Page_navigatedWithinDocument(self, params: dict, session: Optional[str]) -> None:
10521087
"""A tab's page moving to another URL of its document, a fragment or its history API: a command's, when one
1053-
asked for it, else its page's."""
1088+
asked for it, else its page's; a start tab's move before its reload."""
10541089
target = self._sessions.get(session)
1090+
if (load := self._loading.get(session)) is not None and params.get("frameId") == target:
1091+
load.moved.set()
10551092
if self.log is not None and target in self.tabs and params.get("frameId") == target:
10561093
self._spawn(self._log_navigated(self.log.reserve(), log_time(), target, params.get("url", ""), False))
10571094

10581095
def _on_Network_requestWillBeSent(self, params: dict, session: Optional[str]) -> None:
10591096
if (load := self._loading.get(session)) is not None and params.get("type") == "Document":
10601097
load.requests[params["requestId"]] = params.get("loaderId", "")
1098+
if load.reloading and load.loader is None and params.get("frameId") == self._sessions.get(session):
1099+
load.loader = params.get("loaderId") or None
10611100

10621101
def _on_Network_responseReceived(self, params: dict, session: Optional[str]) -> None:
10631102
if (load := self._loading.get(session)) is not None and params.get("type") == "Document" and \
@@ -1228,7 +1267,9 @@ class _StartLoad:
12281267
commits in order, the loads that fired after a commit, and its documents' requests, final responses and failures.
12291268
Its document is the last its main frame committed from its own navigation on, which a page that sends itself
12301269
elsewhere before it loads replaces. It's ``settled`` once its navigation or its document has failed, its document
1231-
has loaded or, on one of the env's ``site``, answered an error."""
1270+
has loaded or, on one of the env's ``site``, answered an error. A ``reloading`` one's navigation stayed in its
1271+
document and was followed by a reload, whose answer names no loader either: its loader is its main frame's first
1272+
document request or commit after, so a reload that fails before it commits fails the load."""
12321273
session: str
12331274
url: str
12341275
site: Optional[str]
@@ -1242,6 +1283,8 @@ class _StartLoad:
12421283
failures: dict[str, str] = field(default_factory=dict)
12431284
settled: asyncio.Event = field(default_factory=asyncio.Event)
12441285
navigating: Optional[asyncio.Future] = None
1286+
reloading: bool = False
1287+
moved: asyncio.Event = field(default_factory=asyncio.Event)
12451288

12461289
def answered(self, navigating: asyncio.Future) -> None:
12471290
"""The browser's answer to its navigation: its loader, or why it couldn't navigate."""
@@ -1255,7 +1298,7 @@ def answered(self, navigating: asyncio.Future) -> None:
12551298
self.error = str(answer["error"].get("message", answer["error"]))
12561299
else:
12571300
self.error = answer.get("result", {}).get("errorText") or None
1258-
self.loader = answer.get("result", {}).get("loaderId")
1301+
self.loader = answer.get("result", {}).get("loaderId") or self.loader
12591302
self.check()
12601303

12611304
def current(self) -> Optional[str]:
@@ -1274,18 +1317,6 @@ def check(self) -> None:
12741317
self.settled.set()
12751318

12761319

1277-
def _landed_at(landed: asyncio.Future, navigating: asyncio.Future) -> None:
1278-
"""``landed`` given the loader a start tab's navigation answered with; cancelled when it failed or named none."""
1279-
if landed.done():
1280-
return
1281-
answer = None if navigating.cancelled() or navigating.exception() is not None else navigating.result()
1282-
loader = ((answer or {}).get("result") or {}).get("loaderId")
1283-
if isinstance(loader, str):
1284-
landed.set_result(loader)
1285-
else:
1286-
landed.cancel()
1287-
1288-
12891320
def _types(event: dict) -> bool:
12901321
"""Whether a page input event types something."""
12911322
return event.get("method") in TYPING_METHODS and any(field in (event.get("params") or {}) for field in KEY_FIELDS)

‎packages/agentenv-protocol/src/agentenv_protocol/website_browser/log.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -563,12 +563,13 @@ async def navigated(self, target: str, loader: Optional[str]) -> Optional[dict]:
563563

564564
async def navigated_within(self, target: str) -> Optional[dict]:
565565
"""What moved ``target`` within its document, once its navigations in flight have answered: a navigation
566-
answered with no loader, else a reload or move in history, which it uses up."""
566+
answered with no loader, else a reload or move in history, which it uses up; one asked for after that
567+
navigation, as a start tab's reload is, is left for the navigation it makes."""
567568
if target not in self._same_document and self._navigating.get(target):
568569
await asyncio.wait(set(self._navigating[target]), timeout=CAUSE_WAIT_S)
569-
pending = self._next.pop(target, None)
570570
if (cause := self._same_document.pop(target, None)) is not None:
571571
return cause
572+
pending = self._next.pop(target, None)
572573
if pending is not None and pending[1] >= asyncio.get_running_loop().time():
573574
return pending[0]
574575
return None

‎packages/agentenv-protocol/tests/test_browser_log.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,21 @@ async def test_a_move_within_a_document_is_the_commands_that_asked_for_it_and_us
281281
assert await causes.navigated_within("T1") is None
282282

283283

284+
@pytest.mark.asyncio
285+
async def test_a_reload_asked_for_once_a_move_within_a_document_answered_is_left_for_the_navigation_it_makes():
286+
causes = browser_log.Causes()
287+
moved, reload = {"seq": 3, "via": "agentenv"}, {"seq": 3, "via": "agentenv", "reload": True}
288+
answer = asyncio.get_running_loop().create_future()
289+
causes.navigating("T1", answer, moved)
290+
within = asyncio.ensure_future(causes.navigated_within("T1"))
291+
await asyncio.sleep(0)
292+
answer.set_result(None)
293+
causes.navigates_next("T1", reload)
294+
295+
assert await within == moved
296+
assert await causes.navigated("T1", "L-reload") == reload
297+
298+
284299
@pytest.mark.asyncio
285300
async def test_a_move_back_with_nowhere_to_go_leaves_no_cause_behind(tmp_path):
286301
pipe = Pipe()

‎packages/agentenv-protocol/tests/test_website_browser.py‎

Lines changed: 114 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import os
1414
import signal
1515
import sys
16+
from urllib.parse import urldefrag, urlsplit, urlunsplit
1617

1718
import pytest
1819

@@ -610,12 +611,24 @@ class Navigating(StubDevTools):
610611
"""A browser whose first tab is ``start`` and whose new tabs are t1, t2, …, each attached as it opens, answering a
611612
navigation as ``pages`` says by URL: a status and the URL it ends on, then committed and, unless ``no-load``
612613
follows, loaded, or with ``("redirect", status, url)`` following, sending itself there by script before it loads;
613-
``refused``, a network error; ``silent``, never answered, as a page that never answers isn't; or ``stale``, a load
614-
event with no commit."""
614+
``refused``, a network error; ``aborted``, a document request that fails before it commits; ``silent``, never
615+
answered, as a page that never answers isn't; or ``stale``, a load event with no commit. A navigation that stays in
616+
a tab's document, to a URL with a fragment that differs from where it is (``at``) in its fragment alone, as a
617+
browser spells both (``_document``), moves within it and names no loader, as a browser's does; a reload loads where
618+
the tab is as a navigation there would, naming no loader either."""
615619

616-
def __init__(self, pages):
620+
#: How long after the browser answers them a move within a document lands, and a reload starts loading.
621+
MOVE_LANDS_S, RELOAD_LOADS_S = 0.05, 0.01
622+
623+
def __init__(self, pages, reload_answers=True):
617624
super().__init__({"Target.getTargets": {"targetInfos": []}})
618-
self.pages, self.opened, self.loaders = pages, 0, 0
625+
self.pages, self.opened, self.loaders, self.urls, self.pending = pages, 0, 0, {}, []
626+
self.reload_answers = reload_answers
627+
628+
def at(self, target, url):
629+
"""The tab ``target`` at ``url``, as the browser tells the control."""
630+
self.urls[target] = url
631+
self.send("Target.targetInfoChanged", _page(target, url))
619632

620633
async def call(self, method, params=None, *, session_id=None, timeout_s=None):
621634
if method == "Target.createTarget":
@@ -626,12 +639,42 @@ async def call(self, method, params=None, *, session_id=None, timeout_s=None):
626639
self.send("Target.attachedToTarget", {"sessionId": f"s-{target}", "waitingForDebugger": True,
627640
**_page(target)})
628641
return {"targetId": target}
642+
if method == "Page.reload":
643+
self.calls.append((method, params, session_id))
644+
if not self.reload_answers:
645+
await asyncio.Event().wait()
646+
self.pending.append(asyncio.ensure_future(self._reload(self.urls[session_id[2:]], session_id)))
647+
return {}
629648
if method != "Page.navigate":
630649
return await super().call(method, params, session_id=session_id, timeout_s=timeout_s)
631650
self.calls.append((method, params, session_id))
632-
page, target = self.pages[params["url"]], session_id[2:]
651+
url, target = params["url"], session_id[2:]
652+
if "#" in url and _document(url) == _document(self.urls.get(target, "")):
653+
self.pending.append(asyncio.ensure_future(self._move_within(target, url, session_id)))
654+
return {"frameId": target}
655+
return await self._load(url, session_id)
656+
657+
async def _move_within(self, target, url, session_id):
658+
"""A move within a tab's document, which the browser answers before it lands."""
659+
await asyncio.sleep(self.MOVE_LANDS_S)
660+
self.send("Page.navigatedWithinDocument", {"frameId": target, "url": url}, session_id)
661+
self.at(target, url)
662+
663+
async def _reload(self, url, session_id):
664+
"""A reload, which the browser answers before it commits."""
665+
await asyncio.sleep(self.RELOAD_LOADS_S)
666+
await self._load(url, session_id)
667+
668+
async def _load(self, url, session_id):
669+
page, target = self.pages[url], session_id[2:]
670+
self.urls[target] = url
633671
self.loaders += 1
634672
loader, request = f"L{self.loaders}", f"r{self.loaders}"
673+
if page == "aborted":
674+
document = {"requestId": request, "loaderId": loader, "type": "Document", "frameId": target}
675+
self.send("Network.requestWillBeSent", document, session_id)
676+
self.send("Network.loadingFailed", {"requestId": request, "errorText": "net::ERR_ABORTED"}, session_id)
677+
return {"frameId": target, "loaderId": loader, "errorText": "net::ERR_ABORTED"}
635678
if page == "refused":
636679
return {"frameId": target, "loaderId": loader, "errorText": "net::ERR_CONNECTION_REFUSED"}
637680
if page == "silent":
@@ -655,6 +698,12 @@ def _document(self, session, target, loader, request, status, url):
655698
self.send("Page.frameNavigated", {"frame": {"id": target, "loaderId": loader, "url": url}}, session)
656699

657700

701+
def _document(url):
702+
"""The document ``url`` names, as a browser spells it: without its fragment, an empty path as "/"."""
703+
parts = urlsplit(urldefrag(url)[0])
704+
return urlunsplit(parts._replace(path=parts.path or "/"))
705+
706+
658707
async def _started(devtools, viewport=(1280, 800)) -> BrowserControl:
659708
"""A control following a browser just launched, on its first tab, ``start``."""
660709
control = BrowserControl(devtools, "agent", viewport)
@@ -736,6 +785,66 @@ async def test_a_start_page_that_sends_itself_elsewhere_before_it_loads_is_judge
736785
assert (tab["final_url"], tab["status"], tab["outcome"]) == (final, status, outcome)
737786

738787

788+
@pytest.mark.asyncio
789+
@pytest.mark.parametrize("at, url", [("https://shop.test/#/orders", "https://shop.test/#/orders"),
790+
("https://shop.test/#/cart", "https://shop.test/#/orders"),
791+
("https://shop.test/#/orders", "https://shop.test#/orders")],
792+
ids=["at-its-url", "at-another-route-of-its-page", "at-its-url-spelled-otherwise"])
793+
async def test_a_start_tab_already_on_its_pages_document_is_reloaded_there_and_loads_again(at, url):
794+
devtools = Navigating({url: (200, url)})
795+
control = await _started(devtools)
796+
devtools.at("start", at)
797+
798+
(tab,) = await control.load_start_tabs([url], _shop, own_timeout_s=2)
799+
800+
assert (tab["outcome"], tab["status"], tab["final_url"], tab["error"]) == ("loaded", 200, url, None)
801+
assert [(method, params) for method, params, on in devtools.calls if on == "s-start" and method in (
802+
"Page.navigate", "Page.reload")] == [("Page.navigate", {"url": url}), ("Page.reload", {})]
803+
assert not control._loading
804+
805+
806+
@pytest.mark.asyncio
807+
async def test_a_start_tab_whose_reload_fails_before_it_commits_fails_at_once_naming_why():
808+
url = "https://shop.test/#/orders"
809+
devtools = Navigating({url: "aborted"})
810+
control = await _started(devtools)
811+
devtools.at("start", url)
812+
loop = asyncio.get_running_loop()
813+
started = loop.time()
814+
815+
(tab,) = await control.load_start_tabs([url], _shop, own_timeout_s=2)
816+
817+
assert (tab["outcome"], tab["error"]) == ("failed", "net::ERR_ABORTED")
818+
assert loop.time() - started < 1
819+
820+
821+
@pytest.mark.asyncio
822+
async def test_a_start_tab_whose_reload_never_answers_times_out_and_leaves_no_cause_for_a_later_navigation():
823+
url = "https://shop.test/#/orders"
824+
devtools = Navigating({url: (200, url)}, reload_answers=False)
825+
control = await _started(devtools)
826+
devtools.at("start", url)
827+
828+
(tab,) = await control.load_start_tabs([url], _shop, own_timeout_s=0.5)
829+
await asyncio.sleep(0)
830+
831+
assert tab["outcome"] == "timed_out"
832+
assert "start" not in control.causes._next
833+
834+
835+
@pytest.mark.asyncio
836+
async def test_a_start_tab_on_another_page_of_its_site_navigates_and_isnt_reloaded():
837+
url = "https://shop.test/#/orders"
838+
devtools = Navigating({url: (200, url)})
839+
control = await _started(devtools)
840+
devtools.at("start", "https://shop.test/account")
841+
842+
(tab,) = await control.load_start_tabs([url], _shop, own_timeout_s=2)
843+
844+
assert (tab["outcome"], tab["final_url"]) == ("loaded", url)
845+
assert (_sent(devtools, "Page.navigate"), _sent(devtools, "Page.reload")) == ([({"url": url}, "s-start")], [])
846+
847+
739848
@pytest.mark.asyncio
740849
async def test_a_start_tab_still_loading_when_the_browser_ends_fails_naming_that_and_is_kept_with_its_pipe_gone():
741850
def gone(params, session):

0 commit comments

Comments
 (0)