diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/svc.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/svc.py index 1581e1acb..febbb94a7 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/svc.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/svc.py @@ -188,13 +188,17 @@ def _utf8(value: str) -> str: async def _both_ways(client: WebSocket, app: ClientConnection) -> None: - """Each message from either side sent to the other until one side closes, then the other closed with its code.""" + """Each message from either side sent to the other until one side closes, then the other closed with its code and + reason.""" async def inbound() -> None: while True: message = await client.receive() if message["type"] == "websocket.disconnect": code = message.get("code") or 1000 - await app.close(1000 if code in UNSENDABLE_CLOSES else code) + if code in UNSENDABLE_CLOSES: + await app.close(1000) + else: + await app.close(code, message.get("reason") or "") return await app.send(message["text"] if message.get("text") is not None else message["bytes"]) @@ -207,7 +211,10 @@ async def outbound() -> None: await client.send_bytes(message) code = app.close_code or 1000 with contextlib.suppress(RuntimeError, WebSocketDisconnect): - await client.close(1000 if code in UNSENDABLE_CLOSES else code) + if code in UNSENDABLE_CLOSES: + await client.close(1000) + else: + await client.close(code, app.close_reason or "") tasks = [asyncio.ensure_future(inbound()), asyncio.ensure_future(outbound())] try: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/cdp.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/cdp.py index 562a8e4e1..5fd0cac85 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/cdp.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/cdp.py @@ -18,8 +18,10 @@ from starlette.requests import Request from starlette.websockets import WebSocket, WebSocketDisconnect +from .devtools import DevToolsError from .log import (ERROR_CHARS, KEY_FIELDS, TYPING_METHODS, BrowserLog, acts, redact_text, redacted_args, redacted_params) +from .scope import ENDS_CONTEXT, MAKES_CONTEXT, WITH_INFO, Scope, Targets if TYPE_CHECKING: from .contexts import Contexts @@ -34,6 +36,8 @@ ROUTE = "/cdp/" #: The close code and reason a connection gets when its browser goes away, which a client may connect again after. RELAUNCHED = (1012, "the browser relaunched: connect again") +#: The close code and reason a connection scoped to a lease gets when the lease ends, which its address ends with. +LEASE_ENDED = (1001, "the lease ended: ask for an address again") #: The close code and reason a connection gets when its client falls behind by more than ``OUTBOX_LIMIT_BYTES``. TOO_SLOW = (1013, "the client fell too far behind: connect again") #: How much a connection's client may have waiting to be sent to it before it is closed; a single answer bigger than @@ -52,8 +56,6 @@ ENDS_BROWSER = frozenset({"Browser.close", "Browser.crash", "Browser.crashGpuProcess"}) #: The commands that set how and where downloads are saved, which agentenv keeps as its own. SETS_DOWNLOADS = frozenset({"Browser.setDownloadBehavior", "Page.setDownloadBehavior"}) -#: The commands that make and end a browser context, which agentenv records as leases. -MAKES_CONTEXT, ENDS_CONTEXT = "Target.createBrowserContext", "Target.disposeBrowserContext" #: The schemes of the browser's own files, settings and tools, which an agent's navigation or new tab may not open. OWN_SCHEMES = frozenset({"file", "chrome", "chrome-extension", "chrome-untrusted", "view-source", "devtools"}) #: The ``about:`` pages that aren't the browser's own under another name. @@ -77,6 +79,8 @@ class Address: http_url: str ws_url: str named: bool = False + #: The label of the lease it's scoped to, None for the browser's default context. + context: Optional[str] = None def reached(request: Request) -> str: @@ -102,19 +106,22 @@ def _hash(token: str) -> str: return hashlib.sha256(token.encode()).hexdigest() -def _address(user: str, base: str, named: bool, token: str) -> Address: +def _address(user: str, base: str, named: bool, token: str, context: Optional[str] = None) -> Address: root = f"{base}{ROUTE}{token}" - return Address(user, f"{root}/", f"{websocket_form(root)}/devtools/browser", named) + return Address(user, f"{root}/", f"{websocket_form(root)}/devtools/browser", named, context) class Tokens: """The tokens minted, one per user: in memory, so asking twice gives the same address, and in ``path`` only by hash, with the user, the base address and whether its caller named it, so an address still works after the server - restarts while the computer's session, which can read the file, can't use one.""" + restarts while the computer's session, which can read the file, can't use one. A token scoped to a lease is one per + user and lease, in memory only, as no lease outlives the server; the lease's end revokes it.""" def __init__(self, path: Path) -> None: self.path = path self._minted: dict[str, Address] = {} + self._scoped: dict[tuple[str, str], Address] = {} + self._scoped_known: dict[str, tuple[str, str, bool, str]] = {} try: stored = json.loads(path.read_text()) self._known = {digest: (entry["user"], entry["base"], entry["named"]) @@ -122,9 +129,15 @@ def __init__(self, path: Path) -> None: except (OSError, ValueError, TypeError, KeyError): self._known = {} - def mint(self, user: str, base: str, *, named: bool = False) -> Address: - """``user``'s address under ``base``, ``named`` when its caller named where its agent reaches the env: the one - minted before, else a new one.""" + def mint(self, user: str, base: str, *, named: bool = False, context: Optional[str] = None) -> Address: + """``user``'s address under ``base``, ``named`` when its caller named where its agent reaches the env, scoped + to the lease ``context`` when it names one: the one minted before, else a new one.""" + if context is not None: + if (user, context) not in self._scoped: + token = secrets.token_urlsafe(TOKEN_BYTES) + self._scoped[(user, context)] = _address(user, base, named, token, context) + self._scoped_known[_hash(token)] = (user, base, named, context) + return self._scoped[(user, context)] if user not in self._minted: token = secrets.token_urlsafe(TOKEN_BYTES) self._minted[user] = _address(user, base, named, token) @@ -134,13 +147,22 @@ def mint(self, user: str, base: str, *, named: bool = False) -> Address: def revoke_all(self) -> None: """Every address minted so far ended, as a reset ends them.""" - self._minted, self._known = {}, {} + self._minted, self._known, self._scoped, self._scoped_known = {}, {}, {}, {} self._save() + def revoke(self, context: str) -> None: + """Every address scoped to the lease ``context`` ended, as the lease has.""" + self._scoped = {key: address for key, address in self._scoped.items() if key[1] != context} + self._scoped_known = {digest: known for digest, known in self._scoped_known.items() if known[3] != context} + def address(self, token: str) -> Optional[Address]: - """The address ``token`` was minted for, or None for one that wasn't.""" - known = self._known.get(_hash(token)) - return _address(*known, token) if known is not None else None + """The address ``token`` was minted for, or None for one that wasn't, or whose lease has ended.""" + digest = _hash(token) + if (known := self._known.get(digest)) is not None: + return _address(*known, token) + if (scoped := self._scoped_known.get(digest)) is not None: + return _address(*scoped[:3], token, scoped[3]) + return None def _save(self) -> None: self.path.parent.mkdir(parents=True, exist_ok=True) @@ -159,6 +181,8 @@ class Verdict: params: dict reason: str = "" changed: bool = False + #: The error a refusal answers with in place of agentenv's own, as for a target the connection doesn't see. + error: Optional[dict] = None def guard(method: str, params: dict, *, dialogs_answered: bool, download_path: Optional[str] = None) -> Verdict: @@ -204,11 +228,14 @@ class _Connection: """One client's WebSocket, its sessions over the pipe (its browser-level one and those it attached), its unanswered commands, what waits to be sent to it, and its entry in the record.""" - def __init__(self, websocket: WebSocket, user: str, session: str, record: dict, number: int) -> None: + def __init__(self, websocket: WebSocket, user: str, session: str, record: dict, number: int, + scope: Optional[Scope] = None) -> None: self.websocket, self.user, self.session, self.record, self.number = websocket, user, session, record, number + #: What it sees and acts in, a lease or the default context; None where the relay keeps no contexts. + self.scope = scope self.sessions = {session} #: The tab each session it attached is on. - self.targets: dict[str, str] = {} + self.session_tabs: dict[str, str] = {} #: Held by each of its commands from when it arrives until it's sent, so they're sent in the order they came, #: whatever one waits on before it's sent. self.order = asyncio.Lock() @@ -251,12 +278,16 @@ class Relay: causes the tab events a command brings. An agent's tool call a driver serves through a CDP user's connections, such as Playwright MCP's, is an entry too, which that user's commands are credited to while it's open. With ``contexts``, a context a connection makes is leased to its user, only one it made may it dispose, and a lease's - downloads stay in its folder.""" + downloads stay in its folder. A connection is scoped to a lease or to the default context, and sees only its scope + and the contexts its user made (``scope``).""" def __init__(self, devtools: DevToolsPipe, *, dialogs_answered: bool = False, log: Optional[BrowserLog] = None, control: Optional[BrowserControl] = None, contexts: Optional[Contexts] = None) -> None: self.devtools, self.dialogs_answered, self.log, self.control = devtools, dialogs_answered, log, control self.contexts = contexts + #: Each target's context, as Chromium reports it. + self.target_contexts = Targets() + self._tasks: set[asyncio.Future] = set() self.connections: list[dict] = [] self._by_session: dict[str, _Connection] = {} self._open: set[_Connection] = set() @@ -291,15 +322,32 @@ def end_call(self, seq: int, ok: bool, error: Optional[str] = None) -> bool: self.log.finish(call.begun, **fields) return True - async def serve(self, websocket: WebSocket, user: str) -> None: - """``websocket``, a client of ``user``'s, relayed over a browser-level session of its own until either ends.""" + async def serve(self, websocket: WebSocket, user: str, context: Optional[str] = None) -> None: + """``websocket``, a client of ``user``'s, relayed over a browser-level session of its own until either ends, + scoped to the lease ``context`` when it names one.""" + lease_id = None + if context is not None: + lease = self.contexts.get(context) if self.contexts is not None else None + if lease is None: + await websocket.accept() + await websocket.close(code=LEASE_ENDED[0], reason=LEASE_ENDED[1]) + return + lease_id = lease.id try: session = (await self.devtools.call("Target.attachToBrowserTarget"))["sessionId"] except Exception as e: await websocket.close(code=1013, reason=f"the browser isn't answering: {type(e).__name__}") return - record = {"user": user, "opened_at": _now(), "guarded": []} - connection = _Connection(websocket, user, session, record, len(self.connections) + 1) + if context is not None and ((lease := self.contexts.get(context)) is None or lease.id != lease_id): + with contextlib.suppress(Exception): + await self.devtools.call("Target.detachFromTarget", {"sessionId": session}) + await websocket.accept() + await websocket.close(code=LEASE_ENDED[0], reason=LEASE_ENDED[1]) + return + record = {"user": user, "opened_at": _now(), "guarded": [], **({"context": context} if context else {})} + scope = Scope(self.contexts, self.target_contexts, user, lease_id, self._frame_context) \ + if self.contexts is not None else None + connection = _Connection(websocket, user, session, record, len(self.connections) + 1, scope) self._by_session[session] = connection self._open.add(connection) self.connections.append(record) @@ -331,6 +379,14 @@ def ended(self) -> None: """The browser has gone: every connection is closed with a reason a client may connect again after.""" for connection in list(self._open): self._close(connection, RELAUNCHED) + self.target_contexts.forget() + + def lease_ended(self, context_id: str) -> None: + """The lease the browser knows as ``context_id`` has ended: each connection scoped to it is closed, saying + so.""" + for connection in list(self._open): + if connection.scope is not None and connection.scope.lease == context_id: + self._close(connection, LEASE_ENDED) async def _command(self, connection: _Connection, message: Any) -> None: if connection.closing: @@ -351,6 +407,8 @@ def reply(answer: dict) -> None: replied = True if sending is not None and isinstance(answer.get("result"), dict): self._answered(connection, method, params, answer["result"]) + if connection.scope is not None: + answer = {**answer, "result": connection.scope.answer(method, answer["result"])} self._put(connection, _reply(message, answer)) try: @@ -360,8 +418,8 @@ def reply(answer: dict) -> None: if session not in connection.sessions: verdict, answer = None, {"error": NO_SESSION} elif verdict.action == "refuse": - answer = {"error": {"code": SERVER_ERROR, - "message": f"agentenv refuses {method}: {verdict.reason}"}} + answer = {"error": verdict.error or {"code": SERVER_ERROR, + "message": f"agentenv refuses {method}: {verdict.reason}"}} elif verdict.action != "send": answer = {"result": {}} else: @@ -387,24 +445,28 @@ def reply(answer: dict) -> None: logged.finish(answer, guarded) def _verdict(self, connection: _Connection, session: str, method: str, params: dict) -> Verdict: - """The guard's verdict on ``connection``'s command, with what its contexts say: where a lease's downloads - go, and that only a context it made may it dispose.""" - if self.contexts is None: + """The guard's verdict on ``connection``'s command, after its scope's (``Scope.verdict``), and with where a + lease's downloads go.""" + if connection.scope is None: return guard(method, params, dialogs_answered=self.dialogs_answered) - if method == ENDS_CONTEXT and (reason := self.contexts.may_dispose(connection.user, - params.get("browserContextId"))): - return Verdict("refuse", params, reason) + ruling = connection.scope.verdict(method, params, on_browser_session=session == connection.session) + if ruling.refused: + return Verdict("refuse", params, ruling.refused, error=ruling.error) + params = ruling.params download_path = None if method in SETS_DOWNLOADS: connection.set_downloads = True context = params.get("browserContextId") if method.startswith("Browser.") else \ - self._context_of(connection.targets.get(session)) + self._context_of(connection.session_tabs.get(session)) download_path = self.contexts.download_path(context) return guard(method, params, dialogs_answered=self.dialogs_answered, download_path=download_path) def _context_of(self, target: Optional[str]) -> Optional[str]: return self.control.tabs.get(target, {}).get("context_id") if self.control is not None and target else None + def _frame_context(self, frame: object) -> Optional[str]: + return self.control.context_of_frame(frame) if self.control is not None else None + def _answered(self, connection: _Connection, method: str, params: dict, result: dict) -> None: """A context ``connection`` made leased to its user before it hears the answer, and one it disposed ended.""" if self.contexts is None: @@ -432,7 +494,7 @@ def _begin(self, connection: _Connection, session: str, method: str, params: dic control.""" if self.log is None: return None - target = connection.targets.get(session) if session != connection.session else None + target = connection.session_tabs.get(session) if session != connection.session else None if target is None and isinstance(params.get("targetId"), str): target = params["targetId"] acting = acts(method) @@ -451,23 +513,53 @@ def _guarded(connection: _Connection, method: str, verdict: Verdict) -> None: guarded.append({"method": method, "did": GUARDED[verdict.action], "at": _now()}) def _event(self, message: dict) -> None: + params = message.get("params") or {} + if message.get("method") in WITH_INFO: + self.target_contexts.saw(params.get("targetInfo")) connection = self._by_session.get(message.get("sessionId")) if connection is None: return - params = message.get("params") or {} + if message.get("sessionId") == connection.session and not self._hears(connection, message): + return if message.get("method") == "Target.attachedToTarget" and params.get("sessionId"): connection.sessions.add(params["sessionId"]) self._by_session[params["sessionId"]] = connection if isinstance(target := (params.get("targetInfo") or {}).get("targetId"), str): - connection.targets[params["sessionId"]] = target + connection.session_tabs[params["sessionId"]] = target elif message.get("method") == "Target.detachedFromTarget" and params.get("sessionId"): connection.sessions.discard(params["sessionId"]) - connection.targets.pop(params["sessionId"], None) + connection.session_tabs.pop(params["sessionId"], None) self._by_session.pop(params["sessionId"], None) if message["sessionId"] == connection.session: message = {key: value for key, value in message.items() if key != "sessionId"} self._put(connection, message) + def _hears(self, connection: _Connection, message: dict) -> bool: + """Whether ``connection`` hears an event on its browser session, as its scope says: never one about a session it + doesn't have, and a target it doesn't see that auto-attached to it let go.""" + if connection.scope is None: + return True + method, params = message.get("method"), message.get("params") or {} + if method == "Target.detachedFromTarget": + return params.get("sessionId") in connection.sessions + heard, let_go = connection.scope.hears(method, params) + if let_go is not None: + self._spawn(self._let_go(connection.session, let_go, params.get("waitingForDebugger"))) + return heard + + async def _let_go(self, owner: str, session: str, waiting: object) -> None: + """A target auto-attached to the browser session ``owner`` as ``session`` that its connection doesn't see: told + to run when it waits for its debugger, so it isn't held, and detached.""" + with contextlib.suppress(ConnectionError, DevToolsError): + if waiting: + await self.devtools.call("Runtime.runIfWaitingForDebugger", session_id=session) + await self.devtools.call("Target.detachFromTarget", {"sessionId": session}, session_id=owner) + + def _spawn(self, work) -> None: + task = asyncio.ensure_future(work) + self._tasks.add(task) + task.add_done_callback(self._tasks.discard) + def _put(self, connection: _Connection, message: dict) -> None: """``message`` waiting to be sent to ``connection``'s client, which is closed instead, with what waited dropped, once it has fallen too far behind.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/contexts.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/contexts.py index f2fe63d5d..9a5e3cbee 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/contexts.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/contexts.py @@ -79,6 +79,8 @@ def __init__(self, devtools: DevToolsPipe, control: BrowserControl, home: Callable[[], Path] = Path.home) -> None: self.devtools, self.control, self.home = devtools, control, home self.leases: dict[str, Lease] = {} + #: What hears of each lease's end, whatever ends it: its release, its disposal or the browser's. + self.on_end: list[Callable[[Lease], None]] = [] self._made: dict[str, int] = {} self._tasks: set[asyncio.Future] = set() self._lock = asyncio.Lock() @@ -135,6 +137,7 @@ async def release(self, label: str) -> bool: if (lease := self.leases.pop(label, None)) is None: return False self.control.closing_context(lease.id) + self._ended(lease) with contextlib.suppress(DevToolsError): await self.devtools.call("Target.disposeBrowserContext", {"browserContextId": lease.id}) await asyncio.to_thread(shutil.rmtree, lease.folder(self.home()), True) @@ -177,6 +180,7 @@ def disposed(self, context_id: Optional[str]) -> None: deleted, so a lease of its label after has a folder of its own.""" if (lease := self.of(context_id)) is not None: self.leases.pop(lease.label, None) + self._ended(lease) folder, gone = lease.folder(self.home()), self.home() / CONTEXTS / f".gone-{lease.id}" with contextlib.suppress(OSError): folder.rename(gone) @@ -202,10 +206,19 @@ def resend_downloads(self) -> None: def ended(self) -> None: """The browser has gone, and every context with it: no lease is left, and their folders are deleted.""" + for lease in list(self.leases.values()): + self._ended(lease) self.leases.clear() self._made.clear() shutil.rmtree(self.home() / CONTEXTS, ignore_errors=True) + def _ended(self, lease: Lease) -> None: + for hook in self.on_end: + try: + hook(lease) + except Exception as e: + logger.warning(f"the end of the lease {lease.label!r}: {type(e).__name__}: {e}") + def _record(self, lease: Lease) -> Lease: self.leases[lease.label] = lease self.control.other_contexts[lease.id] = lease.label diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py index 05a480013..d3687a52f 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/control.py @@ -260,6 +260,12 @@ async def tab_list(self, context: Optional[str] = None, *, every: bool = False) return [OpenTab(**tab, active=tab["id"] == active, context=self.label_of(tab)).model_dump() for tab in self.tabs.values() if self.in_context(tab.get("context_id"), context)] + def context_of_frame(self, frame: object) -> Optional[str]: + """The context of the tab the frame ``frame`` is in, a tab's own frame among them; None for one it hasn't + seen.""" + tab = self._frame_tabs.get(frame, frame) if isinstance(frame, str) else None + return self.tabs.get(tab, {}).get("context_id") if tab is not None else None + def label_of(self, entry: dict) -> Optional[str]: """The label of the lease a tab or download was in, None for the default context.""" return self.other_contexts.get(entry.get("context_id")) @@ -937,6 +943,12 @@ def _on_Page_frameNavigated(self, params: dict, session: Optional[str]) -> None: load.commits.append(frame["loaderId"]) load.check() + def _on_Page_frameAttached(self, params: dict, session: Optional[str]) -> None: + """A frame attached to a page of one of its tabs, kept as that tab's: a cross-origin one goes on in a process + of its own, whose navigations its tab's page doesn't hear of.""" + if session in self._sessions and isinstance(frame := params.get("frameId"), str): + self._frame_tabs[frame] = self._sessions[session] + def _on_Page_navigatedWithinDocument(self, params: dict, session: Optional[str]) -> None: """A tab's page moving to another URL of its document, a fragment or its history API: a command's, when one asked for it, else its page's.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py index ffcf32496..17b439c43 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py @@ -27,12 +27,13 @@ from ..computer import app_data from ..computer.app import SOCKET, agent_acted, forget_agent_acted from ..computer.dataplane import file_parts -from ..computer.errors import DeviceUnavailable, InvalidArgument +from ..computer.errors import DeviceUnavailable, InvalidArgument, PermissionDenied, TargetUnavailable from ..tool_context import DEFAULT_ROLE, ROLE_HEADER, normalize_role from ..transfers import WriteNamespaceGrant from ..types import METHOD_ADD, RPC_PATH, EnvironmentExtension, IntakeDeclaration, IntakeFormat, intake_extension from . import data from .contexts import CONTEXTS, TASK, Contexts, label_problem +from .scope import Scope from .control import AGENT, DOWNLOADS, EXIT, LIVE_START_TAB_TIMEOUT_S, LOADED, STOP, UPLOADS, BrowserControl, _now from .devtools import PIPE_IN, PIPE_OUT, DevToolsError, DevToolsPipe from .recording import BROWSER_RECORDING, MAX_DURATION_S, MAX_FRAMES, BrowserRecordings @@ -168,6 +169,7 @@ def __init__(self) -> None: self.cdp_tokens = Tokens(SOCKET.parent / "cdp-tokens.json") self.relay = Relay(self.devtools, dialogs_answered=self.settings.dialogs != AGENT, log=self.log, control=self.control, contexts=self.contexts) + self.contexts.on_end.append(self._lease_ended) #: Its recordings: frames of the tab an agent last acted in, and its log, uploaded under each one's grant. self.recordings = BrowserRecordings(self.log, self.control) self.devtools.on_close(self._pipe_closed) @@ -350,6 +352,10 @@ async def start_tabs_loaded(self) -> bool: "caller's agent reaches the env, when it names one, else at the address the call reached the browser at, " "with a token agentenv mints in its path: each connection to it gets its own browser-level session behind a " "guard, and record lists them as cdp_connections, each with the commands the guard refused or changed. " + "Naming a lease's label as context, cdp returns an address scoped to it, for a user it's granted to: its " + "connections see that lease as the browser's default context, its targets alone, pages, iframes and workers, " + "with the contexts their user made, and what they open, set or download lands in it; the address ends with " + "the lease. An address with no context sees the default context and its user's own, not the leases. " "sites/set hands it the env's own sites, by name and domains: a start tab on one must load, its document's " "status 2xx, or its launch counts it failed, while one on any other site is waited on 30 s at most. record " "keeps each launch's start_tabs: the URL asked for and ended on, the status, any network error, the load time, " @@ -429,16 +435,28 @@ async def browser_recording_end_all(self, reason: str = "teardown") -> dict: return {"recordings": await self.recordings.end_all(reason)} @extension(BROWSER, op="cdp") - async def browser_cdp(self, user: str, request: Request, origin: Optional[str] = None) -> dict: + async def browser_cdp(self, user: str, request: Request, origin: Optional[str] = None, + context: Optional[str] = None) -> dict: """``user``'s CDP address once the browser's ready, the same one each time it's asked for, as the server that minted it keeps it: under ``origin``, where the caller's agent reaches the env, when the caller names it, as a deploy names the env's other addresses, since a tunnel in front of the env may not say how it was reached; - else at the address this call reached the browser at.""" + else at the address this call reached the browser at. Scoped to the lease ``context`` when it names one, + which must be granted to ``user``, and which the address ends with.""" base = under(_origin(origin), request) if origin is not None else reached(request) await self._until_ready() - address = self.cdp_tokens.mint(user, base, named=origin is not None) + if context is not None: + if (lease := self.contexts.get(context)) is None: + raise TargetUnavailable(f"no context {context!r} is leased; contexts/list names the leased ones") + if user not in lease.grants: + raise PermissionDenied(f"the context {context!r} isn't granted to {user!r}: contexts/grant grants it") + address = self.cdp_tokens.mint(user, base, named=origin is not None, context=context) return {"http_url": address.http_url, "ws_url": address.ws_url} + def _lease_ended(self, lease: Any) -> None: + """A lease ended: the addresses scoped to it end with it, and their connections are closed.""" + self.cdp_tokens.revoke(lease.label) + self.relay.lease_ended(lease.id) + def served_asgi(self, asgi: Any) -> Any: return _LoggedOps(super().served_asgi(asgi), self) @@ -466,7 +484,8 @@ async def _cdp_discovery(self, request: Request) -> Response: "WebKit-Version": version.get("revision"), "webSocketDebuggerUrl": address.ws_url if address.named else f"{websocket_form(reached(request))}{ROUTE}{token}/devtools/browser"}) - tabs = await self.control.tab_list() + tabs = [tab for tab in await self.control.tab_list(every=True) + if self._sees(address, self.control.tabs.get(tab["id"], {}).get("context_id"))] except (ConnectionError, DevToolsError, asyncio.TimeoutError) as e: return JSONResponse({"error": f"the browser isn't answering: {e}"}, status_code=503) return JSONResponse([{"id": tab["id"], "type": "page", "title": tab["title"], "url": tab["url"], @@ -477,7 +496,15 @@ async def _cdp_connect(self, websocket: WebSocket) -> None: if address is None: await websocket.close(code=1008) return - await self.relay.serve(websocket, address.user) + await self.relay.serve(websocket, address.user, address.context) + + def _sees(self, address: Any, context_id: Optional[str]) -> bool: + """Whether an address sees a tab in the browser context ``context_id``, as a connection to it does.""" + lease = self.contexts.get(address.context) if address.context is not None else None + if address.context is not None and lease is None: + return False + return Scope(self.contexts, self.relay.target_contexts, address.user, lease.id if lease else None).sees( + context_id) async def _until_ready(self) -> None: """Back once it's ready, as ``computer-app/v1``'s ``ready`` says, or DeviceUnavailable after its ready diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/scope.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/scope.py new file mode 100644 index 000000000..c6686a0ba --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/scope.py @@ -0,0 +1,155 @@ +"""What a CDP connection sees and acts in, its ``Scope``: a lease or the browser's default context, and every context +its user made. The relay keeps each target's context as Chromium reports it and asks a connection's scope about each +command it sends (``Scope.verdict``), each answer it gets (``Scope.answer``) and each event on its browser session +(``Scope.hears``): it hears only of the targets of the contexts it sees, pages, cross-origin iframes and workers alike; +its commands that act in a context and name none get its lease, and the ones that name a context or a target it +doesn't see are refused. A guardrail that keeps cooperating agents apart; the sandbox stays the boundary.""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Callable, Optional + +if TYPE_CHECKING: + from .contexts import Contexts + +#: The commands that act in a browser context, which name it as ``browserContextId``, the default without one. +ACTS_IN_CONTEXT = frozenset({"Target.createTarget", "Storage.getCookies", "Storage.setCookies", "Storage.clearCookies", + "Browser.grantPermissions", "Browser.resetPermissions", "Browser.setPermission", + "Browser.setDownloadBehavior"}) +#: The commands on a target, which name it as ``targetId``. +ON_TARGET = frozenset({"Target.attachToTarget", "Target.activateTarget", "Target.closeTarget", "Target.getTargetInfo", + "Target.autoAttachRelated", "Browser.getWindowForTarget"}) +#: The domains whose commands on a browser session act across every context, intercepting, tracing or trusting for all +#: of them, which a scoped connection may send only on its own pages' sessions. +BROWSER_WIDE = ("Fetch.", "Tracing.", "Security.") +#: The ``Storage`` commands a scoped connection may send on its browser session: the cookie ones, which get its lease. +COOKIES = frozenset({"Storage.getCookies", "Storage.setCookies", "Storage.clearCookies"}) +#: The ``Browser`` commands on a window, which name it as ``windowId``. +ON_WINDOW = frozenset({"Browser.setWindowBounds", "Browser.getWindowBounds", "Browser.setContentsSize"}) +#: The ``Target`` events about a target, which carry its ``targetInfo``, and those that carry only its ``targetId``. +WITH_INFO = frozenset({"Target.targetCreated", "Target.targetInfoChanged", "Target.attachedToTarget"}) +WITH_ID = frozenset({"Target.targetDestroyed", "Target.targetCrashed"}) +#: Chromium's answer to a command for a target it doesn't know, as a connection is answered for one it doesn't see. +NO_TARGET = {"code": -32602, "message": "No target with given id found"} +#: The commands that make and end a browser context, which agentenv records as leases. +MAKES_CONTEXT, ENDS_CONTEXT = "Target.createBrowserContext", "Target.disposeBrowserContext" + + +class Targets: + """The browser context of each target Chromium has reported, by its id: kept for the browser's life, as each + connection hears of a target's end apart from the others.""" + + def __init__(self) -> None: + self.contexts: dict[str, str] = {} + + def saw(self, info: object) -> None: + if isinstance(info, dict) and isinstance(target := info.get("targetId"), str) and \ + isinstance(context := info.get("browserContextId"), str): + self.contexts[target] = context + + def context_of(self, target: object) -> Optional[str]: + return self.contexts.get(target) if isinstance(target, str) else None + + def forget(self) -> None: + self.contexts.clear() + + +@dataclass(frozen=True) +class Ruling: + """What a connection's scope makes of a command: ``params`` as sent, its scope added when it acts in a context and + names none, else ``refused`` and why, with the ``error`` it's answered with in place of agentenv's own.""" + + params: dict + refused: Optional[str] = None + error: Optional[dict] = None + + +@dataclass +class Scope: + """What one connection of ``user``'s sees and acts in: the lease ``lease`` names, the browser's id for it, else + the default context, and every context its user made. ``target_contexts`` is each target's context, and + ``frame_context`` the context of the tab a frame is in, None for one it hasn't seen.""" + + contexts: Contexts + target_contexts: Targets + user: str + lease: Optional[str] = None + frame_context: Callable[[object], Optional[str]] = lambda frame: None + #: The windows of its tabs, which it learns asking for them, and the downloads begun where it doesn't see. + windows: set[int] = field(default_factory=set) + unseen_downloads: set[str] = field(default_factory=set) + + def sees(self, context: Optional[str]) -> bool: + """Whether it sees the context ``context``: its scope, or one its user made. A target no context is known for + is the default context's.""" + if context is not None and context == self.lease: + return True + if context is not None and (lease := self.contexts.of(context)) is not None and lease.made_by == self.user: + return True + return self.lease is None and (context is None or context not in self.contexts.control.other_contexts) + + def verdict(self, method: str, params: dict, *, on_browser_session: bool) -> Ruling: + """Its verdict on a command, sent on its browser session or on a page's: refused when it would dispose a + context its user didn't make, open a second browser session, touch a target it doesn't see or, scoped to a + lease, act on every context or a window not its tabs'; else as sent, with its lease added where a command acts + in a context and names none, refused where one names a context it doesn't see.""" + if method == ENDS_CONTEXT and (reason := self.contexts.may_dispose(self.user, params.get("browserContextId"))): + return Ruling(params, reason) + if method == "Target.attachToBrowserTarget": + return Ruling(params, "each connection has its browser session; another would see past its scope") + if on_browser_session and method in ON_TARGET and not self.sees( + self.target_contexts.context_of(params["targetId"]) if params.get("targetId") else self.lease): + return Ruling(params, "the target isn't one this address sees", NO_TARGET) + if on_browser_session and self.lease is not None: + if method.startswith(BROWSER_WIDE) or (method.startswith("Storage.") and method not in COOKIES): + return Ruling(params, "on an address scoped to a lease it would act on every context; a page's own " + "session takes it") + if method in ON_WINDOW and params.get("windowId") not in self.windows: + return Ruling(params, "the window isn't one of this address's tabs'") + if method not in ACTS_IN_CONTEXT: + return Ruling(params) + named = params.get("browserContextId") + if named is None: + return Ruling({**params, "browserContextId": self.lease} if self.lease is not None else params) + if self.sees(named): + return Ruling(params) + return Ruling(params, f"the browser context {named} isn't one this address sees") + + def answer(self, method: str, result: dict) -> dict: + """A command's ``result`` as it hears it: the targets of contexts it doesn't see left out, the contexts listed + beside the default only its user's own, the lease shown as the default; the window a tab of its is in kept.""" + if method == "Browser.getWindowForTarget" and isinstance(window := result.get("windowId"), int): + self.windows.add(window) + if method == "Target.getTargets" and isinstance(infos := result.get("targetInfos"), list): + for info in infos: + self.target_contexts.saw(info) + return {**result, "targetInfos": [info for info in infos if isinstance(info, dict) + and self.sees(info.get("browserContextId"))]} + if method == "Target.getBrowserContexts" and isinstance(ids := result.get("browserContextIds"), list): + own = [context for context in ids if context != self.lease and (lease := self.contexts.of(context)) + is not None and lease.made_by == self.user] + shown = {**result, "browserContextIds": own} + if self.lease is not None and "defaultBrowserContextId" in result: + shown["defaultBrowserContextId"] = self.lease + return shown + return result + + def hears(self, method: str, params: dict) -> tuple[bool, Optional[str]]: + """Whether it hears an event on its browser session, a target's only for a context it sees, a download's only + for a frame of one; and the session of a target it doesn't see that auto-attached to it, which the relay + lets go.""" + if method in WITH_INFO or method in WITH_ID: + target = (params.get("targetInfo") or {}).get("targetId") if method in WITH_INFO else params.get("targetId") + if self.sees(self.target_contexts.context_of(target)): + return True, None + attached = params.get("sessionId") if method == "Target.attachedToTarget" else None + return False, attached if isinstance(attached, str) else None + if method == "Browser.downloadWillBegin": + frame = params.get("frameId") + if self.sees(self.frame_context(frame) or self.target_contexts.context_of(frame)): + return True, None + self.unseen_downloads.add(params.get("guid")) + return False, None + if method == "Browser.downloadProgress": + return params.get("guid") not in self.unseen_downloads, None + return True, None diff --git a/packages/agentenv-protocol/tests/test_browser_cdp.py b/packages/agentenv-protocol/tests/test_browser_cdp.py index 648897a04..48412ae9d 100644 --- a/packages/agentenv-protocol/tests/test_browser_cdp.py +++ b/packages/agentenv-protocol/tests/test_browser_cdp.py @@ -112,6 +112,67 @@ async def request(self, method, params=None, *, session_id=None, answered=None): return await super().request(method, params, session_id=session_id, answered=answered) +class ScopePipe(ContextPipe): + """A browser whose default context ``DEF`` has the page ``P0``, and whose task leases ``L1`` and ``L2`` have the + pages ``PB`` and ``PS`` and ``L1`` the worker ``WB``: a connection's setDiscoverTargets announces them, getTargets + and getBrowserContexts answer with them, and an agent's own contexts' pages come as it makes them. Each command + agentenv itself sends is kept in ``called`` with its params.""" + + INFOS = [{"targetId": "P0", "type": "page", "browserContextId": "DEF"}, + {"targetId": "PB", "type": "page", "browserContextId": "L1"}, + {"targetId": "WB", "type": "service_worker", "browserContextId": "L1"}, + {"targetId": "PS", "type": "page", "browserContextId": "L2"}] + + def __init__(self) -> None: + super().__init__() + self.leased, self.called, self.infos = 0, [], list(self.INFOS) + + async def call(self, method, params=None, *, session_id=None, timeout_s=None): + self.called.append((method, params, session_id)) + if method == "Target.createBrowserContext": + self.leased += 1 + return {"browserContextId": f"L{self.leased}"} + return await super().call(method, params, session_id=session_id, timeout_s=timeout_s) + + async def request(self, method, params=None, *, session_id=None, answered=None): + if method == "Target.setDiscoverTargets": + self.sent.append((method, session_id)) + for info in self.infos: + self.emit({"method": "Target.targetCreated", "sessionId": session_id, "params": {"targetInfo": info}}) + return {"result": {}} + if method == "Target.getTargets": + return {"result": {"targetInfos": list(self.infos)}} + if method == "Browser.getWindowForTarget": + self.sent.append((method, session_id)) + return {"result": {"windowId": {"PB": 7, "PS": 9}.get((params or {}).get("targetId"), 1), "bounds": {}}} + if method == "Target.getBrowserContexts": + ids = sorted({info["browserContextId"] for info in self.infos} - {"DEF"}) + return {"result": {"browserContextIds": ids, "defaultBrowserContextId": "DEF"}} + answer = await super().request(method, params, session_id=session_id, answered=answered) + if method == "Target.createBrowserContext": + self.infos.append({"targetId": f"P{answer['result']['browserContextId']}", "type": "page", + "browserContextId": answer["result"]["browserContextId"]}) + return answer + + +async def _hear(socket, message: dict) -> tuple[list[dict], dict]: + """The events ``socket`` hears from sending ``message`` until its answer, and the answer.""" + await socket.send(json.dumps(message)) + heard = [] + while (answer := json.loads(await socket.recv())).get("id") != message["id"]: + heard.append(answer) + return heard, answer + + +async def _scoped(tmp_path): + """A ScopePipe, its contexts with the task's leases buyer for ada and seller for bob, and the relay over both.""" + pipe = ScopePipe() + contexts = Contexts(pipe, BrowserControl(pipe, "agent"), home=lambda: tmp_path) + await contexts.lease("buyer", grants=("ada",)) + await contexts.lease("seller", grants=("bob",)) + return pipe, contexts + + class SlowClient: """A client's WebSocket that takes in nothing sent to it until ``reading`` is set, and records what it got and how it was closed.""" @@ -143,12 +204,12 @@ async def _relay(dialogs_answered: bool = False, pipe=None, **logging): relay = Relay(pipe, dialogs_answered=dialogs_answered, **logging) async def endpoint(websocket): - await relay.serve(websocket, websocket.path_params["user"]) + await relay.serve(websocket, websocket.path_params["user"], websocket.path_params.get("context")) sock = socket.socket() sock.bind(("127.0.0.1", 0)) - server = uvicorn.Server(uvicorn.Config(Starlette(routes=[WebSocketRoute("/{user}", endpoint)]), log_level="warning", - lifespan="off")) + routes = [WebSocketRoute("/{user}", endpoint), WebSocketRoute("/{user}/{context}", endpoint)] + server = uvicorn.Server(uvicorn.Config(Starlette(routes=routes), log_level="warning", lifespan="off")) serving = asyncio.create_task(server.serve(sockets=[sock])) while not server.started: await asyncio.sleep(0.01) @@ -337,6 +398,201 @@ async def test_a_context_an_agent_makes_is_its_lease_keeping_downloads_in_its_fo assert [entry["did"] for entry in relay.connections[0]["guarded"]] == ["changed", "refused"] +@pytest.mark.asyncio +async def test_a_scoped_address_sees_its_lease_as_the_default_and_nothing_of_another_context(tmp_path): + pipe, contexts = await _scoped(tmp_path) + async with _relay(pipe=pipe, contexts=contexts) as (_, relay, base): + async with connect(f"{base}/ada/buyer") as ada, connect(f"{base}/carol") as carol: + heard, _ = await _hear(ada, {"id": 1, "method": "Target.setDiscoverTargets", "params": {"discover": True}}) + listed = await _ask(ada, {"id": 2, "method": "Target.getTargets"}) + contexts_listed = await _ask(ada, {"id": 3, "method": "Target.getBrowserContexts"}) + theirs = await _ask(ada, {"id": 4, "method": "Target.attachToTarget", "params": {"targetId": "PS"}}) + elsewhere = await _ask(ada, {"id": 5, "method": "Target.activateTarget", "params": {"targetId": "P0"}}) + mine = await _ask(ada, {"id": 6, "method": "Target.attachToTarget", "params": {"targetId": "PB"}}) + await _ask(ada, {"id": 7, "method": "Target.createTarget", "params": {"url": "https://shop.test/"}}) + opened_in = pipe.params["Target.createTarget"] + await _ask(ada, {"id": 8, "method": "Storage.getCookies", "params": {}}) + cookies_of = pipe.params["Storage.getCookies"] + reached = await _ask(ada, {"id": 9, "method": "Storage.getCookies", "params": {"browserContextId": "L2"}}) + browser = await _ask(ada, {"id": 10, "method": "Target.attachToBrowserTarget"}) + default_heard, _ = await _hear(carol, {"id": 1, "method": "Target.setDiscoverTargets", + "params": {"discover": True}}) + default_contexts = await _ask(carol, {"id": 2, "method": "Target.getBrowserContexts"}) + default_cookies = await _ask(carol, {"id": 3, "method": "Storage.getCookies", + "params": {"browserContextId": "L1"}}) + + assert [event["params"]["targetInfo"]["targetId"] for event in heard] == ["PB", "WB"] + assert [info["targetId"] for info in listed["result"]["targetInfos"]] == ["PB", "WB"] + assert contexts_listed["result"] == {"browserContextIds": [], "defaultBrowserContextId": "L1"} + assert theirs["error"] == elsewhere["error"] == {"code": -32602, "message": "No target with given id found"} + assert "result" in mine and opened_in["browserContextId"] == "L1" and cookies_of == {"browserContextId": "L1"} + assert "isn't one this address sees" in reached["error"]["message"] + assert "each connection has its browser session" in browser["error"]["message"] + assert [event["params"]["targetInfo"]["targetId"] for event in default_heard] == ["P0"] + assert default_contexts["result"]["browserContextIds"] == [] and "error" in default_cookies + assert relay.connections[0]["context"] == "buyer" and "context" not in relay.connections[1] + + +@pytest.mark.asyncio +async def test_an_address_sees_the_contexts_its_user_made_and_no_one_elses(tmp_path): + pipe, contexts = await _scoped(tmp_path) + async with _relay(pipe=pipe, contexts=contexts) as (_, relay, base): + async with connect(f"{base}/ada") as ada, connect(f"{base}/ada/buyer") as scoped, \ + connect(f"{base}/bob") as bob: + made = (await _ask(ada, {"id": 1, "method": "Target.createBrowserContext", "params": {}}))["result"] + own = made["browserContextId"] + from_scoped, _ = await _hear(scoped, {"id": 1, "method": "Target.setDiscoverTargets", + "params": {"discover": True}}) + listed = await _ask(scoped, {"id": 2, "method": "Target.getBrowserContexts"}) + opened = await _ask(scoped, {"id": 3, "method": "Target.createTarget", + "params": {"url": "about:blank", "browserContextId": own}}) + from_bob, _ = await _hear(bob, {"id": 1, "method": "Target.setDiscoverTargets", + "params": {"discover": True}}) + + assert sorted(event["params"]["targetInfo"]["targetId"] for event in from_scoped) == sorted(["PB", f"P{own}", "WB"]) + assert listed["result"]["browserContextIds"] == [own] and "result" in opened + assert [event["params"]["targetInfo"]["targetId"] for event in from_bob] == ["P0"] + + +@pytest.mark.asyncio +async def test_a_target_of_another_context_auto_attached_to_a_scoped_connection_runs_and_detaches_unheard(tmp_path): + pipe, contexts = await _scoped(tmp_path) + async with _relay(pipe=pipe, contexts=contexts) as (_, relay, base): + async with connect(f"{base}/ada/buyer") as ada: + await _ask(ada, {"id": 1, "method": "Browser.getVersion"}) + session = relay.connections[0] and next(iter(relay._open)).session + pipe.emit({"method": "Target.attachedToTarget", "sessionId": session, "params": { + "sessionId": "PS-s", "targetInfo": {"targetId": "PS2", "type": "page", "browserContextId": "L2"}, + "waitingForDebugger": True}}) + pipe.emit({"method": "Target.attachedToTarget", "sessionId": session, "params": { + "sessionId": "PB-s", "targetInfo": {"targetId": "PB2", "type": "page", "browserContextId": "L1"}, + "waitingForDebugger": True}}) + heard = json.loads(await asyncio.wait_for(ada.recv(), 5)) + for _ in range(100): + if ("Target.detachFromTarget", {"sessionId": "PS-s"}, session) in pipe.called: + break + await asyncio.sleep(0.01) + + assert heard["method"] == "Target.attachedToTarget" and heard["params"]["sessionId"] == "PB-s" + assert ("Runtime.runIfWaitingForDebugger", None, "PS-s") in pipe.called + assert ("Target.detachFromTarget", {"sessionId": "PS-s"}, session) in pipe.called + assert not any(session_id == "PB-s" for _, _, session_id in pipe.called) + + +@pytest.mark.asyncio +async def test_a_leases_end_closes_the_connections_scoped_to_it_and_one_to_an_ended_lease_is_refused(tmp_path): + pipe, contexts = await _scoped(tmp_path) + async with _relay(pipe=pipe, contexts=contexts) as (_, relay, base): + async with connect(f"{base}/ada/buyer") as ada, connect(f"{base}/carol") as carol: + await _ask(ada, {"id": 1, "method": "Browser.getVersion"}) + relay.lease_ended(contexts.get("buyer").id) + with pytest.raises(ConnectionClosedOK) as ended: + await asyncio.wait_for(ada.recv(), 5) + still = await _ask(carol, {"id": 1, "method": "Browser.getVersion"}) + async with connect(f"{base}/ada/nobody") as late: + with pytest.raises(ConnectionClosedOK) as refused: + await asyncio.wait_for(late.recv(), 5) + + assert (ended.value.rcvd.code, ended.value.rcvd.reason) == browser_cdp.LEASE_ENDED + assert (refused.value.rcvd.code, refused.value.rcvd.reason) == browser_cdp.LEASE_ENDED and "result" in still + + +@pytest.mark.asyncio +async def test_a_lease_that_ends_while_a_connection_to_it_attaches_closes_it_saying_so(tmp_path): + pipe, contexts = await _scoped(tmp_path) + gate, attaching = asyncio.Event(), asyncio.Event() + real = pipe.call + + async def call(method, params=None, *, session_id=None, timeout_s=None): + if method == "Target.attachToBrowserTarget": + attaching.set() + await gate.wait() + return await real(method, params, session_id=session_id, timeout_s=timeout_s) + + pipe.call = call + async with _relay(pipe=pipe, contexts=contexts) as (_, relay, base): + opening = asyncio.ensure_future(connect(f"{base}/ada/buyer")) + await asyncio.wait_for(attaching.wait(), 5) + await contexts.release("buyer") + gate.set() + ada = await asyncio.wait_for(opening, 5) + with pytest.raises(ConnectionClosedOK) as ended: + await asyncio.wait_for(ada.recv(), 5) + + assert (ended.value.rcvd.code, ended.value.rcvd.reason) == browser_cdp.LEASE_ENDED + assert ("Target.detachFromTarget", {"sessionId": "b1"}, None) in pipe.called and relay.connections == [] + + +@pytest.mark.asyncio +async def test_a_download_a_cross_origin_iframe_began_is_heard_on_its_leases_connection_only(tmp_path): + pipe, contexts = await _scoped(tmp_path) + async with _relay(pipe=pipe, contexts=contexts) as (_, relay, base): + async with connect(f"{base}/ada/buyer") as ada, connect(f"{base}/carol") as carol: + await _ask(ada, {"id": 1, "method": "Browser.getVersion"}) + await _ask(carol, {"id": 1, "method": "Browser.getVersion"}) + sessions = {connection.user: connection.session for connection in relay._open} + pipe.emit({"method": "Target.attachedToTarget", "sessionId": sessions["ada"], "params": { + "sessionId": "IF-s", "targetInfo": {"targetId": "IF1", "type": "iframe", "browserContextId": "L1"}, + "waitingForDebugger": False}}) + await ada.recv() + for user in ("ada", "carol"): + pipe.emit({"method": "Browser.downloadWillBegin", "sessionId": sessions[user], + "params": {"guid": "g1", "frameId": "IF1", "url": "http://x.test/f", + "suggestedFilename": "f"}}) + heard = json.loads(await asyncio.wait_for(ada.recv(), 5)) + nothing = await _ask(carol, {"id": 2, "method": "Browser.getVersion"}) + + assert heard["method"] == "Browser.downloadWillBegin" and heard["params"]["guid"] == "g1" + assert nothing["id"] == 2 + + +@pytest.mark.asyncio +async def test_a_scoped_connection_reaches_no_further_than_its_lease_on_its_browser_session(tmp_path): + pipe, contexts = await _scoped(tmp_path) + wide = ["Fetch.enable", "Tracing.start", "Security.setIgnoreCertificateErrors", "Storage.clearDataForOrigin"] + async with _relay(pipe=pipe, contexts=contexts) as (_, relay, base): + async with connect(f"{base}/ada/buyer") as ada, connect(f"{base}/carol") as carol: + refused = [await _ask(ada, {"id": n, "method": method, "params": {}}) for n, method in enumerate(wide, 1)] + cookies = await _ask(ada, {"id": 10, "method": "Storage.getCookies"}) + await _ask(ada, {"id": 11, "method": "Target.getTargets"}) + window = await _ask(ada, {"id": 12, "method": "Browser.getWindowForTarget", "params": {"targetId": "PB"}}) + theirs = await _ask(ada, {"id": 13, "method": "Browser.getWindowForTarget", "params": {"targetId": "PS"}}) + own_window = await _ask(ada, {"id": 14, "method": "Browser.setWindowBounds", + "params": {"windowId": 7, "bounds": {}}}) + other_window = await _ask(ada, {"id": 15, "method": "Browser.setWindowBounds", + "params": {"windowId": 9, "bounds": {}}}) + await ada.send(json.dumps({"id": 16, "method": "Target.attachToTarget", + "params": {"targetId": "PB", "flatten": True}})) + while json.loads(await ada.recv()).get("id") != 16: + pass + on_page = await _ask(ada, {"id": 17, "method": "Fetch.enable", "sessionId": "b1-c"}) + on_default = await _ask(carol, {"id": 1, "method": "Fetch.enable"}) + + assert all("act on every context" in answer["error"]["message"] for answer in refused), refused + assert "result" in cookies and pipe.params["Storage.getCookies"] == {"browserContextId": "L1"} + assert window["result"]["windowId"] == 7 and theirs["error"]["message"] == "No target with given id found" + assert "result" in own_window and "isn't one of this address's tabs'" in other_window["error"]["message"] + assert "result" in on_page and "result" in on_default + assert ("Fetch.enable", "b1-c") in pipe.sent and ("Fetch.enable", "b2") in pipe.sent + assert not any(method in wide and session == "b1" for method, session in pipe.sent) + + +def test_a_scoped_token_is_one_per_user_and_lease_kept_in_memory_and_revoked_with_its_lease(tmp_path): + tokens = Tokens(tmp_path / "cdp-tokens.json") + plain = tokens.mint("ada", "https://computer.test") + scoped = tokens.mint("ada", "https://computer.test", context="buyer") + again = tokens.mint("ada", "https://computer.test", context="buyer") + other = tokens.mint("bob", "https://computer.test", context="buyer") + token = scoped.ws_url.split("/cdp/")[1].split("/")[0] + + assert again == scoped and scoped.context == "buyer" and other != scoped and plain.context is None + assert tokens.address(token).context == "buyer" + assert Tokens(tmp_path / "cdp-tokens.json").address(token) is None + tokens.revoke("buyer") + assert tokens.address(token) is None and tokens.address(plain.ws_url.split("/cdp/")[1].split("/")[0]) is not None + assert tokens.mint("ada", "https://computer.test", context="buyer") != scoped + + @pytest.mark.asyncio async def test_a_client_that_falls_too_far_behind_is_closed_and_what_waited_for_it_dropped(monkeypatch): monkeypatch.setattr(browser_cdp, "OUTBOX_LIMIT_BYTES", 10_000) @@ -762,7 +1018,8 @@ async def tabs(context=None, every=False): if e["uri"] == "urn:agentenv:browser/v1") assert computer_app_params["agent_routes"] == ["/cdp/"] assert browser_methods["cdp"]["request"]["properties"] == {"user": {"type": "string"}, - "origin": {"type": "string", "default": None}} + "origin": {"type": "string", "default": None}, + "context": {"type": "string", "default": None}} @pytest.mark.asyncio diff --git a/packages/agentenv-protocol/tests/test_browser_contexts.py b/packages/agentenv-protocol/tests/test_browser_contexts.py index b8207ef20..acf473c09 100644 --- a/packages/agentenv-protocol/tests/test_browser_contexts.py +++ b/packages/agentenv-protocol/tests/test_browser_contexts.py @@ -8,10 +8,11 @@ from pathlib import PurePosixPath import pytest +from starlette.requests import Request from agentenv_protocol import website_browser from agentenv_protocol.computer import Launch, Ready, app, linux -from agentenv_protocol.computer.errors import InvalidArgument, TargetUnavailable +from agentenv_protocol.computer.errors import InvalidArgument, PermissionDenied, TargetUnavailable from agentenv_protocol.computer.platform import Platform from agentenv_protocol.website_browser import BrowserControl, CookieToSet, WebsiteBrowserEnvironment, data from agentenv_protocol.website_browser.contexts import Contexts, label_problem @@ -268,6 +269,20 @@ async def test_every_contexts_tabs_list_together_each_naming_its_context_the_pag await browser.browser_tabs_list(context="buyer", every=True) +@pytest.mark.asyncio +async def test_a_frame_attached_to_a_leases_page_is_its_tab_s_though_it_navigates_in_a_process_of_its_own( + browser_home): + devtools, control, contexts = _contexts(browser_home) + await contexts.lease("buyer") + _tab(devtools, "L1", "C1") + _attached(devtools, "L1", "s-l1") + + devtools.send("Page.frameAttached", {"frameId": "OOPIF", "parentFrameId": "L1"}, "s-l1") + + assert control.context_of_frame("OOPIF") == "C1" and control.context_of_frame("L1") == "C1" + assert control.context_of_frame("elsewhere") is None + + @pytest.mark.asyncio async def test_a_label_an_agent_disposed_and_a_load_leases_again_at_once_keeps_the_new_folder(browser_home): _, _, contexts = _contexts(browser_home) @@ -385,3 +400,35 @@ async def test_the_ops_lease_grant_list_and_release_and_name_a_context_by_its_la with pytest.raises(TargetUnavailable): await browser.browser_cookies_get(context="seller") assert await browser.browser_contexts_release("seller") == {"released": False} + + +@pytest.mark.asyncio +async def test_a_scoped_cdp_address_is_only_for_a_user_its_lease_is_granted_to_and_ends_with_the_lease( + browser_home, monkeypatch): + browser = Browser() + devtools = StubDevTools() + devtools.listen(browser.control._event) + browser.devtools = browser.contexts.devtools = browser.control.devtools = devtools + monkeypatch.setattr(browser.contexts, "home", lambda: browser_home) + + async def ready(): + return None + + monkeypatch.setattr(browser, "_until_ready", ready) + request = Request({"type": "http", "method": "POST", "scheme": "http", "path": "/", "query_string": b"", + "server": ("computer.test", 80), "headers": [(b"host", b"computer.test")]}) + await browser.browser_contexts_lease("buyer") + await browser.browser_contexts_grant("buyer", "ada") + + with pytest.raises(PermissionDenied, match="isn't granted to 'bob'"): + await browser.browser_cdp("bob", request, context="buyer") + with pytest.raises(TargetUnavailable, match="no context 'seller' is leased"): + await browser.browser_cdp("ada", request, context="seller") + scoped = await browser.browser_cdp("ada", request, context="buyer") + token = scoped["ws_url"].split("/cdp/")[1].split("/")[0] + found = browser.cdp_tokens.address(token) + await browser.browser_contexts_release("buyer") + + assert (found.user, found.context) == ("ada", "buyer") + assert browser.cdp_tokens.address(token) is None + diff --git a/packages/agentenv-protocol/tests/test_computer_svc_forward.py b/packages/agentenv-protocol/tests/test_computer_svc_forward.py index c99666c6e..16f456c63 100644 --- a/packages/agentenv-protocol/tests/test_computer_svc_forward.py +++ b/packages/agentenv-protocol/tests/test_computer_svc_forward.py @@ -47,9 +47,9 @@ @linux() class Echo(ComputerAppEnvironment): - """An app whose own server echoes what a WebSocket sends it until "bye", which it closes with 4001; answers with - the headers it got and the size of what it was sent; and streams 20 MB, its first MB at once and the rest once - ``released``.""" + """An app whose own server echoes what a WebSocket sends it until "bye", which it closes with 4001 and a reason; + answers with the headers it got and the size of what it was sent; and streams 20 MB, its first MB at once and the + rest once ``released``.""" agent_routes = ("/raw/",) @@ -104,7 +104,7 @@ async def _echo(self, websocket: WebSocket) -> None: "authorization": websocket.headers.get("authorization")})) while (message := await websocket.receive())["type"] != "websocket.disconnect": if message.get("text") == "bye": - await websocket.close(code=4001) + await websocket.close(code=4001, reason="you said bye") return await (websocket.send_text(message["text"]) if message.get("text") is not None else websocket.send_bytes(message["bytes"][::-1])) @@ -178,7 +178,7 @@ async def test_a_websocket_through_svc_reaches_the_app_both_ways_until_it_closes opened = await socket.recv() assert hello == {"forwarded": "/svc/echo", "authorization": None} and subprotocol == "cdp" - assert (text, data) == ("Page.navigate", b"\x03\x02\x01") and closed.value.rcvd.code == 4001 + assert (text, data) == ("Page.navigate", b"\x03\x02\x01") and (closed.value.rcvd.code, closed.value.rcvd.reason) == (4001, "you said bye") assert refused == [403, 403, 403] and malformed == [b"HTTP/1.1 403 Forbidden", b"HTTP/1.1 403 Forbidden"] assert raw == "/raw/a%3Fb" and opened == "/raw/open" diff --git a/src/agent_env/browsers/README.md b/src/agent_env/browsers/README.md index 5b4ea8b97..20d59398d 100644 --- a/src/agent_env/browsers/README.md +++ b/src/agent_env/browsers/README.md @@ -61,6 +61,12 @@ and the browser's `data/add` loads them, which opens the start tabs again while after the load that seeds the site and again after `reset_env`, which starts the browser afresh. Preflight refuses a step on an env with no browser, none of the name given, or several and none named. On a multi env, the steps reach the browser apps of its computers, each through the computer's own record and control token. +`browser_connect_cdp {context?}` with a `context`, a lease granted to the agent, hands it an address scoped to +that lease: the agent sees the lease as the browser's default context, its pages, cross-origin iframes and workers +alone, with the contexts it made itself, and what it opens, sets or downloads lands there; it can't see or attach to +another context's targets, nor send on its browser session the commands that act on every context (`Fetch`, +`Tracing`, `Security`, `Storage` beyond cookies, or another context's windows), and the address ends with the lease. An address with no context sees the default context +and the agent's own, not the leases. `browser_connect_cdp` hands an agent a CDP address to the same browser, under the origin where the agent reaches the env, so through a tunnel that ends TLS it's https and wss; on a multi env, one at its gateway, asked through the gateway's `/svc//` route, so the agent reaches the browser through the gateway's CDP relay as it reaches the diff --git a/src/agent_env/task_step/task_steps/browser/connect_cdp.py b/src/agent_env/task_step/task_steps/browser/connect_cdp.py index 836b761cf..597b5d916 100644 --- a/src/agent_env/task_step/task_steps/browser/connect_cdp.py +++ b/src/agent_env/task_step/task_steps/browser/connect_cdp.py @@ -39,7 +39,9 @@ class BrowserConnectCdpTaskStep(BrowserStep): """Hand the agent ``agent_name`` a CDP address to the website browser ``browser`` in the env ``env_id``, the env's only one when unnamed: added to its cdp-config/v1 as ``name``, with every tool the env serves hidden from its role unless ``keep_tools``. The address is the browser's own for that agent, as its sandbox reaches it; for a - browser in one of a multi env's computers, the multi env's gateway's, which relays it to the browser.""" + browser in one of a multi env's computers, the multi env's gateway's, which relays it to the browser. With a + ``context``, a lease's label granted to the agent, the address is scoped to that lease, which the agent sees as + the browser's default context, and ends with it.""" type: ClassVar[str] = "browser_connect_cdp" entity_refs = (EntityRef.env("env_id"),) @@ -55,8 +57,10 @@ def __init__( keep_tools: bool = False, depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True, + context: Optional[str] = None, ): - super().__init__(id, version, env_id, browser, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + super().__init__(id, version, env_id, browser, depends_on=depends_on, fail_task_on_error=fail_task_on_error, + context=context) self.agent_name = agent_name or TaskStep.DEFAULT_AGENT_NAME self.name = name self.keep_tools = keep_tools @@ -74,7 +78,7 @@ def to_dict(self) -> dict: @classmethod def from_dict(cls, data: dict) -> BrowserConnectCdpTaskStep: - return cls(**cls._browser_from_dict(data), agent_name=data.get("agent_name"), + return cls(**cls._in_context_from_dict(data), agent_name=data.get("agent_name"), name=data.get("name") or DEFAULT_NAME, keep_tools=data.get("keep_tools", False)) async def execute(self, context: TaskStepContext) -> TaskStepContext: @@ -85,10 +89,14 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: if cdp_config is None or not agent.a2a_url: raise RuntimeError(f"agent {agent.agent_name!r} can't take a CDP address: its card has no " f"{A2AAgent.EXT_CDP_CONFIG}") + if self.context_label is not None and not _takes(card, "context"): + raise RuntimeError(f"the website browser {browser or deployed.env_id!r} scopes no CDP address to a lease: " + "rebuild it on a newer agentenv-framework-protocol") base, theirs = _env_urls(env, agent) - asked = {"user": agent.agent_name, **({"origin": theirs} if _takes_origin(card) else {})} - address = (await self._call(context, "cdp", asked, CDP_TIMEOUT_S) if deployed.env_id == self.env_id - else await _cdp_at_gateway(env, deployed, card, asked)) + asked = {"user": agent.agent_name, **({"origin": theirs} if _takes(card, "origin") else {}), + **({"context": self.context_label} if self.context_label is not None else {})} + address = (await self._call(context, "cdp", asked, CDP_TIMEOUT_S, in_context=False) + if deployed.env_id == self.env_id else await _cdp_at_gateway(env, deployed, card, asked)) http_url = _rebased(address["http_url"], base, theirs) ws_url = _rebased(address["ws_url"], _ws(base), _ws(theirs)) async with httpx.AsyncClient(timeout=ADD_TIMEOUT_S) as client: @@ -101,13 +109,15 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: if hidden: await env.invoke(DISABLE_TOOL_EXTENSION_URI, DISABLE, {"role": role, "tools": hidden}) logger.info(f"agent {agent.agent_name!r} has {self.name!r}, a CDP address to {browser or 'the env'} in env " - f"{env.env_id!r}" + (f"; the env's tools are hidden from role {role!r}" if hidden else "")) + f"{env.env_id!r}" + (f", scoped to the lease {self.context_label!r}" if self.context_label else "") + + (f"; the env's tools are hidden from role {role!r}" if hidden else "")) # The agent's cdp-config now holds this address under its name, so it replaces any connection recorded so. records = context.metadata.setdefault(CONNECTIONS_KEY, []) records[:] = [record for record in records if (record["agent_name"], record["name"]) != (self.agent_name, self.name)] + [{ "step_id": self.id, "env_id": self.env_id, "agent_name": self.agent_name, "browser": browser, - "name": self.name, "keep_tools": self.keep_tools, "role": role, "tools_hidden": hidden}] + "name": self.name, "keep_tools": self.keep_tools, "role": role, "tools_hidden": hidden, + **({"context": self.context_label} if self.context_label is not None else {})}] return context @@ -158,10 +168,11 @@ def _env_urls(deployed: DeployedEnv, agent: DeployedAgent) -> tuple[str, str]: return base, theirs.rstrip("/") -def _takes_origin(card: dict) -> bool: - """Whether the browser's cdp takes an origin: one composed before it did refuses any it doesn't know.""" +def _takes(card: dict, field: str) -> bool: + """Whether the browser's cdp takes ``field``, an origin or a context: one composed before it did refuses any it + doesn't know.""" method = find_extension_method(card, BROWSER, "cdp") or {} - return "origin" in method.get("request", {}).get("properties", {}) + return field in method.get("request", {}).get("properties", {}) def _rebased(url: str, base: str, theirs: str) -> str: diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index 4ad96acb0..f2b68d39b 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -111,6 +111,7 @@ from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep from agent_env.task_step.task_steps.reset_env import ResetEnvTaskStep from tst.integration.computer.browser_contexts import check_browser_contexts +from tst.integration.computer.browser_scoped import check_scoped_addresses from tst.integration.computer.browser_data import SITE as DATA_SITE, check_browser_data, check_browser_data_under_chaos from tst.integration.computer.browser_steps import check_browser_steps from tst.integration.computer.browser_state import check_browser_state, check_browser_state_under_chaos @@ -495,6 +496,26 @@ async def test_its_contexts_are_leased_by_steps_and_by_agents_each_with_its_own_ assert not report.failed +@pytest.mark.asyncio +async def test_an_address_scoped_to_a_lease_sees_it_alone_as_the_default_and_ends_with_it(browser_computer, + cdp_clients): + with namespace_routing(): + deployed = await Env.get(browser_computer.id).deploy() + context = TaskStepContext(deployed_envs=[deployed]) + try: + await _ready(deployed) + card = (await _get(deployed, "/.well-known/agent-env.json")).json() + app = next(child for child in card["children_environments"] if child["name"] == "chromium") + await check_scoped_addresses( + deployed, browser_computer.id, + lambda op: deployed.environment_url + protocol_v1.find_extension_method(app, BROWSER, op)["endpoint"], + cdp_clients) + finally: + report = await teardown_run(context) + + assert not report.failed + + @pytest.mark.asyncio async def test_a_task_pinning_a_version_of_the_browser_passes_preflight_on_it_and_runs_its_steps_through_task_run( browser_computer): @@ -1759,7 +1780,7 @@ async def _answers_again(deployed) -> bool: async def _cdp_at_gateway(deployed, browser: DeployedEnv, *, at: str | None = None, user: str = "ada", - origin: str | None = None, ca: LocalCA | None = None) -> dict: + origin: str | None = None, ca: LocalCA | None = None, context: str | None = None) -> dict: """The CDP address the browser gives ``user``, asked through the gateway's ``/svc/chromium/`` route, at ``at``, a front before it, else its own address, with the browser's own control token, as agentenv asks a computer child, naming ``origin`` or not.""" @@ -1767,7 +1788,8 @@ async def _cdp_at_gateway(deployed, browser: DeployedEnv, *, at: str | None = No path = protocol_v1.find_extension_method(app, BROWSER, "cdp")["endpoint"] async with httpx.AsyncClient(timeout=180, verify=_trusting(ca) if ca else True) as http: answer = await http.post(f"{at or deployed.gateway_url}/svc/chromium{path}", - json={"user": user, **({"origin": origin} if origin else {})}, + json={"user": user, **({"origin": origin} if origin else {}), + **({"context": context} if context else {})}, headers=browser.control_headers) assert answer.status_code == 200, answer.text return answer.json() @@ -1886,11 +1908,11 @@ def step(type: str, **fields): ).execute(context)).deployed_envs browser = DeployedEnv.from_dict(deployed.child_envs["chromium"]) for label, user in (("buyer", "alex"), ("seller", "bo")): - context = await step("browser_lease_context", label=label).execute(context) + context = await step("browser_lease_context", label=label, grants=["ada"]).execute(context) context = await step("browser_sign_in", site=session_shop.id, user=user, context=label).execute(context) context = await step("browser_open_tab", url=SESSION_PAGE, context=label).execute(context) - address = await _cdp_at_gateway(deployed, browser) - seen = {label: await _lease_page_text(browser, address["ws_url"], label) for label in ("buyer", "seller")} + seen = {label: await _lease_page_text(browser, (await _cdp_at_gateway(deployed, browser, context=label))[ + "ws_url"], label) for label in ("buyer", "seller")} default_cookies = (await _browser_op(browser, "cookies/get", {"urls": ["https://shop.test/"]}))["cookies"] finally: report = await teardown_run(context) diff --git a/tst/integration/computer/browser_scoped.py b/tst/integration/computer/browser_scoped.py new file mode 100644 index 000000000..debf78780 --- /dev/null +++ b/tst/integration/computer/browser_scoped.py @@ -0,0 +1,151 @@ +"""CDP addresses scoped to leases on a deployed chromium: steps lease buyer for ada and seller for bob, each signed in +by its own cookie with a tab open; ada's address scoped to buyer sees only buyer's targets, the cross-origin iframe +of a page there among them as its own target, and can't attach to seller's; bob isn't given an address to buyer; +through the real clients, Playwright on buyer's address finds the lease as its default context, its cookie and tabs, +and clicks inside the iframe and downloads a file from it, which the lease lists, Puppeteer on seller's finds seller +alone, and Playwright on the default address none of the leases' tabs; both leases' agents click in their iframes at +once, both getting there; and the lease released closes its connections, saying so, and refuses its address after.""" + +import asyncio +import itertools +import json +import subprocess +import time +from pathlib import Path +from typing import Callable + +import pytest +from websockets.asyncio.client import connect +from websockets.exceptions import ConnectionClosed, InvalidStatus + +from agent_env.env.envs.computer.utils import inside +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.registry import get_task_step_registry +from tst.integration.computer.browser_contexts import _caller + +SITE = "http://127.0.0.1:8791/" +FRAME = "http://127.0.0.2:8792/frame" +SERVE_S, WAIT_S, POLL_S = 900, 90.0, 0.5 +# Serves, on two loopback addresses for the seconds it's given, a page titled "frames" whose iframe is the other +# address's /frame: a button that counts its presses in its body's data-clicked, and a link to /file, a download. +SERVE_FRAMES = r"""import http.server, sys, threading, time +PAGE = b"frames" +FRAME = (b"frameget") +class Page(http.server.BaseHTTPRequestHandler): + def do_GET(self): + if self.path.startswith("/file"): + body = b"a receipt from the iframe\n" + self.send_response(200) + self.send_header("Content-Type", "application/octet-stream") + self.send_header("Content-Disposition", "attachment; filename=receipt.txt") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + return + body = FRAME if self.path.startswith("/frame") else PAGE + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, *args): + pass +for host, port in (("127.0.0.1", 8791), ("127.0.0.2", 8792)): + server = http.server.ThreadingHTTPServer((host, port), Page) + threading.Thread(target=server.serve_forever, daemon=True).start() +time.sleep(float(sys.argv[1])) +""" + + +async def _raw(ws_url: str, commands: list[tuple[str, dict]]) -> list[dict]: + """Each of ``commands`` sent in order over one CDP connection at ``ws_url``; each answer.""" + answers = [] + async with connect(ws_url, max_size=None, open_timeout=60) as ws: + ids = itertools.count(1) + for method, params in commands: + message_id = next(ids) + await ws.send(json.dumps({"id": message_id, "method": method, "params": params})) + while (reply := json.loads(await asyncio.wait_for(ws.recv(), 60))).get("id") != message_id: + pass + answers.append(reply) + return answers + + +async def _until(read, holds, what: str): + last, deadline = None, time.monotonic() + WAIT_S + while time.monotonic() < deadline: + last = await read() + if holds(last): + return last + await asyncio.sleep(POLL_S) + raise AssertionError(f"{what} didn't happen within {WAIT_S:g} s: {last!r}") + + +async def check_scoped_addresses(deployed, env_id: str, op_url: Callable[[str], str], clients: Path) -> None: + """The scoped addresses, as the module says, through browser/v1 at ``op_url(op)`` and the Node clients installed + in ``clients``.""" + call = _caller(deployed, op_url) + context = TaskStepContext(deployed_envs=[deployed]) + + def step(type: str, suffix: str, **fields): + return get_task_step_registry()[type].from_dict({"id": f"{type}-{suffix}", "version": None, "type": type, + "env_id": env_id, **fields}) + + async def cdp(user: str, label: str | None, status: int = 200) -> dict: + return await call("cdp", {"user": user, **({"context": label} if label else {})}, status=status) + + serving = asyncio.ensure_future(inside.run(deployed, ["python3", "-c", SERVE_FRAMES, str(SERVE_S)], + timeout_s=SERVE_S + 60)) + try: + await asyncio.sleep(3) + for label, user in (("buyer", "ada"), ("seller", "bob")): + context = await step("browser_lease_context", label, label=label, grants=[user]).execute(context) + context = await step("browser_set_cookies", label, context=label, cookies=[ + {"name": "sid", "value": label, "url": SITE, "max_age": 3600}]).execute(context) + context = await step("browser_open_tab", label, context=label, url=SITE).execute(context) + leases = {lease["context"]: lease["id"] for lease in (await call("contexts/list", method="GET"))["contexts"]} + seller_tab = (await call("tabs/list", {"context": "seller"}, method="GET"))["tabs"][0]["id"] + buyer, seller, default = await cdp("ada", "buyer"), await cdp("bob", "seller"), await cdp("carol", None) + refused = await cdp("bob", "buyer", status=403) + + async def buyer_targets() -> list[dict]: + [listed] = await _raw(buyer["ws_url"], [("Target.getTargets", {})]) + return listed["result"]["targetInfos"] + + seen = await _until(buyer_targets, lambda infos: any(info["type"] == "iframe" for info in infos), + "the buyer's cross-origin iframe") + [attached] = await _raw(buyer["ws_url"], [("Target.attachToTarget", {"targetId": seller_tab, + "flatten": True})]) + ran = subprocess.run(["node", "scoped.mjs", buyer["ws_url"], seller["ws_url"], default["ws_url"], SITE], + cwd=clients, capture_output=True, text=True, timeout=600) + assert ran.returncode == 0, ran.stderr[-2000:] + clients_saw = json.loads(ran.stdout.strip().splitlines()[-1]) + buyer_downloads = await _until(lambda: call("downloads/list", {"context": "buyer"}, method="GET"), + lambda found: any(d["filename"] == "receipt.txt" and d["state"] == "completed" + for d in found["downloads"]), "the iframe's download") + + async with connect(buyer["ws_url"], max_size=None, open_timeout=60) as held: + await call("contexts/release", {"context": "buyer"}) + with pytest.raises(ConnectionClosed) as closed: + await asyncio.wait_for(held.recv(), 30) + with pytest.raises(InvalidStatus): + await _raw(buyer["ws_url"], [("Browser.getVersion", {})]) + finally: + serving.cancel() + + assert "permission_denied" in json.dumps(refused), refused + assert {info["browserContextId"] for info in seen} == {leases["buyer"]}, seen + assert attached["error"]["message"] == "No target with given id found", attached + by_buyer, by_seller, by_default = (clients_saw[key].get("ok") for key in ("buyer", "seller", "default")) + assert by_buyer and by_seller and by_default, clients_saw + assert by_buyer["contexts"] == 1 and by_buyer["cookies"] == ["sid=buyer"], by_buyer + assert set(by_buyer["pages"]) == {SITE} and by_buyer["clicked"] == "1" and by_buyer["frame_url"] == FRAME + assert by_buyer["download"] == "receipt.txt", by_buyer + assert all(d["context"] == "buyer" for d in buyer_downloads["downloads"]), buyer_downloads + assert by_seller["contexts"] == 1 and by_seller["pages"] == [SITE] and by_seller["cookies"] == ["sid=seller"] + assert SITE not in by_default["pages"], by_default + at_once = clients_saw["at_once"] + assert at_once["buyer"].get("ok", {}).get("clicked") == "10", at_once + assert at_once["seller"].get("ok", {}).get("clicked") == "10", at_once + assert (closed.value.rcvd.code, closed.value.rcvd.reason) == (1001, "the lease ended: ask for an address again") diff --git a/tst/integration/computer/cdp_clients/scoped.mjs b/tst/integration/computer/cdp_clients/scoped.mjs new file mode 100644 index 000000000..104588440 --- /dev/null +++ b/tst/integration/computer/cdp_clients/scoped.mjs @@ -0,0 +1,88 @@ +// Drives a website browser through CDP addresses scoped to leases and prints, as JSON, what each client saw. +// Usage: node scoped.mjs : +// Playwright on the buyer's address: its default context's cookies and pages, then a new page at the frame page, a +// click inside its cross-origin iframe and a download from it; Puppeteer on the seller's: its pages, its contexts +// and a new page's cookies; Playwright on the default address: its pages; then Playwright on the buyer's and the +// seller's addresses at once, each clicking inside the iframe ten times. +import { chromium } from "playwright-core"; +import puppeteer from "puppeteer-core"; + +const [buyerWs, sellerWs, defaultWs, framePage] = process.argv.slice(2); +const TIMEOUT_MS = 60_000; +const CLICKS = 10; + +async function settled(work) { + try { + return { ok: await work }; + } catch (error) { + return { error: String(error.message || error).split("\n")[0] }; + } +} + +async function framed(context) { + const page = await context.newPage(); + await page.goto(framePage, { timeout: TIMEOUT_MS }); + return page; +} + +async function onBuyer() { + const browser = await chromium.connectOverCDP(buyerWs, { timeout: TIMEOUT_MS }); + const context = browser.contexts()[0]; + const seen = { + contexts: browser.contexts().length, + cookies: (await context.cookies()).map((cookie) => `${cookie.name}=${cookie.value}`), + pages: context.pages().map((page) => page.url()), + }; + const page = await framed(context); + const frame = page.frameLocator("iframe"); + await frame.locator("#press").click({ timeout: TIMEOUT_MS }); + seen.clicked = await frame.locator("body").getAttribute("data-clicked"); + seen.frame_url = page.frames()[1]?.url(); + const [download] = await Promise.all([page.waitForEvent("download", { timeout: TIMEOUT_MS }), + frame.locator("#get").click({ timeout: TIMEOUT_MS })]); + seen.download = download.suggestedFilename(); + await page.close(); + return seen; +} + +async function onSeller() { + const browser = await puppeteer.connect({ browserWSEndpoint: sellerWs, protocolTimeout: TIMEOUT_MS }); + const seen = { + contexts: browser.browserContexts().length, + pages: (await browser.pages()).map((page) => page.url()), + }; + const page = await browser.newPage(); + await page.goto(framePage, { timeout: TIMEOUT_MS }); + seen.cookies = (await page.cookies(framePage)).map((cookie) => `${cookie.name}=${cookie.value}`); + await page.close(); + await browser.disconnect(); + return seen; +} + +async function onDefault() { + const browser = await chromium.connectOverCDP(defaultWs, { timeout: TIMEOUT_MS }); + return { contexts: browser.contexts().length, pages: browser.contexts()[0].pages().map((page) => page.url()) }; +} + +async function clicking(endpoint) { + const browser = await chromium.connectOverCDP(endpoint, { timeout: TIMEOUT_MS }); + const page = await framed(browser.contexts()[0]); + const frame = page.frameLocator("iframe"); + const started = Date.now(); + for (let i = 0; i < CLICKS; i++) { + await frame.locator("#press").click({ timeout: TIMEOUT_MS }); + } + const seen = { clicked: await frame.locator("body").getAttribute("data-clicked"), ms: Date.now() - started }; + await page.close(); + return seen; +} + +const result = { + buyer: await settled(onBuyer()), + seller: await settled(onSeller()), + default: await settled(onDefault()), +}; +const [buyerClicks, sellerClicks] = await Promise.all([settled(clicking(buyerWs)), settled(clicking(sellerWs))]); +result.at_once = { buyer: buyerClicks, seller: sellerClicks }; +console.log(JSON.stringify(result)); +process.exit(0); diff --git a/tst/integration/computer/conftest.py b/tst/integration/computer/conftest.py index 720388a17..779f41113 100644 --- a/tst/integration/computer/conftest.py +++ b/tst/integration/computer/conftest.py @@ -140,7 +140,7 @@ def cdp_clients(tmp_path_factory) -> Path: if not npm or not (match := re.match(r"v(\d+)", version)) or int(match.group(1)) < 20: pytest.skip("the CDP clients need Node 20 or later and npm") folder = tmp_path_factory.mktemp("cdp-clients") - for name in ("package.json", "connect.mjs"): + for name in ("package.json", "connect.mjs", "scoped.mjs"): shutil.copy(CDP_CLIENTS / name, folder / name) installed = subprocess.run([npm, "install", "--no-audit", "--no-fund", "--loglevel=error"], cwd=folder, capture_output=True, text=True, timeout=600) diff --git a/tst/unit/task_step/test_browser_connect_cdp.py b/tst/unit/task_step/test_browser_connect_cdp.py index d4c7cba96..ac877537d 100644 --- a/tst/unit/task_step/test_browser_connect_cdp.py +++ b/tst/unit/task_step/test_browser_connect_cdp.py @@ -26,6 +26,12 @@ _BROWSER_TAKING_ORIGIN = {**_BROWSER, "params": {**_BROWSER["params"], "methods": {"cdp": { **_BROWSER["params"]["methods"]["cdp"], "request": {"type": "object", "required": ["user"], "properties": { "user": {"type": "string"}, "origin": {"type": "string", "default": None}}}}}}} +#: The browser's extension as one that scopes an address to a lease serves it. +_BROWSER_SCOPING = {**_BROWSER, "params": {**_BROWSER["params"], "methods": {"cdp": { + **_BROWSER["params"]["methods"]["cdp"], "request": {"type": "object", "required": ["user"], "properties": { + "user": {"type": "string"}, "origin": {"type": "string", "default": None}, + "context": {"type": "string", "default": None}}}}, + "contexts/lease": {"endpoint": "/svc/chromium/agentenv/ext/browser/contexts/lease"}}}} _TOOL_ACCESS = {"uri": "urn:agentenv:disable-tool/v1", "params": {"endpoint": "/tools/disable", "methods": {"disable": {"method": "POST"}}}} _CDP_CONFIG = {"uri": "urn:agentenv:cdp-config/v1", "params": {"endpoint": "/ext/cdp-config"}} @@ -189,11 +195,34 @@ async def test_a_connection_under_a_name_already_connected_replaces_its_record(m ("other", "chromium")] +@pytest.mark.asyncio +async def test_a_step_naming_a_lease_asks_for_an_address_scoped_to_it_and_one_that_scopes_none_is_refused( + monkeypatch): + sent = _mock(monkeypatch, tokens=["t1"]) + scoping = _env([_app("chromium", _BROWSER_SCOPING)]) + context = TaskStepContext(deployed_envs=[scoping], deployed_agents=[_agent()], metadata={}) + step = BrowserConnectCdpTaskStep(id="connect", version=None, env_id="computer", agent_name="solver", + context="buyer") + + await step.execute(context) + with pytest.raises(RuntimeError, match="scopes no CDP address to a lease"): + await step.execute(TaskStepContext(deployed_envs=[_env([_app("chromium", _BROWSER_TAKING_ORIGIN)])], + deployed_agents=[_agent()], metadata={})) + + asked = sent[0] + assert json.loads(asked.content) == {"user": "solver", "origin": "http://host.docker.internal:41000", + "context": "buyer"} + assert context.metadata["browser_connections"][0]["context"] == "buyer" and len(sent) == 3 + + def test_it_round_trips_and_a_blank_name_is_a_config_problem(): step = BrowserConnectCdpTaskStep(id="connect", version=None, env_id="computer", agent_name="solver", browser="chromium", name="shop", keep_tools=True) + scoped = BrowserConnectCdpTaskStep(id="connect", version=None, env_id="computer", context="buyer") assert BrowserConnectCdpTaskStep.from_dict(step.to_dict()).to_dict() == step.to_dict() + assert BrowserConnectCdpTaskStep.from_dict(scoped.to_dict()).context_label == "buyer" + assert "context" not in step.to_dict() assert BrowserConnectCdpTaskStep(id="c", version=None, env_id="computer", name="").config_problems() == [ "step 'c': name must be a non-empty string"]