From 8b628fe567e94e79ff08d1a40cf93615175277eb Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 12:27:35 -0800 Subject: [PATCH 1/9] feat(computers): Chromium on the desktop goes out through the egress proxy by policy, comes back after a crash, and X never blanks (FD-5481) - The ubuntu-24.04 base gives every Chromium build an app brings a managed policy (agentenv-egress.json in Chrome for Testing's, Chromium's, the snap's and Chrome's folders): ProxySettings fixed_servers at the egress proxy, bypassing what the session's NO_PROXY does, and WebRtcIPHandling disable_non_proxied_udp. It's the base's, not the chromium app's, since the same app runs on the browser computer, whose site proxy its own flags name. - A website browser that a signal or a failing code ends is launched again by its server (AppRuntime relaunch_on_crash, cause "crash"), up to CRASH_RELAUNCHES (5) times a boot; one that exits cleanly was closed and stays so, and one its server stopped is its server's to start. - Xvfb runs with no screensaver and no DPMS (-s 0 -dpms), whose timeouts run on real time. - Chromium's network time queries were off already (QUIET_POLICIES). Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/computer/app.py | 6 ++ .../agentenv_protocol/computer/app_runtime.py | 19 +++- .../website_browser/environment.py | 5 + .../tests/test_computer_app_relaunch.py | 102 ++++++++++++++++++ .../tests/test_website_browser.py | 20 ++-- .../artifacts/ubuntu-24.04/Dockerfile | 6 ++ .../ubuntu-24.04/egress/chromium-policy.json | 8 ++ .../artifacts/ubuntu-24.04/start-desktop | 3 +- .../computer/chromium_universe_test.py | 97 +++++++++++++++++ tst/unit/bundle/shipped_computers_test.py | 6 +- .../env/envs/test_computer_egress_base.py | 21 +++- 11 files changed, 278 insertions(+), 15 deletions(-) create mode 100644 packages/agentenv-protocol/tests/test_computer_app_relaunch.py create mode 100644 src/agent_env/computers/artifacts/ubuntu-24.04/egress/chromium-policy.json create mode 100644 tst/integration/computer/chromium_universe_test.py diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py index e594e33f9..c440a975d 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py @@ -480,6 +480,7 @@ def __init__(self) -> None: if hasattr(fn, _READY_CHECK_ATTR) and applies(getattr(fn, _READY_CHECK_ATTR), on)}, ready.timeout_s if ready is not None else DEFAULT_TIMEOUT_S, desktop=self._on_desktop(), launch_fds=self._launch_fds, relaunch_on_start=self._relaunch_on_start(), + relaunch_on_crash=self._relaunch_on_crash(), launch_running=self._launch_running()) self._computer_app_data = AppDataPlane( self._computer_app, {op: getattr(self, name) for op, name in cls._data_hooks.items()}, self.paths, @@ -503,6 +504,11 @@ def _relaunch_on_start(self) -> bool: """Whether only the server that launched it can talk to it, so a server that restarts launches it again.""" return False + def _relaunch_on_crash(self) -> bool: + """Whether a crash of the app its server launched, which its server sees end (``AppRuntime.exited``), launches + it again: no.""" + return False + def _launch_running(self) -> Optional[Callable[[], Awaitable[None]]]: """What a launch does while it runs instead of starting a second copy: nothing, so another copy starts.""" return None diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py index 44319a0b5..341cf56dc 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py @@ -40,6 +40,9 @@ LAUNCHES = Path(".agentenv/launches") #: How long an app that ended on its own may take to be reaped, for its exit code. EXIT_WAIT_S = 5.0 +#: How many times in a boot an app that relaunches after a crash is launched again, so one that can't start doesn't +#: loop. +CRASH_RELAUNCHES = 5 #: How long a stopped app may take to exit before it's killed. STOP_TIMEOUT_S = 10.0 HOLD_BOOT_LAUNCH_ENV = "AGENTENV_HOLD_BOOT_LAUNCH" @@ -65,7 +68,7 @@ class AppRuntime: def __init__(self, launch: Optional[Launch], ready_checks: dict[str, Callable[[], Any]], timeout_s: float, *, desktop: bool = True, launch_fds: Optional[Callable[[], Awaitable[dict[int, int]]]] = None, - relaunch_on_start: bool = False, + relaunch_on_start: bool = False, relaunch_on_crash: bool = False, launch_running: Optional[Callable[[], Awaitable[None]]] = None) -> None: self.launch = launch self.ready_checks = ready_checks @@ -73,6 +76,7 @@ def __init__(self, launch: Optional[Launch], ready_checks: dict[str, Callable[[] self.desktop = desktop self.launch_fds = launch_fds self.relaunch_on_start = relaunch_on_start + self.relaunch_on_crash = relaunch_on_crash self.launch_running = launch_running self.exits: set[asyncio.Future] = set() self.started = time.time() @@ -147,7 +151,9 @@ def paused(self) -> contextlib.AbstractContextManager: def exited(self) -> bool: """Log that the app this server launched last ended on its own, with its exit code or the signal that ended - it, unless the server stopped it; whether it did.""" + it, unless the server stopped it; whether it did. With ``relaunch_on_crash``, one a signal or a failing code + ended is launched again, up to ``CRASH_RELAUNCHES`` times a boot; one that exited cleanly was closed, and stays + so.""" if (pid := _recorded_pid()) is None: return False noting = asyncio.ensure_future(self._log_exit(pid)) @@ -159,6 +165,15 @@ async def _log_exit(self, pid: int) -> None: code = await session.returncode(pid, EXIT_WAIT_S) self.log("exit", pid, **({} if code is None else {"signal": signal.Signals(-code).name} if code < 0 else {"code": code})) + if not (self.relaunch_on_crash and code and _recorded_pid() == pid): + return + if (relaunched := sum(entry.get("cause") == "crash" for entry in self.launches())) >= CRASH_RELAUNCHES: + self.log("crash_relaunches_spent", pid, relaunched=relaunched) + return + try: + await self.launch_app("crash") + except Exception as e: + self.launch_failed = f"{type(e).__name__}: {e}" def log(self, event: str, pid: int, **fields: Any) -> None: """Append ``event`` for the app's ``pid`` to its launch log, which outlives its server for the boot.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py index 67cbb3d94..f2b5a9d51 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py @@ -296,6 +296,11 @@ def _write_prompt_answers(self) -> None: def _relaunch_on_start(self) -> bool: return True + def _relaunch_on_crash(self) -> bool: + """A browser that crashes is launched again, as the agent's main window onto sites: one a page or a disposed + context crashes would otherwise stay gone until someone launched it.""" + return True + def _launch_running(self) -> Callable[[], Awaitable[None]]: return self._open_window diff --git a/packages/agentenv-protocol/tests/test_computer_app_relaunch.py b/packages/agentenv-protocol/tests/test_computer_app_relaunch.py new file mode 100644 index 000000000..cf19e337c --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_app_relaunch.py @@ -0,0 +1,102 @@ +"""An app whose server sees it end, a website browser's, launched again when a signal or a failing code ended it, up to +``CRASH_RELAUNCHES`` times a boot; not when it exited cleanly, when its server stopped it, or when it doesn't ask.""" +from __future__ import annotations + +import asyncio +import signal +from types import SimpleNamespace + +import pytest + +from agentenv_protocol.computer import Launch, app_runtime, session +from agentenv_protocol.computer.app_runtime import CRASH_RELAUNCHES, AppRuntime + + +@pytest.fixture +def desktop(tmp_path, monkeypatch): + """The app's folder as the current directory, and a session whose launches each get a new pid that runs until + ``ended`` gives it its code; the pids launched, in order.""" + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("ENVIRONMENT_NAME", "browser") + monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") + state = SimpleNamespace(launched=[], running=set(), codes={}) + + async def launch(argv, cwd, env, fds=None): + pid = 100 + len(state.launched) + state.launched.append(pid) + state.running.add(pid) + return pid + + async def returncode(pid, timeout_s): + return state.codes.get(pid) + + async def stop(pid, timeout_s): + state.running.discard(pid) + + monkeypatch.setattr(session, "launch", launch) + monkeypatch.setattr(session, "stop", stop) + monkeypatch.setattr(session, "alive", lambda pid: pid in state.running) + monkeypatch.setattr(session, "returncode", returncode) + return state + + +def _runtime(relaunch_on_crash: bool = True) -> AppRuntime: + return AppRuntime(Launch(at="boot", command=["chrome"]), {}, 30, desktop=False, relaunch_on_crash=relaunch_on_crash) + + +async def _ended(runtime: AppRuntime, desktop, code: int) -> bool: + """The app last launched ended with ``code``, as its server sees it end; whether it ended on its own.""" + pid = desktop.launched[-1] + desktop.running.discard(pid) + desktop.codes[pid] = code + on_its_own = runtime.exited() + await asyncio.gather(*runtime.exits) + return on_its_own + + +@pytest.mark.asyncio +async def test_a_browser_a_signal_ends_is_launched_again_and_says_why(desktop): + runtime = _runtime() + await runtime.launch_app("boot") + + assert await _ended(runtime, desktop, -signal.SIGKILL) + + assert desktop.launched == [100, 101] and runtime.running() + assert [(entry["event"], entry["pid"], entry.get("cause") or entry.get("signal")) for entry in runtime.launches()] == [ + ("launch", 100, "boot"), ("exit", 100, "SIGKILL"), ("launch", 101, "crash")] + + +@pytest.mark.asyncio +async def test_a_failing_code_is_a_crash_and_a_clean_exit_is_a_close(desktop): + runtime = _runtime() + await runtime.launch_app("boot") + + await _ended(runtime, desktop, 1) + await _ended(runtime, desktop, 0) + + assert desktop.launched == [100, 101] and not runtime.running() + + +@pytest.mark.asyncio +async def test_one_its_server_stopped_or_that_doesnt_ask_stays_down(desktop): + stopped, unasked = _runtime(), _runtime(relaunch_on_crash=False) + await stopped.launch_app("boot") + await stopped.stop_app() + + assert not stopped.exited() + await unasked.launch_app("boot") + await _ended(unasked, desktop, -signal.SIGTRAP) + + assert desktop.launched == [100, 101] and not unasked.running() + + +@pytest.mark.asyncio +async def test_it_is_launched_again_at_most_crash_relaunches_times_a_boot(desktop): + runtime = _runtime() + await runtime.launch_app("boot") + + for _ in range(CRASH_RELAUNCHES + 2): + await _ended(runtime, desktop, -signal.SIGSEGV) + + assert len(desktop.launched) == 1 + CRASH_RELAUNCHES and not runtime.running() + assert runtime.launches()[-1]["event"] == "crash_relaunches_spent" diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 816337d61..33b1a4cd3 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -1212,8 +1212,8 @@ async def test_its_control_says_the_tab_whose_page_has_focus_is_active_else_the_ @pytest.mark.asyncio -async def test_its_connection_hears_events_its_server_logs_each_launch_and_an_exit_it_didnt_ask_for(monkeypatch, - tmp_path): +async def test_its_connection_hears_events_its_server_logs_each_launch_and_a_crash_it_launches_it_again_after( + monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") browser = FakeBrowser() @@ -1227,15 +1227,19 @@ async def test_its_connection_hears_events_its_server_logs_each_launch_and_an_ex await _answering(browser) os.killpg(second, signal.SIGKILL) for _ in range(int(WAIT_S / POLL_S)): - if any(entry["event"] == "exit" for entry in runtime.launches()): + if any(entry.get("cause") == "crash" for entry in runtime.launches()): break await asyncio.sleep(POLL_S) - after_exit = await browser.browser_record() + relaunched = await _answering(browser) + after_crash = await browser.browser_record() + launches = runtime.launches() + await runtime.stop_app() browser.devtools.close() assert [tab["id"] for tab in record["tabs"]] == ["start"] - assert after_exit["tabs"] == [] and [(tab["id"], tab["closed_by"]) for tab in after_exit["closed_tabs"]] == [ - ("start", "stop"), ("start", "exit")] - assert [(entry["event"], entry.get("cause"), entry["pid"]) for entry in runtime.launches()] == [ + closed = [(tab["id"], tab["closed_by"]) for tab in after_crash["closed_tabs"]] + assert closed[:2] == [("start", "stop"), ("start", "exit")] + assert [(entry["event"], entry.get("cause"), entry["pid"]) for entry in launches[:4]] == [ ("launch", "boot", first), ("stop", None, first), ("launch", "data/reset", second), ("exit", None, second)] - assert runtime.launches()[-1]["signal"] == "SIGKILL" + assert launches[3]["signal"] == "SIGKILL" and relaunched + assert (launches[4]["event"], launches[4]["cause"]) == ("launch", "crash") and launches[4]["pid"] != second diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/Dockerfile b/src/agent_env/computers/artifacts/ubuntu-24.04/Dockerfile index 8124c124b..7ac094a1b 100644 --- a/src/agent_env/computers/artifacts/ubuntu-24.04/Dockerfile +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/Dockerfile @@ -56,6 +56,12 @@ RUN sed -i -e "s/^AGENTENV_SESSION_UID=.*/AGENTENV_SESSION_UID=$(id -u user)/" \ > /etc/ld.so.preload COPY --from=egress /out/egress /usr/local/lib/agentenv-egress/egress COPY egress/agentenv-egress.sh /etc/profile.d/agentenv-egress.sh +# Every Chromium build an app brings goes out through the egress proxy by a managed policy, however it's launched. +COPY egress/chromium-policy.json /usr/local/lib/agentenv-egress/chromium-policy.json +RUN for build in /etc/opt/chrome_for_testing /etc/chromium /etc/chromium-browser /etc/opt/chrome; do \ + install -D -m 0644 /usr/local/lib/agentenv-egress/chromium-policy.json \ + "$build/policies/managed/agentenv-egress.json"; \ + done COPY root.conf /etc/supervisor/root.conf COPY supervisord.conf /etc/supervisor/supervisord.conf COPY xfwm4.xml /etc/xdg/xfce4/xfconf/xfce-perchannel-xml/xfwm4.xml diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/egress/chromium-policy.json b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/chromium-policy.json new file mode 100644 index 000000000..ef2d43a7e --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/egress/chromium-policy.json @@ -0,0 +1,8 @@ +{ + "ProxySettings": { + "ProxyMode": "fixed_servers", + "ProxyServer": "http://127.0.0.1:3128", + "ProxyBypassList": "localhost;127.0.0.1;[::1];host.docker.internal" + }, + "WebRtcIPHandling": "disable_non_proxied_udp" +} diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/start-desktop b/src/agent_env/computers/artifacts/ubuntu-24.04/start-desktop index 423f143ac..321f69cd1 100755 --- a/src/agent_env/computers/artifacts/ubuntu-24.04/start-desktop +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/start-desktop @@ -26,7 +26,8 @@ case "${1:-}" in # supervisord runs one X at a time, so a lock here is one a killed X left, and X refuses to start while the pid # it names is taken, which after a restart it often is. rm -f "/tmp/.X${DISPLAY#:}-lock" - exec Xvfb "$DISPLAY" -screen 0 "${resolution}x24" -nolisten tcp -ac + # No screensaver and no DPMS: their timeouts run on real time, and would blank an unattended screen. + exec Xvfb "$DISPLAY" -screen 0 "${resolution}x24" -nolisten tcp -ac -s 0 -dpms ;; with-x) shift diff --git a/tst/integration/computer/chromium_universe_test.py b/tst/integration/computer/chromium_universe_test.py new file mode 100644 index 000000000..3ae8a5ff5 --- /dev/null +++ b/tst/integration/computer/chromium_universe_test.py @@ -0,0 +1,97 @@ +"""agentenv's chromium on the ubuntu-24.04 desktop, on the universe clock: its boot browser, never launched again, goes +out through the computer's egress proxy by the base's policy and loads live sites it hasn't visited with the universe +182 days back, 730 days on and six years on, and right after a jump back; killed, it comes back by itself; and X runs +no screensaver. Needs Docker, the local registry on localhost:5000, and the internet.""" + +import asyncio +import json +import re +import subprocess +import time + +import pytest +from agentenv_protocol import client as protocol_v1 +from agentenv_protocol.types import WELL_KNOWN_PATH +from agentenv_protocol.website_browser import BROWSER + +from agent_env.env import Env +from agent_env.env.envs.computer.utils import inside +from agent_env.store.routing import namespace_routing +from agent_env.task.teardown import teardown_run +from agent_env.task_step.context import TaskStepContext +from tst.integration.computer.browser_state import _client +from tst.integration.computer.conftest import UniverseClock, computer_container +from tst.integration.computer.website_browser_test import _app, _browser_process, _ready + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] + +DAY = 86400 +#: Live sites the browser hasn't visited, each with its page's title, opened in turn with the universe that many days +#: from now: the last right after the jump back. +STOPS = [(-182, "https://example.com/", "Example Domain"), (730, "https://example.org/", "Example Domain"), + (6 * 365, "https://example.net/", "Example Domain"), (-182, "https://www.wikipedia.org/", "Wikipedia")] +#: How long a page may take to load before its tab counts as not loaded. +LOAD_S = 45 +LAUNCHES = "/opt/agentenv/apps/chromium/.agentenv/launches" +POLICY = "/etc/opt/chrome_for_testing/policies/managed/agentenv-egress.json" + + +async def _loaded(ok, url: str, title: str) -> str: + """The title of the active tab once it's ``title`` after navigating to ``url``, else the tabs it last saw.""" + await ok("navigate", {"url": url}) + tabs, deadline = [], time.monotonic() + LOAD_S + while time.monotonic() < deadline: + tabs = (await ok("tabs/list", method="GET"))["tabs"] + if (active := next((tab for tab in tabs if tab["active"]), None)) and active["title"] == title: + return title + await asyncio.sleep(0.5) + return json.dumps(tabs) + + +def _egress_rows(deployed) -> list[dict]: + log = subprocess.run(["docker", "exec", computer_container(deployed.sandbox_id), "cat", + "/var/log/agentenv-root/egress.log"], capture_output=True, text=True).stdout + return [json.loads(line) for line in log.splitlines() if line.startswith("{")] + + +@pytest.mark.asyncio +async def test_its_boot_browser_loads_live_sites_at_any_universe_date_comes_back_when_killed_and_x_never_blanks( + chromium_desktop): + with namespace_routing(): + deployed = await Env.get(chromium_desktop.id).deploy() + context = TaskStepContext(deployed_envs=[deployed]) + try: + card = await _app(deployed, WELL_KNOWN_PATH, "chromium") + _, ok = _client(deployed, lambda op: f"{deployed.environment_url}/svc/chromium" + f"{protocol_v1.find_extension_method(card, BROWSER, op)['endpoint']}") + booted = await _browser_process(deployed) + clock, loaded = UniverseClock(deployed), [] + for days, url, title in STOPS: + clock.set(time.time() + days * DAY, 1) + await asyncio.sleep(2) # each program maps the page the next time it reads the clock, within a second + loaded.append(await _loaded(ok, url, title)) + still = await _browser_process(deployed) + rows = _egress_rows(deployed) + + await inside.run(deployed, ["kill", "-9", str(booted["pid"])], timeout_s=30) + back = await _browser_process(deployed, other_than=booted["pid"]) + ready = await _ready(deployed, card, "chromium") + launches = [json.loads(line) for line in (await inside.run(deployed, ["cat", LAUNCHES], timeout_s=30))[ + "stdout"].splitlines() if line] + xset = (await inside.run(deployed, ["xset", "q"], timeout_s=30))["stdout"] + policy = json.loads((await inside.run(deployed, ["cat", POLICY], timeout_s=30))["stdout"]) + finally: + report = await teardown_run(context) + + assert loaded == [title for _, _, title in STOPS], loaded + assert still["pid"] == booted["pid"] + hosts = {url.split("/")[2] for _, url, _ in STOPS} + intercepted = {row["host"] for row in rows if row.get("host") in hosts and row.get("mode") == "intercept"} + assert intercepted == hosts, [row for row in rows if row.get("host") in hosts][-8:] + assert back["pid"] != booted["pid"] and ready["ready"], ready + assert [(entry["event"], entry.get("cause") or entry.get("signal")) for entry in launches[-2:]] == [ + ("exit", "SIGKILL"), ("launch", "crash")], launches + assert re.search(r"Screen Saver:.*?timeout:\s+0\b", xset, re.S), xset + assert "DPMS is Disabled" in xset or "does not have the DPMS Extension" in xset, xset + assert policy["ProxySettings"]["ProxyServer"] == "http://127.0.0.1:3128" + assert not report.failed diff --git a/tst/unit/bundle/shipped_computers_test.py b/tst/unit/bundle/shipped_computers_test.py index 47860103e..7df4b7f05 100644 --- a/tst/unit/bundle/shipped_computers_test.py +++ b/tst/unit/bundle/shipped_computers_test.py @@ -48,9 +48,9 @@ def test_a_task_naming_ubuntu_24_04_writes_the_shipped_base_over_its_folders_ima assert built[1] == ["Dockerfile", "artifact.toml", *(f"clock/{name}" for name in ( "COPYING", "README.md", "agentenv_clock_page.c", "agentenv_clock_waits.c", "build.sh", "clock-follower", "clock.conf", "clockpage.py", "libfaketime.patch", "tls_realtime.c")), *(f"egress/{name}" for name in ( - "README.md", "agentenv-egress.sh", "boot.go", "boot_test.go", "clock.go", "clock_test.go", "go.mod", "go.sum", - "hello.go", "hello_test.go", "leaves.go", "main.go", "proxy.go", "proxy_test.go", "testdata/page-armed.hex", - "testdata/page-unarmed.hex", "upstream.go")), "root.conf", + "README.md", "agentenv-egress.sh", "boot.go", "boot_test.go", "chromium-policy.json", "clock.go", + "clock_test.go", "go.mod", "go.sum", "hello.go", "hello_test.go", "leaves.go", "main.go", "proxy.go", + "proxy_test.go", "testdata/page-armed.hex", "testdata/page-unarmed.hex", "upstream.go")), "root.conf", "start-desktop", "supervisord.conf", "xfwm4.xml"] assert computer.metadata["validated_images"]["container"]["arm64"]["status"] == "uninspected" diff --git a/tst/unit/env/envs/test_computer_egress_base.py b/tst/unit/env/envs/test_computer_egress_base.py index 2678fbe34..d3ead3027 100644 --- a/tst/unit/env/envs/test_computer_egress_base.py +++ b/tst/unit/env/envs/test_computer_egress_base.py @@ -7,6 +7,7 @@ import configparser import importlib.util +import json import re import shlex from datetime import UTC, datetime @@ -14,7 +15,7 @@ from urllib.parse import urlsplit import agent_env -from agent_env.env.envs.computer.utils import sites +from agent_env.env.envs.computer.utils import site_proxy, sites from agent_env.providers.sandbox_providers.local_docker.provider import LOCAL_TRUST_ENV BASE = Path(agent_env.__file__).parent / "computers" / "artifacts" / "ubuntu-24.04" @@ -87,3 +88,21 @@ def test_the_pages_the_proxy_is_tested_on_are_the_ones_the_followers_writer_writ assert golden == { "armed": page.encode(4, 1577836800_000000000, 1794000000_123456789, 2.0, 1, 7, 5_000_000_000, 5_000_000_000), "unarmed": page.encode(2, 0, 0, 1.0, 0, 3, 0, 0)} + + +def test_every_chromium_build_goes_out_through_the_proxy_by_a_policy_that_bypasses_what_the_session_does(): + policy = json.loads((EGRESS / "chromium-policy.json").read_text()) + session = _session_environment() + loop = re.search(r"for build in (.+?); do", (BASE / "Dockerfile").read_text()).group(1).split() + + assert policy["ProxySettings"] == { + "ProxyMode": "fixed_servers", "ProxyServer": session["HTTPS_PROXY"], + "ProxyBypassList": ";".join(f"[{host}]" if ":" in host else host for host in session["NO_PROXY"].split(","))} + assert policy["WebRtcIPHandling"] == "disable_non_proxied_udp" + assert {str(path.parents[2]) for path in site_proxy.Paths().chromium_policies} == set(loop) + + +def test_x_has_no_screensaver_or_dpms_whose_timeouts_run_on_real_time(): + xvfb = shlex.split(next(line for line in (BASE / "start-desktop").read_text().splitlines() if "exec Xvfb" in line)) + + assert xvfb[xvfb.index("-s") + 1] == "0" and "-dpms" in xvfb From fc84011d2d198144feeff3e7f46ee88fad787d64 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 12:45:07 -0800 Subject: [PATCH 2/9] fix(computers): a crashed browser's leftovers are stopped before it's launched again, so it starts afresh On the host, a SIGKILLed Chromium was never launched again: its helpers outlived it for a moment, so its session still counted as the app running, and the relaunch reached for a window over its closed pipe instead of starting a browser. What's left of the crashed session is now stopped first. The integration test waits for the crash's launch in the app's log, and the browser probe's pgrep pattern no longer matches the probe's own command line, which had passed it off as a browser when none ran. Co-Authored-By: Claude Opus 5.5 --- .../agentenv_protocol/computer/app_runtime.py | 6 ++++-- .../tests/test_computer_app_relaunch.py | 21 +++++++++++++++++-- .../computer/chromium_universe_test.py | 15 +++++++++++-- .../computer/website_browser_test.py | 2 +- 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py index 341cf56dc..62f1a2727 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py @@ -152,8 +152,9 @@ def paused(self) -> contextlib.AbstractContextManager: def exited(self) -> bool: """Log that the app this server launched last ended on its own, with its exit code or the signal that ended it, unless the server stopped it; whether it did. With ``relaunch_on_crash``, one a signal or a failing code - ended is launched again, up to ``CRASH_RELAUNCHES`` times a boot; one that exited cleanly was closed, and stays - so.""" + ended is launched again, up to ``CRASH_RELAUNCHES`` times a boot, once what's left of it is stopped: a browser's + helpers outlive it for a moment, and while they run its session counts as the app running. One that exited + cleanly was closed, and stays so.""" if (pid := _recorded_pid()) is None: return False noting = asyncio.ensure_future(self._log_exit(pid)) @@ -171,6 +172,7 @@ async def _log_exit(self, pid: int) -> None: self.log("crash_relaunches_spent", pid, relaunched=relaunched) return try: + await session.stop(pid, STOP_TIMEOUT_S) await self.launch_app("crash") except Exception as e: self.launch_failed = f"{type(e).__name__}: {e}" diff --git a/packages/agentenv-protocol/tests/test_computer_app_relaunch.py b/packages/agentenv-protocol/tests/test_computer_app_relaunch.py index cf19e337c..583f1fd49 100644 --- a/packages/agentenv-protocol/tests/test_computer_app_relaunch.py +++ b/packages/agentenv-protocol/tests/test_computer_app_relaunch.py @@ -19,7 +19,7 @@ def desktop(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) monkeypatch.setenv("ENVIRONMENT_NAME", "browser") monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") - state = SimpleNamespace(launched=[], running=set(), codes={}) + state = SimpleNamespace(launched=[], running=set(), codes={}, stopped=[]) async def launch(argv, cwd, env, fds=None): pid = 100 + len(state.launched) @@ -31,6 +31,7 @@ async def returncode(pid, timeout_s): return state.codes.get(pid) async def stop(pid, timeout_s): + state.stopped.append(pid) state.running.discard(pid) monkeypatch.setattr(session, "launch", launch) @@ -61,11 +62,27 @@ async def test_a_browser_a_signal_ends_is_launched_again_and_says_why(desktop): assert await _ended(runtime, desktop, -signal.SIGKILL) - assert desktop.launched == [100, 101] and runtime.running() + assert desktop.launched == [100, 101] and desktop.stopped == [100] and runtime.running() assert [(entry["event"], entry["pid"], entry.get("cause") or entry.get("signal")) for entry in runtime.launches()] == [ ("launch", 100, "boot"), ("exit", 100, "SIGKILL"), ("launch", 101, "crash")] +@pytest.mark.asyncio +async def test_what_a_crash_left_running_is_stopped_so_the_app_is_launched_afresh_not_reached_through_it(desktop): + runtime, reached = _runtime(), [] + + async def open_window(): + reached.append("window") + + runtime.launch_running = open_window + await runtime.launch_app("boot") + desktop.codes[100] = -signal.SIGKILL + assert runtime.exited() + await asyncio.gather(*runtime.exits) + + assert (desktop.stopped, desktop.launched, reached) == ([100], [100, 101], []) + + @pytest.mark.asyncio async def test_a_failing_code_is_a_crash_and_a_clean_exit_is_a_close(desktop): runtime = _runtime() diff --git a/tst/integration/computer/chromium_universe_test.py b/tst/integration/computer/chromium_universe_test.py index 3ae8a5ff5..4234eb9d6 100644 --- a/tst/integration/computer/chromium_universe_test.py +++ b/tst/integration/computer/chromium_universe_test.py @@ -48,6 +48,18 @@ async def _loaded(ok, url: str, title: str) -> str: return json.dumps(tabs) +async def _launches_until_relaunched(deployed) -> list[dict]: + """The chromium app's launch log once it records a launch after a crash, or as it last was within a minute.""" + launches: list[dict] = [] + for _ in range(60): + out = await inside.run(deployed, ["cat", LAUNCHES], timeout_s=30) + launches = [json.loads(line) for line in out["stdout"].splitlines() if line] + if any(entry.get("cause") == "crash" for entry in launches): + break + await asyncio.sleep(1) + return launches + + def _egress_rows(deployed) -> list[dict]: log = subprocess.run(["docker", "exec", computer_container(deployed.sandbox_id), "cat", "/var/log/agentenv-root/egress.log"], capture_output=True, text=True).stdout @@ -74,10 +86,9 @@ async def test_its_boot_browser_loads_live_sites_at_any_universe_date_comes_back rows = _egress_rows(deployed) await inside.run(deployed, ["kill", "-9", str(booted["pid"])], timeout_s=30) + launches = await _launches_until_relaunched(deployed) back = await _browser_process(deployed, other_than=booted["pid"]) ready = await _ready(deployed, card, "chromium") - launches = [json.loads(line) for line in (await inside.run(deployed, ["cat", LAUNCHES], timeout_s=30))[ - "stdout"].splitlines() if line] xset = (await inside.run(deployed, ["xset", "q"], timeout_s=30))["stdout"] policy = json.loads((await inside.run(deployed, ["cat", POLICY], timeout_s=30))["stdout"]) finally: diff --git a/tst/integration/computer/website_browser_test.py b/tst/integration/computer/website_browser_test.py index a2da177f0..5a80b0d4f 100644 --- a/tst/integration/computer/website_browser_test.py +++ b/tst/integration/computer/website_browser_test.py @@ -73,7 +73,7 @@ def log_message(self, *args): # The browser's pid and argv, and how many TCP sockets anything in its session listens on: its oldest process with # agentenv's pipe flag, whose title Chromium rewrites as one line. BROWSER_PROCESS = """import glob, json, os, subprocess -pid = int(subprocess.run(["pgrep", "-o", "-f", "--", "--remote-debugging-pipe"], capture_output=True, text=True, +pid = int(subprocess.run(["pgrep", "-o", "-f", "--", "[-]-remote-debugging-pipe"], capture_output=True, text=True, check=True).stdout) listening = set() for table in ("/proc/net/tcp", "/proc/net/tcp6"): From 824946cc1cdb6f3f83f4004d2ba4c2e3025c350d Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 12:54:11 -0800 Subject: [PATCH 3/9] fix(computers): a browser's crash relaunch takes the app's turn, so no data call changes its profile as it starts The relaunch after a crash ran beside the data plane's calls (Greptile): a data/reset or load arriving as it started found the dead pid, counted the app stopped and replaced the profile while the new browser read it. The app's turn, the lock those calls and launches take, moves from AppDataPlane to AppRuntime, and the relaunch holds it, skipping when anything launched the app since the crash (by the launch log: a data call that found it gone clears the pid file without launching it). Co-Authored-By: Claude Opus 5.5 --- .../agentenv_protocol/computer/app_data.py | 2 +- .../agentenv_protocol/computer/app_runtime.py | 30 ++++++++++------- .../tests/test_computer_app_relaunch.py | 33 +++++++++++++++++++ 3 files changed, 53 insertions(+), 12 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py index bd81cf97b..9935a8401 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py @@ -63,7 +63,7 @@ def __init__(self, runtime: AppRuntime, hooks: dict[str, Callable], paths: list[ self.live, self.keeps_paths, self.resets_failed_add = live, keeps_paths, resets_failed_add #: What a call that stopped the app awaits once it has launched it again, whose error fails the call. self.relaunched = relaunched - self.turn = asyncio.Lock() + self.turn = runtime.turn self.changing: set[asyncio.Task] = set() def keep_initial(self) -> None: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py index 62f1a2727..ac11a7bfe 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py @@ -79,6 +79,9 @@ def __init__(self, launch: Optional[Launch], ready_checks: dict[str, Callable[[] self.relaunch_on_crash = relaunch_on_crash self.launch_running = launch_running self.exits: set[asyncio.Future] = set() + #: Held by each launch or change of the app's data that must not overlap another (its data plane's ``turn``), + #: and by a relaunch after a crash. + self.turn = asyncio.Lock() self.started = time.time() self.launch_failed: Optional[str] = None self.checking: dict[str, asyncio.Future] = {} @@ -152,8 +155,9 @@ def paused(self) -> contextlib.AbstractContextManager: def exited(self) -> bool: """Log that the app this server launched last ended on its own, with its exit code or the signal that ended it, unless the server stopped it; whether it did. With ``relaunch_on_crash``, one a signal or a failing code - ended is launched again, up to ``CRASH_RELAUNCHES`` times a boot, once what's left of it is stopped: a browser's - helpers outlive it for a moment, and while they run its session counts as the app running. One that exited + ended is launched again, up to ``CRASH_RELAUNCHES`` times a boot: holding the ``turn``, so no data call changes + its files meanwhile, unless something launched it since, and once what's left of it is stopped, as a browser's + helpers outlive it for a moment and while they run its session counts as the app running. One that exited cleanly was closed, and stays so.""" if (pid := _recorded_pid()) is None: return False @@ -166,16 +170,20 @@ async def _log_exit(self, pid: int) -> None: code = await session.returncode(pid, EXIT_WAIT_S) self.log("exit", pid, **({} if code is None else {"signal": signal.Signals(-code).name} if code < 0 else {"code": code})) - if not (self.relaunch_on_crash and code and _recorded_pid() == pid): + if not (self.relaunch_on_crash and code): return - if (relaunched := sum(entry.get("cause") == "crash" for entry in self.launches())) >= CRASH_RELAUNCHES: - self.log("crash_relaunches_spent", pid, relaunched=relaunched) - return - try: - await session.stop(pid, STOP_TIMEOUT_S) - await self.launch_app("crash") - except Exception as e: - self.launch_failed = f"{type(e).__name__}: {e}" + async with self.turn: + launches = self.launches() + if next((entry["pid"] for entry in reversed(launches) if entry["event"] == "launch"), None) != pid: + return + if (relaunched := sum(entry.get("cause") == "crash" for entry in launches)) >= CRASH_RELAUNCHES: + self.log("crash_relaunches_spent", pid, relaunched=relaunched) + return + try: + await session.stop(pid, STOP_TIMEOUT_S) + await self.launch_app("crash") + except Exception as e: + self.launch_failed = f"{type(e).__name__}: {e}" def log(self, event: str, pid: int, **fields: Any) -> None: """Append ``event`` for the app's ``pid`` to its launch log, which outlives its server for the boot.""" diff --git a/packages/agentenv-protocol/tests/test_computer_app_relaunch.py b/packages/agentenv-protocol/tests/test_computer_app_relaunch.py index 583f1fd49..b9e54bf90 100644 --- a/packages/agentenv-protocol/tests/test_computer_app_relaunch.py +++ b/packages/agentenv-protocol/tests/test_computer_app_relaunch.py @@ -117,3 +117,36 @@ async def test_it_is_launched_again_at_most_crash_relaunches_times_a_boot(deskto assert len(desktop.launched) == 1 + CRASH_RELAUNCHES and not runtime.running() assert runtime.launches()[-1]["event"] == "crash_relaunches_spent" + + +@pytest.mark.asyncio +async def test_a_relaunch_after_a_crash_waits_for_the_data_call_that_holds_the_turn(desktop): + runtime = _runtime() + await runtime.launch_app("boot") + await runtime.turn.acquire() + desktop.running.discard(100) + desktop.codes[100] = -signal.SIGKILL + + assert runtime.exited() + await asyncio.sleep(0.05) + during = list(desktop.launched) + runtime.turn.release() + await asyncio.gather(*runtime.exits) + + assert (during, desktop.launched) == ([100], [100, 101]) + + +@pytest.mark.asyncio +async def test_one_launched_again_since_its_crash_isnt_launched_once_more_for_it(desktop): + runtime = _runtime() + await runtime.launch_app("boot") + await runtime.turn.acquire() + desktop.running.discard(100) + desktop.codes[100] = -signal.SIGKILL + + assert runtime.exited() + await runtime.launch_app("data/reset") + runtime.turn.release() + await asyncio.gather(*runtime.exits) + + assert (desktop.launched, desktop.stopped) == ([100, 101], []) From 750c28361493454428e32d199490938550fa1016 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 13:14:32 -0800 Subject: [PATCH 4/9] test: the browser tests that killed Chromium expect it back, launched again for the crash chromium_app_test's launch-cause test and the browser computer's two chaos tests killed the browser and expected it to stay down, to be launched from the menu or to fail every call after. It now comes back by itself, so the launch-cause test reads boot, the kill's exit, then a launch for the crash; and the chaos tests hold that calls with the browser killed answer at once, failing while it's gone, and work again once it's launched again. Co-Authored-By: Claude Opus 5.5 --- tst/integration/computer/browser_computer_test.py | 4 ++-- tst/integration/computer/browser_data.py | 15 ++++++++------- tst/integration/computer/browser_state.py | 8 ++++++-- tst/integration/computer/chromium_app_test.py | 11 +++++------ 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index 37b09f7cf..a45e4cd3a 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -389,7 +389,7 @@ async def test_its_cookies_local_storage_and_downloads_are_read_and_set_through_ @pytest.mark.asyncio -async def test_its_state_operations_hold_up_at_once_through_a_server_restart_and_fail_at_once_with_the_browser_gone( +async def test_its_state_operations_hold_up_at_once_through_a_server_restart_and_answer_at_once_with_the_browser_killed( browser_computer): with namespace_routing(): deployed = await Env.get(browser_computer.id).deploy() @@ -555,7 +555,7 @@ async def test_each_start_tabs_page_sees_the_windows_size_from_its_first_script_ @pytest.mark.asyncio -async def test_its_data_calls_take_turns_a_server_restart_takes_a_load_and_with_the_browser_gone_they_fail_at_once( +async def test_its_data_calls_take_turns_a_server_restart_takes_a_load_and_with_the_browser_killed_they_answer_at_once( browser_computer): with namespace_routing(): deployed = await Env.get(browser_computer.id).deploy() diff --git a/tst/integration/computer/browser_data.py b/tst/integration/computer/browser_data.py index e0561ece4..31b556ccf 100644 --- a/tst/integration/computer/browser_data.py +++ b/tst/integration/computer/browser_data.py @@ -4,10 +4,11 @@ files; once an agent has acted, a load can't change its tabs but sets its cookies; data/reset puts it back signed out at its start tab with its addresses ended; the snapshot taken mid-run loads back whole, its cookies, local storage, tabs and history; and reset_env, a reboot, brings back its start tab with no cookies or local storage. Under chaos: data calls at once each get their answer, its server killed and back takes a load, and -with chromium killed a snapshot fails at once rather than wait.""" +with chromium killed a snapshot answers at once rather than wait, and works again once chromium is launched again.""" import asyncio import base64 +import contextlib import json import time import uuid @@ -228,16 +229,16 @@ def cookie(value: str) -> list: after = await protocol_v1.get_data(base, timeout=120, headers=headers) await kill("[-]-remote-debugging-pipe") started = time.monotonic() - try: + with contextlib.suppress(Exception): await protocol_v1.get_data(base, timeout=120, headers=headers) - failed = None - except Exception as e: - failed = type(e).__name__ - failed_s = time.monotonic() - started + answered_s = time.monotonic() - started + await ready() + relaunched = await protocol_v1.get_data(base, timeout=120, headers=headers) for answer in (at_once[0], at_once[2], after): assert "browser-state.json" in [uploaded_object_path(part) for part in answer.parts] state = next(json.loads(base64.b64decode(part.file.bytes)) for part in after.parts if uploaded_object_path(part) == "browser-state.json") assert [c["value"] for c in state["cookies"] if c["name"] == "c"] == ["3"], state["cookies"] - assert failed is not None and failed_s < 60, (failed, failed_s) + assert answered_s < 60, answered_s + assert "browser-state.json" in [uploaded_object_path(part) for part in relaunched.parts] diff --git a/tst/integration/computer/browser_state.py b/tst/integration/computer/browser_state.py index c4dc5e01d..c358ca323 100644 --- a/tst/integration/computer/browser_state.py +++ b/tst/integration/computer/browser_state.py @@ -4,7 +4,8 @@ a partitioned cookie read with its key and deleted beside an unpartitioned one of its name; a site's service worker, which answers its pages, kept from agentenv's; a download with its file; and an operation refused without the control token. Under chaos: calls at once each get their answer; the pages agentenv serves itself never show in the record; with the browser's server killed and back, -the operations work again; and with the browser killed, they fail at once rather than wait.""" +the operations work again; and with the browser killed, they answer at once rather than wait, failing while it's +gone, and work again once it's launched again.""" import asyncio import base64 @@ -207,6 +208,8 @@ async def check_browser_state_under_chaos(deployed, endpoint: Callable[[str], st started = time.monotonic() down = [await call("cookies/get", {"urls": [SITE]}), await call("local_storage/get", {"origins": [ORIGIN]})] down_s = time.monotonic() - started + await ready() + relaunched = await ok("cookies/get", {"urls": [SITE]}) finally: serving.cancel() @@ -216,5 +219,6 @@ async def check_browser_state_under_chaos(deployed, endpoint: Callable[[str], st assert {"name": "k2", "value": "2"} in after[0]["localStorage"], after for record in (recorded, recorded_back): assert [tab["url"] for tab in record["tabs"]] == ["about:blank"] and record["closed_tabs"] == [], record - assert all(answer.status_code >= 400 for answer in down) and down_s < 30, ([a.text for a in down], down_s) + assert down_s < 30, ([a.text for a in down], down_s) + assert "cookies" in relaunched, relaunched return {"cookie_kept": "kept" in cookies_back, "item_kept": {"name": "k", "value": "1"} in storage_back} diff --git a/tst/integration/computer/chromium_app_test.py b/tst/integration/computer/chromium_app_test.py index 8ea752da3..4f279f71f 100644 --- a/tst/integration/computer/chromium_app_test.py +++ b/tst/integration/computer/chromium_app_test.py @@ -293,8 +293,7 @@ async def test_its_control_records_tabs_a_download_a_dialog_and_each_launch_with await serving browser = await _browser_process(deployed) await inside.run(deployed, ["kill", "-9", str(browser["pid"])], timeout_s=30) - exited = await _record(deployed, card, lambda r: r["launches"][-1]["event"] == "exit") - await inside.run(deployed, ["sh", "-c", LAUNCHER], timeout_s=60) + crashed = await _record(deployed, card, lambda r: r["launches"][-1].get("cause") == "crash") relaunched = await _ready(deployed, card, "chromium") await inside.run(deployed, ["pkill", "-f", "computer.serve_app chromium.py"], timeout_s=30) restarted = await _record(deployed, card, lambda r: r["launches"][-1].get("cause") == "server_restart") @@ -315,11 +314,11 @@ async def test_its_control_records_tabs_a_download_a_dialog_and_each_launch_with hashlib.sha256(_report(n)).hexdigest() for n in range(1, len(downloads) + 1)), recorded assert [tab["closed_by"] for tab in recorded["closed_tabs"]] == ["unknown"], recorded assert [(tab["url"], tab["active"]) for tab in recorded["tabs"]] == [(RECORDED, True)], recorded - assert [entry["event"] for entry in exited["launches"]] == ["launch", "exit"], exited - assert exited["launches"][1]["signal"] == "SIGKILL" and recorded["since"], exited - assert exited["launches"][0]["cause"] == "boot" and relaunched["ready"] is True, relaunched + assert [(entry["event"], entry.get("cause") or entry.get("signal")) for entry in crashed["launches"]] == [ + ("launch", "boot"), ("exit", "SIGKILL"), ("launch", "crash")], crashed + assert recorded["since"] and relaunched["ready"] is True, relaunched events = [(entry["event"], entry.get("cause")) for entry in restarted["launches"]] - assert events[:3] == [("launch", "boot"), ("exit", None), ("launch", "launch")] + assert events[:3] == [("launch", "boot"), ("exit", None), ("launch", "crash")] assert events[3:] in ([("stop", None), ("launch", "server_restart")], [("launch", "server_restart")]), events assert not report.failed From 6d3078f101fb49002c3f4cca79b98a506a500e7d Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 13:31:26 -0800 Subject: [PATCH 5/9] fix(computers): a browser its server relaunches after a crash isn't gone to readiness An exit the app's server launches it again after is logged relaunching, and apps.ended_on_its_own reads only the latest launch's exit, so the page driver's final readiness check waits for the relaunch instead of failing at the crash; a relaunch that fails is logged relaunch_failed and fails it at once. Co-Authored-By: Claude Opus 5.5 --- .../agentenv_protocol/computer/app_runtime.py | 24 +++++----- .../src/agentenv_protocol/computer/apps.py | 23 ++++++---- .../tests/test_computer_app_relaunch.py | 46 +++++++++++++++++-- 3 files changed, 71 insertions(+), 22 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py index ac11a7bfe..b60ea9784 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py @@ -155,10 +155,10 @@ def paused(self) -> contextlib.AbstractContextManager: def exited(self) -> bool: """Log that the app this server launched last ended on its own, with its exit code or the signal that ended it, unless the server stopped it; whether it did. With ``relaunch_on_crash``, one a signal or a failing code - ended is launched again, up to ``CRASH_RELAUNCHES`` times a boot: holding the ``turn``, so no data call changes - its files meanwhile, unless something launched it since, and once what's left of it is stopped, as a browser's - helpers outlive it for a moment and while they run its session counts as the app running. One that exited - cleanly was closed, and stays so.""" + ended is launched again, up to ``CRASH_RELAUNCHES`` times a boot (``_log_exit``): holding the ``turn``, so no + data call changes its files meanwhile, unless something launched it since, and once what's left of it is + stopped, as a browser's helpers outlive it for a moment and while they run its session counts as the app + running. One that exited cleanly was closed, and stays so.""" if (pid := _recorded_pid()) is None: return False noting = asyncio.ensure_future(self._log_exit(pid)) @@ -167,23 +167,25 @@ def exited(self) -> bool: return True async def _log_exit(self, pid: int) -> None: + """The exit logged, marked ``relaunching`` when it's a crash its server launches it again after, which those + reading the log (``apps.ended_on_its_own``) don't take for the app gone; and that relaunch, or why it failed.""" code = await session.returncode(pid, EXIT_WAIT_S) + relaunching = bool(self.relaunch_on_crash and code and sum( + entry.get("cause") == "crash" for entry in self.launches()) < CRASH_RELAUNCHES) self.log("exit", pid, **({} if code is None else - {"signal": signal.Signals(-code).name} if code < 0 else {"code": code})) - if not (self.relaunch_on_crash and code): + {"signal": signal.Signals(-code).name} if code < 0 else {"code": code}), + **({"relaunching": True} if relaunching else {})) + if not relaunching: return async with self.turn: - launches = self.launches() - if next((entry["pid"] for entry in reversed(launches) if entry["event"] == "launch"), None) != pid: - return - if (relaunched := sum(entry.get("cause") == "crash" for entry in launches)) >= CRASH_RELAUNCHES: - self.log("crash_relaunches_spent", pid, relaunched=relaunched) + if next((entry["pid"] for entry in reversed(self.launches()) if entry["event"] == "launch"), None) != pid: return try: await session.stop(pid, STOP_TIMEOUT_S) await self.launch_app("crash") except Exception as e: self.launch_failed = f"{type(e).__name__}: {e}" + self.log("relaunch_failed", pid, error=self.launch_failed) def log(self, event: str, pid: int, **fields: Any) -> None: """Append ``event`` for the app's ``pid`` to its launch log, which outlives its server for the boot.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py index 437dd639c..f4d53d356 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py @@ -125,15 +125,22 @@ async def served_card(folder: Path) -> Optional[dict]: def ended_on_its_own(folder: Path) -> Optional[str]: - """Why the app in ``folder`` can't be ready again where nothing launches it but its server: its launch log says it - ended on its own, an infrastructure failure; None while it hasn't.""" + """Why the app in ``folder`` can't be ready again where nothing launches it but its server: its launch log says + its last launch ended on its own, an infrastructure failure, and its server isn't launching it again after a crash + (an exit marked ``relaunching``), or failed to; None while it hasn't.""" + ended = None for entry in launch_log(folder): - if entry.get("event") == "exit": - how = (f"by signal {entry['signal']}" if "signal" in entry else - f"with code {entry['code']}" if "code" in entry else "in a way that wasn't recorded") - return (f"it ended on its own at {entry.get('at')}, {how}, and nothing launches it again: an " - "infrastructure failure") - return None + if entry.get("event") == "launch": + ended = None + elif entry.get("event") in ("exit", "relaunch_failed"): + ended = entry + if ended is None or ended.get("relaunching"): + return None + if ended["event"] == "relaunch_failed": + return f"it crashed, and its launch after the crash failed: {ended.get('error')}: an infrastructure failure" + how = (f"by signal {ended['signal']}" if "signal" in ended else + f"with code {ended['code']}" if "code" in ended else "in a way that wasn't recorded") + return f"it ended on its own at {ended.get('at')}, {how}, and nothing launches it again: an infrastructure failure" def launch_log(folder: Path) -> list[dict]: diff --git a/packages/agentenv-protocol/tests/test_computer_app_relaunch.py b/packages/agentenv-protocol/tests/test_computer_app_relaunch.py index b9e54bf90..9834e74ab 100644 --- a/packages/agentenv-protocol/tests/test_computer_app_relaunch.py +++ b/packages/agentenv-protocol/tests/test_computer_app_relaunch.py @@ -1,14 +1,16 @@ """An app whose server sees it end, a website browser's, launched again when a signal or a failing code ended it, up to -``CRASH_RELAUNCHES`` times a boot; not when it exited cleanly, when its server stopped it, or when it doesn't ask.""" +``CRASH_RELAUNCHES`` times a boot; not when it exited cleanly, when its server stopped it, or when it doesn't ask. Its +launch log says so, and what reads it (``apps.ended_on_its_own``) takes it for gone only once nothing will launch it.""" from __future__ import annotations import asyncio import signal +from pathlib import Path from types import SimpleNamespace import pytest -from agentenv_protocol.computer import Launch, app_runtime, session +from agentenv_protocol.computer import Launch, app_runtime, apps, session from agentenv_protocol.computer.app_runtime import CRASH_RELAUNCHES, AppRuntime @@ -116,7 +118,8 @@ async def test_it_is_launched_again_at_most_crash_relaunches_times_a_boot(deskto await _ended(runtime, desktop, -signal.SIGSEGV) assert len(desktop.launched) == 1 + CRASH_RELAUNCHES and not runtime.running() - assert runtime.launches()[-1]["event"] == "crash_relaunches_spent" + assert runtime.launches()[-1]["event"] == "exit" and "relaunching" not in runtime.launches()[-1] + assert "nothing launches it again" in apps.ended_on_its_own(Path.cwd()) @pytest.mark.asyncio @@ -150,3 +153,40 @@ async def test_one_launched_again_since_its_crash_isnt_launched_once_more_for_it await asyncio.gather(*runtime.exits) assert (desktop.launched, desktop.stopped) == ([100, 101], []) + + +@pytest.mark.asyncio +async def test_a_crash_its_server_launches_it_again_after_isnt_the_app_gone_while_or_after(desktop): + runtime = _runtime() + await runtime.launch_app("boot") + await runtime.turn.acquire() + desktop.running.discard(100) + desktop.codes[100] = -signal.SIGKILL + + assert runtime.exited() + await asyncio.sleep(0.05) + while_relaunching = apps.ended_on_its_own(Path.cwd()) + runtime.turn.release() + await asyncio.gather(*runtime.exits) + + assert while_relaunching is None and apps.ended_on_its_own(Path.cwd()) is None + assert runtime.launches()[1]["relaunching"] is True + + +@pytest.mark.asyncio +async def test_a_clean_exit_or_a_relaunch_that_fails_is_the_app_gone(desktop, monkeypatch): + closed, failing = _runtime(), _runtime() + await closed.launch_app("boot") + await _ended(closed, desktop, 0) + gone = apps.ended_on_its_own(Path.cwd()) + + async def refused(argv, cwd, env, fds=None): + raise OSError("no such file: chrome") + + await failing.launch_app("boot") + monkeypatch.setattr(session, "launch", refused) + await _ended(failing, desktop, -signal.SIGTRAP) + + assert "with code 0, and nothing launches it again" in gone + assert apps.ended_on_its_own(Path.cwd()) == ( + "it crashed, and its launch after the crash failed: OSError: no such file: chrome: an infrastructure failure") From 1708b502a40bfa69b28decdbac519722281b1bc1 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 13:50:06 -0800 Subject: [PATCH 6/9] fix(computers): crash relaunches back off, so a browser whose X server restarts comes back with it A website browser's relaunches after a crash wait 1 s, doubling with each crash, instead of following at once: killing the X server took the browser with it, and its five relaunches, each failing without X, could be spent before supervisord had X back or not, so the computer came back or failed for good by a race. The browser computer's kill tests now expect a killed browser and a killed X server to come back, and one killed past its relaunches to fail it. Co-Authored-By: Claude Opus 5.5 --- .../agentenv_protocol/computer/app_runtime.py | 17 +++--- .../tests/test_computer_app_relaunch.py | 21 ++++++-- .../computer/browser_computer_test.py | 53 +++++++++++++------ 3 files changed, 64 insertions(+), 27 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py index b60ea9784..bcfa93ec1 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py @@ -43,6 +43,9 @@ #: How many times in a boot an app that relaunches after a crash is launched again, so one that can't start doesn't #: loop. CRASH_RELAUNCHES = 5 +#: How long after its first crash such an app is launched again, doubling with each crash after, so one that crashed for +#: want of something its supervisor brings back, as a browser whose X server is restarting, finds it there. +CRASH_RELAUNCH_BACKOFF_S = 1.0 #: How long a stopped app may take to exit before it's killed. STOP_TIMEOUT_S = 10.0 HOLD_BOOT_LAUNCH_ENV = "AGENTENV_HOLD_BOOT_LAUNCH" @@ -155,10 +158,11 @@ def paused(self) -> contextlib.AbstractContextManager: def exited(self) -> bool: """Log that the app this server launched last ended on its own, with its exit code or the signal that ended it, unless the server stopped it; whether it did. With ``relaunch_on_crash``, one a signal or a failing code - ended is launched again, up to ``CRASH_RELAUNCHES`` times a boot (``_log_exit``): holding the ``turn``, so no - data call changes its files meanwhile, unless something launched it since, and once what's left of it is - stopped, as a browser's helpers outlive it for a moment and while they run its session counts as the app - running. One that exited cleanly was closed, and stays so.""" + ended is launched again, up to ``CRASH_RELAUNCHES`` times a boot, after a wait that doubles with each crash + (``CRASH_RELAUNCH_BACKOFF_S``, ``_log_exit``): holding the ``turn``, so no data call changes its files + meanwhile, unless something launched it since, and once what's left of it is stopped, as a browser's helpers + outlive it for a moment and while they run its session counts as the app running. One that exited cleanly was + closed, and stays so.""" if (pid := _recorded_pid()) is None: return False noting = asyncio.ensure_future(self._log_exit(pid)) @@ -170,13 +174,14 @@ async def _log_exit(self, pid: int) -> None: """The exit logged, marked ``relaunching`` when it's a crash its server launches it again after, which those reading the log (``apps.ended_on_its_own``) don't take for the app gone; and that relaunch, or why it failed.""" code = await session.returncode(pid, EXIT_WAIT_S) - relaunching = bool(self.relaunch_on_crash and code and sum( - entry.get("cause") == "crash" for entry in self.launches()) < CRASH_RELAUNCHES) + crashes = sum(entry.get("cause") == "crash" for entry in self.launches()) + relaunching = bool(self.relaunch_on_crash and code and crashes < CRASH_RELAUNCHES) self.log("exit", pid, **({} if code is None else {"signal": signal.Signals(-code).name} if code < 0 else {"code": code}), **({"relaunching": True} if relaunching else {})) if not relaunching: return + await asyncio.sleep(CRASH_RELAUNCH_BACKOFF_S * 2 ** crashes) async with self.turn: if next((entry["pid"] for entry in reversed(self.launches()) if entry["event"] == "launch"), None) != pid: return diff --git a/packages/agentenv-protocol/tests/test_computer_app_relaunch.py b/packages/agentenv-protocol/tests/test_computer_app_relaunch.py index 9834e74ab..0a7216f24 100644 --- a/packages/agentenv-protocol/tests/test_computer_app_relaunch.py +++ b/packages/agentenv-protocol/tests/test_computer_app_relaunch.py @@ -1,5 +1,5 @@ """An app whose server sees it end, a website browser's, launched again when a signal or a failing code ended it, up to -``CRASH_RELAUNCHES`` times a boot; not when it exited cleanly, when its server stopped it, or when it doesn't ask. Its +``CRASH_RELAUNCHES`` times a boot, each a wait that doubles after the last; not when it exited cleanly, when its server stopped it, or when it doesn't ask. Its launch log says so, and what reads it (``apps.ended_on_its_own``) takes it for gone only once nothing will launch it.""" from __future__ import annotations @@ -13,12 +13,15 @@ from agentenv_protocol.computer import Launch, app_runtime, apps, session from agentenv_protocol.computer.app_runtime import CRASH_RELAUNCHES, AppRuntime +REAL_SLEEP = asyncio.sleep + @pytest.fixture def desktop(tmp_path, monkeypatch): - """The app's folder as the current directory, and a session whose launches each get a new pid that runs until - ``ended`` gives it its code; the pids launched, in order.""" + """The app's folder as the current directory, a session whose launches each get a new pid that runs until + ``ended`` gives it its code, and no wait before a relaunch; the pids launched, in order.""" monkeypatch.chdir(tmp_path) + monkeypatch.setattr(app_runtime, "CRASH_RELAUNCH_BACKOFF_S", 0) monkeypatch.setenv("ENVIRONMENT_NAME", "browser") monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") state = SimpleNamespace(launched=[], running=set(), codes={}, stopped=[]) @@ -110,14 +113,22 @@ async def test_one_its_server_stopped_or_that_doesnt_ask_stays_down(desktop): @pytest.mark.asyncio -async def test_it_is_launched_again_at_most_crash_relaunches_times_a_boot(desktop): - runtime = _runtime() +async def test_it_is_launched_again_at_most_crash_relaunches_times_a_boot_each_wait_twice_the_last(desktop, monkeypatch): + runtime, waits = _runtime(), [] + + async def sleep(seconds): + waits.append(seconds) + await REAL_SLEEP(0) + + monkeypatch.setattr(app_runtime, "CRASH_RELAUNCH_BACKOFF_S", 1.5) + monkeypatch.setattr(app_runtime.asyncio, "sleep", sleep) await runtime.launch_app("boot") for _ in range(CRASH_RELAUNCHES + 2): await _ended(runtime, desktop, -signal.SIGSEGV) assert len(desktop.launched) == 1 + CRASH_RELAUNCHES and not runtime.running() + assert waits == [1.5 * 2 ** crash for crash in range(CRASH_RELAUNCHES)] assert runtime.launches()[-1]["event"] == "exit" and "relaunching" not in runtime.launches()[-1] assert "nothing launches it again" in apps.ended_on_its_own(Path.cwd()) diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index a45e4cd3a..0dc610fae 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -6,9 +6,9 @@ that run code hidden until tool access names one, as the installed Playwright MCP lists them, and back after Playwright MCP or the driver is killed, and its page tools navigate, move in history, answer a dialog, use tabs and upload; the browser's X server runs but no desktop, and nothing listens but the driver and the site proxy; a -killed app server or driver is started again by supervisord and the computer is ready again; chromium killed fails it -for good, computer-ready/v1 saying why and the tools answering with it, and so does its X server killed, which takes -chromium with it though supervisord starts X again. Through its cdp address, Playwright and Puppeteer drive it two at once, +killed app server or driver is started again by supervisord, and chromium killed by its app server, and the computer is +ready again; chromium killed once more than its app server relaunches it fails it for good, computer-ready/v1 saying why +and the tools answering with it; its X server killed takes chromium with it, and both come back. Through its cdp address, Playwright and Puppeteer drive it two at once, each on its own session, by http_url and by ws_url; a wrong token is refused, and so is code in the computer without one; one client's close leaves the browser up for the other, which can't open file: URLs; the address outlives a relaunch by its app server's restart, and the browser's reset and the computer's end it. Behind a front that ends TLS, as a remote provider's tunnel does, @@ -64,6 +64,7 @@ from agentenv_protocol import client as protocol_v1 from agentenv_protocol.client import find_extension_method, invoke_extension from agentenv_protocol.computer import COMPUTER_PAGE, COMPUTER_READY +from agentenv_protocol.computer.app_runtime import CRASH_RELAUNCHES from agentenv_protocol.computer.conformance.client import image from agentenv_protocol.tool_access import ENABLE, ENABLE_TOOL_EXTENSION_URI from agentenv_protocol.tool_context import ROLE_HEADER @@ -257,7 +258,8 @@ async def test_its_tools_see_and_drive_the_page_that_is_its_screen(browser_compu @pytest.mark.asyncio -async def test_a_killed_app_server_or_driver_comes_back_and_a_killed_browser_fails_it_for_good(browser_computer): +async def test_a_killed_app_server_driver_or_browser_comes_back_and_a_browser_killed_past_its_relaunches_fails_it_for_good( + browser_computer): with namespace_routing(): deployed = await Env.get(browser_computer.id).deploy( env_params={"app_settings": {"chromium": {"start_urls": [PAGE_URL]}}}) @@ -265,12 +267,17 @@ async def test_a_killed_app_server_or_driver_comes_back_and_a_killed_browser_fai try: await _kill(deployed, "[s]erve_app") server_back = await _ready(deployed) - launches = (await _record(deployed))["launches"] await _kill(deployed, "[c]omputer.serve driver.py:PageDriver") driver_back = await _ready(deployed) - shot = await _screenshot(deployed) await _kill(deployed, "[-]-remote-debugging-pipe", oldest=True) + await _crash_launches(deployed, 1) + browser_back = await _ready(deployed) + shot = await _screenshot(deployed) + for crashes in range(1, CRASH_RELAUNCHES + 1): + await _crash_launches(deployed, crashes) + await _kill(deployed, "[-]-remote-debugging-pipe", oldest=True) failed = await _until(deployed, lambda status: "failed" in status) + launches = (await _record(deployed))["launches"] async with streamablehttp_client(deployed.mcp_url) as (read, write, _): async with ClientSession(read, write) as session: await session.initialize() @@ -278,18 +285,19 @@ async def test_a_killed_app_server_or_driver_comes_back_and_a_killed_browser_fai finally: report = await teardown_run(context) - assert server_back["ready"] and driver_back["ready"] and shot.size == (1280, 800) - assert [entry.get("cause") for entry in launches if entry["event"] == "launch"] == ["boot", "server_restart"] + assert server_back["ready"] and driver_back["ready"] and browser_back["ready"] and shot.size == (1280, 800) + assert [entry.get("cause") for entry in launches if entry["event"] == "launch"] == [ + "boot", "server_restart", *["crash"] * CRASH_RELAUNCHES] browser = next(check for check in failed["checks"] if check["name"] == "browser") assert failed["ready"] is False and browser["final"] is True and failed["failed"] == f"browser: {browser['detail']}" - assert "by signal SIGKILL" in browser["detail"] and "infrastructure failure" in browser["detail"] + assert "by signal SIGKILL" in browser["detail"] and "nothing launches it again" in browser["detail"] + assert "infrastructure failure" in browser["detail"] assert refused.isError and "has failed" in refused.content[0].text and "SIGKILL" in refused.content[0].text assert not report.failed @pytest.mark.asyncio -async def test_a_killed_x_server_comes_back_but_takes_the_browser_with_it_which_fails_the_computer_for_good( - browser_computer): +async def test_a_killed_x_server_comes_back_and_so_does_the_browser_it_took_with_it(browser_computer): with namespace_routing(): deployed = await Env.get(browser_computer.id).deploy( env_params={"app_settings": {"chromium": {"start_urls": [PAGE_URL]}}}) @@ -297,15 +305,16 @@ async def test_a_killed_x_server_comes_back_but_takes_the_browser_with_it_which_ try: before = await _screenshot(deployed) await _kill(deployed, "^Xvfb :99", oldest=True) - failed = await _until(deployed, lambda status: "failed" in status) x_back = await _until_x(deployed) + launches = await _crash_launches(deployed, 1) + back = await _ready(deployed) + after = await _screenshot(deployed) finally: report = await teardown_run(context) - assert before.size == (1280, 800) - browser = next(check for check in failed["checks"] if check["name"] == "browser") - assert failed["ready"] is False and browser["final"] is True and "infrastructure failure" in browser["detail"] - assert x_back + assert before.size == after.size == (1280, 800) + assert x_back and back["ready"], back + assert [entry.get("cause") for entry in launches if entry["event"] == "launch"][:2] == ["boot", "crash"], launches assert not report.failed @@ -748,6 +757,18 @@ async def _state(deployed, holds) -> dict: raise AssertionError(f"the page didn't get there within {WAIT_S:g} s; it last said {last!r}") +async def _crash_launches(deployed, count: int) -> list[dict]: + """The browser's launch log once ``count`` of its launches were after a crash, within ``WAIT_S``.""" + launches, deadline = [], time.monotonic() + WAIT_S + while time.monotonic() < deadline: + with contextlib.suppress(httpx.HTTPError, ValueError, KeyError): + launches = (await _record(deployed))["launches"] + if sum(entry.get("cause") == "crash" for entry in launches) >= count: + return launches + await asyncio.sleep(POLL_S) + raise AssertionError(f"the browser wasn't launched after crash {count} within {WAIT_S:g} s; its log: {launches!r}") + + async def _status(deployed) -> dict: card = (await _get(deployed, "/.well-known/agent-env.json")).json() path = protocol_v1.find_extension_method(card, COMPUTER_READY, "status")["endpoint"] From 9a8e0f32e286e9feaf47582055c7fbbdea987256 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 14:07:17 -0800 Subject: [PATCH 7/9] revert(computers): crash relaunches don't back off; a browser goes with its X server for good Chromium exits with code 0 when its X server goes, a close, so its server never relaunched it there and the X server kill test was never a race: the backoff fixed nothing. The kill test keeps its fix: a killed browser comes back, and one killed past its relaunches fails the computer. Co-Authored-By: Claude Opus 5.5 --- .../agentenv_protocol/computer/app_runtime.py | 17 ++++++--------- .../tests/test_computer_app_relaunch.py | 21 +++++-------------- .../computer/browser_computer_test.py | 17 ++++++++------- 3 files changed, 20 insertions(+), 35 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py index bcfa93ec1..b60ea9784 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py @@ -43,9 +43,6 @@ #: How many times in a boot an app that relaunches after a crash is launched again, so one that can't start doesn't #: loop. CRASH_RELAUNCHES = 5 -#: How long after its first crash such an app is launched again, doubling with each crash after, so one that crashed for -#: want of something its supervisor brings back, as a browser whose X server is restarting, finds it there. -CRASH_RELAUNCH_BACKOFF_S = 1.0 #: How long a stopped app may take to exit before it's killed. STOP_TIMEOUT_S = 10.0 HOLD_BOOT_LAUNCH_ENV = "AGENTENV_HOLD_BOOT_LAUNCH" @@ -158,11 +155,10 @@ def paused(self) -> contextlib.AbstractContextManager: def exited(self) -> bool: """Log that the app this server launched last ended on its own, with its exit code or the signal that ended it, unless the server stopped it; whether it did. With ``relaunch_on_crash``, one a signal or a failing code - ended is launched again, up to ``CRASH_RELAUNCHES`` times a boot, after a wait that doubles with each crash - (``CRASH_RELAUNCH_BACKOFF_S``, ``_log_exit``): holding the ``turn``, so no data call changes its files - meanwhile, unless something launched it since, and once what's left of it is stopped, as a browser's helpers - outlive it for a moment and while they run its session counts as the app running. One that exited cleanly was - closed, and stays so.""" + ended is launched again, up to ``CRASH_RELAUNCHES`` times a boot (``_log_exit``): holding the ``turn``, so no + data call changes its files meanwhile, unless something launched it since, and once what's left of it is + stopped, as a browser's helpers outlive it for a moment and while they run its session counts as the app + running. One that exited cleanly was closed, and stays so.""" if (pid := _recorded_pid()) is None: return False noting = asyncio.ensure_future(self._log_exit(pid)) @@ -174,14 +170,13 @@ async def _log_exit(self, pid: int) -> None: """The exit logged, marked ``relaunching`` when it's a crash its server launches it again after, which those reading the log (``apps.ended_on_its_own``) don't take for the app gone; and that relaunch, or why it failed.""" code = await session.returncode(pid, EXIT_WAIT_S) - crashes = sum(entry.get("cause") == "crash" for entry in self.launches()) - relaunching = bool(self.relaunch_on_crash and code and crashes < CRASH_RELAUNCHES) + relaunching = bool(self.relaunch_on_crash and code and sum( + entry.get("cause") == "crash" for entry in self.launches()) < CRASH_RELAUNCHES) self.log("exit", pid, **({} if code is None else {"signal": signal.Signals(-code).name} if code < 0 else {"code": code}), **({"relaunching": True} if relaunching else {})) if not relaunching: return - await asyncio.sleep(CRASH_RELAUNCH_BACKOFF_S * 2 ** crashes) async with self.turn: if next((entry["pid"] for entry in reversed(self.launches()) if entry["event"] == "launch"), None) != pid: return diff --git a/packages/agentenv-protocol/tests/test_computer_app_relaunch.py b/packages/agentenv-protocol/tests/test_computer_app_relaunch.py index 0a7216f24..9834e74ab 100644 --- a/packages/agentenv-protocol/tests/test_computer_app_relaunch.py +++ b/packages/agentenv-protocol/tests/test_computer_app_relaunch.py @@ -1,5 +1,5 @@ """An app whose server sees it end, a website browser's, launched again when a signal or a failing code ended it, up to -``CRASH_RELAUNCHES`` times a boot, each a wait that doubles after the last; not when it exited cleanly, when its server stopped it, or when it doesn't ask. Its +``CRASH_RELAUNCHES`` times a boot; not when it exited cleanly, when its server stopped it, or when it doesn't ask. Its launch log says so, and what reads it (``apps.ended_on_its_own``) takes it for gone only once nothing will launch it.""" from __future__ import annotations @@ -13,15 +13,12 @@ from agentenv_protocol.computer import Launch, app_runtime, apps, session from agentenv_protocol.computer.app_runtime import CRASH_RELAUNCHES, AppRuntime -REAL_SLEEP = asyncio.sleep - @pytest.fixture def desktop(tmp_path, monkeypatch): - """The app's folder as the current directory, a session whose launches each get a new pid that runs until - ``ended`` gives it its code, and no wait before a relaunch; the pids launched, in order.""" + """The app's folder as the current directory, and a session whose launches each get a new pid that runs until + ``ended`` gives it its code; the pids launched, in order.""" monkeypatch.chdir(tmp_path) - monkeypatch.setattr(app_runtime, "CRASH_RELAUNCH_BACKOFF_S", 0) monkeypatch.setenv("ENVIRONMENT_NAME", "browser") monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") state = SimpleNamespace(launched=[], running=set(), codes={}, stopped=[]) @@ -113,22 +110,14 @@ async def test_one_its_server_stopped_or_that_doesnt_ask_stays_down(desktop): @pytest.mark.asyncio -async def test_it_is_launched_again_at_most_crash_relaunches_times_a_boot_each_wait_twice_the_last(desktop, monkeypatch): - runtime, waits = _runtime(), [] - - async def sleep(seconds): - waits.append(seconds) - await REAL_SLEEP(0) - - monkeypatch.setattr(app_runtime, "CRASH_RELAUNCH_BACKOFF_S", 1.5) - monkeypatch.setattr(app_runtime.asyncio, "sleep", sleep) +async def test_it_is_launched_again_at_most_crash_relaunches_times_a_boot(desktop): + runtime = _runtime() await runtime.launch_app("boot") for _ in range(CRASH_RELAUNCHES + 2): await _ended(runtime, desktop, -signal.SIGSEGV) assert len(desktop.launched) == 1 + CRASH_RELAUNCHES and not runtime.running() - assert waits == [1.5 * 2 ** crash for crash in range(CRASH_RELAUNCHES)] assert runtime.launches()[-1]["event"] == "exit" and "relaunching" not in runtime.launches()[-1] assert "nothing launches it again" in apps.ended_on_its_own(Path.cwd()) diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index 0dc610fae..989dcc6a1 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -8,7 +8,8 @@ and upload; the browser's X server runs but no desktop, and nothing listens but the driver and the site proxy; a killed app server or driver is started again by supervisord, and chromium killed by its app server, and the computer is ready again; chromium killed once more than its app server relaunches it fails it for good, computer-ready/v1 saying why -and the tools answering with it; its X server killed takes chromium with it, and both come back. Through its cdp address, Playwright and Puppeteer drive it two at once, +and the tools answering with it, and so does its X server killed, though supervisord starts X again: chromium exits +cleanly without it, as a closed browser does, so nothing launches it again. Through its cdp address, Playwright and Puppeteer drive it two at once, each on its own session, by http_url and by ws_url; a wrong token is refused, and so is code in the computer without one; one client's close leaves the browser up for the other, which can't open file: URLs; the address outlives a relaunch by its app server's restart, and the browser's reset and the computer's end it. Behind a front that ends TLS, as a remote provider's tunnel does, @@ -297,7 +298,8 @@ async def test_a_killed_app_server_driver_or_browser_comes_back_and_a_browser_ki @pytest.mark.asyncio -async def test_a_killed_x_server_comes_back_and_so_does_the_browser_it_took_with_it(browser_computer): +async def test_a_killed_x_server_comes_back_but_takes_the_browser_with_it_which_fails_the_computer_for_good( + browser_computer): with namespace_routing(): deployed = await Env.get(browser_computer.id).deploy( env_params={"app_settings": {"chromium": {"start_urls": [PAGE_URL]}}}) @@ -305,16 +307,15 @@ async def test_a_killed_x_server_comes_back_and_so_does_the_browser_it_took_with try: before = await _screenshot(deployed) await _kill(deployed, "^Xvfb :99", oldest=True) + failed = await _until(deployed, lambda status: "failed" in status) x_back = await _until_x(deployed) - launches = await _crash_launches(deployed, 1) - back = await _ready(deployed) - after = await _screenshot(deployed) finally: report = await teardown_run(context) - assert before.size == after.size == (1280, 800) - assert x_back and back["ready"], back - assert [entry.get("cause") for entry in launches if entry["event"] == "launch"][:2] == ["boot", "crash"], launches + assert before.size == (1280, 800) + browser = next(check for check in failed["checks"] if check["name"] == "browser") + assert failed["ready"] is False and browser["final"] is True and "infrastructure failure" in browser["detail"] + assert x_back assert not report.failed From 25481b4aa4327198bc613f09e27ae2ddc6f66ceb Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 14:17:02 -0800 Subject: [PATCH 8/9] revert(computers): a crashed browser isn't relaunched; it fails the browser computer as before The relaunch after a crash, and the readiness and tests it changed, come out: a crash is an infrastructure failure, better failed and retried than run on with a fresh browser, and it was most of this change's complexity. The browser computer fails for good when its browser crashes, saying why, as on main; on the desktop the browser stays gone until its launcher opens it again. The proxy policy, network time off and the screensaver off stay. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/computer/app.py | 6 - .../agentenv_protocol/computer/app_data.py | 2 +- .../agentenv_protocol/computer/app_runtime.py | 33 +-- .../src/agentenv_protocol/computer/apps.py | 23 +-- .../website_browser/environment.py | 5 - .../tests/test_computer_app_relaunch.py | 192 ------------------ .../tests/test_website_browser.py | 20 +- .../computer/browser_computer_test.py | 44 +--- tst/integration/computer/browser_data.py | 15 +- tst/integration/computer/browser_state.py | 8 +- tst/integration/computer/chromium_app_test.py | 11 +- .../computer/chromium_universe_test.py | 29 +-- 12 files changed, 50 insertions(+), 338 deletions(-) delete mode 100644 packages/agentenv-protocol/tests/test_computer_app_relaunch.py diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py index c440a975d..e594e33f9 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py @@ -480,7 +480,6 @@ def __init__(self) -> None: if hasattr(fn, _READY_CHECK_ATTR) and applies(getattr(fn, _READY_CHECK_ATTR), on)}, ready.timeout_s if ready is not None else DEFAULT_TIMEOUT_S, desktop=self._on_desktop(), launch_fds=self._launch_fds, relaunch_on_start=self._relaunch_on_start(), - relaunch_on_crash=self._relaunch_on_crash(), launch_running=self._launch_running()) self._computer_app_data = AppDataPlane( self._computer_app, {op: getattr(self, name) for op, name in cls._data_hooks.items()}, self.paths, @@ -504,11 +503,6 @@ def _relaunch_on_start(self) -> bool: """Whether only the server that launched it can talk to it, so a server that restarts launches it again.""" return False - def _relaunch_on_crash(self) -> bool: - """Whether a crash of the app its server launched, which its server sees end (``AppRuntime.exited``), launches - it again: no.""" - return False - def _launch_running(self) -> Optional[Callable[[], Awaitable[None]]]: """What a launch does while it runs instead of starting a second copy: nothing, so another copy starts.""" return None diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py index 9935a8401..bd81cf97b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py @@ -63,7 +63,7 @@ def __init__(self, runtime: AppRuntime, hooks: dict[str, Callable], paths: list[ self.live, self.keeps_paths, self.resets_failed_add = live, keeps_paths, resets_failed_add #: What a call that stopped the app awaits once it has launched it again, whose error fails the call. self.relaunched = relaunched - self.turn = runtime.turn + self.turn = asyncio.Lock() self.changing: set[asyncio.Task] = set() def keep_initial(self) -> None: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py index b60ea9784..44319a0b5 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py @@ -40,9 +40,6 @@ LAUNCHES = Path(".agentenv/launches") #: How long an app that ended on its own may take to be reaped, for its exit code. EXIT_WAIT_S = 5.0 -#: How many times in a boot an app that relaunches after a crash is launched again, so one that can't start doesn't -#: loop. -CRASH_RELAUNCHES = 5 #: How long a stopped app may take to exit before it's killed. STOP_TIMEOUT_S = 10.0 HOLD_BOOT_LAUNCH_ENV = "AGENTENV_HOLD_BOOT_LAUNCH" @@ -68,7 +65,7 @@ class AppRuntime: def __init__(self, launch: Optional[Launch], ready_checks: dict[str, Callable[[], Any]], timeout_s: float, *, desktop: bool = True, launch_fds: Optional[Callable[[], Awaitable[dict[int, int]]]] = None, - relaunch_on_start: bool = False, relaunch_on_crash: bool = False, + relaunch_on_start: bool = False, launch_running: Optional[Callable[[], Awaitable[None]]] = None) -> None: self.launch = launch self.ready_checks = ready_checks @@ -76,12 +73,8 @@ def __init__(self, launch: Optional[Launch], ready_checks: dict[str, Callable[[] self.desktop = desktop self.launch_fds = launch_fds self.relaunch_on_start = relaunch_on_start - self.relaunch_on_crash = relaunch_on_crash self.launch_running = launch_running self.exits: set[asyncio.Future] = set() - #: Held by each launch or change of the app's data that must not overlap another (its data plane's ``turn``), - #: and by a relaunch after a crash. - self.turn = asyncio.Lock() self.started = time.time() self.launch_failed: Optional[str] = None self.checking: dict[str, asyncio.Future] = {} @@ -154,11 +147,7 @@ def paused(self) -> contextlib.AbstractContextManager: def exited(self) -> bool: """Log that the app this server launched last ended on its own, with its exit code or the signal that ended - it, unless the server stopped it; whether it did. With ``relaunch_on_crash``, one a signal or a failing code - ended is launched again, up to ``CRASH_RELAUNCHES`` times a boot (``_log_exit``): holding the ``turn``, so no - data call changes its files meanwhile, unless something launched it since, and once what's left of it is - stopped, as a browser's helpers outlive it for a moment and while they run its session counts as the app - running. One that exited cleanly was closed, and stays so.""" + it, unless the server stopped it; whether it did.""" if (pid := _recorded_pid()) is None: return False noting = asyncio.ensure_future(self._log_exit(pid)) @@ -167,25 +156,9 @@ def exited(self) -> bool: return True async def _log_exit(self, pid: int) -> None: - """The exit logged, marked ``relaunching`` when it's a crash its server launches it again after, which those - reading the log (``apps.ended_on_its_own``) don't take for the app gone; and that relaunch, or why it failed.""" code = await session.returncode(pid, EXIT_WAIT_S) - relaunching = bool(self.relaunch_on_crash and code and sum( - entry.get("cause") == "crash" for entry in self.launches()) < CRASH_RELAUNCHES) self.log("exit", pid, **({} if code is None else - {"signal": signal.Signals(-code).name} if code < 0 else {"code": code}), - **({"relaunching": True} if relaunching else {})) - if not relaunching: - return - async with self.turn: - if next((entry["pid"] for entry in reversed(self.launches()) if entry["event"] == "launch"), None) != pid: - return - try: - await session.stop(pid, STOP_TIMEOUT_S) - await self.launch_app("crash") - except Exception as e: - self.launch_failed = f"{type(e).__name__}: {e}" - self.log("relaunch_failed", pid, error=self.launch_failed) + {"signal": signal.Signals(-code).name} if code < 0 else {"code": code})) def log(self, event: str, pid: int, **fields: Any) -> None: """Append ``event`` for the app's ``pid`` to its launch log, which outlives its server for the boot.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py index f4d53d356..437dd639c 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py @@ -125,22 +125,15 @@ async def served_card(folder: Path) -> Optional[dict]: def ended_on_its_own(folder: Path) -> Optional[str]: - """Why the app in ``folder`` can't be ready again where nothing launches it but its server: its launch log says - its last launch ended on its own, an infrastructure failure, and its server isn't launching it again after a crash - (an exit marked ``relaunching``), or failed to; None while it hasn't.""" - ended = None + """Why the app in ``folder`` can't be ready again where nothing launches it but its server: its launch log says it + ended on its own, an infrastructure failure; None while it hasn't.""" for entry in launch_log(folder): - if entry.get("event") == "launch": - ended = None - elif entry.get("event") in ("exit", "relaunch_failed"): - ended = entry - if ended is None or ended.get("relaunching"): - return None - if ended["event"] == "relaunch_failed": - return f"it crashed, and its launch after the crash failed: {ended.get('error')}: an infrastructure failure" - how = (f"by signal {ended['signal']}" if "signal" in ended else - f"with code {ended['code']}" if "code" in ended else "in a way that wasn't recorded") - return f"it ended on its own at {ended.get('at')}, {how}, and nothing launches it again: an infrastructure failure" + if entry.get("event") == "exit": + how = (f"by signal {entry['signal']}" if "signal" in entry else + f"with code {entry['code']}" if "code" in entry else "in a way that wasn't recorded") + return (f"it ended on its own at {entry.get('at')}, {how}, and nothing launches it again: an " + "infrastructure failure") + return None def launch_log(folder: Path) -> list[dict]: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py index f2b5a9d51..67cbb3d94 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py @@ -296,11 +296,6 @@ def _write_prompt_answers(self) -> None: def _relaunch_on_start(self) -> bool: return True - def _relaunch_on_crash(self) -> bool: - """A browser that crashes is launched again, as the agent's main window onto sites: one a page or a disposed - context crashes would otherwise stay gone until someone launched it.""" - return True - def _launch_running(self) -> Callable[[], Awaitable[None]]: return self._open_window diff --git a/packages/agentenv-protocol/tests/test_computer_app_relaunch.py b/packages/agentenv-protocol/tests/test_computer_app_relaunch.py deleted file mode 100644 index 9834e74ab..000000000 --- a/packages/agentenv-protocol/tests/test_computer_app_relaunch.py +++ /dev/null @@ -1,192 +0,0 @@ -"""An app whose server sees it end, a website browser's, launched again when a signal or a failing code ended it, up to -``CRASH_RELAUNCHES`` times a boot; not when it exited cleanly, when its server stopped it, or when it doesn't ask. Its -launch log says so, and what reads it (``apps.ended_on_its_own``) takes it for gone only once nothing will launch it.""" -from __future__ import annotations - -import asyncio -import signal -from pathlib import Path -from types import SimpleNamespace - -import pytest - -from agentenv_protocol.computer import Launch, app_runtime, apps, session -from agentenv_protocol.computer.app_runtime import CRASH_RELAUNCHES, AppRuntime - - -@pytest.fixture -def desktop(tmp_path, monkeypatch): - """The app's folder as the current directory, and a session whose launches each get a new pid that runs until - ``ended`` gives it its code; the pids launched, in order.""" - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("ENVIRONMENT_NAME", "browser") - monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") - state = SimpleNamespace(launched=[], running=set(), codes={}, stopped=[]) - - async def launch(argv, cwd, env, fds=None): - pid = 100 + len(state.launched) - state.launched.append(pid) - state.running.add(pid) - return pid - - async def returncode(pid, timeout_s): - return state.codes.get(pid) - - async def stop(pid, timeout_s): - state.stopped.append(pid) - state.running.discard(pid) - - monkeypatch.setattr(session, "launch", launch) - monkeypatch.setattr(session, "stop", stop) - monkeypatch.setattr(session, "alive", lambda pid: pid in state.running) - monkeypatch.setattr(session, "returncode", returncode) - return state - - -def _runtime(relaunch_on_crash: bool = True) -> AppRuntime: - return AppRuntime(Launch(at="boot", command=["chrome"]), {}, 30, desktop=False, relaunch_on_crash=relaunch_on_crash) - - -async def _ended(runtime: AppRuntime, desktop, code: int) -> bool: - """The app last launched ended with ``code``, as its server sees it end; whether it ended on its own.""" - pid = desktop.launched[-1] - desktop.running.discard(pid) - desktop.codes[pid] = code - on_its_own = runtime.exited() - await asyncio.gather(*runtime.exits) - return on_its_own - - -@pytest.mark.asyncio -async def test_a_browser_a_signal_ends_is_launched_again_and_says_why(desktop): - runtime = _runtime() - await runtime.launch_app("boot") - - assert await _ended(runtime, desktop, -signal.SIGKILL) - - assert desktop.launched == [100, 101] and desktop.stopped == [100] and runtime.running() - assert [(entry["event"], entry["pid"], entry.get("cause") or entry.get("signal")) for entry in runtime.launches()] == [ - ("launch", 100, "boot"), ("exit", 100, "SIGKILL"), ("launch", 101, "crash")] - - -@pytest.mark.asyncio -async def test_what_a_crash_left_running_is_stopped_so_the_app_is_launched_afresh_not_reached_through_it(desktop): - runtime, reached = _runtime(), [] - - async def open_window(): - reached.append("window") - - runtime.launch_running = open_window - await runtime.launch_app("boot") - desktop.codes[100] = -signal.SIGKILL - assert runtime.exited() - await asyncio.gather(*runtime.exits) - - assert (desktop.stopped, desktop.launched, reached) == ([100], [100, 101], []) - - -@pytest.mark.asyncio -async def test_a_failing_code_is_a_crash_and_a_clean_exit_is_a_close(desktop): - runtime = _runtime() - await runtime.launch_app("boot") - - await _ended(runtime, desktop, 1) - await _ended(runtime, desktop, 0) - - assert desktop.launched == [100, 101] and not runtime.running() - - -@pytest.mark.asyncio -async def test_one_its_server_stopped_or_that_doesnt_ask_stays_down(desktop): - stopped, unasked = _runtime(), _runtime(relaunch_on_crash=False) - await stopped.launch_app("boot") - await stopped.stop_app() - - assert not stopped.exited() - await unasked.launch_app("boot") - await _ended(unasked, desktop, -signal.SIGTRAP) - - assert desktop.launched == [100, 101] and not unasked.running() - - -@pytest.mark.asyncio -async def test_it_is_launched_again_at_most_crash_relaunches_times_a_boot(desktop): - runtime = _runtime() - await runtime.launch_app("boot") - - for _ in range(CRASH_RELAUNCHES + 2): - await _ended(runtime, desktop, -signal.SIGSEGV) - - assert len(desktop.launched) == 1 + CRASH_RELAUNCHES and not runtime.running() - assert runtime.launches()[-1]["event"] == "exit" and "relaunching" not in runtime.launches()[-1] - assert "nothing launches it again" in apps.ended_on_its_own(Path.cwd()) - - -@pytest.mark.asyncio -async def test_a_relaunch_after_a_crash_waits_for_the_data_call_that_holds_the_turn(desktop): - runtime = _runtime() - await runtime.launch_app("boot") - await runtime.turn.acquire() - desktop.running.discard(100) - desktop.codes[100] = -signal.SIGKILL - - assert runtime.exited() - await asyncio.sleep(0.05) - during = list(desktop.launched) - runtime.turn.release() - await asyncio.gather(*runtime.exits) - - assert (during, desktop.launched) == ([100], [100, 101]) - - -@pytest.mark.asyncio -async def test_one_launched_again_since_its_crash_isnt_launched_once_more_for_it(desktop): - runtime = _runtime() - await runtime.launch_app("boot") - await runtime.turn.acquire() - desktop.running.discard(100) - desktop.codes[100] = -signal.SIGKILL - - assert runtime.exited() - await runtime.launch_app("data/reset") - runtime.turn.release() - await asyncio.gather(*runtime.exits) - - assert (desktop.launched, desktop.stopped) == ([100, 101], []) - - -@pytest.mark.asyncio -async def test_a_crash_its_server_launches_it_again_after_isnt_the_app_gone_while_or_after(desktop): - runtime = _runtime() - await runtime.launch_app("boot") - await runtime.turn.acquire() - desktop.running.discard(100) - desktop.codes[100] = -signal.SIGKILL - - assert runtime.exited() - await asyncio.sleep(0.05) - while_relaunching = apps.ended_on_its_own(Path.cwd()) - runtime.turn.release() - await asyncio.gather(*runtime.exits) - - assert while_relaunching is None and apps.ended_on_its_own(Path.cwd()) is None - assert runtime.launches()[1]["relaunching"] is True - - -@pytest.mark.asyncio -async def test_a_clean_exit_or_a_relaunch_that_fails_is_the_app_gone(desktop, monkeypatch): - closed, failing = _runtime(), _runtime() - await closed.launch_app("boot") - await _ended(closed, desktop, 0) - gone = apps.ended_on_its_own(Path.cwd()) - - async def refused(argv, cwd, env, fds=None): - raise OSError("no such file: chrome") - - await failing.launch_app("boot") - monkeypatch.setattr(session, "launch", refused) - await _ended(failing, desktop, -signal.SIGTRAP) - - assert "with code 0, and nothing launches it again" in gone - assert apps.ended_on_its_own(Path.cwd()) == ( - "it crashed, and its launch after the crash failed: OSError: no such file: chrome: an infrastructure failure") diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 33b1a4cd3..816337d61 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -1212,8 +1212,8 @@ async def test_its_control_says_the_tab_whose_page_has_focus_is_active_else_the_ @pytest.mark.asyncio -async def test_its_connection_hears_events_its_server_logs_each_launch_and_a_crash_it_launches_it_again_after( - monkeypatch, tmp_path): +async def test_its_connection_hears_events_its_server_logs_each_launch_and_an_exit_it_didnt_ask_for(monkeypatch, + tmp_path): monkeypatch.chdir(tmp_path) monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") browser = FakeBrowser() @@ -1227,19 +1227,15 @@ async def test_its_connection_hears_events_its_server_logs_each_launch_and_a_cra await _answering(browser) os.killpg(second, signal.SIGKILL) for _ in range(int(WAIT_S / POLL_S)): - if any(entry.get("cause") == "crash" for entry in runtime.launches()): + if any(entry["event"] == "exit" for entry in runtime.launches()): break await asyncio.sleep(POLL_S) - relaunched = await _answering(browser) - after_crash = await browser.browser_record() - launches = runtime.launches() - await runtime.stop_app() + after_exit = await browser.browser_record() browser.devtools.close() assert [tab["id"] for tab in record["tabs"]] == ["start"] - closed = [(tab["id"], tab["closed_by"]) for tab in after_crash["closed_tabs"]] - assert closed[:2] == [("start", "stop"), ("start", "exit")] - assert [(entry["event"], entry.get("cause"), entry["pid"]) for entry in launches[:4]] == [ + assert after_exit["tabs"] == [] and [(tab["id"], tab["closed_by"]) for tab in after_exit["closed_tabs"]] == [ + ("start", "stop"), ("start", "exit")] + assert [(entry["event"], entry.get("cause"), entry["pid"]) for entry in runtime.launches()] == [ ("launch", "boot", first), ("stop", None, first), ("launch", "data/reset", second), ("exit", None, second)] - assert launches[3]["signal"] == "SIGKILL" and relaunched - assert (launches[4]["event"], launches[4]["cause"]) == ("launch", "crash") and launches[4]["pid"] != second + assert runtime.launches()[-1]["signal"] == "SIGKILL" diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index 989dcc6a1..37b09f7cf 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -6,10 +6,9 @@ that run code hidden until tool access names one, as the installed Playwright MCP lists them, and back after Playwright MCP or the driver is killed, and its page tools navigate, move in history, answer a dialog, use tabs and upload; the browser's X server runs but no desktop, and nothing listens but the driver and the site proxy; a -killed app server or driver is started again by supervisord, and chromium killed by its app server, and the computer is -ready again; chromium killed once more than its app server relaunches it fails it for good, computer-ready/v1 saying why -and the tools answering with it, and so does its X server killed, though supervisord starts X again: chromium exits -cleanly without it, as a closed browser does, so nothing launches it again. Through its cdp address, Playwright and Puppeteer drive it two at once, +killed app server or driver is started again by supervisord and the computer is ready again; chromium killed fails it +for good, computer-ready/v1 saying why and the tools answering with it, and so does its X server killed, which takes +chromium with it though supervisord starts X again. Through its cdp address, Playwright and Puppeteer drive it two at once, each on its own session, by http_url and by ws_url; a wrong token is refused, and so is code in the computer without one; one client's close leaves the browser up for the other, which can't open file: URLs; the address outlives a relaunch by its app server's restart, and the browser's reset and the computer's end it. Behind a front that ends TLS, as a remote provider's tunnel does, @@ -65,7 +64,6 @@ from agentenv_protocol import client as protocol_v1 from agentenv_protocol.client import find_extension_method, invoke_extension from agentenv_protocol.computer import COMPUTER_PAGE, COMPUTER_READY -from agentenv_protocol.computer.app_runtime import CRASH_RELAUNCHES from agentenv_protocol.computer.conformance.client import image from agentenv_protocol.tool_access import ENABLE, ENABLE_TOOL_EXTENSION_URI from agentenv_protocol.tool_context import ROLE_HEADER @@ -259,8 +257,7 @@ async def test_its_tools_see_and_drive_the_page_that_is_its_screen(browser_compu @pytest.mark.asyncio -async def test_a_killed_app_server_driver_or_browser_comes_back_and_a_browser_killed_past_its_relaunches_fails_it_for_good( - browser_computer): +async def test_a_killed_app_server_or_driver_comes_back_and_a_killed_browser_fails_it_for_good(browser_computer): with namespace_routing(): deployed = await Env.get(browser_computer.id).deploy( env_params={"app_settings": {"chromium": {"start_urls": [PAGE_URL]}}}) @@ -268,17 +265,12 @@ async def test_a_killed_app_server_driver_or_browser_comes_back_and_a_browser_ki try: await _kill(deployed, "[s]erve_app") server_back = await _ready(deployed) + launches = (await _record(deployed))["launches"] await _kill(deployed, "[c]omputer.serve driver.py:PageDriver") driver_back = await _ready(deployed) - await _kill(deployed, "[-]-remote-debugging-pipe", oldest=True) - await _crash_launches(deployed, 1) - browser_back = await _ready(deployed) shot = await _screenshot(deployed) - for crashes in range(1, CRASH_RELAUNCHES + 1): - await _crash_launches(deployed, crashes) - await _kill(deployed, "[-]-remote-debugging-pipe", oldest=True) + await _kill(deployed, "[-]-remote-debugging-pipe", oldest=True) failed = await _until(deployed, lambda status: "failed" in status) - launches = (await _record(deployed))["launches"] async with streamablehttp_client(deployed.mcp_url) as (read, write, _): async with ClientSession(read, write) as session: await session.initialize() @@ -286,13 +278,11 @@ async def test_a_killed_app_server_driver_or_browser_comes_back_and_a_browser_ki finally: report = await teardown_run(context) - assert server_back["ready"] and driver_back["ready"] and browser_back["ready"] and shot.size == (1280, 800) - assert [entry.get("cause") for entry in launches if entry["event"] == "launch"] == [ - "boot", "server_restart", *["crash"] * CRASH_RELAUNCHES] + assert server_back["ready"] and driver_back["ready"] and shot.size == (1280, 800) + assert [entry.get("cause") for entry in launches if entry["event"] == "launch"] == ["boot", "server_restart"] browser = next(check for check in failed["checks"] if check["name"] == "browser") assert failed["ready"] is False and browser["final"] is True and failed["failed"] == f"browser: {browser['detail']}" - assert "by signal SIGKILL" in browser["detail"] and "nothing launches it again" in browser["detail"] - assert "infrastructure failure" in browser["detail"] + assert "by signal SIGKILL" in browser["detail"] and "infrastructure failure" in browser["detail"] assert refused.isError and "has failed" in refused.content[0].text and "SIGKILL" in refused.content[0].text assert not report.failed @@ -399,7 +389,7 @@ async def test_its_cookies_local_storage_and_downloads_are_read_and_set_through_ @pytest.mark.asyncio -async def test_its_state_operations_hold_up_at_once_through_a_server_restart_and_answer_at_once_with_the_browser_killed( +async def test_its_state_operations_hold_up_at_once_through_a_server_restart_and_fail_at_once_with_the_browser_gone( browser_computer): with namespace_routing(): deployed = await Env.get(browser_computer.id).deploy() @@ -565,7 +555,7 @@ async def test_each_start_tabs_page_sees_the_windows_size_from_its_first_script_ @pytest.mark.asyncio -async def test_its_data_calls_take_turns_a_server_restart_takes_a_load_and_with_the_browser_killed_they_answer_at_once( +async def test_its_data_calls_take_turns_a_server_restart_takes_a_load_and_with_the_browser_gone_they_fail_at_once( browser_computer): with namespace_routing(): deployed = await Env.get(browser_computer.id).deploy() @@ -758,18 +748,6 @@ async def _state(deployed, holds) -> dict: raise AssertionError(f"the page didn't get there within {WAIT_S:g} s; it last said {last!r}") -async def _crash_launches(deployed, count: int) -> list[dict]: - """The browser's launch log once ``count`` of its launches were after a crash, within ``WAIT_S``.""" - launches, deadline = [], time.monotonic() + WAIT_S - while time.monotonic() < deadline: - with contextlib.suppress(httpx.HTTPError, ValueError, KeyError): - launches = (await _record(deployed))["launches"] - if sum(entry.get("cause") == "crash" for entry in launches) >= count: - return launches - await asyncio.sleep(POLL_S) - raise AssertionError(f"the browser wasn't launched after crash {count} within {WAIT_S:g} s; its log: {launches!r}") - - async def _status(deployed) -> dict: card = (await _get(deployed, "/.well-known/agent-env.json")).json() path = protocol_v1.find_extension_method(card, COMPUTER_READY, "status")["endpoint"] diff --git a/tst/integration/computer/browser_data.py b/tst/integration/computer/browser_data.py index 31b556ccf..e0561ece4 100644 --- a/tst/integration/computer/browser_data.py +++ b/tst/integration/computer/browser_data.py @@ -4,11 +4,10 @@ files; once an agent has acted, a load can't change its tabs but sets its cookies; data/reset puts it back signed out at its start tab with its addresses ended; the snapshot taken mid-run loads back whole, its cookies, local storage, tabs and history; and reset_env, a reboot, brings back its start tab with no cookies or local storage. Under chaos: data calls at once each get their answer, its server killed and back takes a load, and -with chromium killed a snapshot answers at once rather than wait, and works again once chromium is launched again.""" +with chromium killed a snapshot fails at once rather than wait.""" import asyncio import base64 -import contextlib import json import time import uuid @@ -229,16 +228,16 @@ def cookie(value: str) -> list: after = await protocol_v1.get_data(base, timeout=120, headers=headers) await kill("[-]-remote-debugging-pipe") started = time.monotonic() - with contextlib.suppress(Exception): + try: await protocol_v1.get_data(base, timeout=120, headers=headers) - answered_s = time.monotonic() - started - await ready() - relaunched = await protocol_v1.get_data(base, timeout=120, headers=headers) + failed = None + except Exception as e: + failed = type(e).__name__ + failed_s = time.monotonic() - started for answer in (at_once[0], at_once[2], after): assert "browser-state.json" in [uploaded_object_path(part) for part in answer.parts] state = next(json.loads(base64.b64decode(part.file.bytes)) for part in after.parts if uploaded_object_path(part) == "browser-state.json") assert [c["value"] for c in state["cookies"] if c["name"] == "c"] == ["3"], state["cookies"] - assert answered_s < 60, answered_s - assert "browser-state.json" in [uploaded_object_path(part) for part in relaunched.parts] + assert failed is not None and failed_s < 60, (failed, failed_s) diff --git a/tst/integration/computer/browser_state.py b/tst/integration/computer/browser_state.py index c358ca323..c4dc5e01d 100644 --- a/tst/integration/computer/browser_state.py +++ b/tst/integration/computer/browser_state.py @@ -4,8 +4,7 @@ a partitioned cookie read with its key and deleted beside an unpartitioned one of its name; a site's service worker, which answers its pages, kept from agentenv's; a download with its file; and an operation refused without the control token. Under chaos: calls at once each get their answer; the pages agentenv serves itself never show in the record; with the browser's server killed and back, -the operations work again; and with the browser killed, they answer at once rather than wait, failing while it's -gone, and work again once it's launched again.""" +the operations work again; and with the browser killed, they fail at once rather than wait.""" import asyncio import base64 @@ -208,8 +207,6 @@ async def check_browser_state_under_chaos(deployed, endpoint: Callable[[str], st started = time.monotonic() down = [await call("cookies/get", {"urls": [SITE]}), await call("local_storage/get", {"origins": [ORIGIN]})] down_s = time.monotonic() - started - await ready() - relaunched = await ok("cookies/get", {"urls": [SITE]}) finally: serving.cancel() @@ -219,6 +216,5 @@ async def check_browser_state_under_chaos(deployed, endpoint: Callable[[str], st assert {"name": "k2", "value": "2"} in after[0]["localStorage"], after for record in (recorded, recorded_back): assert [tab["url"] for tab in record["tabs"]] == ["about:blank"] and record["closed_tabs"] == [], record - assert down_s < 30, ([a.text for a in down], down_s) - assert "cookies" in relaunched, relaunched + assert all(answer.status_code >= 400 for answer in down) and down_s < 30, ([a.text for a in down], down_s) return {"cookie_kept": "kept" in cookies_back, "item_kept": {"name": "k", "value": "1"} in storage_back} diff --git a/tst/integration/computer/chromium_app_test.py b/tst/integration/computer/chromium_app_test.py index 4f279f71f..8ea752da3 100644 --- a/tst/integration/computer/chromium_app_test.py +++ b/tst/integration/computer/chromium_app_test.py @@ -293,7 +293,8 @@ async def test_its_control_records_tabs_a_download_a_dialog_and_each_launch_with await serving browser = await _browser_process(deployed) await inside.run(deployed, ["kill", "-9", str(browser["pid"])], timeout_s=30) - crashed = await _record(deployed, card, lambda r: r["launches"][-1].get("cause") == "crash") + exited = await _record(deployed, card, lambda r: r["launches"][-1]["event"] == "exit") + await inside.run(deployed, ["sh", "-c", LAUNCHER], timeout_s=60) relaunched = await _ready(deployed, card, "chromium") await inside.run(deployed, ["pkill", "-f", "computer.serve_app chromium.py"], timeout_s=30) restarted = await _record(deployed, card, lambda r: r["launches"][-1].get("cause") == "server_restart") @@ -314,11 +315,11 @@ async def test_its_control_records_tabs_a_download_a_dialog_and_each_launch_with hashlib.sha256(_report(n)).hexdigest() for n in range(1, len(downloads) + 1)), recorded assert [tab["closed_by"] for tab in recorded["closed_tabs"]] == ["unknown"], recorded assert [(tab["url"], tab["active"]) for tab in recorded["tabs"]] == [(RECORDED, True)], recorded - assert [(entry["event"], entry.get("cause") or entry.get("signal")) for entry in crashed["launches"]] == [ - ("launch", "boot"), ("exit", "SIGKILL"), ("launch", "crash")], crashed - assert recorded["since"] and relaunched["ready"] is True, relaunched + assert [entry["event"] for entry in exited["launches"]] == ["launch", "exit"], exited + assert exited["launches"][1]["signal"] == "SIGKILL" and recorded["since"], exited + assert exited["launches"][0]["cause"] == "boot" and relaunched["ready"] is True, relaunched events = [(entry["event"], entry.get("cause")) for entry in restarted["launches"]] - assert events[:3] == [("launch", "boot"), ("exit", None), ("launch", "crash")] + assert events[:3] == [("launch", "boot"), ("exit", None), ("launch", "launch")] assert events[3:] in ([("stop", None), ("launch", "server_restart")], [("launch", "server_restart")]), events assert not report.failed diff --git a/tst/integration/computer/chromium_universe_test.py b/tst/integration/computer/chromium_universe_test.py index 4234eb9d6..ee127de05 100644 --- a/tst/integration/computer/chromium_universe_test.py +++ b/tst/integration/computer/chromium_universe_test.py @@ -1,7 +1,7 @@ """agentenv's chromium on the ubuntu-24.04 desktop, on the universe clock: its boot browser, never launched again, goes out through the computer's egress proxy by the base's policy and loads live sites it hasn't visited with the universe -182 days back, 730 days on and six years on, and right after a jump back; killed, it comes back by itself; and X runs -no screensaver. Needs Docker, the local registry on localhost:5000, and the internet.""" +182 days back, 730 days on and six years on, and right after a jump back; and X runs no screensaver. Needs Docker, the +local registry on localhost:5000, and the internet.""" import asyncio import json @@ -21,7 +21,7 @@ from agent_env.task_step.context import TaskStepContext from tst.integration.computer.browser_state import _client from tst.integration.computer.conftest import UniverseClock, computer_container -from tst.integration.computer.website_browser_test import _app, _browser_process, _ready +from tst.integration.computer.website_browser_test import _app, _browser_process pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] @@ -32,7 +32,6 @@ (6 * 365, "https://example.net/", "Example Domain"), (-182, "https://www.wikipedia.org/", "Wikipedia")] #: How long a page may take to load before its tab counts as not loaded. LOAD_S = 45 -LAUNCHES = "/opt/agentenv/apps/chromium/.agentenv/launches" POLICY = "/etc/opt/chrome_for_testing/policies/managed/agentenv-egress.json" @@ -48,18 +47,6 @@ async def _loaded(ok, url: str, title: str) -> str: return json.dumps(tabs) -async def _launches_until_relaunched(deployed) -> list[dict]: - """The chromium app's launch log once it records a launch after a crash, or as it last was within a minute.""" - launches: list[dict] = [] - for _ in range(60): - out = await inside.run(deployed, ["cat", LAUNCHES], timeout_s=30) - launches = [json.loads(line) for line in out["stdout"].splitlines() if line] - if any(entry.get("cause") == "crash" for entry in launches): - break - await asyncio.sleep(1) - return launches - - def _egress_rows(deployed) -> list[dict]: log = subprocess.run(["docker", "exec", computer_container(deployed.sandbox_id), "cat", "/var/log/agentenv-root/egress.log"], capture_output=True, text=True).stdout @@ -67,7 +54,7 @@ def _egress_rows(deployed) -> list[dict]: @pytest.mark.asyncio -async def test_its_boot_browser_loads_live_sites_at_any_universe_date_comes_back_when_killed_and_x_never_blanks( +async def test_its_boot_browser_loads_live_sites_at_any_universe_date_and_x_never_blanks( chromium_desktop): with namespace_routing(): deployed = await Env.get(chromium_desktop.id).deploy() @@ -84,11 +71,6 @@ async def test_its_boot_browser_loads_live_sites_at_any_universe_date_comes_back loaded.append(await _loaded(ok, url, title)) still = await _browser_process(deployed) rows = _egress_rows(deployed) - - await inside.run(deployed, ["kill", "-9", str(booted["pid"])], timeout_s=30) - launches = await _launches_until_relaunched(deployed) - back = await _browser_process(deployed, other_than=booted["pid"]) - ready = await _ready(deployed, card, "chromium") xset = (await inside.run(deployed, ["xset", "q"], timeout_s=30))["stdout"] policy = json.loads((await inside.run(deployed, ["cat", POLICY], timeout_s=30))["stdout"]) finally: @@ -99,9 +81,6 @@ async def test_its_boot_browser_loads_live_sites_at_any_universe_date_comes_back hosts = {url.split("/")[2] for _, url, _ in STOPS} intercepted = {row["host"] for row in rows if row.get("host") in hosts and row.get("mode") == "intercept"} assert intercepted == hosts, [row for row in rows if row.get("host") in hosts][-8:] - assert back["pid"] != booted["pid"] and ready["ready"], ready - assert [(entry["event"], entry.get("cause") or entry.get("signal")) for entry in launches[-2:]] == [ - ("exit", "SIGKILL"), ("launch", "crash")], launches assert re.search(r"Screen Saver:.*?timeout:\s+0\b", xset, re.S), xset assert "DPMS is Disabled" in xset or "does not have the DPMS Extension" in xset, xset assert policy["ProxySettings"]["ProxyServer"] == "http://127.0.0.1:3128" From ea4454d646dfa51a6d59d980d82ef08fd9d487d5 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 14:28:17 -0800 Subject: [PATCH 9/9] test(browser): the nudge test times the second capture from before the frame, not from the first capture The control arms its NUDGE_S wait before its first capture's task sends it, so the first capture is sent a moment after the wait starts and the second can follow it by a hair under NUDGE_S (0.49966 s on a loaded runner). Timing the second from before the frame call is what the control promises. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/tests/test_browser_recording.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/agentenv-protocol/tests/test_browser_recording.py b/packages/agentenv-protocol/tests/test_browser_recording.py index e3376c88f..2da359a0a 100644 --- a/packages/agentenv-protocol/tests/test_browser_recording.py +++ b/packages/agentenv-protocol/tests/test_browser_recording.py @@ -417,10 +417,11 @@ async def test_a_tab_behind_the_active_one_is_framed_once_a_second_capture_nudge pipe.emit("Target.targetCreated", {"targetInfo": {"targetId": "T1", "type": "page", "url": "about:blank"}}) pipe.emit("Target.attachedToTarget", {"sessionId": "s-T1", "targetInfo": {"targetId": "T1", "type": "page"}}) + asked = asyncio.get_running_loop().time() data, _, tab, _ = await control.frame("T1", load_wait_s=0) - (_, _, first), (_, _, second) = pipe.captured - assert (base64.b64decode(data), tab) == (b"behind", "T1") and second - first >= NUDGE_S + (_, _, _), (_, _, second) = pipe.captured + assert (base64.b64decode(data), tab) == (b"behind", "T1") and second - asked >= NUDGE_S @pytest.mark.asyncio