From 632ceaf92fe14ff8f0696a84357a7485906204fc Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 8 Nov 2026 08:40:14 -0800 Subject: [PATCH 1/4] feat(computer): a browser alone routing sites goes out only through the site proxy's forward port (FD-4934) - The shared site proxy listens on 127.0.0.1:3128 as a forward proxy. A CONNECT to a site's name tunnels into TLS with the site's certificate from the deploy's CA and on to its route, as the transparent listeners send it; a request naming a site's URL is served as the HTTP listener serves it, a real name sent to HTTPS and a .localhost one kept on HTTP; responses stream; any other host, loopback among them, is refused at once with 403 and a log line. - A browser computer that routes sites boots with AGENTENV_SITE_PROXY naming the port, which runs out of the agent's reach as the rest of the site proxy does. The headless base's stand-in moves to 3129 and serves only a browser computer that routes no sites. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/computers/README.md | 11 ++- .../ubuntu-24.04-headless/Dockerfile | 5 +- .../ubuntu-24.04-headless/site-proxy.py | 7 +- .../ubuntu-24.04-headless/supervisord.conf | 4 +- .../env/envs/computer/utils/site_proxy.py | 98 +++++++++++++----- .../env/envs/computer/utils/sites.py | 15 ++- .../env_providers/env_computer_provider.py | 4 +- .../computer/browser_computer_test.py | 99 ++++++++++++++++++- tst/integration/computer/start_tabs.py | 16 ++- tst/unit/env/envs/test_computer_env.py | 1 + tst/unit/env/envs/test_computer_site_proxy.py | 98 ++++++++++++++++++ tst/unit/env/envs/test_computer_sites.py | 5 + 12 files changed, 320 insertions(+), 43 deletions(-) diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index dc929d354..5d2210074 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -13,10 +13,13 @@ building it. Name it from another bundle as `ubuntu-24.04`, the alias `aliases.t `artifacts/ubuntu-24.04-headless/` is Ubuntu 24.04 (`amd64` and `arm64`) with no display server or desktop (`display = "none"`), for one website browser, which runs headless there, and the `page` driver, whose screen is the browser's page: a browser computer, which agentenv's `chromium` deploys alone as. supervisord runs every program as the -session user, among them a stand-in site proxy on `127.0.0.1:3128`, the browser's only route out, which forwards every -request to wherever it names, refusing nothing: the sandbox's network policy is all that limits it until the shared -site proxy replaces it. It has a container form only. Name it from another bundle as `ubuntu-24.04-headless`, or by -its id, `@local/agentenv-framework/computers/ubuntu-24.04-headless`. +session user. +The browser goes out only through a site proxy. A computer that routes sites names the shared site proxy's forward port, +`127.0.0.1:3128`, to it: that port serves the run's sites and refuses every other host, and it runs out of the agent's +reach. A computer that routes no sites keeps the base's stand-in on `127.0.0.1:3129`, which forwards every request to +wherever it names, refusing nothing: the sandbox's network policy is all that limits it. It has a container form only. +Name it from another bundle as `ubuntu-24.04-headless`, or by its id, +`@local/agentenv-framework/computers/ubuntu-24.04-headless`. It also has a disk form, a disk a VM boots from UEFI, which is the container form's root filesystem on the Linux boot recipe in `disk/linux/`: a GPT disk of an EFI system partition holding one UKI, Ubuntu's kernel with a busybox initramfs, diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile index 2314fdd2b..396851eba 100644 --- a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile +++ b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile @@ -1,6 +1,7 @@ # The ubuntu-24.04-headless base computer: Ubuntu 24.04 with no display server or desktop (XDG_SESSION_TYPE=none), # for one website browser, which runs headless there, and a page driver whose screen is the browser's page. supervisord -# runs every program as the session user, and the browser goes out only through the site proxy on 127.0.0.1:3128. +# runs every program as the session user. The browser goes out only through a site proxy: the shared one's forward port +# (127.0.0.1:3128) when the computer routes sites, else this base's stand-in on 127.0.0.1:3129. # Files a page's file chooser may be given go in ~/Uploads. Composing a driver or an app onto it adds its services as # programs in /etc/supervisor/conf.d. The trust files agentenv adds are in /etc/agentenv from boot: ca.pem, empty until # a deploy adds a CA, and ca-bundle.pem, the public roots and those CAs. @@ -21,6 +22,6 @@ COPY start-computer /usr/local/bin/start-computer COPY site-proxy.py /usr/local/bin/agentenv-site-proxy USER user WORKDIR /home/user -ENV LANG=C.UTF-8 XDG_SESSION_TYPE=none AGENTENV_SITE_PROXY=http://127.0.0.1:3128 +ENV LANG=C.UTF-8 XDG_SESSION_TYPE=none AGENTENV_SITE_PROXY=http://127.0.0.1:3129 ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["/usr/local/bin/start-computer"] diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/site-proxy.py b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/site-proxy.py index 5eacb8004..cea102020 100644 --- a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/site-proxy.py +++ b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/site-proxy.py @@ -1,12 +1,13 @@ -"""A stand-in for the shared site proxy in this base: it forwards every request the browser sends it, absolute-form -HTTP and CONNECT tunnels alike, to wherever it names, refusing nothing, on 127.0.0.1:3128.""" +"""A stand-in for the shared site proxy in this base, for a browser computer that routes no sites: it forwards every +request the browser sends it, absolute-form HTTP and CONNECT tunnels alike, to wherever it names, refusing nothing, on +127.0.0.1:3129. A computer routing sites names the shared site proxy's forward port (3128) to the browser instead.""" import http.server import select import socket import socketserver import urllib.parse -PORT = 3128 +PORT = 3129 IDLE_S = 60 HOP = {"connection", "keep-alive", "proxy-connection", "proxy-authorization"} diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/supervisord.conf b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/supervisord.conf index da2860323..dfb8f8437 100644 --- a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/supervisord.conf +++ b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/supervisord.conf @@ -1,5 +1,5 @@ -; The base computer's own program, the site proxy, run as the session user. A composed driver or app adds its own in -; conf.d. +; The base computer's own program, the stand-in site proxy for a computer that routes no sites, run as the session user. +; A composed driver or app adds its own in conf.d. [supervisord] nodaemon=true logfile=/var/log/agentenv/supervisord.log diff --git a/src/agent_env/env/envs/computer/utils/site_proxy.py b/src/agent_env/env/envs/computer/utils/site_proxy.py index 76c97c566..96912cfda 100644 --- a/src/agent_env/env/envs/computer/utils/site_proxy.py +++ b/src/agent_env/env/envs/computer/utils/site_proxy.py @@ -4,16 +4,18 @@ ``python3``, so it uses the standard library alone. ``install``, as root, maps each exact name to loopback in the hosts file, trusts the deployment's CA in the system -store, Firefox's policies, Chromium's managed policies and the session user's NSS databases, starts ``serve`` and checks, as the session user, that -each name loads. ``serve`` listens on loopback: on 443 for a name served over HTTPS, on 80 to redirect those to HTTPS -and to serve ``.localhost`` names over HTTP, and on 53 for DNS, over UDP and TCP, when a name is a wildcard, which a -hosts file can't list; it then drops to ``SERVE_USER`` (``nobody``), else the session user, so the agent, which runs as -the session user, can't stop it, and ``keep``, as root, starts it again should it end. The site sees itself at ``/`` under its own name: the path goes upstream under -the route's prefix, ``X-Forwarded-Host`` and ``X-Forwarded-Proto`` say the name and scheme the browser used, -``AgentEnv-Site-Root`` has the gateway tell the site no prefix, so the links it makes are rooted at ``/`` too, and a -``Location`` or cookie naming the route is rewritten back to the name. Hop-by-hop headers and the browser's own -``X-Forwarded-*`` never go upstream; ``Transfer-Encoding`` does, since bodies are relayed as framed, and an -``Alt-Svc`` offering HTTP/3, which the proxy doesn't serve, never reaches the browser. A WebSocket's upgrade is +store, Firefox's policies, Chromium's managed policies and the session user's NSS databases, starts ``serve`` and +checks, as the session user, that each name loads. ``serve`` listens on loopback: on 443 for a name served over HTTPS, +on 80 to redirect those to HTTPS and to serve ``.localhost`` names over HTTP, on 53 for DNS, over UDP and TCP, when a +name is a wildcard, which a hosts file can't list, and on ``proxy_port`` (3128) as a forward proxy, a browser alone's +only route out, which serves the same sites to ``CONNECT`` and to requests naming their URLs and refuses any other host +at once; it then drops to ``SERVE_USER`` (``nobody``), else the session user, so the agent, which runs as the session +user, can't stop it, and ``keep``, as root, starts it again should it end. The site sees itself at ``/`` under its own +name: the path goes upstream under the route's prefix, ``X-Forwarded-Host`` and ``X-Forwarded-Proto`` say the name and +scheme the browser used, ``AgentEnv-Site-Root`` has the gateway tell the site no prefix, so the links it makes are +rooted at ``/`` too, and a ``Location`` or cookie naming the route is rewritten back to the name. Hop-by-hop headers and +the browser's own ``X-Forwarded-*`` never go upstream; ``Transfer-Encoding`` does, since bodies are relayed as framed, +and an ``Alt-Svc`` offering HTTP/3, which the proxy doesn't serve, never reaches the browser. A WebSocket's upgrade is relayed, for an upstream that answers it.""" from __future__ import annotations @@ -143,6 +145,7 @@ class Config: listen: str = LOOPBACK https_port: int = 443 http_port: int = 80 + proxy_port: Optional[int] = None dns_port: Optional[int] = None nameservers: list[str] = field(default_factory=list) @@ -162,7 +165,8 @@ def load(cls, path: str) -> Config: return cls(user=data.get("user"), sites=sites, serve_user=data.get("serve_user"), listen=data.get("listen", LOOPBACK), https_port=data.get("https_port", 443), http_port=data.get("http_port", 80), - dns_port=data.get("dns_port"), nameservers=list(data.get("nameservers") or [])) + proxy_port=data.get("proxy_port"), dns_port=data.get("dns_port"), + nameservers=list(data.get("nameservers") or [])) def site_for(self, host: str) -> Optional[Site]: """The site ``host`` names: one naming it exactly before one whose wildcard covers it.""" @@ -361,15 +365,61 @@ class SiteProxy: """The proxy's HTTP side: one connection's requests, each forwarded on a connection of its own to its site's route.""" - def __init__(self, config: Config): + def __init__(self, config: Config, tls: Optional[ssl.SSLContext] = None): self.config = config + self.tls = tls self.upstream_tls = ssl.create_default_context() async def handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, scheme: str) -> None: + await self._serving(reader, writer, scheme, self._requests) + + async def handle_forward(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + """The forward-proxy port's connection: a ``CONNECT`` to a site's name tunnels to the site as the listeners + serve it, over TLS with its certificate for a name served over HTTPS; a request naming a site's URL is served + as the HTTP listener serves it; any other host, loopback among them, is refused at once and logged.""" + await self._serving(reader, writer, "proxy", self._proxied) + + async def _requests(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, scheme: str, + peer: str) -> None: + while await self.exchange(reader, writer, scheme, peer): + pass + + async def _proxied(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, _: str, peer: str) -> None: + while (head := await read_head(reader)) is not None: + start, headers = parse_head(head) + method, _, target = start.partition(" ") + target = target.rpartition(" ")[0] + if method == "CONNECT": + host = split_host(target) + if (site := self.config.site_for(host)) is None: + self._refuse(writer, method, target) + return + writer.write(b"HTTP/1.1 200 Connection established\r\n\r\n") + await writer.drain() + tls = site.tls(host.lower()) + if tls: + if self.tls is None: + raise BadRequest(f"{host} is served over HTTPS, and this proxy has no certificate for it") + await writer.start_tls(self.tls) + await self._requests(reader, writer, "https" if tls else "http", peer) + return + named = urlsplit(target).netloc if "://" in target else (header(headers, "host") or "") + if self.config.site_for(split_host(named)) is None: + self._refuse(writer, method, named or target) + return + if not await self.exchange(reader, writer, "http", peer, head=head): + return + + @staticmethod + def _refuse(writer: asyncio.StreamWriter, method: str, host: str) -> None: + print(f"site proxy: refused {method} {host}: no site of this computer", file=sys.stderr, flush=True) + writer.write(simple_response("403 Forbidden", f"{host} is no site of this computer, and the browser reaches " + "its sites alone\n", FAILED)) + + async def _serving(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, scheme: str, run) -> None: peer = (writer.get_extra_info("peername") or (LOOPBACK,))[0] try: - while await self.exchange(reader, writer, scheme, peer): - pass + await run(reader, writer, scheme, peer) except BadRequest as e: writer.write(simple_response("400 Bad Request", f"{e}\n")) except (ConnectionError, asyncio.IncompleteReadError, ssl.SSLError, OSError): @@ -385,9 +435,9 @@ async def handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWrite pass async def exchange(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, scheme: str, - peer: str) -> bool: - """One request and its answer; whether the connection takes another.""" - head = await read_head(reader) + peer: str, head: Optional[bytes] = None) -> bool: + """One request, read unless its ``head`` is given, and its answer; whether the connection takes another.""" + head = head or await read_head(reader) if head is None: return False start, headers = parse_head(head) @@ -395,7 +445,7 @@ async def exchange(self, reader: asyncio.StreamReader, writer: asyncio.StreamWri method, target, version = start.split(" ") except ValueError: raise BadRequest(f"a malformed request line {start!r}") from None - host = header(headers, "host") or (urlsplit(target).netloc if "://" in target else "") + host = (urlsplit(target).netloc if "://" in target else "") or header(headers, "host") or "" site = self.config.site_for(split_host(host)) if host else None if site is None: writer.write(simple_response("421 Misdirected Request", f"{host or 'no host'} is no site of this " @@ -602,13 +652,17 @@ def pick(connection: ssl.SSLObject, name: Optional[str], _: ssl.SSLContext) -> N async def serve(config: Config) -> None: """Listen, drop to the user it serves as, and serve until cancelled.""" - proxy = SiteProxy(config) + context = tls_context(config) + proxy = SiteProxy(config, tls=context) servers = [] - if (context := tls_context(config)) is not None: + if context is not None: servers.append(await asyncio.start_server(lambda r, w: proxy.handle(r, w, "https"), config.listen, config.https_port, ssl=context, limit=HEAD_LIMIT)) servers.append(await asyncio.start_server(lambda r, w: proxy.handle(r, w, "http"), config.listen, config.http_port, limit=HEAD_LIMIT)) + if config.proxy_port: + servers.append(await asyncio.start_server(proxy.handle_forward, config.listen, config.proxy_port, + limit=HEAD_LIMIT)) transport = None if config.dns_port: resolver = Resolver(config) @@ -932,8 +986,8 @@ def install(config_path: str, paths: Paths = Paths()) -> dict: write_in_place(paths.profile, "".join(f"export {name}={value}\n" for name, value in given["trust_environment"].items())) trust["environment"] = str(paths.profile) - serving = {"user": user, "serve_user": serve_user(user), "sites": sites, **{key: given[key] for key in ("listen", "https_port", "http_port") - if key in given}} + serving = {"user": user, "serve_user": serve_user(user), "sites": sites, + **{key: given[key] for key in ("listen", "https_port", "http_port", "proxy_port") if key in given}} if wildcard: serving.update(dns_port=given.get("dns_port", 53), nameservers=nameservers) if upstream_headers: diff --git a/src/agent_env/env/envs/computer/utils/sites.py b/src/agent_env/env/envs/computer/utils/sites.py index e8c3d5ca8..7483dae38 100644 --- a/src/agent_env/env/envs/computer/utils/sites.py +++ b/src/agent_env/env/envs/computer/utils/sites.py @@ -33,6 +33,7 @@ import httpx from agentenv_protocol import client as protocol_client from agentenv_protocol.website import WEBSITE_EXTENSION_URI as WEBSITE +from agentenv_protocol.website_browser import SITE_PROXY_ENV from cryptography import x509 from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import ec @@ -55,6 +56,9 @@ #: By the OS type of each computer site routing reaches, the variables naming the trust files agentenv adds, which a #: computer routing sites boots with. TRUST_ENVIRONMENTS = {"linux": dict(LOCAL_TRUST_ENV)} +#: The site proxy's forward port, a browser alone's only route out when its computer routes sites. +FORWARD_PROXY_PORT = 3128 +FORWARD_PROXY = f"http://127.0.0.1:{FORWARD_PROXY_PORT}" LOG = FOLDER / "site-proxy.log" SYNTHETIC, ALLOWLIST = "synthetic", "allowlist" RANKS = (SYNTHETIC, ALLOWLIST) @@ -284,12 +288,18 @@ def trust_environment(os_type: str, sites: list[Site]) -> dict[str, str]: return dict(TRUST_ENVIRONMENTS[os_type]) if any(route.tls_names for route in site_table(sites).routes) else {} +def browser_environment(display: str) -> dict[str, str]: + """For a computer routing sites, the variable naming the forward port to a browser alone, which runs on a computer + with no display; none on one with a display, whose apps reach the sites by name on the transparent listeners.""" + return {SITE_PROXY_ENV: FORWARD_PROXY} if display == "none" else {} + + def proxy_config(table: SiteTable, issued: Issued, *, user: Optional[str], os_type: str, upstream_headers: Optional[dict[str, dict[str, str]]] = None) -> dict: """What ``site_proxy.py install`` reads: the session user, the CA, the trust files on an ``os_type`` computer and the variables naming them, which its profile script sets as the computer's boot environment does, each route with its certificate and key, and the headers its gateway's sandbox provider asks requests to it to carry - (``upstream_headers``, by site), which the proxy sends upstream alone.""" + (``upstream_headers``, by site), which the proxy sends upstream alone, and its forward port.""" sites = [] for route in table.routes: site = {"name": route.site, "upstream": route.url, "domains": list(route.domains)} @@ -299,7 +309,8 @@ def proxy_config(table: SiteTable, issued: Issued, *, user: Optional[str], os_ty site["upstream_headers"] = dict(upstream_headers[route.site]) sites.append(site) return {"user": user, "ca_pem": issued.ca_pem, "sites": sites, "log": str(LOG), - "trust_files": trust_files(os_type), "trust_environment": dict(TRUST_ENVIRONMENTS[os_type])} + "trust_files": trust_files(os_type), "trust_environment": dict(TRUST_ENVIRONMENTS[os_type]), + "proxy_port": FORWARD_PROXY_PORT} def reachable_problem(policy: Optional[NetworkPolicy], routes: list[Route]) -> Optional[str]: diff --git a/src/agent_env/providers/env_providers/env_computer_provider.py b/src/agent_env/providers/env_providers/env_computer_provider.py index 4796be3b6..7f01bc6e2 100644 --- a/src/agent_env/providers/env_providers/env_computer_provider.py +++ b/src/agent_env/providers/env_providers/env_computer_provider.py @@ -319,8 +319,8 @@ def _environment(env: ComputerEnv, computer: ComputerArtifact, values: dict, driver_settings=params.driver_settings, tool_prefix=tool_prefix), **app_settings_environment(env.composed_apps(), computer_params.app_settings(env, params.app_settings), env.computer.os.type), - **({HOLD_BOOT_LAUNCH_ENV: "1", **sites.trust_environment(env.computer.os.type, params.sites)} - if params.site_routing else {})} + **({HOLD_BOOT_LAUNCH_ENV: "1", **sites.trust_environment(env.computer.os.type, params.sites), + **sites.browser_environment(env.computer.os.display)} if params.site_routing else {})} def _require_name(env: ComputerEnv, card: dict) -> None: diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index 9934ae261..b9005ce1e 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -15,7 +15,10 @@ snapshot back and holds up under chaos; the browser task steps set its cookies and storage and open, activate and close its tabs by URL; a task whose deploy_env pins a version of the browser, its latest one with none, passes preflight on the version pinned and runs its browser steps through task.run; and each start tab's page sees the -window's size from its first script. Needs Docker and the local registry on +window's size from its first script. Routing sites, it goes out only through the shared site proxy's forward port: +start tabs on a site over HTTPS and on a .localhost one load through it, an event stream comes through as it's sent, +any other host and loopback are refused and logged, the session user can't stop the port, and killed it comes back. +Needs Docker and the local registry on localhost:5000, and Node 20 or later and npm for the CDP clients.""" import asyncio @@ -23,6 +26,7 @@ import platform import re import shlex +import subprocess import time import urllib.parse import uuid @@ -55,7 +59,7 @@ from tst.integration.computer.browser_data import SITE as DATA_SITE, check_browser_data, check_browser_data_under_chaos from tst.integration.computer.browser_steps import check_browser_steps from tst.integration.computer.browser_state import check_browser_state, check_browser_state_under_chaos -from tst.integration.computer.start_tabs import tabs_of +from tst.integration.computer.start_tabs import host_pages, tabs_of from tst.util.a2a_test_agent import put_test_agent pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] @@ -197,7 +201,7 @@ async def test_its_tools_see_and_drive_the_page_that_is_its_screen(browser_compu assert clicked["c"] == 1 and doubled["d"] == 1 and typed["v"] == "Hello, World!" and replaced["v"] == "x" assert "C-a" in replaced["k"] and "Enter" in entered["k"] and scrolled["y"] >= 300 assert after.getpixel((200, 150)) != (255, 0, 0) - assert json.loads(listening) == [3128, 18765] and x_server.strip() == "False" + assert json.loads(listening) == [3129, 18765] and x_server.strip() == "False" assert "--headless=new" in command and "--window-size=1280,800" in command assert not report.failed @@ -516,6 +520,95 @@ async def _ready(deployed) -> dict: return await _until(deployed, lambda status: status["ready"]) +#: Loads its URL through the proxy it's given, as a browser alone does, printing the status, the seconds it took and the +#: first line of the body, or the error. +THROUGH_PROXY = """import sys, time, urllib.request +proxy = urllib.request.ProxyHandler({"http": sys.argv[2], "https": sys.argv[2]}) +began = time.monotonic() +try: + with urllib.request.build_opener(proxy).open(sys.argv[1], timeout=20) as answer: + print(answer.status, round(time.monotonic() - began), answer.readline().decode().strip()) +except Exception as e: + print("error", getattr(e, "code", ""), e) +""" + + +async def _through_port(deployed, url: str) -> str: + done = await inside.run(deployed, ["python3", "-c", THROUGH_PROXY, url, "http://127.0.0.1:3128"], timeout_s=60) + return done["stdout"].strip() + + +def _routing_container() -> str: + """The container whose computer runs the shared site proxy, found from this machine's Docker.""" + names = subprocess.run(["docker", "ps", "--format", "{{.Names}}"], capture_output=True, text=True, + check=True).stdout.split() + return next(name for name in names if subprocess.run( + ["docker", "exec", name, "pgrep", "-f", "[s]ite_proxy.py keep"], capture_output=True).returncode == 0) + + +async def _serve_pid(deployed) -> str: + return (await inside.run(deployed, ["pgrep", "-f", "[s]ite_proxy.py serve"], timeout_s=30))["stdout"].strip() + + +def _as_root(container: str, script: str) -> str: + return subprocess.run(["docker", "exec", "-u", "0", container, "sh", "-c", script], capture_output=True, + text=True).stdout + + +@pytest.mark.asyncio +async def test_routing_sites_it_goes_out_only_through_the_forward_port_which_the_agent_cant_stop_and_comes_back( + browser_computer): + with host_pages() as port, namespace_routing(): + params = {"site_routing": True, + "sites": [{"name": "shop", "url": f"http://127.0.0.1:{port}/website/shop/", "domains": ["shop.test"]}, + {"name": "notes", "url": f"http://127.0.0.1:{port}/website/notes/", + "domains": ["notes.localhost"]}], + "app_settings": {"chromium": {"start_urls": ["https://shop.test/", "http://notes.localhost/"]}}} + deployed = await Env.get(browser_computer.id).deploy(env_params=params) + context = TaskStepContext(deployed_envs=[deployed]) + try: + tabs = await tabs_of(deployed) + named = (await inside.run(deployed, ["sh", "-c", "echo $AGENTENV_SITE_PROXY"], timeout_s=30))["stdout"] + pages = {url: await _through_port(deployed, url) for url in ( + "https://shop.test/alpha", "http://notes.localhost/bravo", "http://shop.test/charlie", + "https://shop.test/events", "https://example.com/", "http://127.0.0.1:18765/", + "https://127.0.0.1:18765/")} + container = _routing_container() + logged = _as_root(container, "cat /etc/agentenv/sites/site-proxy.log") + serving = await _serve_pid(deployed) + stopped = await inside.run(deployed, ["pkill", "-f", "[s]ite_proxy.py serve"], timeout_s=30) + after_stop = await _through_port(deployed, "https://shop.test/alpha") + still = await _serve_pid(deployed) + _as_root(container, "pkill -9 -f '[s]ite_proxy.py serve'") + deadline, back = time.monotonic() + 30, "" + while time.monotonic() < deadline and not back.startswith("200 "): + await asyncio.sleep(1) + back = await _through_port(deployed, "https://shop.test/alpha") + restarted = await _serve_pid(deployed) + finally: + report = await teardown_run(context) + + start_tabs = deployed.metadata["computer"]["start_tabs"]["chromium"] + assert [(tab["url"], tab["site"], tab["outcome"]) for tab in start_tabs] == [ + ("https://shop.test/", "shop", "loaded"), ("http://notes.localhost/", "notes", "loaded")], start_tabs + assert [(tab["url"], tab["title"]) for tab in tabs] == [("https://shop.test/landed", "landed"), + ("http://notes.localhost/", "notes")], tabs + assert named.strip() == "http://127.0.0.1:3128" + assert pages["https://shop.test/alpha"].startswith("200 ") and pages["https://shop.test/alpha"].endswith( + "alpha"), pages + assert pages["http://notes.localhost/bravo"].endswith("bravo"), pages + assert pages["http://shop.test/charlie"].endswith("charlie"), pages + status, seconds, line = pages["https://shop.test/events"].split(" ", 2) + assert (status, line) == ("200", "data: first") and int(seconds) < 15, pages + assert all(pages[url].startswith("error") and "403" in pages[url] for url in ( + "https://example.com/", "http://127.0.0.1:18765/", "https://127.0.0.1:18765/")), pages + assert "site proxy: refused CONNECT example.com:443: no site of this computer" in logged + assert "site proxy: refused GET 127.0.0.1:18765: no site of this computer" in logged + assert stopped["exit_code"] != 0 and still == serving and after_stop.startswith("200 "), (stopped, after_stop) + assert back.startswith("200 ") and restarted and restarted != serving, (back, restarted) + assert not report.failed + + async def _kill(deployed, pattern: str, oldest: bool = False) -> None: """SIGKILL the newest, or oldest, process matching ``pattern``, a second after computer-exec has answered, since the driver that runs the command may be the one killed.""" diff --git a/tst/integration/computer/start_tabs.py b/tst/integration/computer/start_tabs.py index ffb236f58..82678c765 100644 --- a/tst/integration/computer/start_tabs.py +++ b/tst/integration/computer/start_tabs.py @@ -1,6 +1,6 @@ """Pages a computer's start tabs open, served on this machine, which a computer reaches as ``host.docker.internal``: a -site's route, whose home page sends itself elsewhere before it loads, or answers an error; two pages titled by name; -and one that never answers.""" +site's route, whose home page sends itself elsewhere before it loads, or answers an error; pages titled by name; an +event stream that sends one event and holds the response open; and one that never answers.""" import contextlib import http.server @@ -24,13 +24,23 @@ def host_pages(site_status: int = 200) -> Iterator[int]: """The pages served on every address of this machine while the block runs, at the port it yields: under ``/website/shop/``, a site's route, its home page, which sends itself to ``landed`` before it loads, or with any - other ``site_status`` answers it, and ``landed``; ``/alpha`` and ``/bravo``; and ``/hang``, which never answers.""" + other ``site_status`` answers it, and ``landed``; ``/alpha`` and ``/bravo``, and any other path, titled by its last + part; an ``events`` path, an event stream that sends one event and holds the response open for ``SLOW_S``; and + ``/hang``, which never answers.""" class Page(http.server.BaseHTTPRequestHandler): def do_GET(self): if self.path in ("/hang", "/website/shop/slow"): time.sleep(HANG_S if self.path == "/hang" else SLOW_S) return + if self.path.endswith("/events"): + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.end_headers() + self.wfile.write(b"data: first\n\n") + self.wfile.flush() + time.sleep(SLOW_S) + return home = self.path == "/website/shop/" status = site_status if home else 200 title = self.path.strip("/").rsplit("/", 1)[-1] diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 9ad340725..b33fd0704 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -1056,6 +1056,7 @@ async def require_display(record, resolution): assert boots.boots[0]["held"] is True and order == ["route", ("ready", {}), "display"] assert env_computer_provider.sites.TRUST_ENVIRONMENTS["linux"].items() <= boots.boots[0]["environment"].items() + assert "AGENTENV_SITE_PROXY" not in boots.boots[0]["environment"] @pytest.mark.asyncio diff --git a/tst/unit/env/envs/test_computer_site_proxy.py b/tst/unit/env/envs/test_computer_site_proxy.py index e520b8447..d3df8bb15 100644 --- a/tst/unit/env/envs/test_computer_site_proxy.py +++ b/tst/unit/env/envs/test_computer_site_proxy.py @@ -89,6 +89,16 @@ async def _proxy(config, scheme="https"): server.close() +@contextlib.asynccontextmanager +async def _forward_port(config): + proxy = SiteProxy(config, tls=site_proxy.tls_context(config)) + server = await asyncio.start_server(proxy.handle_forward, "127.0.0.1", 0) + try: + yield server.sockets[0].getsockname()[1] + finally: + server.close() + + async def _exchange(port, request: bytes, *, server_hostname=None, cadata=None, reader_writer=None): if reader_writer is None: context = None @@ -301,6 +311,93 @@ async def respond(start, headers, body, reader, writer): assert answer.endswith(b"\r\n\r\nall of it") +@pytest.mark.asyncio +async def test_the_forward_port_tunnels_a_connect_to_a_sites_name_through_its_certificate_to_its_route(tmp_path): + async with Upstream() as upstream: + config, issued = _config(tmp_path, upstream.port) + async with _forward_port(config) as port: + reader, writer = await asyncio.open_connection("127.0.0.1", port) + writer.write(b"CONNECT notes.test:443 HTTP/1.1\r\nHost: notes.test:443\r\n\r\n") + established = await reader.readuntil(b"\r\n\r\n") + await writer.start_tls(ssl.create_default_context(cadata=issued.ca_pem), server_hostname="notes.test") + first = await _exchange(port, b"GET /a?b=1 HTTP/1.1\r\nHost: notes.test\r\n\r\n", + reader_writer=(reader, writer)) + second = await _exchange(port, b"GET /c HTTP/1.1\r\nHost: notes.test\r\n\r\n", + reader_writer=(reader, writer)) + writer.close() + + assert established.startswith(b"HTTP/1.1 200 ") + assert (first[0], first[2], second[0]) == ("HTTP/1.1 200 OK", b"hello", "HTTP/1.1 200 OK") + assert [request[0] for request in upstream.requests] == ["GET /website/notes/a?b=1 HTTP/1.1", + "GET /website/notes/c HTTP/1.1"] + assert (upstream.requests[0][1]["x-forwarded-host"], upstream.requests[0][1]["x-forwarded-proto"]) == ( + "notes.test", "https") + + +@pytest.mark.asyncio +async def test_the_forward_port_serves_a_localhost_name_named_by_its_url_and_sends_a_real_name_to_https(tmp_path): + async with Upstream() as upstream: + config, _ = _config(tmp_path, upstream.port, notes=["notes.localhost"], shop=["shop.test"]) + async with _forward_port(config) as port: + local = await _exchange(port, b"GET http://notes.localhost/a?b=1 HTTP/1.1\r\nHost: notes.localhost\r\n\r\n") + local[3][1].close() + real = await _exchange(port, b"GET http://shop.test/cart HTTP/1.1\r\nHost: shop.test\r\n\r\n") + real[3][1].close() + + assert (local[0], local[2]) == ("HTTP/1.1 200 OK", b"hello") + assert upstream.requests[0][0] == "GET /website/notes/a?b=1 HTTP/1.1" + assert upstream.requests[0][1]["x-forwarded-proto"] == "http" and len(upstream.requests) == 1 + assert (real[0], real[1]["location"]) == ("HTTP/1.1 308 Permanent Redirect", "https://shop.test/cart") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_line, named", [ + (b"CONNECT example.com:443 HTTP/1.1", "example.com:443"), + (b"CONNECT 127.0.0.1:18765 HTTP/1.1", "127.0.0.1:18765"), + (b"GET http://127.0.0.1:18765/ HTTP/1.1", "127.0.0.1:18765"), + (b"GET http://example.com/ HTTP/1.1", "example.com"), +], ids=["connect", "connect-loopback", "url-loopback", "url"]) +async def test_the_forward_port_refuses_any_other_host_at_once_and_logs_it(tmp_path, capsys, request_line, named): + async with Upstream() as upstream: + config, _ = _config(tmp_path, upstream.port) + async with _forward_port(config) as port: + start, headers, body, conn = await asyncio.wait_for(_exchange( + port, request_line + b"\r\nHost: " + named.encode() + b"\r\n\r\n"), 5) + conn[1].close() + + assert start == "HTTP/1.1 403 Forbidden" and headers["x-agentenv-site-proxy"] == "failed" + assert body == f"{named} is no site of this computer, and the browser reaches its sites alone\n".encode() + assert upstream.requests == [] + method = request_line.split(b" ")[0].decode() + assert f"site proxy: refused {method} {named}: no site of this computer" in capsys.readouterr().err + + +@pytest.mark.asyncio +async def test_the_forward_port_streams_an_event_stream_as_it_comes(tmp_path): + release = asyncio.Event() + + async def respond(start, headers, body, reader, writer): + writer.write(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n" + b"d\r\ndata: first\n\n\r\n") + await writer.drain() + await release.wait() + writer.write(b"0\r\n\r\n") + + async with Upstream(respond) as upstream: + config, _ = _config(tmp_path, upstream.port, notes=["notes.localhost"]) + async with _forward_port(config) as port: + reader, writer = await asyncio.open_connection("127.0.0.1", port) + writer.write(b"GET http://notes.localhost/events HTTP/1.1\r\nHost: notes.localhost\r\n\r\n") + head = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), 5) + first = await asyncio.wait_for(reader.readuntil(b"\n\n\r\n"), 5) + release.set() + rest = await asyncio.wait_for(reader.readuntil(b"0\r\n\r\n"), 5) + writer.close() + + assert head.startswith(b"HTTP/1.1 200 OK\r\n") and b"text/event-stream" in head + assert first == b"d\r\ndata: first\n\n\r\n" and rest == b"0\r\n\r\n" + + @pytest.mark.asyncio async def test_a_websocket_upgrade_is_relayed_both_ways(tmp_path): async def respond(start, headers, body, reader, writer): @@ -595,6 +692,7 @@ def test_install_writes_the_sites_files_names_and_trust_then_starts_the_proxy_on assert [(s.name, s.domains, bool(s.cert)) for s in config.sites] == [ ("notes", ["notes.test"], True), ("chat", ["*.chat.test"], True), ("bare", ["bare.localhost"], False)] assert (config.dns_port, config.nameservers) == (53, ["127.0.0.11"]) + assert config.proxy_port == sites.FORWARD_PROXY_PORT == 3128 assert [(folder / name).stat().st_mode & 0o777 for name in ("notes.key", "config.json")] == [0o600, 0o600] assert folder.stat().st_mode & 0o777 == 0o755 assert "key_pem" not in (folder / "serve.json").read_text() diff --git a/tst/unit/env/envs/test_computer_sites.py b/tst/unit/env/envs/test_computer_sites.py index c151b467b..5c489f6ad 100644 --- a/tst/unit/env/envs/test_computer_sites.py +++ b/tst/unit/env/envs/test_computer_sites.py @@ -572,6 +572,11 @@ def test_the_trust_a_routing_linux_computer_boots_with_is_the_local_providers_so assert sites.trust_files("linux") == {"ca": "/etc/agentenv/ca.pem", "bundle": "/etc/agentenv/ca-bundle.pem"} +def test_a_browser_alone_routing_sites_is_named_the_forward_port_and_a_computer_with_a_display_is_named_none(): + assert sites.browser_environment("none") == {"AGENTENV_SITE_PROXY": "http://127.0.0.1:3128"} + assert sites.browser_environment("x11") == {} + + def test_a_computer_boots_with_the_trust_variables_only_when_a_site_it_routes_is_served_over_https(): local = [Site(name="notes", url="http://gw/website/notes/", domains=["notes.localhost"]), Site(name="bare", url="http://gw/website/bare/")] From 9babd1e54622079d8f75e51263fd9522cb1d8e16 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 8 Nov 2026 09:06:09 -0800 Subject: [PATCH 2/4] fix(computer): a real name's redirect to HTTPS keeps its query, and the e2e acts on its own computer - A request through the forward port naming a site's URL over plain HTTP was sent to HTTPS without its query string. - The forward port e2e found the computer to kill the proxy in by scanning Docker; it now takes the deployed computer's own container from the local provider. Co-Authored-By: Claude Opus 5.5 --- .../env/envs/computer/utils/site_proxy.py | 4 +++- .../computer/browser_computer_test.py | 16 +++++++--------- tst/unit/env/envs/test_computer_site_proxy.py | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/agent_env/env/envs/computer/utils/site_proxy.py b/src/agent_env/env/envs/computer/utils/site_proxy.py index 96912cfda..0c024168d 100644 --- a/src/agent_env/env/envs/computer/utils/site_proxy.py +++ b/src/agent_env/env/envs/computer/utils/site_proxy.py @@ -452,7 +452,9 @@ async def exchange(self, reader: asyncio.StreamReader, writer: asyncio.StreamWri "computer\n", FAILED)) return False if scheme == "http" and site.tls(split_host(host).lower()): - location = f"https://{split_host(host)}{target if target.startswith('/') else urlsplit(target).path}" + parts = urlsplit(target) + path = target if target.startswith("/") else urlunsplit(("", "", parts.path or "/", parts.query, "")) + location = f"https://{split_host(host)}{path}" writer.write(simple_response("308 Permanent Redirect", f"{location}\n", (("Location", location),))) return False upgrade = header(headers, "upgrade") if "upgrade" in tokens(headers, "connection") else None diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index b9005ce1e..d30adf74a 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -48,6 +48,7 @@ from agent_env.env import Env from agent_env.env.envs.computer.utils import inside from agent_env.env.store import get_env_store +from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider from agent_env.store.routing import namespace_routing from agent_env.task import Task from agent_env.task.teardown import teardown_run @@ -538,12 +539,9 @@ async def _through_port(deployed, url: str) -> str: return done["stdout"].strip() -def _routing_container() -> str: - """The container whose computer runs the shared site proxy, found from this machine's Docker.""" - names = subprocess.run(["docker", "ps", "--format", "{{.Names}}"], capture_output=True, text=True, - check=True).stdout.split() - return next(name for name in names if subprocess.run( - ["docker", "exec", name, "pgrep", "-f", "[s]ite_proxy.py keep"], capture_output=True).returncode == 0) +async def _container_of(deployed) -> str: + """The Docker container ``deployed``'s computer runs in, on this machine's local provider.""" + return (await LocalSandboxProvider().get_sandbox(deployed.sandbox_id)).container_name async def _serve_pid(deployed) -> str: @@ -570,10 +568,10 @@ async def test_routing_sites_it_goes_out_only_through_the_forward_port_which_the tabs = await tabs_of(deployed) named = (await inside.run(deployed, ["sh", "-c", "echo $AGENTENV_SITE_PROXY"], timeout_s=30))["stdout"] pages = {url: await _through_port(deployed, url) for url in ( - "https://shop.test/alpha", "http://notes.localhost/bravo", "http://shop.test/charlie", + "https://shop.test/alpha", "http://notes.localhost/bravo", "http://shop.test/charlie?x=1", "https://shop.test/events", "https://example.com/", "http://127.0.0.1:18765/", "https://127.0.0.1:18765/")} - container = _routing_container() + container = await _container_of(deployed) logged = _as_root(container, "cat /etc/agentenv/sites/site-proxy.log") serving = await _serve_pid(deployed) stopped = await inside.run(deployed, ["pkill", "-f", "[s]ite_proxy.py serve"], timeout_s=30) @@ -597,7 +595,7 @@ async def test_routing_sites_it_goes_out_only_through_the_forward_port_which_the assert pages["https://shop.test/alpha"].startswith("200 ") and pages["https://shop.test/alpha"].endswith( "alpha"), pages assert pages["http://notes.localhost/bravo"].endswith("bravo"), pages - assert pages["http://shop.test/charlie"].endswith("charlie"), pages + assert pages["http://shop.test/charlie?x=1"].endswith("charlie?x=1"), pages status, seconds, line = pages["https://shop.test/events"].split(" ", 2) assert (status, line) == ("200", "data: first") and int(seconds) < 15, pages assert all(pages[url].startswith("error") and "403" in pages[url] for url in ( diff --git a/tst/unit/env/envs/test_computer_site_proxy.py b/tst/unit/env/envs/test_computer_site_proxy.py index d3df8bb15..e81fca430 100644 --- a/tst/unit/env/envs/test_computer_site_proxy.py +++ b/tst/unit/env/envs/test_computer_site_proxy.py @@ -341,13 +341,13 @@ async def test_the_forward_port_serves_a_localhost_name_named_by_its_url_and_sen async with _forward_port(config) as port: local = await _exchange(port, b"GET http://notes.localhost/a?b=1 HTTP/1.1\r\nHost: notes.localhost\r\n\r\n") local[3][1].close() - real = await _exchange(port, b"GET http://shop.test/cart HTTP/1.1\r\nHost: shop.test\r\n\r\n") + real = await _exchange(port, b"GET http://shop.test/cart?q=book&n=2 HTTP/1.1\r\nHost: shop.test\r\n\r\n") real[3][1].close() assert (local[0], local[2]) == ("HTTP/1.1 200 OK", b"hello") assert upstream.requests[0][0] == "GET /website/notes/a?b=1 HTTP/1.1" assert upstream.requests[0][1]["x-forwarded-proto"] == "http" and len(upstream.requests) == 1 - assert (real[0], real[1]["location"]) == ("HTTP/1.1 308 Permanent Redirect", "https://shop.test/cart") + assert (real[0], real[1]["location"]) == ("HTTP/1.1 308 Permanent Redirect", "https://shop.test/cart?q=book&n=2") @pytest.mark.asyncio From 28105edf35a34f62bc982d9435dfc870762334eb Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 8 Nov 2026 10:21:08 -0800 Subject: [PATCH 3/4] fix(computer): the forward port listens only on a browser computer, and its e2e covers a reset - The site proxy listens on the forward port only for a computer with no display, a browser alone's. A desktop computer, where nothing points at it, no longer binds 3128, so an app of its own there can't stop the whole proxy from starting. - The forward port e2e resets the browser computer and checks that it serves the sites, refuses the rest and starts its tabs again. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/computers/README.md | 11 +++--- .../env/envs/computer/utils/site_proxy.py | 20 +++++------ .../env/envs/computer/utils/sites.py | 21 +++++++----- .../env_providers/env_computer_provider.py | 2 +- .../computer/browser_computer_test.py | 15 +++++++- tst/unit/env/envs/test_computer_env.py | 2 +- tst/unit/env/envs/test_computer_site_proxy.py | 10 +++--- tst/unit/env/envs/test_computer_sites.py | 34 +++++++++++++------ 8 files changed, 73 insertions(+), 42 deletions(-) diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index 5d2210074..85cbe9a66 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -13,12 +13,11 @@ building it. Name it from another bundle as `ubuntu-24.04`, the alias `aliases.t `artifacts/ubuntu-24.04-headless/` is Ubuntu 24.04 (`amd64` and `arm64`) with no display server or desktop (`display = "none"`), for one website browser, which runs headless there, and the `page` driver, whose screen is the browser's page: a browser computer, which agentenv's `chromium` deploys alone as. supervisord runs every program as the -session user. -The browser goes out only through a site proxy. A computer that routes sites names the shared site proxy's forward port, -`127.0.0.1:3128`, to it: that port serves the run's sites and refuses every other host, and it runs out of the agent's -reach. A computer that routes no sites keeps the base's stand-in on `127.0.0.1:3129`, which forwards every request to -wherever it names, refusing nothing: the sandbox's network policy is all that limits it. It has a container form only. -Name it from another bundle as `ubuntu-24.04-headless`, or by its id, +session user. The browser goes out only through a site proxy. A browser computer that routes sites names the shared site +proxy's forward port, `127.0.0.1:3128`, to it: that port serves the run's sites and refuses every other host, and it +runs out of the agent's reach. A computer that routes no sites keeps the base's stand-in on `127.0.0.1:3129`, which +forwards every request to wherever it names, refusing nothing: the sandbox's network policy is all that limits it. It +has a container form only. Name it from another bundle as `ubuntu-24.04-headless`, or by its id, `@local/agentenv-framework/computers/ubuntu-24.04-headless`. It also has a disk form, a disk a VM boots from UEFI, which is the container form's root filesystem on the Linux boot diff --git a/src/agent_env/env/envs/computer/utils/site_proxy.py b/src/agent_env/env/envs/computer/utils/site_proxy.py index 0c024168d..7324924cd 100644 --- a/src/agent_env/env/envs/computer/utils/site_proxy.py +++ b/src/agent_env/env/envs/computer/utils/site_proxy.py @@ -7,16 +7,16 @@ store, Firefox's policies, Chromium's managed policies and the session user's NSS databases, starts ``serve`` and checks, as the session user, that each name loads. ``serve`` listens on loopback: on 443 for a name served over HTTPS, on 80 to redirect those to HTTPS and to serve ``.localhost`` names over HTTP, on 53 for DNS, over UDP and TCP, when a -name is a wildcard, which a hosts file can't list, and on ``proxy_port`` (3128) as a forward proxy, a browser alone's -only route out, which serves the same sites to ``CONNECT`` and to requests naming their URLs and refuses any other host -at once; it then drops to ``SERVE_USER`` (``nobody``), else the session user, so the agent, which runs as the session -user, can't stop it, and ``keep``, as root, starts it again should it end. The site sees itself at ``/`` under its own -name: the path goes upstream under the route's prefix, ``X-Forwarded-Host`` and ``X-Forwarded-Proto`` say the name and -scheme the browser used, ``AgentEnv-Site-Root`` has the gateway tell the site no prefix, so the links it makes are -rooted at ``/`` too, and a ``Location`` or cookie naming the route is rewritten back to the name. Hop-by-hop headers and -the browser's own ``X-Forwarded-*`` never go upstream; ``Transfer-Encoding`` does, since bodies are relayed as framed, -and an ``Alt-Svc`` offering HTTP/3, which the proxy doesn't serve, never reaches the browser. A WebSocket's upgrade is -relayed, for an upstream that answers it.""" +name is a wildcard, which a hosts file can't list, and, on a browser computer, on ``proxy_port`` (3128) as a forward +proxy, a browser alone's only route out, which serves the same sites to ``CONNECT`` and to requests naming their URLs +and refuses any other host at once; it then drops to ``SERVE_USER`` (``nobody``), else the session user, so the agent, +which runs as the session user, can't stop it, and ``keep``, as root, starts it again should it end. The site sees +itself at ``/`` under its own name: the path goes upstream under the route's prefix, ``X-Forwarded-Host`` and +``X-Forwarded-Proto`` say the name and scheme the browser used, ``AgentEnv-Site-Root`` has the gateway tell the site no +prefix, so the links it makes are rooted at ``/`` too, and a ``Location`` or cookie naming the route is rewritten back +to the name. Hop-by-hop headers and the browser's own ``X-Forwarded-*`` never go upstream; ``Transfer-Encoding`` does, +since bodies are relayed as framed, and an ``Alt-Svc`` offering HTTP/3, which the proxy doesn't serve, never reaches the +browser. A WebSocket's upgrade is relayed, for an upstream that answers it.""" from __future__ import annotations diff --git a/src/agent_env/env/envs/computer/utils/sites.py b/src/agent_env/env/envs/computer/utils/sites.py index 7483dae38..b19fc7af0 100644 --- a/src/agent_env/env/envs/computer/utils/sites.py +++ b/src/agent_env/env/envs/computer/utils/sites.py @@ -56,9 +56,11 @@ #: By the OS type of each computer site routing reaches, the variables naming the trust files agentenv adds, which a #: computer routing sites boots with. TRUST_ENVIRONMENTS = {"linux": dict(LOCAL_TRUST_ENV)} -#: The site proxy's forward port, a browser alone's only route out when its computer routes sites. +#: The site proxy's forward port, a browser alone's only route out when its computer routes sites, and the display of +#: the computer a browser runs alone on. FORWARD_PROXY_PORT = 3128 FORWARD_PROXY = f"http://127.0.0.1:{FORWARD_PROXY_PORT}" +ALONE_DISPLAY = "none" LOG = FOLDER / "site-proxy.log" SYNTHETIC, ALLOWLIST = "synthetic", "allowlist" RANKS = (SYNTHETIC, ALLOWLIST) @@ -291,15 +293,16 @@ def trust_environment(os_type: str, sites: list[Site]) -> dict[str, str]: def browser_environment(display: str) -> dict[str, str]: """For a computer routing sites, the variable naming the forward port to a browser alone, which runs on a computer with no display; none on one with a display, whose apps reach the sites by name on the transparent listeners.""" - return {SITE_PROXY_ENV: FORWARD_PROXY} if display == "none" else {} + return {SITE_PROXY_ENV: FORWARD_PROXY} if display == ALONE_DISPLAY else {} -def proxy_config(table: SiteTable, issued: Issued, *, user: Optional[str], os_type: str, +def proxy_config(table: SiteTable, issued: Issued, *, user: Optional[str], os_type: str, forward_port: bool, upstream_headers: Optional[dict[str, dict[str, str]]] = None) -> dict: """What ``site_proxy.py install`` reads: the session user, the CA, the trust files on an ``os_type`` computer and the variables naming them, which its profile script sets as the computer's boot environment does, each route with its certificate and key, and the headers its gateway's sandbox provider asks requests to it to carry - (``upstream_headers``, by site), which the proxy sends upstream alone, and its forward port.""" + (``upstream_headers``, by site), which the proxy sends upstream alone, and with ``forward_port``, for a browser + alone, the forward port.""" sites = [] for route in table.routes: site = {"name": route.site, "upstream": route.url, "domains": list(route.domains)} @@ -310,7 +313,7 @@ def proxy_config(table: SiteTable, issued: Issued, *, user: Optional[str], os_ty sites.append(site) return {"user": user, "ca_pem": issued.ca_pem, "sites": sites, "log": str(LOG), "trust_files": trust_files(os_type), "trust_environment": dict(TRUST_ENVIRONMENTS[os_type]), - "proxy_port": FORWARD_PROXY_PORT} + **({"proxy_port": FORWARD_PROXY_PORT} if forward_port else {})} def reachable_problem(policy: Optional[NetworkPolicy], routes: list[Route]) -> Optional[str]: @@ -415,10 +418,11 @@ async def _exec(self, argv: list[str]) -> str: async def route(sandbox: Sandbox, sites: list[Site], *, user: Optional[str], network: Optional[dict], - valid_for: timedelta, label: str, os_type: str) -> dict: + valid_for: timedelta, label: str, os_type: str, display: str) -> dict: """Have the ``os_type`` computer on ``sandbox`` reach each of ``sites`` by its names: the table made, the CA and - certificates issued, the proxy installed and started as root, and each name checked as ``user``; what the deploy - record keeps of it. Raises naming a name that didn't load.""" + certificates issued, the proxy installed and started as root, with its forward port on a computer with no + ``display``, a browser alone's, and each name checked as ``user``; what the deploy record keeps of it. Raises + naming a name that didn't load.""" requested = NetworkPolicy.from_dict(network) if network else None table = site_table(sites, requested.allow_hosts if requested else ()) table.routes = [Route(r.site, sandbox.url_from_sandbox(r.url), r.domains) for r in table.routes] @@ -435,6 +439,7 @@ async def route(sandbox: Sandbox, sites: list[Site], *, user: Optional[str], net await channel.private_folder(FOLDER) await channel.write(PROXY, _PROXY_SOURCE.read_text()) await channel.write_secret(CONFIG, json.dumps(proxy_config(table, issued, user=user, os_type=os_type, + forward_port=display == ALONE_DISPLAY, upstream_headers=upstream_headers))) await channel.run(["chmod", "0600", str(CONFIG)]) output = await channel.run(["python3", str(PROXY), "install", str(CONFIG)]) diff --git a/src/agent_env/providers/env_providers/env_computer_provider.py b/src/agent_env/providers/env_providers/env_computer_provider.py index 7f01bc6e2..20c860eb3 100644 --- a/src/agent_env/providers/env_providers/env_computer_provider.py +++ b/src/agent_env/providers/env_providers/env_computer_provider.py @@ -229,7 +229,7 @@ async def _route_sites(env: ComputerEnv, sandbox: Sandbox, params: computer_para record keeps of it.""" routed = await sites.route(sandbox, params.sites, user=env.computer.session.user, network=env.network, valid_for=timedelta(seconds=valid_for_s), label=env.environment_name, - os_type=env.computer.os.type) + os_type=env.computer.os.type, display=env.computer.os.display) return {**routed, "valid_for_s": valid_for_s} diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index d30adf74a..fe62e6177 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -17,7 +17,8 @@ preflight on the version pinned and runs its browser steps through task.run; and each start tab's page sees the window's size from its first script. Routing sites, it goes out only through the shared site proxy's forward port: start tabs on a site over HTTPS and on a .localhost one load through it, an event stream comes through as it's sent, -any other host and loopback are refused and logged, the session user can't stop the port, and killed it comes back. +any other host and loopback are refused and logged, the session user can't stop the port, killed it comes back, and +after a reset it serves the sites and refuses the rest again. Needs Docker and the local registry on localhost:5000, and Node 20 or later and npm for the CDP clients.""" @@ -583,6 +584,13 @@ async def test_routing_sites_it_goes_out_only_through_the_forward_port_which_the await asyncio.sleep(1) back = await _through_port(deployed, "https://shop.test/alpha") restarted = await _serve_pid(deployed) + await ResetEnvTaskStep(id="reset", version=None, env_id=browser_computer.id, + fail_task_on_error=True).execute(context) + after_reset = await _through_port(deployed, "https://shop.test/alpha") + refused_after_reset = await _through_port(deployed, "https://example.com/") + named_after_reset = (await inside.run(deployed, ["sh", "-c", "echo $AGENTENV_SITE_PROXY"], + timeout_s=30))["stdout"] + tabs_after_reset = await tabs_of(deployed) finally: report = await teardown_run(context) @@ -604,6 +612,11 @@ async def test_routing_sites_it_goes_out_only_through_the_forward_port_which_the assert "site proxy: refused GET 127.0.0.1:18765: no site of this computer" in logged assert stopped["exit_code"] != 0 and still == serving and after_stop.startswith("200 "), (stopped, after_stop) assert back.startswith("200 ") and restarted and restarted != serving, (back, restarted) + assert after_reset.startswith("200 ") and after_reset.endswith("alpha"), after_reset + assert refused_after_reset.startswith("error") and "403" in refused_after_reset, refused_after_reset + assert named_after_reset.strip() == "http://127.0.0.1:3128" + assert [(tab["url"], tab["title"]) for tab in tabs_after_reset] == [("https://shop.test/landed", "landed"), + ("http://notes.localhost/", "notes")] assert not report.failed diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index b33fd0704..274256a76 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -999,7 +999,7 @@ async def test_a_computer_with_site_routing_reaches_the_websites_the_run_deploye assert [site.model_dump() for site in sites] == [{"name": "notes", "url": "http://localhost:8080/website/notes/", "domains": ["notes.test"]}] assert options == {"user": "user", "network": None, "valid_for": timedelta(seconds=3600), "label": "desktop", - "os_type": "linux"} + "os_type": "linux", "display": "x11"} assert boots.boots[0]["environment"][HOLD_BOOT_LAUNCH_ENV] == "1" record = context.deployed_envs[-1].metadata["computer"] assert record["sites"] == {"names": {"routes": {"notes": ["notes.test"]}}, "ca": "CA PEM", "valid_for_s": 3600} diff --git a/tst/unit/env/envs/test_computer_site_proxy.py b/tst/unit/env/envs/test_computer_site_proxy.py index e81fca430..922e636f4 100644 --- a/tst/unit/env/envs/test_computer_site_proxy.py +++ b/tst/unit/env/envs/test_computer_site_proxy.py @@ -679,7 +679,8 @@ def test_install_writes_the_sites_files_names_and_trust_then_starts_the_proxy_on SiteParams(name="chat", url="http://gw/website/chat/", domains=["*.chat.test"]), SiteParams(name="bare", url="http://gw/website/bare/")]) issued = sites.issue(table.routes, valid_for=timedelta(hours=1), label="t") - (folder / "config.json").write_text(json.dumps(sites.proxy_config(table, issued, user=None, os_type="linux"))) + (folder / "config.json").write_text(json.dumps(sites.proxy_config(table, issued, user=None, os_type="linux", + forward_port=True))) started = [] monkeypatch.setattr(site_proxy, "start", lambda path, log: started.append(Config.load(path))) monkeypatch.setattr(site_proxy, "checks_as", lambda user, path: {"sites": {"notes.test": {"status": 200}}, @@ -724,7 +725,7 @@ def test_install_keeps_the_upstream_headers_in_a_file_root_alone_reads_and_serve folder.chmod(0o700) table = sites.site_table([SiteParams(name="notes", url="http://gw/website/notes/", domains=["notes.test"])]) issued = sites.issue(table.routes, valid_for=timedelta(hours=1), label="t") - given = sites.proxy_config(table, issued, user=None, os_type="linux", + given = sites.proxy_config(table, issued, user=None, os_type="linux", forward_port=False, upstream_headers={"notes": {"CF-Access-Client-Secret": "s3cret"}}) (folder / "config.json").write_text(json.dumps(given)) started = [] @@ -955,7 +956,8 @@ def test_a_wildcard_name_starts_dns_even_when_resolv_conf_names_no_other_nameser (root / "etc/resolv.conf").write_text(resolv) table = sites.site_table([SiteParams(name="chat", url="http://gw/website/chat/", domains=["*.chat.test"])]) issued = sites.issue(table.routes, valid_for=timedelta(hours=1), label="t") - (folder / "config.json").write_text(json.dumps(sites.proxy_config(table, issued, user=None, os_type="linux"))) + (folder / "config.json").write_text(json.dumps(sites.proxy_config(table, issued, user=None, os_type="linux", + forward_port=False))) started = [] monkeypatch.setattr(site_proxy, "start", lambda path, log: started.append(Config.load(path))) monkeypatch.setattr(site_proxy, "checks_as", lambda user, path: {"sites": {}, "runtimes": {}}) @@ -1003,7 +1005,7 @@ def sleep(seconds): def test_the_log_is_root_owned_beside_the_config_and_a_link_at_its_path_is_refused(tmp_path): assert sites.proxy_config(sites.site_table([]), SimpleNamespace(ca_pem=None, leaves={}), user=None, - os_type="linux")["log"] == str(sites.FOLDER / "site-proxy.log") + forward_port=False, os_type="linux")["log"] == str(sites.FOLDER / "site-proxy.log") target, log = tmp_path / "elsewhere", tmp_path / "site-proxy.log" target.write_text("kept") log.symlink_to(target) diff --git a/tst/unit/env/envs/test_computer_sites.py b/tst/unit/env/envs/test_computer_sites.py index 5c489f6ad..ffb631726 100644 --- a/tst/unit/env/envs/test_computer_sites.py +++ b/tst/unit/env/envs/test_computer_sites.py @@ -185,7 +185,7 @@ async def test_route_installs_the_proxy_as_root_with_each_sites_route_as_the_com sandbox = FakeVm(_REPORT) routed = await sites.route(sandbox, [_site("notes", "notes.test")], user="user", network=None, - valid_for=timedelta(hours=1), label="desktop", os_type="linux") + valid_for=timedelta(hours=1), label="desktop", os_type="linux", display="x11") assert set(sandbox.written) == {"/etc/agentenv/sites/site_proxy.py", "/etc/agentenv/sites/config.json"} config = json.loads(sandbox.written["/etc/agentenv/sites/config.json"]) @@ -199,17 +199,29 @@ async def test_route_installs_the_proxy_as_root_with_each_sites_route_as_the_com "docker exec -u 0 computer-1 chmod 0600 /etc/agentenv/sites/config.json", "docker exec -u 0 computer-1 python3 /etc/agentenv/sites/site_proxy.py install " "/etc/agentenv/sites/config.json"] - assert "upstream_headers" not in site + assert "upstream_headers" not in site and "proxy_port" not in config assert routed["checks"] == _REPORT["checks"] and routed["names"]["routes"] == {"notes": ["notes.test"]} assert "PRIVATE KEY" not in json.dumps(routed) +@pytest.mark.asyncio +async def test_route_gives_a_browser_alone_the_forward_port_and_a_computer_with_a_display_none(): + alone, desktop = FakeVm(_REPORT), FakeVm(_REPORT) + + for sandbox, display in ((alone, "none"), (desktop, "x11")): + await sites.route(sandbox, [_site("notes", "notes.test")], user="user", network=None, + valid_for=timedelta(hours=1), label="desktop", os_type="linux", display=display) + + configs = [json.loads(sandbox.written["/etc/agentenv/sites/config.json"]) for sandbox in (alone, desktop)] + assert [config.get("proxy_port") for config in configs] == [3128, None] + + @pytest.mark.asyncio async def test_the_config_holding_the_keys_is_written_over_stdin_into_a_private_staging_folder_never_as_text(): sandbox = FakeVm(_REPORT) await sites.route(sandbox, [_site("notes", "notes.test")], user="user", network=None, - valid_for=timedelta(hours=1), label="desktop", os_type="linux") + valid_for=timedelta(hours=1), label="desktop", os_type="linux", display="x11") assert ("write", "/etc/agentenv/sites/config.json") not in sandbox.events secret = sandbox.events.index(("secret", "/tmp/secret-1/config.json")) @@ -227,7 +239,7 @@ async def test_the_headers_the_gateways_provider_asks_for_go_in_the_private_conf "CF-Access-Client-Secret": "s3cret"} if url.startswith("http://host.docker.internal:8080/") else {}) routed = await sites.route(sandbox, [_site("notes", "notes.test")], user="user", network=None, - valid_for=timedelta(hours=1), label="desktop", os_type="linux") + valid_for=timedelta(hours=1), label="desktop", os_type="linux", display="x11") (site,) = json.loads(sandbox.written["/etc/agentenv/sites/config.json"])["sites"] assert site["upstream_headers"] == {"CF-Access-Client-Secret": "s3cret"} @@ -242,7 +254,7 @@ async def test_the_sites_keys_land_in_a_folder_root_alone_can_enter(): sandbox = FakeVm(_REPORT) await sites.route(sandbox, [_site("notes", "notes.test")], user="user", network=None, - valid_for=timedelta(hours=1), label="desktop", os_type="linux") + valid_for=timedelta(hours=1), label="desktop", os_type="linux", display="x11") private = sandbox.events.index("docker exec -u 0 computer-1 chmod 0700 /etc/agentenv/sites") assert private < sandbox.events.index(("cp", "/tmp/secret-1/config.json", "computer-1:/etc/agentenv/sites/config.json")) @@ -255,7 +267,7 @@ async def test_a_name_that_doesnt_load_in_the_computer_fails_the_route(): with pytest.raises(RuntimeError, match="https://notes.test/ didn't load in the computer: certificate verify"): await sites.route(FakeVm(report), [_site("notes", "notes.test")], user="user", network=None, - valid_for=timedelta(hours=1), label="desktop", os_type="linux") + valid_for=timedelta(hours=1), label="desktop", os_type="linux", display="x11") @pytest.mark.asyncio @@ -266,7 +278,7 @@ async def test_a_network_that_keeps_the_computer_from_the_gateway_is_refused_bef "host.docker.internal"): await sites.route(sandbox, [_site("notes", "notes.test")], user="user", network={"mode": "allowlist", "allow_hosts": ["api.example"]}, valid_for=timedelta(hours=1), - label="desktop", os_type="linux") + label="desktop", os_type="linux", display="x11") assert sandbox.scripts == [] and sandbox.written == {} @@ -280,7 +292,7 @@ async def test_a_gateway_the_sandboxs_applied_network_allows_by_its_address_rang routed = await sites.route(sandbox, [_site("notes", "notes.test")], user="user", network={"mode": "allowlist", "allow_hosts": ["api.example"]}, - valid_for=timedelta(hours=1), label="desktop", os_type="linux") + valid_for=timedelta(hours=1), label="desktop", os_type="linux", display="x11") assert routed["checks"] == _REPORT["checks"] routes = [sites.Route("notes", "http://10.0.0.7:8080/website/notes/", ("notes.test",))] @@ -305,7 +317,7 @@ async def write_file_from_text(content, path): write_file_from_text=write_file_from_text) await sites.route(sandbox, [_site("notes", "notes.test")], user="user", network=None, - valid_for=timedelta(hours=1), label="desktop", os_type="linux") + valid_for=timedelta(hours=1), label="desktop", os_type="linux", display="x11") assert calls == [("id", "-u"), ("mkdir", "-p", "/etc/agentenv/sites"), ("chmod", "0700", "/etc/agentenv/sites"), ("mkdir", "-p", "/etc/agentenv/sites"), ("write", "/etc/agentenv/sites/site_proxy.py"), @@ -330,7 +342,7 @@ async def write_file_from_text(content, path): with pytest.raises(RuntimeError, match="the coreweave sandbox runs the computer's commands as uid 1000"): await sites.route(sandbox, [_site("notes", "notes.test")], user="user", network=None, - valid_for=timedelta(hours=1), label="desktop", os_type="linux") + valid_for=timedelta(hours=1), label="desktop", os_type="linux", display="x11") assert calls == [("id", "-u")] @@ -343,7 +355,7 @@ async def test_a_site_naming_the_gateway_its_routes_are_on_is_refused_before_any with pytest.raises(RuntimeError, match=r"the gateway at gw.sites.test \(named by notes's .*sites.test\) would"): await sites.route(sandbox, [_site("notes", *domains)], user="user", network=None, - valid_for=timedelta(hours=1), label="desktop", os_type="linux") + valid_for=timedelta(hours=1), label="desktop", os_type="linux", display="x11") assert sandbox.scripts == [] and sandbox.written == {} From d76843d6732eb883d1c844f13bfad45e158696a9 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 8 Nov 2026 10:30:44 -0800 Subject: [PATCH 4/4] refactor(computer): the forward port takes 3129, and the headless base's stand-in stays as it was on 3128 A browser computer that routes no sites now changes not at all, and the base image isn't touched: the stand-in keeps 3128, and a computer that routes sites names the forward port, 127.0.0.1:3129, to its browser through AGENTENV_SITE_PROXY. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/computers/README.md | 10 +++++----- .../artifacts/ubuntu-24.04-headless/Dockerfile | 5 ++--- .../artifacts/ubuntu-24.04-headless/site-proxy.py | 7 +++---- .../artifacts/ubuntu-24.04-headless/supervisord.conf | 4 ++-- src/agent_env/env/envs/computer/utils/site_proxy.py | 2 +- src/agent_env/env/envs/computer/utils/sites.py | 6 +++--- tst/integration/computer/browser_computer_test.py | 8 ++++---- tst/unit/env/envs/test_computer_site_proxy.py | 2 +- tst/unit/env/envs/test_computer_sites.py | 4 ++-- 9 files changed, 23 insertions(+), 25 deletions(-) diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index 85cbe9a66..23828e71c 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -13,11 +13,11 @@ building it. Name it from another bundle as `ubuntu-24.04`, the alias `aliases.t `artifacts/ubuntu-24.04-headless/` is Ubuntu 24.04 (`amd64` and `arm64`) with no display server or desktop (`display = "none"`), for one website browser, which runs headless there, and the `page` driver, whose screen is the browser's page: a browser computer, which agentenv's `chromium` deploys alone as. supervisord runs every program as the -session user. The browser goes out only through a site proxy. A browser computer that routes sites names the shared site -proxy's forward port, `127.0.0.1:3128`, to it: that port serves the run's sites and refuses every other host, and it -runs out of the agent's reach. A computer that routes no sites keeps the base's stand-in on `127.0.0.1:3129`, which -forwards every request to wherever it names, refusing nothing: the sandbox's network policy is all that limits it. It -has a container form only. Name it from another bundle as `ubuntu-24.04-headless`, or by its id, +session user, among them a stand-in site proxy on `127.0.0.1:3128`, the browser's only route out when the computer +routes no sites, which forwards every request to wherever it names, refusing nothing: the sandbox's network policy is +all that limits it. A computer that routes sites names the shared site proxy's forward port, `127.0.0.1:3129`, to the +browser instead: it serves the run's sites, refuses every other host and runs out of the agent's reach. It has a +container form only. Name it from another bundle as `ubuntu-24.04-headless`, or by its id, `@local/agentenv-framework/computers/ubuntu-24.04-headless`. It also has a disk form, a disk a VM boots from UEFI, which is the container form's root filesystem on the Linux boot diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile index 396851eba..2314fdd2b 100644 --- a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile +++ b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile @@ -1,7 +1,6 @@ # The ubuntu-24.04-headless base computer: Ubuntu 24.04 with no display server or desktop (XDG_SESSION_TYPE=none), # for one website browser, which runs headless there, and a page driver whose screen is the browser's page. supervisord -# runs every program as the session user. The browser goes out only through a site proxy: the shared one's forward port -# (127.0.0.1:3128) when the computer routes sites, else this base's stand-in on 127.0.0.1:3129. +# runs every program as the session user, and the browser goes out only through the site proxy on 127.0.0.1:3128. # Files a page's file chooser may be given go in ~/Uploads. Composing a driver or an app onto it adds its services as # programs in /etc/supervisor/conf.d. The trust files agentenv adds are in /etc/agentenv from boot: ca.pem, empty until # a deploy adds a CA, and ca-bundle.pem, the public roots and those CAs. @@ -22,6 +21,6 @@ COPY start-computer /usr/local/bin/start-computer COPY site-proxy.py /usr/local/bin/agentenv-site-proxy USER user WORKDIR /home/user -ENV LANG=C.UTF-8 XDG_SESSION_TYPE=none AGENTENV_SITE_PROXY=http://127.0.0.1:3129 +ENV LANG=C.UTF-8 XDG_SESSION_TYPE=none AGENTENV_SITE_PROXY=http://127.0.0.1:3128 ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["/usr/local/bin/start-computer"] diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/site-proxy.py b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/site-proxy.py index cea102020..5eacb8004 100644 --- a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/site-proxy.py +++ b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/site-proxy.py @@ -1,13 +1,12 @@ -"""A stand-in for the shared site proxy in this base, for a browser computer that routes no sites: it forwards every -request the browser sends it, absolute-form HTTP and CONNECT tunnels alike, to wherever it names, refusing nothing, on -127.0.0.1:3129. A computer routing sites names the shared site proxy's forward port (3128) to the browser instead.""" +"""A stand-in for the shared site proxy in this base: it forwards every request the browser sends it, absolute-form +HTTP and CONNECT tunnels alike, to wherever it names, refusing nothing, on 127.0.0.1:3128.""" import http.server import select import socket import socketserver import urllib.parse -PORT = 3129 +PORT = 3128 IDLE_S = 60 HOP = {"connection", "keep-alive", "proxy-connection", "proxy-authorization"} diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/supervisord.conf b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/supervisord.conf index dfb8f8437..da2860323 100644 --- a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/supervisord.conf +++ b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/supervisord.conf @@ -1,5 +1,5 @@ -; The base computer's own program, the stand-in site proxy for a computer that routes no sites, run as the session user. -; A composed driver or app adds its own in conf.d. +; The base computer's own program, the site proxy, run as the session user. A composed driver or app adds its own in +; conf.d. [supervisord] nodaemon=true logfile=/var/log/agentenv/supervisord.log diff --git a/src/agent_env/env/envs/computer/utils/site_proxy.py b/src/agent_env/env/envs/computer/utils/site_proxy.py index 7324924cd..64fdc3728 100644 --- a/src/agent_env/env/envs/computer/utils/site_proxy.py +++ b/src/agent_env/env/envs/computer/utils/site_proxy.py @@ -7,7 +7,7 @@ store, Firefox's policies, Chromium's managed policies and the session user's NSS databases, starts ``serve`` and checks, as the session user, that each name loads. ``serve`` listens on loopback: on 443 for a name served over HTTPS, on 80 to redirect those to HTTPS and to serve ``.localhost`` names over HTTP, on 53 for DNS, over UDP and TCP, when a -name is a wildcard, which a hosts file can't list, and, on a browser computer, on ``proxy_port`` (3128) as a forward +name is a wildcard, which a hosts file can't list, and, on a browser computer, on ``proxy_port`` (3129) as a forward proxy, a browser alone's only route out, which serves the same sites to ``CONNECT`` and to requests naming their URLs and refuses any other host at once; it then drops to ``SERVE_USER`` (``nobody``), else the session user, so the agent, which runs as the session user, can't stop it, and ``keep``, as root, starts it again should it end. The site sees diff --git a/src/agent_env/env/envs/computer/utils/sites.py b/src/agent_env/env/envs/computer/utils/sites.py index b19fc7af0..f3b1a28a7 100644 --- a/src/agent_env/env/envs/computer/utils/sites.py +++ b/src/agent_env/env/envs/computer/utils/sites.py @@ -56,9 +56,9 @@ #: By the OS type of each computer site routing reaches, the variables naming the trust files agentenv adds, which a #: computer routing sites boots with. TRUST_ENVIRONMENTS = {"linux": dict(LOCAL_TRUST_ENV)} -#: The site proxy's forward port, a browser alone's only route out when its computer routes sites, and the display of -#: the computer a browser runs alone on. -FORWARD_PROXY_PORT = 3128 +#: The site proxy's forward port, a browser alone's only route out when its computer routes sites (the base's own +#: stand-in, for a computer that routes none, keeps 3128), and the display of the computer a browser runs alone on. +FORWARD_PROXY_PORT = 3129 FORWARD_PROXY = f"http://127.0.0.1:{FORWARD_PROXY_PORT}" ALONE_DISPLAY = "none" LOG = FOLDER / "site-proxy.log" diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index fe62e6177..ab03d9cad 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -203,7 +203,7 @@ async def test_its_tools_see_and_drive_the_page_that_is_its_screen(browser_compu assert clicked["c"] == 1 and doubled["d"] == 1 and typed["v"] == "Hello, World!" and replaced["v"] == "x" assert "C-a" in replaced["k"] and "Enter" in entered["k"] and scrolled["y"] >= 300 assert after.getpixel((200, 150)) != (255, 0, 0) - assert json.loads(listening) == [3129, 18765] and x_server.strip() == "False" + assert json.loads(listening) == [3128, 18765] and x_server.strip() == "False" assert "--headless=new" in command and "--window-size=1280,800" in command assert not report.failed @@ -536,7 +536,7 @@ async def _ready(deployed) -> dict: async def _through_port(deployed, url: str) -> str: - done = await inside.run(deployed, ["python3", "-c", THROUGH_PROXY, url, "http://127.0.0.1:3128"], timeout_s=60) + done = await inside.run(deployed, ["python3", "-c", THROUGH_PROXY, url, "http://127.0.0.1:3129"], timeout_s=60) return done["stdout"].strip() @@ -599,7 +599,7 @@ async def test_routing_sites_it_goes_out_only_through_the_forward_port_which_the ("https://shop.test/", "shop", "loaded"), ("http://notes.localhost/", "notes", "loaded")], start_tabs assert [(tab["url"], tab["title"]) for tab in tabs] == [("https://shop.test/landed", "landed"), ("http://notes.localhost/", "notes")], tabs - assert named.strip() == "http://127.0.0.1:3128" + assert named.strip() == "http://127.0.0.1:3129" assert pages["https://shop.test/alpha"].startswith("200 ") and pages["https://shop.test/alpha"].endswith( "alpha"), pages assert pages["http://notes.localhost/bravo"].endswith("bravo"), pages @@ -614,7 +614,7 @@ async def test_routing_sites_it_goes_out_only_through_the_forward_port_which_the assert back.startswith("200 ") and restarted and restarted != serving, (back, restarted) assert after_reset.startswith("200 ") and after_reset.endswith("alpha"), after_reset assert refused_after_reset.startswith("error") and "403" in refused_after_reset, refused_after_reset - assert named_after_reset.strip() == "http://127.0.0.1:3128" + assert named_after_reset.strip() == "http://127.0.0.1:3129" assert [(tab["url"], tab["title"]) for tab in tabs_after_reset] == [("https://shop.test/landed", "landed"), ("http://notes.localhost/", "notes")] assert not report.failed diff --git a/tst/unit/env/envs/test_computer_site_proxy.py b/tst/unit/env/envs/test_computer_site_proxy.py index 922e636f4..4708de923 100644 --- a/tst/unit/env/envs/test_computer_site_proxy.py +++ b/tst/unit/env/envs/test_computer_site_proxy.py @@ -693,7 +693,7 @@ def test_install_writes_the_sites_files_names_and_trust_then_starts_the_proxy_on assert [(s.name, s.domains, bool(s.cert)) for s in config.sites] == [ ("notes", ["notes.test"], True), ("chat", ["*.chat.test"], True), ("bare", ["bare.localhost"], False)] assert (config.dns_port, config.nameservers) == (53, ["127.0.0.11"]) - assert config.proxy_port == sites.FORWARD_PROXY_PORT == 3128 + assert config.proxy_port == sites.FORWARD_PROXY_PORT == 3129 assert [(folder / name).stat().st_mode & 0o777 for name in ("notes.key", "config.json")] == [0o600, 0o600] assert folder.stat().st_mode & 0o777 == 0o755 assert "key_pem" not in (folder / "serve.json").read_text() diff --git a/tst/unit/env/envs/test_computer_sites.py b/tst/unit/env/envs/test_computer_sites.py index ffb631726..b00572abf 100644 --- a/tst/unit/env/envs/test_computer_sites.py +++ b/tst/unit/env/envs/test_computer_sites.py @@ -213,7 +213,7 @@ async def test_route_gives_a_browser_alone_the_forward_port_and_a_computer_with_ valid_for=timedelta(hours=1), label="desktop", os_type="linux", display=display) configs = [json.loads(sandbox.written["/etc/agentenv/sites/config.json"]) for sandbox in (alone, desktop)] - assert [config.get("proxy_port") for config in configs] == [3128, None] + assert [config.get("proxy_port") for config in configs] == [3129, None] @pytest.mark.asyncio @@ -585,7 +585,7 @@ def test_the_trust_a_routing_linux_computer_boots_with_is_the_local_providers_so def test_a_browser_alone_routing_sites_is_named_the_forward_port_and_a_computer_with_a_display_is_named_none(): - assert sites.browser_environment("none") == {"AGENTENV_SITE_PROXY": "http://127.0.0.1:3128"} + assert sites.browser_environment("none") == {"AGENTENV_SITE_PROXY": "http://127.0.0.1:3129"} assert sites.browser_environment("x11") == {}