diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py index 00856062f..bd81cf97b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_data.py @@ -1,7 +1,9 @@ """An app's data plane as its own server serves it (``AppDataPlane``), by the app's data hooks or, with none, the defaults on the paths it keeps its data in, under the session user's home: a load writes the files it's given under them, an export returns the files there, and a reset puts back the copy its server kept when it first started on this -boot, which is the app as installed, since every boot starts from the computer's image.""" +boot, which is the app as installed, since every boot starts from the computer's image. An export copies the files with +the app paused, or, where the app holds one so that no other process may read it, as Windows lets an app hold a file +and a pause doesn't release, with it stopped and relaunched after.""" from __future__ import annotations import asyncio @@ -17,7 +19,7 @@ from ..agent_env_environment import OP_ADD_DATA, OP_GET_DATA, OP_RESET_DATA from ..transfers import WriteNamespaceGrant -from ..types import METHOD_ADD, METHOD_RESET, FilePart, uploaded_object_path +from ..types import METHOD_ADD, METHOD_GET, METHOD_RESET, FilePart, uploaded_object_path from .app_runtime import paused_app from .dataplane import data_files, file_parts, part_bytes @@ -27,15 +29,24 @@ logger = logging.getLogger(__name__) #: The cause a launch after each data call logs. -_CAUSES = {OP_ADD_DATA: METHOD_ADD, OP_RESET_DATA: METHOD_RESET} +_CAUSES = {OP_ADD_DATA: METHOD_ADD, OP_GET_DATA: METHOD_GET, OP_RESET_DATA: METHOD_RESET} #: Where an app's server keeps its paths as installed, in the app's folder. INITIAL = Path(".agentenv/initial") +#: Windows' errors for a file another process holds so that no other may read it: a sharing violation, and a lock on +#: the part read. +_HELD = (32, 33) + + +class HeldByApp(OSError): + """A file under the app's paths that the running app holds so that no other process may read it, which stopping + the app releases and pausing it doesn't.""" class AppDataPlane: """An app's data plane, served around its contract: one data call or launch at a time; a load and a reset stop the running app first and relaunch it after, but for those the app serves live, and a load that fails is followed by a - reset, when the app says so. A call fails once it runs + reset, when the app says so; an export the app holds a file of (``HeldByApp``) stops it and relaunches it too. A + call fails once it runs past ``timeout_s`` and is cancelled: what awaits stops at once, but a thread can't be stopped, so a sync hook's or a default's thread is waited for, as are the reset and relaunch after it and the next call.""" @@ -90,7 +101,7 @@ async def _change(self, op: str, served: Callable, args: tuple, answer: asyncio. running = op != OP_GET_DATA and op not in self.live and await self.runtime.stop_app() ran = op in self.live and self.runtime.running() try: - result = await self._within(op, served, args, answer) + result = await self._within_held(op, served, args, answer) except Exception: if op == OP_ADD_DATA and self.resets_failed_add(): if op in self.live: @@ -110,6 +121,24 @@ async def _change(self, op: str, served: Callable, args: tuple, answer: asyncio. else: answer.set_result(result) + async def _within_held(self, op: str, served: Callable, args: tuple, answer: asyncio.Future) -> Any: + """``served``'s result within ``timeout_s``; where the app holds a file of its paths (``HeldByApp``), again with + the app stopped, in what's left of it, and the app relaunched after, as a load's is.""" + loop = asyncio.get_running_loop() + deadline = loop.time() + self.timeout_s + try: + return await self._within(op, served, args, answer) + except HeldByApp: + stopped = await self.runtime.stop_app() + try: + result = await self._within(op, served, args, answer, deadline - loop.time()) + finally: + if stopped: + await self.runtime.launch_app(_CAUSES[op]) + if stopped and self.relaunched is not None: + await self.relaunched(op) + return result + async def _undo(self, answer: asyncio.Future) -> None: """Reset the app's data after a load that failed.""" if (reset := self._served(OP_RESET_DATA)) is None: @@ -140,12 +169,14 @@ def _served(self, op: str) -> Optional[Callable]: OP_GET_DATA: functools.partial(get, self.paths), OP_RESET_DATA: functools.partial(reset, self.paths)}[op] - async def _within(self, op: str, served: Callable, args: tuple, answer: asyncio.Future) -> Any: - """``served``'s result within ``timeout_s``; past it, ``answer`` fails at once and the call is cancelled, and - this raises the same once it has ended: at once for what awaits, after any thread it runs.""" + async def _within(self, op: str, served: Callable, args: tuple, answer: asyncio.Future, + left_s: Optional[float] = None) -> Any: + """``served``'s result within ``left_s``, ``timeout_s`` by default; past it, ``answer`` fails at once and the + call is cancelled, and this raises the same once it has ended: at once for what awaits, after any thread it + runs.""" work = asyncio.ensure_future(_called(served, *args)) try: - return await asyncio.wait_for(asyncio.shield(work), self.timeout_s) + return await asyncio.wait_for(asyncio.shield(work), max(0.0, self.timeout_s if left_s is None else left_s)) except TimeoutError: late = TimeoutError(f"{op} ran past the app's data_timeout_s, {self.timeout_s:g} s") if not answer.done(): @@ -185,11 +216,17 @@ async def add(parts: list, paths: list[str]) -> None: async def get(paths: list[str], write_namespace: Optional[WriteNamespaceGrant]) -> list: """A part for each file under ``paths``, copied with the app this server launched paused (``paused_app``), so they - are of one moment, and sent from the copy once it runs again; an app's own data/get hook calls it so too.""" + are of one moment, and sent from the copy once it runs again; an app's own data/get hook calls it so too. A file + the paused app holds raises ``HeldByApp``, for its data plane to copy them with it stopped.""" INITIAL.parent.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(dir=INITIAL.parent) as staging: - with paused_app(): - await in_thread(_stage, Path.home(), paths, Path(staging)) + try: + with paused_app(): + await in_thread(_stage, Path.home(), paths, Path(staging)) + except OSError as e: + if getattr(e, "winerror", None) in _HELD: + raise HeldByApp(f"the running app holds {e.filename or 'a file'} so that nothing else may read it") from e + raise return await file_parts(Path(staging), paths, write_namespace) diff --git a/packages/agentenv-protocol/tests/test_computer_app_data.py b/packages/agentenv-protocol/tests/test_computer_app_data.py index 00b601504..06b6e2ba6 100644 --- a/packages/agentenv-protocol/tests/test_computer_app_data.py +++ b/packages/agentenv-protocol/tests/test_computer_app_data.py @@ -2,8 +2,9 @@ the defaults: a load writes files under them, through no link out of them, an export returns them, and a reset puts back the copy its server kept before it served; its card declares a file tree. Hooks override paths, and read the parts they get with part_bytes. Around a load and a reset the app is stopped and relaunched, a failed load is followed by a reset, -and data calls take turns; an app that loads live is left running through a load, and a failed one is reset with it -stopped, or left as it is when the app says so; an app with hooks may keep its paths as installed. A call past +and data calls take turns; an export of a file the paused app holds stops it, copies it in what's left of its +timeout, and relaunches it after; an app that loads live is left running through a load, and a failed one is reset +with it stopped, or left as it is when the app says so; an app with hooks may keep its paths as installed. A call past data_timeout_s fails then and is cancelled, but a sync hook runs in a thread, which its reset and relaunch, and the next call, wait for. The app serves its skill file.""" from __future__ import annotations @@ -20,7 +21,7 @@ import httpx # noqa: E402 -from agentenv_protocol import add_data, client, reset_data # noqa: E402 +from agentenv_protocol import add_data, client, get_data, reset_data # noqa: E402 from agentenv_protocol.agent_env_environment import OP_ADD_DATA # noqa: E402 from agentenv_protocol.computer import ComputerAppEnvironment, Launch, linux, part_bytes # noqa: E402 from agentenv_protocol.computer import app, app_data, app_runtime, session # noqa: E402 @@ -106,8 +107,10 @@ async def stop(pid, timeout_s): @contextlib.contextmanager def paused(pid): events.append(("pause", pid)) - yield - events.append(("continue", pid)) + try: + yield + finally: + events.append(("continue", pid)) monkeypatch.setattr(session, "launch", launch) monkeypatch.setattr(session, "stop", stop) @@ -210,6 +213,32 @@ async def test_a_load_and_a_reset_stop_the_running_app_and_relaunch_it_and_an_ex ("stop", 4244), ("launch", 4248)] +@pytest.mark.asyncio +async def test_an_export_of_a_file_the_paused_app_holds_stops_the_app_copies_it_and_relaunches_it(home, desktop, + monkeypatch): + """As Windows lets an app hold a file that no other process may read, which a pause doesn't release.""" + (home / "Mail").mkdir() + (home / "Mail/inbox.txt").write_text("held") + mail = PathsMail() + mail._computer_app_data_plane().keep_initial() + await mail._computer_app.launch_app() + stage = app_data._stage + + def held_while_running(*args): + if desktop[-1][0] == "pause": + held = PermissionError(13, "The process cannot access the file", str(home / "Mail/inbox.txt")) + held.winerror = 32 + raise held + stage(*args) + + monkeypatch.setattr(app_data, "_stage", held_while_running) + async with _http(mail) as http: + exported = _files(await _rpc(http, "data/get", {})) + + assert [event[0] for event in desktop] == ["launch", "pause", "continue", "stop", "launch"] + assert desktop[3] == ("stop", 4242) and exported["Mail/inbox.txt"] == b"held" + + @pytest.mark.asyncio async def test_an_export_copies_the_paths_with_the_app_paused_and_sends_the_copy_once_it_runs_again(home, desktop, monkeypatch): @@ -414,6 +443,32 @@ def wipe(self) -> None: ("launch", 4248), ("stop", 4248), ("load", 1), ("loaded", 1), ("launch", 4252)] +@pytest.mark.asyncio +async def test_an_export_retried_with_the_app_stopped_gets_only_what_is_left_of_its_timeout(home, desktop): + @linux(launch=Launch(command=["held"])) + class Held(ComputerAppEnvironment): + data_timeout_s = 1.0 + + @get_data + async def export(self) -> list: + if desktop[-1][0] == "launch": + await asyncio.sleep(0.8) + raise app_data.HeldByApp("the running app holds a file") + await asyncio.Event().wait() + + held = Held() + await held._computer_app.launch_app() + started = time.monotonic() + async with _http(held) as http: + late = await http.post(RPC_PATH, json=_call("data/get", {})) + answered = time.monotonic() - started + await asyncio.gather(*held._computer_app_data_plane().changing) + + assert late.json()["error"]["message"] == "get_data ran past the app's data_timeout_s, 1 s" + assert answered < 1.5 + assert desktop == [("launch", 4242), ("stop", 4242), ("launch", 4244)] + + @pytest.mark.asyncio async def test_it_serves_its_skill_file_and_no_skill_without_one(home, tmp_path): (tmp_path / "folder" / "SKILL.md").write_text("---\ndescription: Reading and sending mail.\n---\n\n# Mail\nUse it.\n") diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index ef1868fb9..671428765 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -39,7 +39,10 @@ a reset boots the disk afresh on a new overlay in the same launcher, at the same before reset in place started is refused a reset, to be redeployed. It boots on local Docker where Docker can pass it `/dev/kvm`, the disk image's files mounted read-only into the launcher where Docker mounts images (28 and later, on the containerd image store), so no copy of a disk, nor of a Windows disk's base, is made for each launcher, else copied into -an image of the launcher it keeps for the next boot; and on `modal_vm`, amd64 only, whose sandbox runs the launcher as +an image of the launcher it keeps for the next boot. There its computer uses a local object store's grants as a container +does, at `host.docker.internal`: Linux's init and Windows' setup map that name to 10.0.2.2, QEMU's address for the +launcher, which forwards the grant server's port to this machine (`VM_HOST_PORTS`), and the local transfer CA's trust +files ride in the seed (`VM_TRUST`), where the init and agentenv-session point TLS clients at them; and on `modal_vm`, amd64 only, whose sandbox runs the launcher as its image under nested virtualization: the provider exports the disk from the image store with this machine's Docker, compressed, once for each composed disk, into the per-user cache, and mounts it on the sandbox at create, which Modal uploads once for each content and the launcher expands onto the sandbox's disk; a thin disk, a Windows one, carries its @@ -181,7 +184,8 @@ with 4 vCPUs and 8 GB, on local Docker with KVM and on `modal_vm`; it has comput protocol's Windows defaults, so the probe runs its fixture there as on Linux, with Python one-liners for the commands it would give a shell. On `modal_vm` it runs in your own Modal workspace, under the evaluation's terms or a licence of your own; whether they allow it there is yours to check. Composing leaves `C:\agentenv` the admins', as `/opt/agentenv` is -root's on Linux: every user reads and runs what's there, and the session user writes only the services' logs and each +root's on Linux: every user, and every sandboxed process (Chrome's, as Program Files allows it), reads and runs +what's there, and the session user writes only the services' logs and each app's `.agentenv` folder. An app's own server runs in the session as its services do, on a loopback port it writes with a token to that folder, which only the session user, SYSTEM and Administrators reach; it launches the app in a Job object, which ends whatever the app started. Composing signs in as `agentadmin`, so diff --git a/src/agent_env/computers/disk/linux/boot/init b/src/agent_env/computers/disk/linux/boot/init index 886180daf..07d38b977 100755 --- a/src/agent_env/computers/disk/linux/boot/init +++ b/src/agent_env/computers/disk/linux/boot/init @@ -2,6 +2,8 @@ # The disk form's first process: mounts the root filesystem the computer's image became, brings up the network, reads # the image's start (/etc/agentenv/vm/image.env, from its OCI config) and the deploy's environment (the NoCloud seed's # user-data), then becomes that start, as its user, as the container runtime would: PID 1 is the image's entrypoint. +# As a container finds this machine at host.docker.internal, it finds the launcher at QEMU's address for it, which +# forwards a local grant server's port, and trusts the local transfer CA the seed carries, as a container does. /bin/busybox --install -s /bin export PATH=/bin mount -t proc proc /proc @@ -35,11 +37,17 @@ while read -r line; do workdir) decode "$value"; workdir=$decoded ;; esac done < "$config" +if [ -f /seed/trust/ca-bundle.pem ]; then + cp /seed/trust/ca-bundle.pem /seed/trust/ca.pem /newroot/etc/agentenv/ || fail "can't copy the seed's trust files" + export SSL_CERT_FILE=/etc/agentenv/ca-bundle.pem REQUESTS_CA_BUNDLE=/etc/agentenv/ca-bundle.pem \ + NODE_EXTRA_CA_CERTS=/etc/agentenv/ca.pem +fi [ -n "$seed" ] && { seed_environment /seed || fail "$refused"; } hostname=$( [ -f /seed/meta-data ] && sed -n 's/^local-hostname: *//p' /seed/meta-data) hostname "${hostname:-computer}" hostname > /newroot/etc/hostname -printf '127.0.0.1\tlocalhost\n::1\tlocalhost ip6-localhost ip6-loopback\n127.0.1.1\t%s\n' "$(hostname)" > /newroot/etc/hosts +printf '127.0.0.1\tlocalhost\n::1\tlocalhost ip6-localhost ip6-loopback\n127.0.1.1\t%s\n10.0.2.2\thost.docker.internal\n' \ + "$(hostname)" > /newroot/etc/hosts [ -s /resolv.conf ] && cp /resolv.conf /newroot/etc/resolv.conf entry=$(awk -F: -v u="${user%%:*}" '$1 == u || $3 == u' /newroot/etc/passwd | head -n 1) [ -n "$entry" ] || fail "the image's user ${user%%:*} isn't in /etc/passwd" diff --git a/src/agent_env/computers/disk/linux/launcher/launch b/src/agent_env/computers/disk/linux/launcher/launch index d3acc6516..9780e3ace 100755 --- a/src/agent_env/computers/disk/linux/launcher/launch +++ b/src/agent_env/computers/disk/linux/launcher/launch @@ -14,6 +14,9 @@ # VM_PORTS guest TCP ports, each served on the same port here by socat, whose listen backlog takes a burst of # connections, in front of QEMU's forward on 127.0.0.1 at the port plus 10000, whose backlog is 1; # socat retries a connection that forward drops every half second, not on the kernel's SYN backoff +# VM_HOST_PORTS ports of this machine's host the guest reaches, as host.docker.internal, at 10.0.2.2, QEMU's address +# for here: socat serves each on 127.0.0.1 here, through to the host, as a local grant server's +# VM_TRUST a folder of trust files, a local transfer CA's, that go into the seed as its trust/ folder # VM_ARCH the disk's arch, amd64 or arm64; this machine's by default # VM_CPUS, VM_MEMORY_MB its size: 2 and 4096 by default # VM_REQUIRE_KVM when set, fail rather than emulate where KVM can't run the disk @@ -26,7 +29,16 @@ # VM_FILES for a windows guest, a folder whose files it finds on a removable FAT drive # launch reset: boot the running computer afresh on a new overlay, its environment the VM_ENV_= lines on # stdin, returning once its new QEMU has started; its ports stay up throughout. +# launch forward PORT: serve PORT of this machine's host to the running computer as VM_HOST_PORTS does, for another +# process's grant server, until stopped. set -euo pipefail + +# forward PORT: PORT of this machine's host served on 127.0.0.1 here, where the guest reaches it at 10.0.2.2. +forward() { exec socat "TCP-LISTEN:$1,bind=127.0.0.1,fork,reuseaddr" "TCP:host.docker.internal:$1"; } + +if [ "${1:-}" = forward ]; then + forward "$2" +fi state=${VM_STATE:-/var/lib/agentenv-vm} # A reset and the boot loop meet here: the reset writes next.env, creates reset and stops the QEMU qemu.pid names; the # loop, finding reset, takes next.env's environment, removes reset and boots afresh; the reset returns once qemu.pid @@ -96,9 +108,10 @@ case "$guest/$arch" in echo "launch: no architecture $arch" >&2 exit 2 ;; esac -forwards="" fronts=() +forwards="" fronts=() taken="" for port in ${VM_PORTS:-}; do behind=$((port + 10000)) + taken="$taken $port $behind" if [ "$behind" -gt 65535 ] || [[ " $VM_PORTS " == *" $behind "* ]]; then echo "launch: port $port can't be forwarded: QEMU's forward behind its proxy would take port $behind, which" \ "is $([ "$behind" -gt 65535 ] && echo "above 65535" || echo "another of VM_PORTS")" >&2 @@ -108,6 +121,14 @@ for port in ${VM_PORTS:-}; do socat "TCP-LISTEN:$port,fork,reuseaddr,backlog=1024" "TCP:127.0.0.1:$behind,connect-timeout=0.5,retry=600,interval=0.05" & fronts+=("$!") done +for port in ${VM_HOST_PORTS:-}; do + if [[ "$taken " == *" $port "* ]]; then + echo "launch: host port $port can't be forwarded: a guest port's proxy or its forward here takes it" >&2 + exit 2 + fi + forward "$port" & + fronts+=("$!") +done trap 'kill ${fronts[@]+"${fronts[@]}"} ${presser:-} 2>/dev/null || true' EXIT trap 'kill "$(cat "$control/qemu.pid" 2>/dev/null)" 2>/dev/null; exit 143' TERM INT if [[ "$disk" == *.zst ]]; then @@ -135,8 +156,9 @@ boot() { qemu-img create -q -f qcow2 -b "$disk" -F qcow2 "$booted" fi seed "$work/seed" + [ -z "${VM_TRUST:-}" ] || cp -r "$VM_TRUST" "$work/seed/trust" mkfs.vfat -C -n CIDATA "$work/seed.img" 2048 >/dev/null - mcopy -i "$work/seed.img" "$work/seed/"* ::/ + mcopy -s -i "$work/seed.img" "$work/seed/"* ::/ if [ "$guest" = windows ]; then cp /usr/share/OVMF/OVMF_VARS_4M.ms.fd "$work/vars.fd" mkdir "$work/tpm" diff --git a/src/agent_env/computers/disk/windows/agentenv-session.py b/src/agent_env/computers/disk/windows/agentenv-session.py index e29fa2f26..0cb7d89f1 100644 --- a/src/agent_env/computers/disk/windows/agentenv-session.py +++ b/src/agent_env/computers/disk/windows/agentenv-session.py @@ -3,7 +3,8 @@ in its prefix with the build-phase settings the disk fixes, the deploy's environment over them, read from the NoCloud seed the launcher attaches (the CIDATA volume; agentenv_protocol.computer.seed, version 1), and its own environment over that; one that exits, or couldn't start, starts again after a pause, what kept it from starting written to its log. It -waits for the seed, and refuses one that carries only versions it doesn't read, as an init does.""" +waits for the seed, and refuses one that carries only versions it doesn't read, as an init does. A seed's trust folder, +a local transfer CA's, is what TLS clients are pointed at, as a container's are.""" import base64 import ctypes @@ -64,7 +65,7 @@ def main() -> None: raise SystemExit(f"agentenv-session: no CIDATA seed after {WAIT_S} s") time.sleep(1) fixed = json.loads(BUILD_SETTINGS.read_text()) if BUILD_SETTINGS.is_file() else {} - base = {**os.environ, **fixed, **environment(folder)} + base = {**os.environ, **fixed, **trust(folder / "trust"), **environment(folder)} specs = [json.loads(path.read_text()) for path in sorted(SERVICES.glob("*.json"))] running = {spec["name"]: start(spec, base) for spec in specs} while True: @@ -74,6 +75,14 @@ def main() -> None: running[spec["name"]] = start(spec, base) +def trust(folder: Path) -> dict[str, str]: + """The variables that point TLS clients at the trust files in ``folder``, none when it has none.""" + if not (folder / "ca-bundle.pem").is_file(): + return {} + return {"SSL_CERT_FILE": str(folder / "ca-bundle.pem"), "REQUESTS_CA_BUNDLE": str(folder / "ca-bundle.pem"), + "NODE_EXTRA_CA_CERTS": str(folder / "ca.pem")} + + def start(spec: dict, base: dict[str, str]) -> subprocess.Popen | None: """``spec``'s service started, its output to its log; None when it couldn't be, the reason in its log if that opens.""" diff --git a/src/agent_env/computers/disk/windows/setup.ps1 b/src/agent_env/computers/disk/windows/setup.ps1 index 1be6b9142..373e6ad02 100644 --- a/src/agent_env/computers/disk/windows/setup.ps1 +++ b/src/agent_env/computers/disk/windows/setup.ps1 @@ -41,6 +41,8 @@ try { icacls C:\ProgramData\ssh\administrators_authorized_keys /inheritance:r /grant '*S-1-5-32-544:F' /grant 'SYSTEM:F' | Out-Null # QEMU's user network reaches the guest at the ports the launcher forwards alone, which the layers' services listen on. Set-NetFirewallProfile -Profile Domain, Private, Public -Enabled False + # The launcher, at 10.0.2.2, forwards a local grant server's port to the machine it runs on, as host.docker.internal. + Add-Content -Path "$env:SystemRoot\System32\drivers\etc\hosts" -Value "`r`n10.0.2.2 host.docker.internal" -Encoding ASCII Set-Service sshd -StartupType Automatic Start-Service sshd diff --git a/src/agent_env/env/envs/computer/utils/windows_disk.py b/src/agent_env/env/envs/computer/utils/windows_disk.py index 3e5526397..1721e20b1 100644 --- a/src/agent_env/env/envs/computer/utils/windows_disk.py +++ b/src/agent_env/env/envs/computer/utils/windows_disk.py @@ -75,6 +75,9 @@ #: The groups and the account the folders under ``ROOT`` grant access to, by their well-known SIDs, as icacls takes #: them. ADMINISTRATORS, SYSTEM, USERS = "*S-1-5-32-544", "*S-1-5-18", "*S-1-5-32-545" +#: ALL APPLICATION PACKAGES and ALL RESTRICTED APPLICATION PACKAGES: the identities a sandboxed process runs as, Chrome's +#: and its kind's, which read and run their own program where they're installed, as Program Files lets them. +APP_PACKAGES, RESTRICTED_APP_PACKAGES = "*S-1-15-2-1", "*S-1-15-2-2" #: What every service runs with: Python reading and writing files as UTF-8, as on Linux, not in Windows' code page. UTF8 = {"PYTHONUTF8": "1"} #: What a base's install prints once, as it fetches Microsoft's media. @@ -350,10 +353,12 @@ def write(build: Path, parts: list[Part], *, user: str, build_settings: dict[str def _locked(user: str) -> list[str]: - """The lines that leave ``ROOT`` the admins', as on Linux ``/opt/agentenv`` is root's: every user reads and runs - what's there, none of the access ``C:\\`` gives every user to change it is left, and the session user ``user`` - writes only the services' logs, and each app's own folder, which keeps its own access.""" - return [_granted_alone(ROOT, {ADMINISTRATORS: "F", SYSTEM: "F", USERS: "RX"}), + """The lines that leave ``ROOT`` the admins', as on Linux ``/opt/agentenv`` is root's: every user, and every + sandboxed process, reads and runs what's there, none of the access ``C:\\`` gives every user to change it is left, + and the session user ``user`` writes only the services' logs, and each app's own folder, which keeps its own + access.""" + return [_granted_alone(ROOT, {ADMINISTRATORS: "F", SYSTEM: "F", USERS: "RX", APP_PACKAGES: "RX", + RESTRICTED_APP_PACKAGES: "RX"}), f"Checked {{ icacls '{ROOT / 'logs'}' /grant {_ps(f'{user}:(OI)(CI)M')} | Out-Null }}"] diff --git a/src/agent_env/providers/sandbox_providers/disk_launcher.py b/src/agent_env/providers/sandbox_providers/disk_launcher.py index d0812b707..ee20a2b60 100644 --- a/src/agent_env/providers/sandbox_providers/disk_launcher.py +++ b/src/agent_env/providers/sandbox_providers/disk_launcher.py @@ -18,6 +18,13 @@ RESET = ("sh", "-c", f"[ -d {CONTROL} ] || exit {UNRESETTABLE}; exec {LAUNCH} reset") +def forward(port: int) -> tuple[str, ...]: + """The command that serves ``port`` of this machine's host to the running computer, as ``VM_HOST_PORTS`` does: the + launcher's ``launch forward``, which a launcher older than it would take for a second boot, so it isn't run there, + whose computer has no use for it.""" + return ("sh", "-c", f"grep -q '^forward()' {LAUNCH} || exit 0; exec {LAUNCH} forward {port}") + + def launcher_variables(ports: list[int], env: dict[str, str], cpu: float, memory: int) -> dict[str, str]: """The environment that has the launcher size its VM, forward ``ports``, refuse to boot without KVM and hand the computer ``env``, each value base64'd, so one of several lines reaches the computer whole.""" diff --git a/src/agent_env/providers/sandbox_providers/local_docker/provider.py b/src/agent_env/providers/sandbox_providers/local_docker/provider.py index ab4efc79e..7a3c32c0f 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/provider.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/provider.py @@ -39,7 +39,7 @@ from agent_env.attribution import Attribution from agent_env.config.paths import state_root from agent_env.providers.sandbox_providers.host_images import build_here, built_here, host_platform, present, pull_here -from agent_env.providers.sandbox_providers.disk_launcher import RESET, UNRESETTABLE, reset_input, unresettable +from agent_env.providers.sandbox_providers.disk_launcher import RESET, UNRESETTABLE, forward, reset_input, unresettable from agent_env.providers.sandbox_providers.local_docker.cli import DockerFailed, docker from agent_env.providers.sandbox_providers.local_docker.disk import kvm_platforms, launcher_args, machine_for from agent_env.providers.sandbox_providers.local_docker.relay import relay_image @@ -599,12 +599,22 @@ def _remove_staged(sandbox_id: str) -> None: def local_grant_trust() -> Path | None: """The trust files a container on this host needs to use the configured object store's grants: the local transfer CA's, when the store is a local one that hands out grants; None otherwise.""" + return local_ca().trust_dir if _granting_store() is not None else None + + +def local_grants() -> Optional[tuple[int, Path]]: + """The port of the configured object store's grant server, started, and the local transfer CA's trust files, when + the store is a local one handing out grants; None otherwise.""" + store = _granting_store() + return (store.grant_port(), local_ca().trust_dir) if store is not None else None + + +def _granting_store() -> Optional[LocalFilesystemObjectStore]: + """The configured object store, or the local one a run routes to, when it's a local one that hands out grants.""" store = config.get_config().get_object_store() if isinstance(store, LocalRunObjectStore): store = store.local - if isinstance(store, LocalFilesystemObjectStore) and store.supports_transfer_grants: - return local_ca().trust_dir - return None + return store if isinstance(store, LocalFilesystemObjectStore) and store.supports_transfer_grants else None async def start_trusting(sandbox: VmSandbox, container: str, trust_dir: Path) -> None: @@ -733,10 +743,11 @@ async def _boot_disk(self, sandbox: LocalSandbox, *, image: str, env: dict[str, ports = list(sandbox._port_map) publish = [arg for port in ports for spec in port_bindings(sandbox.host_ips, sandbox.host_port(port), port) for arg in ("-p", spec)] + grants = await asyncio.to_thread(local_grants) running = asyncio.ensure_future(asyncio.to_thread( docker, "run", "-d", "--name", sandbox.container_name, "--label", f"{SANDBOX_LABEL}={sandbox.sandbox_id}", - *publish, *machine.args, *launcher_args(ports, env, size["cpu"], size["memory"]), machine.image, - timeout=_DOCKER_QUERY_SECONDS)) + *publish, *machine.args, *launcher_args(ports, env, size["cpu"], size["memory"]), + *(_grant_args(*grants) if grants else ()), machine.image, timeout=_DOCKER_QUERY_SECONDS)) try: await asyncio.shield(running) except asyncio.CancelledError: # the container docker is still starting must exist before teardown removes it @@ -896,13 +907,17 @@ async def create_sandbox( ) async def get_sandbox(self, sandbox_id: str) -> LocalSandbox: - work_dir = LocalSandbox.find_work_dir(sandbox_id) - if work_dir is None: - raise RuntimeError(f"Local sandbox work directory not found for sandbox_id={sandbox_id!r}") - port_map = work_dir / _PORT_MAP_FILE - ports = {int(c): h for c, h in json.loads(port_map.read_text()).items()} if port_map.exists() else None - # VM mode even for a container it owns: exec runs on this host, so steps must `docker exec` into it. - return LocalSandbox(sandbox_id=sandbox_id, work_dir=work_dir, port_map=ports) + """The sandbox ``sandbox_id``; a disk's, with this process's grant server forwarded into its launcher, as + lume's VMs get theirs as another process reconnects: each process that hands it grants serves them on a port + of its own.""" + sandbox = _found(sandbox_id) + if _is_disk(sandbox.work_dir) and (grants := await asyncio.to_thread(local_grants)) is not None: + await asyncio.to_thread(_forward_grants, sandbox.container_name, grants[0]) + return sandbox + + async def terminate_sandbox(self, sandbox_id: str) -> None: + """Terminate the sandbox ``sandbox_id``, forwarding no grant port into a disk's launcher it is removing.""" + await _found(sandbox_id).terminate() @classmethod def shares_network_with(cls, sandbox_type: Optional[str]) -> bool: @@ -959,6 +974,43 @@ def _host_ips() -> tuple[str, ...]: timeout=_DOCKER_QUERY_SECONDS)) +def _grant_args(port: int, trust: Path) -> list[str]: + """The ``docker run`` arguments that let a disk's computer use this machine's local grants, as lume's VMs do: it + finds ``host.docker.internal`` at QEMU's address for the launcher, where the launcher forwards the grant server's + ``port`` to this machine, and trusts the local transfer CA, whose ``trust`` files the launcher puts in its seed.""" + return [*(arg for entry in _EXTRA_HOSTS for arg in ("--add-host", entry)), + "--mount", f"type=bind,src={trust},dst={_LAUNCHER_TRUST},readonly", + "-e", f"VM_HOST_PORTS={port}", "-e", f"VM_TRUST={_LAUNCHER_TRUST}"] + + +def _found(sandbox_id: str) -> LocalSandbox: + """The sandbox ``sandbox_id`` from its work directory, in VM mode even for a container it owns: exec runs on this + host, so steps must ``docker exec`` into it.""" + work_dir = LocalSandbox.find_work_dir(sandbox_id) + if work_dir is None: + raise RuntimeError(f"Local sandbox work directory not found for sandbox_id={sandbox_id!r}") + port_map = work_dir / _PORT_MAP_FILE + ports = {int(c): h for c, h in json.loads(port_map.read_text()).items()} if port_map.exists() else None + return LocalSandbox(sandbox_id=sandbox_id, work_dir=work_dir, port_map=ports) + + +def _is_disk(work_dir: Path) -> bool: + """Whether the sandbox in ``work_dir`` is a disk's computer: ``create_vm`` records the size of the VM it boots.""" + return (work_dir / _VM_SIZE_FILE).exists() + + +def _forward_grants(container: str, port: int) -> None: + """``port`` forwarded in a disk's launcher ``container`` (``launch forward``), unless it is already, as a second + listener on it exits; one that can't be is logged, the computer then reaching no grant of this process's.""" + try: + docker("exec", "-d", container, *forward(port), timeout=_DOCKER_QUERY_SECONDS) + except DockerFailed as e: + logger.warning("Couldn't forward grant port %s into %s, so grants from this process won't reach its computer: " + "%s", port, container, e) + + _DOCKER_QUERY_SECONDS = 30 #: How long a reset may take: longer than the minute the launcher's reset waits for the computer to start again. _RESET_SECONDS = 90 +#: Where a disk's launcher finds the local transfer CA's trust files. +_LAUNCHER_TRUST = "/agentenv-trust" diff --git a/src/agent_env/providers/sandbox_providers/lume/provider.py b/src/agent_env/providers/sandbox_providers/lume/provider.py index f1e2da52f..da032a1c9 100644 --- a/src/agent_env/providers/sandbox_providers/lume/provider.py +++ b/src/agent_env/providers/sandbox_providers/lume/provider.py @@ -38,7 +38,7 @@ from agent_env.config.paths import state_root from agent_env.providers.sandbox_providers.local_docker.provider import ( LocalSandboxProvider, - local_grant_trust, + local_grants, trust_env, ) from agent_env.providers.sandbox_providers.sandbox import NetworkPolicy, VmSandbox @@ -49,7 +49,6 @@ ) from agent_env.store.image_store.registry_api import pin_digest from agent_env.store.local_state import awaiting_lock -from agent_env.store.routing import LocalRunObjectStore logger = logging.getLogger(__name__) @@ -203,7 +202,7 @@ async def reset_vm(self, sandbox: Any, *, image: str, env: dict[str, str]) -> No async def get_sandbox(self, sandbox_id: str) -> LumeSandbox: """The VM ``sandbox_id``, with this process's grant server forwarded into it, as another process reconnects.""" - grants = await asyncio.to_thread(_local_grants) + grants = await asyncio.to_thread(local_grants) async with self._changing(sandbox_id): state = self._state(sandbox_id) if "address" in state and grants is not None: @@ -349,7 +348,7 @@ async def _started(self, name: str, state: dict, env: dict[str, str]) -> None: if sizes: await self._lume("set", name, *[part for pair in zip(sizes[::2], sizes[1::2]) for part in pair]) folder = self._folder(name) - grants = None if state.get("build") else await asyncio.to_thread(_local_grants) + grants = None if state.get("build") else await asyncio.to_thread(local_grants) if grants is not None: env = {**trust_env(f"{SHARED_FILES}/{SEED_FOLDER}/{TRUST_FOLDER}"), **env} share = (f"{state['build']}:rw" if state.get("build") @@ -586,13 +585,3 @@ def _alive(pid: Optional[int]) -> bool: def _kill(pid: int) -> None: with contextlib.suppress(ProcessLookupError, PermissionError): os.killpg(pid, signal.SIGTERM) - - -def _local_grants() -> Optional[tuple[int, Path]]: - """The port of the configured object store's grant server, started, and the local transfer CA's trust files, when - the store is a local one handing out grants; None otherwise.""" - trust = local_grant_trust() - if trust is None: - return None - store = get_config().get_object_store() - return (store.local if isinstance(store, LocalRunObjectStore) else store).grant_port(), trust diff --git a/tst/unit/env/envs/test_computer_windows.py b/tst/unit/env/envs/test_computer_windows.py index 3803fc587..b359995a4 100644 --- a/tst/unit/env/envs/test_computer_windows.py +++ b/tst/unit/env/envs/test_computer_windows.py @@ -7,6 +7,7 @@ agentenv-session. The scripts the Windows test bundle's verifiers run inside the computer are Python.""" import ast +import importlib.util import json import threading import time @@ -189,7 +190,7 @@ def test_a_windows_env_composes_a_thin_disk_on_its_base_here_the_driver_installe "driver.py"], "env": {"PYTHONUTF8": "1", "MCP_PORT": "18765"}} assert lines[-3:] == [ "Checked { icacls 'C:\\agentenv' /inheritance:r /grant:r '*S-1-5-32-544:(OI)(CI)F' '*S-1-5-18:(OI)(CI)F' " - "'*S-1-5-32-545:(OI)(CI)RX' | Out-Null }", + "'*S-1-5-32-545:(OI)(CI)RX' '*S-1-15-2-1:(OI)(CI)RX' '*S-1-15-2-2:(OI)(CI)RX' | Out-Null }", "Checked { icacls 'C:\\agentenv\\logs' /grant 'user:(OI)(CI)M' | Out-Null }", ""] assert {"provision.ps1", "context/driver/driver.py", "context/driver/install.ps1"} <= set(files) computer = env.computer_artifact() @@ -443,3 +444,18 @@ def test_the_scripts_the_windows_bundles_verifiers_run_inside_the_computer_are_p if isinstance(node, ast.Assign) and [target.id for target in node.targets] == [script]] ast.parse(text) + + +def test_the_session_points_tls_clients_at_a_seeds_trust_folder_and_at_nothing_without_one(tmp_path): + spec = importlib.util.spec_from_file_location("agentenv_session", windows_disk.RECIPE / "agentenv-session.py") + session = importlib.util.module_from_spec(spec) + spec.loader.exec_module(session) + trust = tmp_path / "trust" + + assert session.trust(trust) == {} + trust.mkdir() + (trust / "ca-bundle.pem").write_text("roots and the local CA") + (trust / "ca.pem").write_text("the local CA") + assert session.trust(trust) == {"SSL_CERT_FILE": str(trust / "ca-bundle.pem"), + "REQUESTS_CA_BUNDLE": str(trust / "ca-bundle.pem"), + "NODE_EXTRA_CA_CERTS": str(trust / "ca.pem")} diff --git a/tst/unit/env/envs/test_disk_recipe.py b/tst/unit/env/envs/test_disk_recipe.py index e85d4ddd4..744040ffa 100644 --- a/tst/unit/env/envs/test_disk_recipe.py +++ b/tst/unit/env/envs/test_disk_recipe.py @@ -7,6 +7,7 @@ import asyncio import base64 import hashlib +import importlib.util import json import os import re @@ -26,7 +27,10 @@ from agent_env.providers.sandbox_providers.local_docker import disk as local_disk from agent_env.providers.sandbox_providers.local_docker import provider as local_sandbox from agent_env.providers.sandbox_providers.local_docker.cli import DockerFailed -from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox, LocalSandboxProvider +from agent_env.env.envs.computer.utils import windows_disk +from agent_env.providers.sandbox_providers import disk_launcher +from agent_env.providers.sandbox_providers.local_docker.provider import (LOCAL_TRUST_DIR, LocalSandbox, + LocalSandboxProvider, trust_env) from agent_env.providers.sandbox_providers.sandbox import SANDBOX_LABEL TRICKY = "a value with spaces, = signs, 'quotes', $dollars and\na second line" @@ -222,6 +226,60 @@ def test_the_launcher_serves_each_port_through_socat_in_front_of_qemus_forward_o assert running.wait(10) == 143 +def test_the_launcher_forwards_each_host_port_from_loopback_here_to_the_host_and_seeds_the_trust_files(launcher, + tmp_path): + """The guest finds host.docker.internal at 10.0.2.2, which QEMU's network gives this container's loopback, and + finds a local transfer CA's trust files in its seed's trust folder.""" + launch, calls = launcher + trust = tmp_path / "trust" + trust.mkdir() + + running = launch(VM_PORTS="18765", VM_HOST_PORTS="41234", VM_TRUST=str(trust)) + + assert "socat TCP-LISTEN:41234,bind=127.0.0.1,fork,reuseaddr TCP:host.docker.internal:41234" in _logged( + calls, "socat", 2) + assert f"cp -r {trust} {tmp_path}/state/boot/seed/trust" in _logged(calls, "cp -r") + assert _logged(calls, "mcopy")[0].startswith(f"mcopy -s -i {tmp_path}/state/boot/seed.img {tmp_path}/state/boot/seed/") + running.terminate() + assert running.wait(10) == 143 + + +@pytest.mark.parametrize("host_ports", ["18765", "28765"], ids=["a-guest-ports-proxy", "its-forward"]) +def test_the_launcher_refuses_a_host_port_a_guest_ports_proxy_or_forward_takes(launcher, host_ports): + launch, calls = launcher + + refused = launch(VM_PORTS="18765", VM_HOST_PORTS=host_ports) + + assert refused.wait(10) == 2 + assert f"host port {host_ports} can't be forwarded: a guest port's proxy or its forward here takes it" in ( + refused.stderr.read()) + + +def test_launch_forward_serves_a_host_port_to_the_running_computer_as_the_boot_does(launcher): + launch, calls = launcher + + forwarding = launch("forward", "52345") + + assert _logged(calls, "socat") == [ + "socat TCP-LISTEN:52345,bind=127.0.0.1,fork,reuseaddr TCP:host.docker.internal:52345"] + forwarding.terminate() + + +def test_a_disks_guests_point_tls_clients_at_the_trust_files_a_containers_variables_name(tmp_path): + """Neither guest can import the framework, so each keeps its own copy of trust_env's variables and files.""" + spec = importlib.util.spec_from_file_location("agentenv_session", windows_disk.RECIPE / "agentenv-session.py") + session = importlib.util.module_from_spec(spec) + spec.loader.exec_module(session) + init = (DISK / "boot" / "init").read_text() + expected = trust_env(LOCAL_TRUST_DIR) + + assert all(f"{name}={value}" in init for name, value in expected.items()) + assert all(f"/seed/trust/{PurePosixPath(value).name}" in init for value in expected.values()) + (tmp_path / "ca-bundle.pem").write_text("") + assert {name: Path(value).name for name, value in session.trust(tmp_path).items()} == { + name: PurePosixPath(value).name for name, value in expected.items()} + + @pytest.mark.parametrize("ports, why", [("55536", "above 65535"), ("6080 16080", "another of VM_PORTS")]) def test_the_launcher_refuses_a_port_whose_forward_behind_the_proxy_has_no_port_of_its_own(launcher, ports, why): launch, calls = launcher @@ -359,7 +417,7 @@ def test_a_windows_install_boots_its_iso_first_pressing_enter_and_finds_its_answ assert "-device ide-hd,drive=disk,bus=ide.0,bootindex=1,rotation_rate=1 " in qemu assert "-device usb-storage,bus=usb.0,drive=files,removable=on " in qemu assert _logged(calls, "socat - UNIX-CONNECT:") - assert _logged(calls, "mcopy -s")[0].startswith(f"mcopy -s -i {tmp_path}/state/boot/files.img {answers}/") + assert f"mcopy -s -i {tmp_path}/state/boot/files.img {answers}/autounattend.xml ::/" in _logged(calls, "mcopy -s", 2) running.terminate() assert running.wait(10) == 143 pressed = calls.read_text().count("socat - UNIX-CONNECT:") @@ -444,6 +502,7 @@ def launched(monkeypatch): monkeypatch.setattr(local_sandbox, "machine_for", AsyncMock(return_value=local_disk.Machine("agentenv-vm:0123"))) monkeypatch.setattr(local_sandbox, "_free_host_port", lambda: 20001) monkeypatch.setattr(local_sandbox, "_host_ips", lambda: ("127.0.0.1",)) + monkeypatch.setattr(local_sandbox, "local_grants", lambda: None) monkeypatch.setattr(local_sandbox, "docker", lambda *args, input=None, timeout=None: ran.append( args if input is None else (*args, input, timeout)) or "") monkeypatch.setattr(LocalSandbox, "pull_images", AsyncMock()) @@ -471,6 +530,34 @@ async def test_a_disk_boots_in_the_launcher_with_kvm_its_ports_published_and_a_r assert sandbox.tunnel_urls == {18765: "http://127.0.0.1:20001"} and rebuilt.owns_container +@pytest.mark.asyncio +async def test_a_disk_gets_this_processs_grant_port_and_the_trust_and_a_reconnect_its_own_but_not_a_teardown( + launched, monkeypatch): + monkeypatch.setattr(local_sandbox, "local_grants", lambda: (41234, Path("/state/tls/trust/ca"))) + + sandbox = await LocalSandboxProvider().create_vm(image="localhost:5000/disk@sha256:" + "a" * 64, + exposed_ports=[18765], env={}, cpu=2, memory=4096) + + monkeypatch.setattr(local_sandbox, "local_grants", lambda: (52345, Path("/state/tls/trust/ca"))) + await LocalSandboxProvider().get_sandbox(sandbox.sandbox_id) + + run, forward = launched + hosts = [arg for entry in local_sandbox._EXTRA_HOSTS for arg in ("--add-host", entry)] + assert run == ("run", "-d", "--name", sandbox.container_name, "--label", f"{SANDBOX_LABEL}={sandbox.sandbox_id}", + "-p", "127.0.0.1:20001:18765", *local_disk.launcher_args([18765], {}, 2, 4096), *hosts, + "--mount", "type=bind,src=/state/tls/trust/ca,dst=/agentenv-trust,readonly", + "-e", "VM_HOST_PORTS=41234", "-e", "VM_TRUST=/agentenv-trust", "agentenv-vm:0123") + assert forward == ("exec", "-d", sandbox.container_name, *disk_launcher.forward(52345)) + assert disk_launcher.forward(52345)[-1] == (f"grep -q '^forward()' {LAUNCH} || exit 0; " + f"exec {LAUNCH} forward 52345") + + launched.clear() + monkeypatch.setattr(LocalSandbox, "terminate", AsyncMock()) + await LocalSandboxProvider().terminate_sandbox(sandbox.sandbox_id) + + assert launched == [] and LocalSandbox.terminate.await_count == 1 + + @pytest.mark.asyncio @pytest.mark.parametrize("returncode, refused", [ (UNRESETTABLE, "the computer in container .* was booted by an older agent-env, whose launcher can't reset it in " diff --git a/tst/unit/providers/sandbox_providers/lume_sandbox_provider_test.py b/tst/unit/providers/sandbox_providers/lume_sandbox_provider_test.py index 1e1979367..f7a37ff2c 100644 --- a/tst/unit/providers/sandbox_providers/lume_sandbox_provider_test.py +++ b/tst/unit/providers/sandbox_providers/lume_sandbox_provider_test.py @@ -122,7 +122,7 @@ def mac(tmp_path, monkeypatch): monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "state")) monkeypatch.setattr(lume, "_free_port", lambda: 48123) monkeypatch.setattr(lume, "_POLL_S", 0.05) - monkeypatch.setattr(lume, "_local_grants", lambda: None) + monkeypatch.setattr(lume, "local_grants", lambda: None) provider = LumeSandboxProvider(lume=str(bin_ / "lume"), ssh=str(bin_ / "ssh"), relay_python=str(bin_ / "relay"), ssh_password="s3cret-pw", boot_timeout_s=10) provider.fake = tmp_path @@ -183,7 +183,7 @@ async def test_a_local_stores_grants_reach_a_vm_at_its_loopback_through_ssh_forw trust.mkdir() (trust / "ca-bundle.pem").write_text("roots and the CA") (trust / "ca.pem").write_text("the CA") - monkeypatch.setattr(lume, "_local_grants", lambda: (54321, trust)) + monkeypatch.setattr(lume, "local_grants", lambda: (54321, trust)) sandbox = await mac.create_vm(image="local:bench", env={"ENVIRONMENT_NAME": "desk"}, exposed_ports=[18765]) @@ -200,7 +200,7 @@ async def test_a_local_stores_grants_reach_a_vm_at_its_loopback_through_ssh_forw assert LumeSandboxProvider.REACHES_HOST_AT == "127.0.0.1" ports = iter([54322, 54323]) - monkeypatch.setattr(lume, "_local_grants", lambda: (next(ports), trust)) + monkeypatch.setattr(lume, "local_grants", lambda: (next(ports), trust)) await asyncio.gather(mac.get_sandbox(sandbox.sandbox_id), mac.get_sandbox(sandbox.sandbox_id)) assert {call["args"][-2] for call in _waited(lambda: _forwards(mac.fake, 3))[1:]} == { @@ -208,7 +208,7 @@ async def test_a_local_stores_grants_reach_a_vm_at_its_loopback_through_ssh_forw forwards = json.loads((folder.parent / "state.json").read_text())["forwards"] assert set(forwards) == {"54321", "54322", "54323"} - monkeypatch.setattr(lume, "_local_grants", lambda: (54324, trust)) + monkeypatch.setattr(lume, "local_grants", lambda: (54324, trust)) monkeypatch.setenv("FAKE_LUME_SLOW_DELETE", "1") _, reconnect = await asyncio.gather(sandbox.terminate(), mac.get_sandbox(sandbox.sandbox_id), return_exceptions=True)