diff --git a/src/agent_env/task_step/task_steps/run_code.py b/src/agent_env/task_step/task_steps/run_code.py index e1ae345bc..6695a0e18 100644 --- a/src/agent_env/task_step/task_steps/run_code.py +++ b/src/agent_env/task_step/task_steps/run_code.py @@ -30,7 +30,6 @@ import json import logging import os -import re import shlex from dataclasses import dataclass, field from pathlib import Path, PurePosixPath @@ -55,14 +54,14 @@ _COMPUTER_RUN_DIR = ".agentenv/run_code" # How much longer than the script's timeout a computer lets the runner run, so the runner stops the script first. _RUNNER_SLACK_S = 10 -# What a POSIX extended regular expression, as `pkill -f` reads one, gives a meaning to outside brackets. -_ERE_SPECIAL = re.compile(r"[.\[\\()*+?{|^$]") - # The bootstrap we run in the sandbox: `python3 runner.py `. _RUNNER_SOURCE = (Path(__file__).parent / "run_code_runner.py").read_text(encoding="utf-8") +# Inside a computer, the runner's own `--clear` and `--size` stand in for the shell tools its OS may not have, isolated +# from the home it runs in. +_RUNNER_TOOL = ("python3", "-I", "-c", _RUNNER_SOURCE) # A universe member with one of these names would shadow run_code's own staging files. -_RESERVED_MEMBERS = frozenset({"runner.py", "input.json", "output.json"}) +_RESERVED_MEMBERS = frozenset({"runner.py", "input.json", "output.json", "runner.lock", "runner.pid"}) # Where a single-file FileArtifact is always staged, regardless of its own filename. _STAGED_SCRIPT = "script.py" @@ -373,10 +372,11 @@ async def _read_output(self, target: _ExecTarget, run_dir: str) -> Any: # rejected here rather than slurped into the worker — and we never decode a # truncated file (which would crash strict UTF-8 decode mid-codepoint). path = f"{run_dir}/output.json" - exit_code, size_out, stderr = await target.run("wc", "-c", path) - if exit_code != 0: - raise self._exec_error(f"wrote no result file: {self._stderr_tail(stderr)}") - if int(size_out.split()[0]) > self.MAX_OUTPUT_BYTES: + try: + size = await target.size(path) + except OSError as exc: + raise self._exec_error(f"wrote no result file: {self._stderr_tail(str(exc))}") from exc + if size > self.MAX_OUTPUT_BYTES: raise self._validation_error( f"result exceeds {self.MAX_OUTPUT_BYTES}-byte cap" ) @@ -554,12 +554,20 @@ async def _cat(target: _ExecTarget, path: str) -> str: return stdout +async def _size(target: _ExecTarget, *command: str) -> int: + exit_code, stdout, stderr = await target.run(*command) + if exit_code != 0: + raise OSError(stderr) + return int(stdout.split()[0]) + + class _ExecTarget(Protocol): def run_dir(self, step_id: str) -> str: ... def endpoints(self, deployed: DeployedSandboxEnv) -> dict: ... async def reset_dir(self, path: str) -> None: ... async def write_file(self, path: str, content: str) -> None: ... async def read_file(self, path: str) -> str: ... + async def size(self, path: str) -> int: ... async def run(self, *command: str) -> tuple[int, str, str]: ... @@ -587,6 +595,9 @@ async def write_file(self, path: str, content: str) -> None: async def read_file(self, path: str) -> str: return await _cat(self, path) + async def size(self, path: str) -> int: + return await _size(self, "wc", "-c", path) + async def run(self, *command: str) -> tuple[int, str, str]: # Files are written as root, so run as root too. return await self.sandbox.exec_with_output("sudo", *command) @@ -618,6 +629,9 @@ async def write_file(self, path: str, content: str) -> None: async def read_file(self, path: str) -> str: return await _cat(self, path) + async def size(self, path: str) -> int: + return await _size(self, "wc", "-c", path) + async def run(self, *command: str) -> tuple[int, str, str]: return await self.sandbox.exec_with_output(*self.prefix, *command) @@ -639,15 +653,17 @@ def endpoints(self, deployed: DeployedSandboxEnv) -> dict: return inside.loopback(deployed) async def reset_dir(self, path: str) -> None: - """End whatever still runs from the folder, as a run whose driver died leaves going, then remove it.""" - await self.run("pkill", "-f", _ERE_SPECIAL.sub(r"\\\g<0>", f"{path}/runner.py")) await self.remove_dir(path) async def remove_dir(self, path: str) -> None: - exit_code, _, stderr = await self.run("rm", "-rf", path) + """End whatever still runs from the folder, as a run whose driver died leaves going, then remove it.""" + exit_code, _, stderr = await self.run(*_RUNNER_TOOL, "--clear", path) if exit_code != 0: raise OSError(stderr) + async def size(self, path: str) -> int: + return await _size(self, *_RUNNER_TOOL, "--size", path) + async def write_file(self, path: str, content: str) -> None: await inside.add_files(self.deployed, {path: content.encode("utf-8")}) diff --git a/src/agent_env/task_step/task_steps/run_code_runner.py b/src/agent_env/task_step/task_steps/run_code_runner.py index e78fe7294..c373f1510 100644 --- a/src/agent_env/task_step/task_steps/run_code_runner.py +++ b/src/agent_env/task_step/task_steps/run_code_runner.py @@ -5,7 +5,13 @@ Given a fourth argument, `timeout_seconds`, it first supervises: it reruns itself without it in a new session and stops that whole process group at the limit, so no target needs -GNU `timeout`. +GNU `timeout`. Windows has no process groups to signal, so there it ends the rerun's process +tree with `taskkill`. While it supervises, it holds a lock on `runner.lock` in `run_dir`, its pid +in `runner.pid`. + +`--clear ` ends the supervisor holding that lock, with all it started, then removes the +folder; `--size ` prints a file's size. run_code runs these inside a computer, as its OS +may have no POSIX tools to do it with. Both the entry script's own directory and `run_dir` go on sys.path, so a multi-file artifact can import its siblings whether it is laid out flat or under a subdirectory. @@ -18,9 +24,17 @@ _TIMEOUT_EXIT_CODE = 124 # GNU timeout's, which run_code reports as a timeout _GRACE_SECONDS = 5 # between SIGTERM and SIGKILL _POLL_SECONDS = 0.1 +_LOCK, _PID = "runner.lock", "runner.pid" + + +def _end_tree(pid): + subprocess.run(["taskkill", "/T", "/F", "/PID", str(pid)], capture_output=True) def _signal_group(pgid, signum): + if os.name == "nt": + _end_tree(pgid) + return try: os.killpg(pgid, signum) except (ProcessLookupError, PermissionError): # gone, or (macOS) only zombies are left @@ -49,8 +63,9 @@ def forward(signum, _frame): # Caught before the child starts, as GNU timeout does, so it gets their defaults: one ignored here (a background # job's SIGINT) would stay ignored in it. - for signum in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP): - signal.signal(signum, forward) + for name in ("SIGTERM", "SIGINT", "SIGHUP"): # Windows has no SIGHUP + if hasattr(signal, name): + signal.signal(getattr(signal, name), forward) child = subprocess.Popen(command, start_new_session=True) for signum in early: _signal_group(child.pid, signum) @@ -67,13 +82,78 @@ def forward(signum, _frame): except subprocess.TimeoutExpired: pass if kill_at is not None: - _signal_group(child.pid, signal.SIGKILL) + _signal_group(child.pid, getattr(signal, "SIGKILL", signal.SIGTERM)) status = child.wait() if timed_out: return _TIMEOUT_EXIT_CODE return status if status >= 0 else 128 - status +def _hold(run_dir): + """Lock ``run_dir`` for this supervisor's life, its pid beside the lock.""" + with open(os.path.join(run_dir, _PID), "w") as pid: + pid.write(str(os.getpid())) + lock = open(os.path.join(run_dir, _LOCK), "wb") + if os.name == "nt": + msvcrt.locking(lock.fileno(), msvcrt.LK_LOCK, 1) + else: + fcntl.flock(lock, fcntl.LOCK_EX) + return lock + + +def _held(path): + """Whether another process holds the lock at ``path``.""" + with open(path, "r+b") as lock: + try: + if os.name == "nt": + msvcrt.locking(lock.fileno(), msvcrt.LK_NBLCK, 1) + msvcrt.locking(lock.fileno(), msvcrt.LK_UNLCK, 1) + else: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + return True + return False + + +def _clear(run_dir): + """End the supervisor still running from ``run_dir``, as a run whose caller died leaves one, with all it started, + then remove the folder. A supervisor passes a SIGTERM on to its group; Windows ends the tree at once.""" + lock = os.path.join(run_dir, _LOCK) + if os.path.exists(lock) and _held(lock): + with open(os.path.join(run_dir, _PID)) as pid_file: + pid = int(pid_file.read()) + if os.name == "nt": + _end_tree(pid) + else: + try: + os.kill(pid, signal.SIGTERM) + except (ProcessLookupError, PermissionError): + pass + deadline = time.monotonic() + _GRACE_SECONDS * 2 + while _held(lock) and time.monotonic() < deadline: + time.sleep(_POLL_SECONDS) + if os.path.isdir(run_dir): + shutil.rmtree(run_dir) + + +if sys.argv[1:2] in (["--clear"], ["--size"]): + import os + import shutil + import signal + import subprocess + import time + + if os.name == "nt": + import msvcrt + else: + import fcntl + + if sys.argv[1] == "--size": + print(os.path.getsize(sys.argv[2])) + else: + _clear(sys.argv[2]) + sys.exit(0) + if len(sys.argv) == 5: # Only the supervisor imports these, and not from the run dir (sys.path[0]): an artifact member named after one # would otherwise run here, outside the limit. The script's own process imports just what it always has. @@ -86,6 +166,12 @@ def forward(signum, _frame): import subprocess import time + if os.name == "nt": + import msvcrt + else: + import fcntl + + lock = _hold(sys.argv[1]) sys.exit(_supervise([sys.executable] + sys.argv[:4], float(sys.argv[4]))) # Below the supervisor, which must not import these. diff --git a/tst/integration/computer/windows/artifacts/check_inside/run.py b/tst/integration/computer/windows/artifacts/check_inside/run.py new file mode 100644 index 000000000..b1e54991a --- /dev/null +++ b/tst/integration/computer/windows/artifacts/check_inside/run.py @@ -0,0 +1,20 @@ +"""A run_code script inside the Windows computer: it runs in the session user's home, and from there reaches the driver +on loopback, whose card says Windows, and its readiness with the computer's token.""" +import json +import os +import urllib.request +from pathlib import Path + + +def run(input): + with urllib.request.urlopen(f"{input['env']['url']}/.well-known/agent-env.json", timeout=10) as response: + card = json.load(response) + computer = next(ext["params"] for ext in card["capabilities"]["extensions"] + if ext["uri"] == "urn:agentenv:computer/v1") + status = urllib.request.Request(f"{input['env']['url']}/agentenv/ext/computer-ready/status", + headers=input["env"]["headers"]) + with urllib.request.urlopen(status, timeout=30) as response: + ready = json.load(response) + if computer["platform"]["os"] != "windows" or not Path.cwd().samefile(Path.home()) or not ready.get("ready"): + raise AssertionError(f"os {computer['platform']['os']}, cwd {Path.cwd()}, ready {ready}") + return {"user": os.environ.get("USERNAME"), "home": str(Path.home())} diff --git a/tst/integration/computer/windows/tasks/notes.json b/tst/integration/computer/windows/tasks/notes.json index 3cc8b8d93..063f87656 100644 --- a/tst/integration/computer/windows/tasks/notes.json +++ b/tst/integration/computer/windows/tasks/notes.json @@ -5,7 +5,9 @@ "verifier_id": "before"}, {"id": "session", "type": "env_outcome_verifier", "env_id": "notes-desk", "file_artifact_id": "check_session", "verifier_id": "session", "depends_on": ["before"]}, - {"id": "reset", "type": "reset_env", "env_id": "notes-desk", "depends_on": ["session"]}, + {"id": "inside", "type": "run_code", "env_id": "notes-desk", "script_artifact_id": "check_inside", + "depends_on": ["session"]}, + {"id": "reset", "type": "reset_env", "env_id": "notes-desk", "depends_on": ["inside"]}, {"id": "after", "type": "env_outcome_verifier", "env_id": "notes-desk", "file_artifact_id": "check_notes", "verifier_id": "after", "depends_on": ["reset"]} ] diff --git a/tst/integration/computer/windows_test.py b/tst/integration/computer/windows_test.py index 0cfdc0b4b..0c1b3e681 100644 --- a/tst/integration/computer/windows_test.py +++ b/tst/integration/computer/windows_test.py @@ -3,8 +3,8 @@ passes the conformance probe at full, cua-driver's known failure on Windows waived, before and after a reset in place; and an agent drives Notepad on it through cua-driver's tools. And the bundle's notes app, composed onto it as a layer, runs on its deploy's settings, its window up and its install done as its Windows install says, its own folder and -server the session user's alone, before and after a reset; and on it the protocol's Windows session launches, pauses -and stops apps, and all they start, as an app's server does. Needs Docker, KVM and the ISO; the agent's task, a model +server the session user's alone, before and after a reset; on it the protocol's Windows session launches, pauses +and stops apps, and all they start, as an app's server does; and run_code runs a script inside it. Needs Docker, KVM and the ISO; the agent's task, a model endpoint too.""" from pathlib import Path diff --git a/tst/unit/computer_home.py b/tst/unit/computer_home.py index 9564ca2fd..5f7d8f086 100644 --- a/tst/unit/computer_home.py +++ b/tst/unit/computer_home.py @@ -1,7 +1,6 @@ """A computer at its driver's boundary, for steps that reach inside one: its home a folder on this machine, ``data/add`` writing there, ``data/get`` reading there, and computer-exec/v1 ``run`` running commands there; each call recorded, -with the headers it sent, and the address each ``data/add`` is sent to. -``pkill`` matches nothing, so one test never ends another's processes on this machine.""" +with the headers it sent, and the address each ``data/add`` is sent to.""" from __future__ import annotations @@ -74,8 +73,6 @@ async def invoke_extension(self, base_url, card, uri, params=None, timeout=30, v return await asyncio.to_thread(self._run, params) def _run(self, params: dict) -> dict: - if params["argv"][0] == "pkill": - return {"exit_code": 1, "stdout": "", "stderr": "", "truncated": False, "timed_out": False} try: done = subprocess.run(params["argv"], cwd=self.home, env={**os.environ, **(params.get("env") or {})}, capture_output=True, text=True, timeout=params["timeout_s"]) diff --git a/tst/unit/task_step/test_run_code.py b/tst/unit/task_step/test_run_code.py index de5865b58..a972dd17a 100644 --- a/tst/unit/task_step/test_run_code.py +++ b/tst/unit/task_step/test_run_code.py @@ -14,6 +14,7 @@ import time from contextlib import contextmanager from pathlib import Path +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -838,6 +839,16 @@ def test_runner_parses_as_python_3_9(): ast.parse(_RUNNER.read_text(), feature_version=(3, 9)) +def test_on_windows_the_runner_ends_the_scripts_process_tree_as_it_has_no_process_groups(): + ran = [] + runner = {"os": SimpleNamespace(name="nt"), "subprocess": SimpleNamespace(run=lambda argv, **_: ran.append(argv))} + exec(_RUNNER.read_text().partition("\nif sys.argv[1:2] in")[0], runner) + + runner["_signal_group"](4242, signal.SIGTERM) + + assert ran == [["taskkill", "/T", "/F", "/PID", "4242"]] + + def _runner_args(run_dir, body, timeout): """Stage the runner, script and input as run_code does, the runner inside the run dir.""" (run_dir / "runner.py").write_text(_RUNNER.read_text()) @@ -953,6 +964,20 @@ def test_a_forwarded_signal_the_script_ignores_is_followed_by_sigkill(tmp_path): assert _gone_within(child) +def test_clearing_a_folder_ends_the_runner_still_running_from_it_with_all_it_started_then_removes_it(tmp_path): + run_dir = tmp_path / "run" + run_dir.mkdir() + proc = subprocess.Popen(_runner_args(run_dir, _STARTS_A_CHILD, 60)) + child = _wait_for_pid(run_dir) + + cleared = subprocess.run([sys.executable, "-I", "-c", _RUNNER.read_text(), "--clear", str(run_dir)], + capture_output=True, text=True, timeout=60) + + assert cleared.returncode == 0, cleared.stderr + assert proc.wait(timeout=30) == 128 + signal.SIGTERM + assert _gone_within(child) and not run_dir.exists() + + def test_artifact_files_named_after_the_supervisors_imports_never_run_in_it(tmp_path): run_dir = tmp_path / "run" run_dir.mkdir() @@ -1322,9 +1347,12 @@ async def test_a_script_for_a_computer_runs_inside_it_and_reaches_its_driver_on_ run_dir = ".agentenv/run_code/run-code-1" assert [call[1] for call in computer.calls if call[0] == "data/add"] == [ [{"path": f"{run_dir}/script.py"}], [{"path": f"{run_dir}/input.json"}], [{"path": f"{run_dir}/runner.py"}]] - (script,) = [params for kind, params, *_ in computer.calls if kind == "run" and params["argv"][0] == "python3"] - assert script["argv"] == ["python3", f"{run_dir}/runner.py", run_dir, "run", "script.py", "600"] - assert script["timeout_s"] == 610 + tool = ["python3", "-I", "-c", _RUNNER.read_text(encoding="utf-8")] + runs = [params for kind, params, *_ in computer.calls if kind == "run"] + assert [params["argv"] for params in runs] == [ + [*tool, "--clear", run_dir], ["python3", f"{run_dir}/runner.py", run_dir, "run", "script.py", "600"], + [*tool, "--size", f"{run_dir}/output.json"], [*tool, "--clear", run_dir]] + assert runs[1]["timeout_s"] == 610 assert ("data/get", [f"{run_dir}/output.json"]) in computer.calls assert not (computer.home / run_dir).exists() @@ -1341,7 +1369,7 @@ async def test_a_failed_script_in_a_computer_is_reported_and_leaves_no_run_folde async def test_a_run_the_computer_stopped_at_its_timeout_is_reported_as_one(computer, monkeypatch): monkeypatch.setattr(computer, "_run", lambda params: {"exit_code": None, "stdout": "", "stderr": "", "truncated": False, "timed_out": True} - if params["argv"][0] == "python3" else Computer._run(computer, params)) + if _runs_the_script(params) else Computer._run(computer, params)) with pytest.raises(RunCodeExecutionError, match="timed out after 30s"): await _run_in(computer, _DEFAULT_BODY, timeout_seconds=30) @@ -1352,7 +1380,7 @@ async def test_a_run_whose_call_drops_fails_naming_why_and_the_next_one_runs_afr invoke, dropped = computer.invoke_extension, [] async def drop_once(base_url, card, uri, params=None, timeout=30, verify=True, *, method=None, headers=None): - if params["argv"][0] == "python3" and not dropped: + if _runs_the_script(params) and not dropped: dropped.append(params["request_id"]) raise httpx.RemoteProtocolError("the driver went away") return await invoke(base_url, card, uri, params, timeout, verify, method=method) @@ -1365,13 +1393,16 @@ async def drop_once(base_url, card, uri, params=None, timeout=30, verify=True, * await _run_in(computer, _DEFAULT_BODY, id="read.v1") runs = [params for kind, params, *_ in computer.calls if kind == "run"] - staged = [params["argv"] for params in runs if params["argv"][0] == "pkill"] - assert staged == [["pkill", "-f", r"\.agentenv/run_code/read\.v1/runner\.py"]] * 2 - assert not re.search(staged[0][2], "python3 /home/u/.agentenv/run_code/read-v1/runner.py") - scripts = [params["request_id"] for params in runs if params["argv"][0] == "python3"] + cleared = [params["argv"][-1] for params in runs if params["argv"][-2:-1] == ["--clear"]] + assert cleared == [".agentenv/run_code/read.v1"] * 4 + scripts = [params["request_id"] for params in runs if _runs_the_script(params)] assert len(scripts) == 1 and scripts[0] != dropped[0] +def _runs_the_script(params): + return params["argv"][1].endswith("/runner.py") + + @pytest.mark.asyncio @pytest.mark.parametrize("card, step, problem", [ ({**CARD, "capabilities": {**CARD["capabilities"], "operations": ["data/get"]}}, {},