Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/local-backends.yml
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ jobs:
run: |
.venv/bin/python .github/scripts/check_skip_policy.py junit-integration-slow.xml \
--allow model_endpoint_configured remote_sandbox default_a2a_agent mcp_server_sources windows_media \
omarchy_image
omarchy_image osworld_sources
# The tests' times, for .github/scripts/slow_test_durations.py to rebalance the shards with.
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
if: always()
Expand Down
5 changes: 4 additions & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,10 @@ all four must pass, and so must `plugin-api` (see "Making changes"). The `instal
the lockfile, and must pass when it runs. The gateway's virtual-clock tests build MCP servers
from sources outside this repository, found through `AGENT_ENV_TEST_MCP_SERVERS_DIR` (one
`<server>/Dockerfile` per server), and skip without them; when your change touches the virtual
clock, run `make int-test-slow` with it set, and say so in the pull request. A test may skip only when a capability is genuinely unavailable, and the skip reason
clock, run `make int-test-slow` with it set, and say so in the pull request. The OSWorld end-to-end test
(`tst/integration/computer/osworld_test.py`) needs scale-cua's OSWorld guest server source at `OSWORLD_SERVER_SOURCE`
and a controller image at `OSWORLD_CONTROLLER_IMAGE`, and skips without them; when your change touches
`src/agent_env/task_step/task_steps/computer/osworld`, run it with them set, and say so in the pull request. A test may skip only when a capability is genuinely unavailable, and the skip reason
must name it (`agentenv-capability-missing: <name>`, see `tst/util/capabilities.py`); the
`check_skip_policy` step fails the run otherwise. The `clean-install` job runs the same script as
`make clean-install-test` on every pull request, and must pass too, as must the `ui` job, which
Expand Down
3 changes: 2 additions & 1 deletion src/agent_env/task/episode/rules.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,8 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s
"browser_release_context", "browser_set_cookies", "browser_sign_in", "browser_set_local_storage", "build_mcp_cli",
"combine_universe_verdicts", "configure_scheduled_checks", "deploy_env", "deploy_sandbox", "end_browser_recording",
"end_computer_recording", "start_browser_recording", "start_computer_recording", "env_outcome_verifier",
"modify_env_tool_access", "reset_env", "review",
"modify_env_tool_access", "osworld_deploy_controller", "osworld_evaluate", "osworld_initialize", "reset_env",
"review",
"run_container_unit_tests_verifier", "run_docker_container", "snapshot_env", "snapshot_computer_env", "sync_env_clock",
"validation_gate_aggregator", "verify_env_card", "verify_env_core_protocol", "verify_mcp_tool_schema",
"verify_spec_conformance", "verify_universe_load_export_roundtrip",
Expand Down
6 changes: 6 additions & 0 deletions src/agent_env/task_step/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,9 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]:
from agent_env.task_step.task_steps.snapshot_agent_state import SnapshotAgentStateTaskStep
from agent_env.task_step.task_steps.snapshot_env import SnapshotEnvTaskStep
from agent_env.task_step.task_steps.computer.snapshot_computer_env import SnapshotComputerEnvTaskStep
from agent_env.task_step.task_steps.computer.osworld.deploy_controller import OSWorldDeployControllerTaskStep
from agent_env.task_step.task_steps.computer.osworld.evaluate import OSWorldEvaluateTaskStep
from agent_env.task_step.task_steps.computer.osworld.initialize import OSWorldInitializeTaskStep
from agent_env.task_step.task_steps.peer_agents import PeerAgentsTaskStep
from agent_env.task_step.task_steps.verifiers.verify_sandbox import VerifySandboxTaskStep
from agent_env.task_step.task_steps.verifiers.aggregate_verifiers import AggregateVerifiersTaskStep
Expand Down Expand Up @@ -168,6 +171,9 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]:
_get_type(SnapshotAgentStateTaskStep): SnapshotAgentStateTaskStep,
_get_type(SnapshotEnvTaskStep): SnapshotEnvTaskStep,
_get_type(SnapshotComputerEnvTaskStep): SnapshotComputerEnvTaskStep,
_get_type(OSWorldDeployControllerTaskStep): OSWorldDeployControllerTaskStep,
_get_type(OSWorldInitializeTaskStep): OSWorldInitializeTaskStep,
_get_type(OSWorldEvaluateTaskStep): OSWorldEvaluateTaskStep,
_get_type(RunCodeTaskStep): RunCodeTaskStep,
_get_type(PeerAgentsTaskStep): PeerAgentsTaskStep,
_get_type(VerifyA2ASnapshotStep): VerifyA2ASnapshotStep,
Expand Down
41 changes: 41 additions & 0 deletions src/agent_env/task_step/task_steps/computer/osworld/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
"""OSWorld tasks on agent-env computers.

``osworld_deploy_controller`` deploys the OSWorld controller beside a computer that runs the ``osworld-server`` app,
``osworld_initialize`` sets a task up through it and ``osworld_evaluate`` grades it there. Beside the steps live what
they share: the ``osworld-server`` computer app (``apps/``), which runs the OSWorld guest server on a computer, how a
step finds that server (:mod:`.guest`) and how it calls the controller (:mod:`.controller`).

The guest server's source isn't part of agent-env; :func:`stage_server` copies the app's folder with that source into a
folder ``agent-env env put`` takes."""
from __future__ import annotations

import shutil
from pathlib import Path

#: The app's ``environment_name``.
APP_NAME = "osworld-server"
#: The extension the app's card advertises at the prefix its server forwards to the guest server under. A computer
#: nests the app's card under its ``children_environments`` with the endpoint rooted where it serves the app, so a step
#: finds the guest server by this URI in the computer's card.
GUEST_EXTENSION = "urn:agentenv:osworld-guest/v1"
#: The prefix the app's server forwards to the guest server under.
PROXY_PREFIX = "/osworld"
#: The app's folder.
APP_DIR = Path(__file__).parent / "apps" / APP_NAME
#: Where a staged app keeps the guest server's source, and the files of that source it runs.
PAYLOAD_DIR = "payload"
PAYLOAD_FILES = ("main.py", "pyxcursor.py")


def stage_server(source: Path, dest: Path) -> Path:
"""Copy the app's folder to ``dest`` with the guest server's source from ``source``, a folder holding
``PAYLOAD_FILES``, under ``payload/``; returns ``dest``."""
if missing := [name for name in PAYLOAD_FILES if not (source / name).is_file()]:
raise FileNotFoundError(f"{source} is not the OSWorld guest server's source: no {missing}")
if dest.exists():
raise FileExistsError(f"{dest} exists")
shutil.copytree(APP_DIR, dest, ignore=shutil.ignore_patterns("__pycache__", "*.pyc"))
(dest / PAYLOAD_DIR).mkdir()
for name in PAYLOAD_FILES:
shutil.copy2(source / name, dest / PAYLOAD_DIR / name)
return dest
29 changes: 29 additions & 0 deletions src/agent_env/task_step/task_steps/computer/osworld/__main__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
"""``python -m agent_env.task_step.task_steps.computer.osworld stage --source <guest server source> --out <folder>``: the ``osworld-server`` app's
folder with the guest server's source, for ``agent-env env put``."""
from __future__ import annotations

import argparse
import sys
from pathlib import Path

from agent_env.task_step.task_steps.computer.osworld import stage_server


def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(prog="python -m agent_env.task_step.task_steps.computer.osworld")
commands = parser.add_subparsers(dest="command", required=True)
stage = commands.add_parser("stage", help="stage the osworld-server app with the guest server's source")
stage.add_argument("--source", required=True, type=Path, help="the folder holding the guest server's main.py")
stage.add_argument("--out", required=True, type=Path, help="the folder to write, which must not exist")
args = parser.parse_args(argv)
try:
out = stage_server(args.source, args.out)
except (FileNotFoundError, FileExistsError) as e:
print(e, file=sys.stderr)
return 1
print(f"Staged {out}; put it with: agent-env env put {out}")
return 0


if __name__ == "__main__":
sys.exit(main())
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
type = "computer_app"
environment_name = "osworld-server"
module = "osworld_server_app.py:OSWorldServer"
computer = "ubuntu-24.04"
driver = "reference"
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
"""The ``osworld-server`` computer app: the OSWorld guest server, which a CUA VM runs on port 5000, run on a computer
as the session user from the source staged under ``payload/`` (``agent_env.task_step.task_steps.computer.osworld.stage_server``), so the
``osworld_*`` steps set up and grade OSWorld tasks there. Its own server forwards ``/osworld/<route>`` to the guest
server, and its card advertises that prefix as the endpoint of the ``osworld-guest/v1`` extension, which the computer
roots where it serves the app and guards with the env's control token. The OSWorld controller, deployed beside the
computer by ``osworld_deploy_controller``, reads the endpoint from the computer's card; an agent can't reach it there.

This file runs on the computer, in the app's folder: it imports only what the app's server has."""
from __future__ import annotations

import socket
from typing import Any

import httpx
from agentenv_protocol import extension
from agentenv_protocol.computer import (ComputerAppEnvironment, Packages, ReadyCheck, Service, computer_app, linux,
ready_check)
from starlette.requests import Request
from starlette.responses import Response

GUEST_EXTENSION = "urn:agentenv:osworld-guest/v1"
PREFIX = "/osworld"
HOST, PORT = "127.0.0.1", 5000
UPSTREAM = f"http://{HOST}:{PORT}"
METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE"]
#: Setup downloads task assets and evaluation runs long checks; the controller allows each up to 1800 s.
TIMEOUT_S = 1800.0
#: Not forwarded: the hop's own headers, and the control token, which is the computer's.
_DROPPED_REQUEST = frozenset({"host", "content-length", "connection", "transfer-encoding", "authorization"})
_DROPPED_RESPONSE = frozenset({"content-length", "connection", "transfer-encoding", "content-encoding"})
#: What a CUA VM installs for the guest server that Ubuntu 24.04's desktop computer lacks: AT-SPI's Python bindings
#: for its accessibility tree, Tk for pyautogui, scrot for pyautogui's screenshots on X11, the window and input tools its
#: setup routes run, ffmpeg for recordings.
PACKAGES = ["python3-pyatspi", "python3-gi", "gir1.2-atspi-2.0", "at-spi2-core", "python3-tk", "scrot", "ffmpeg",
"wmctrl", "xdotool", "x11-xkb-utils", "x11-xserver-utils"]


async def forward(request: Request, client: httpx.AsyncClient, upstream: str = UPSTREAM) -> Response:
"""``request`` under ``PREFIX``, sent to the guest server with its method, query and body, and its answer
returned as the guest server gave it."""
headers = {k: v for k, v in request.headers.items() if k.lower() not in _DROPPED_REQUEST}
try:
answer = await client.request(request.method, f"{upstream}/{request.path_params.get('path', '')}",
params=list(request.query_params.multi_items()), headers=headers,
content=await request.body(), timeout=TIMEOUT_S)
except httpx.HTTPError as e:
return Response(f"osworld-server unreachable: {type(e).__name__}", status_code=502)
return Response(answer.content, status_code=answer.status_code,
headers={k: v for k, v in answer.headers.items() if k.lower() not in _DROPPED_RESPONSE})


def guest_server_check() -> ReadyCheck:
try:
with socket.create_connection((HOST, PORT), timeout=2):
return ReadyCheck("osworld-server", True)
except OSError as e:
return ReadyCheck("osworld-server", False, f"{HOST}:{PORT}: {e}")


@computer_app(supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}])
@linux(packages=Packages(apt=PACKAGES), python="requirements.txt",
services=[Service(name="osworld-server", command="bash start-osworld-server.sh")])
class OSWorldServer(ComputerAppEnvironment):
"""The OSWorld guest server, forwarded under ``/osworld/`` to the controller beside the computer."""

def create_app(self) -> Any:
app = super().create_app()
client = httpx.AsyncClient()

async def _forward(request: Request) -> Response:
return await forward(request, client)

app.custom_route(f"{PREFIX}/{{path:path}}", methods=METHODS)(_forward)
return app

@extension(GUEST_EXTENSION, method="GET", path=PREFIX,
description="The OSWorld guest server's routes, under this endpoint; a GET says whether it's up.")
async def guest(self) -> dict:
check = guest_server_check()
return {"ready": check.ok, "detail": check.detail}

@ready_check(os="linux")
def guest_server(self) -> ReadyCheck:
return guest_server_check()
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# The app's own server forwards /osworld/* to the guest server with httpx; the rest is what the guest server imports
# on Linux. AT-SPI's bindings come from the OS's packages (python3-pyatspi, python3-gi).
httpx>=0.27.0
python3-xlib==0.15
PyAutoGUI==0.9.54
Pillow==13.0.0
requests
flask
numpy
lxml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#!/usr/bin/env bash
# Runs the OSWorld guest server on 127.0.0.1:5000 with the app's venv; the computer's supervisord restarts it when it
# exits (its app services are autorestart). The OS's dist-packages go after the venv's, for AT-SPI's bindings, which
# pip can't install.
cd "$(dirname "$0")/payload"
export DISPLAY="${DISPLAY:-:0}"
export PYTHONUNBUFFERED=1
# python-xlib refuses to connect without an Xauthority file; the computer's X server takes connections without one.
[ -e "${XAUTHORITY:-$HOME/.Xauthority}" ] || touch "${XAUTHORITY:-$HOME/.Xauthority}"
exec ../venv/bin/python -c 'import runpy, sys; sys.path.append("/usr/lib/python3/dist-packages"); sys.argv = ["main.py"]; runpy.run_path("main.py", run_name="__main__")'
Loading