From 5390455d399291721746a9803c55f185085a6632 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 8 Nov 2026 20:14:33 -0800 Subject: [PATCH 1/5] feat(drivers): codex-computer-use, OpenAI's Codex Computer Use as a macOS driver (FD-5356) The ChatGPT app's unified-computer-use plugin as a computer driver on macOS. ChatGPT 26.930.51102, OpenAI's versioned download pinned by sha256, is unpacked beside the driver and never run; its helper app is copied to a fixed path and granted Accessibility, Post Event and Screen Recording. The driver starts the plugin's cua_repl as the app would, one server per agent session, and forwards its js and js_reset, hiding the two tools it serves only the ChatGPT app. It answers Codex's per-app approvals from the allowed_apps setting, by name or bundle identifier (every app when unset, audio recording never), and runs each call under Codex's managed read-only sandbox unless js_sandbox is off. No ChatGPT sign-in is needed. computer/v1 is MacOSDriver's. On a Mac host with lume, it composes onto macos-26 and passes gate 3's full probe (53 passed, 1 not applicable); a verifier driving the deployed driver over MCP opened Calculator, computed 7 x 6 = 42, was refused TextEdit, which allowed_apps didn't name, and took a computer/v1 screenshot. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/drivers/README.md | 20 ++ src/agent_env/drivers/aliases.toml | 1 + .../drivers/envs/codex-computer-use/driver.py | 270 ++++++++++++++++++ .../drivers/envs/codex-computer-use/env.toml | 15 + .../envs/codex-computer-use/install.sh | 12 + .../drivers/envs/codex-computer-use/mcp.json | 105 +++++++ .../codex-computer-use/requirements-macos.txt | 4 + tst/unit/bundle/shipped_bundles_test.py | 1 + .../drivers/codex_computer_use_driver_test.py | 246 ++++++++++++++++ 9 files changed, 674 insertions(+) create mode 100644 src/agent_env/drivers/envs/codex-computer-use/driver.py create mode 100644 src/agent_env/drivers/envs/codex-computer-use/env.toml create mode 100755 src/agent_env/drivers/envs/codex-computer-use/install.sh create mode 100644 src/agent_env/drivers/envs/codex-computer-use/mcp.json create mode 100644 src/agent_env/drivers/envs/codex-computer-use/requirements-macos.txt create mode 100644 tst/unit/drivers/codex_computer_use_driver_test.py diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index b24c86cd6..2d789fa6a 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -125,5 +125,25 @@ cuts a server's instructions at 4,096 characters, so the skill itself isn't in t (`DO_NOT_TRACK`). `computer-ready` checks its version, that it prints the pinned skill, and the TCC permissions. Name it as `macos-harness`. +`envs/codex-computer-use/` is OpenAI's Codex Computer Use as a driver, for the same macOS computers. It is the ChatGPT +app's `unified-computer-use` plugin, whose MCP server, `cua_repl`, runs the JavaScript an agent sends against OpenAI's +`cua` API, which reads apps' accessibility trees and acts on them in the background through the plugin's helper app. +ChatGPT 26.930.51102, OpenAI's versioned download checked by sha256, is unpacked beside the driver by `install.sh` and +never run, so it never updates itself, and its helper app is copied to `/opt/agentenv/codex-computer-use/`, where +`[install.macos]` grants it Accessibility, Post Event and Screen Recording, as it grants the driver's own Python. The +driver starts `cua_repl` as the app would, on the app's own Node with the environment the app builds and the computer +surface alone, one server for each agent session, since the REPL keeps its bindings between calls; one that stops is +started afresh on the next call. Agents get the server's instructions and its `js` and `js_reset` tools, as `mcp.json` +pins them; `js_add_node_module_dir` and `turn_ended`, which it serves the ChatGPT app, are hidden. Codex asks its host +before each action on an app, and the driver answers: it approves the apps its `allowed_apps` setting names, by name or +bundle identifier, case aside, or every app when it names none, and nothing else, recording the computer's audio +included. Each call runs under Codex's own sandbox, read-only but for `~/.codex` and the temporary folders and with no +network, unless its `js_sandbox` setting is off; the apps it drives are outside it, and it still lets the JavaScript +read files and run commands, so a bench that wants the GUI alone guards for that. No ChatGPT sign-in is needed. Its +`computer_*` tools are agent-desktop's, from `MacOSDriver`. `computer-ready` checks the executables' sha256, that +`cua_repl` lists the tools `mcp.json` pins and answers `cua.getState()`, and the TCC permissions. To move it to another +ChatGPT version, pin the zip's URL and sha256 in `env.toml` and the executables' in `driver.py`'s `EXECUTABLES`, and +take `mcp.json` again from what that version's `cua_repl` says at connect and lists. Name it as `codex-computer-use`. + This bundle has no tasks of its own: other bundles pull its drivers by alias, and `agent-env run drivers` has nothing to run. diff --git a/src/agent_env/drivers/aliases.toml b/src/agent_env/drivers/aliases.toml index f53cc16c2..a93822b44 100644 --- a/src/agent_env/drivers/aliases.toml +++ b/src/agent_env/drivers/aliases.toml @@ -5,3 +5,4 @@ cua-driver = ["cua-driver"] page = ["page"] agent-desktop = ["agent-desktop"] macos-harness = ["macos-harness"] +codex-computer-use = ["codex-computer-use"] diff --git a/src/agent_env/drivers/envs/codex-computer-use/driver.py b/src/agent_env/drivers/envs/codex-computer-use/driver.py new file mode 100644 index 000000000..5284a00c5 --- /dev/null +++ b/src/agent_env/drivers/envs/codex-computer-use/driver.py @@ -0,0 +1,270 @@ +"""OpenAI's Codex Computer Use as a computer driver on macOS: the ChatGPT app's ``unified-computer-use`` plugin, whose +MCP server, ``cua_repl``, runs the JavaScript an agent sends against OpenAI's ``cua`` API, which reads apps' +accessibility trees and acts on them in the background through the plugin's helper app. The pinned app is unpacked +beside the driver and never run, so it never updates itself; the driver starts ``cua_repl`` as the app would, on the +app's own Node with the environment the app builds, one server for each agent session, since the REPL keeps its +bindings between calls. Agents get the server's instructions and its ``js`` and ``js_reset`` tools as pinned +(``mcp.json``); the two it serves only the ChatGPT app are hidden. Codex asks its host before each action on an app, +and the driver answers: it approves the apps the ``allowed_apps`` setting names, by name or bundle identifier, or every +app when it names none, and nothing else, recording the computer's audio included. Each call runs under Codex's own +sandbox, read-only but for Codex's folder and the temporary ones and with no network, unless the ``js_sandbox`` setting +is off; the apps it drives are outside it. No ChatGPT sign-in is needed. The computer/v1 operations are MacOSDriver's; +this process's Python is granted Accessibility, Post Event and Screen Recording, as the helper app is. It needs +``agentenv-framework-protocol[computer,macos]`` and the pinned ChatGPT app.""" +from __future__ import annotations + +import asyncio +import contextlib +import functools +import hashlib +import json +import logging +import os +import re +from pathlib import Path +from typing import Any, AsyncIterator, Callable + +from mcp import ClientSession, McpError, StdioServerParameters +from mcp.client.stdio import stdio_client +from mcp.shared.context import RequestContext +from mcp.types import CONNECTION_CLOSED, CallToolResult, ElicitRequestParams, ElicitResult, TextContent + +from agentenv_protocol import EnvironmentTool, ForwardedTool, ToolContext, server_instructions +from agentenv_protocol.computer import ComputerSettings, MacOSDriver, ReadyCheck, computer_driver + +logger = logging.getLogger(__name__) + +HERE = Path(__file__).resolve().parent +#: The pinned ChatGPT app, as install.sh unpacks it, and what of it starts Codex's REPL. +VERSION = "26.930.51102" +APP = HERE / "ChatGPT.app" +RESOURCES = APP / "Contents" / "Resources" +NODE = RESOURCES / "cua_node" / "bin" / "node" +NODE_REPL = RESOURCES / "cua_node" / "bin" / "node_repl" +NODE_MODULES = RESOURCES / "cua_node" / "lib" / "node_modules" +LAUNCHER = NODE_MODULES / "@oai" / "cua-repl" / "bin" / "cua-repl.mjs" +CODEX_CLI = RESOURCES / "codex-cli" / "CodexCLI.app" / "Contents" / "MacOS" / "codex" +#: Codex Computer Use's helper app, where install.sh copies it and the layer's grant names it. +HELPER = Path("/opt/agentenv/codex-computer-use/Codex Computer Use.app") +#: The sha256 of each executable at the pinned version, by its path under the app or the helper. +EXECUTABLES = { + "ChatGPT.app/Contents/MacOS/ChatGPT": "1f2989ac3d702e48c740fe4c54b1b6279f9e48e0e4c0ead7d4501464e2ea976e", + "ChatGPT.app/Contents/Resources/cua_node/bin/node": + "59f603e9e6d81c8b6bdaf6bb35b1990e7b8048762ca07a524ac7e87ade6fea15", + "ChatGPT.app/Contents/Resources/cua_node/bin/node_repl": + "bbd1640851593a724e2a6bc558f3a16078f40b09d4fe76f2f6d89f8b895eb5ed", + "Codex Computer Use.app/Contents/MacOS/SkyComputerUseService": + "fa5b5d685f550af82902cff1a345db558a1da3eb526cf78cd4659fa0dcde01a5", +} +#: What ``cua_repl`` says at connect and lists at the pinned version: its instructions and its tools. +MANIFEST = HERE / "mcp.json" +#: The tools agents get; the others are the ChatGPT app's own. +FORWARDED = ("js", "js_reset") +#: What of the driver's environment ``cua_repl`` gets: the session user's own; none of a deploy's values. +KEPT = ("HOME", "USER", "LOGNAME", "TMPDIR", "LANG") +#: The PATH it runs, and launches apps, with. +PATH = "/usr/bin:/bin:/usr/sbin:/sbin" +#: The JavaScript the readiness check runs: an inventory, which needs no app's approval. +PROBE = "await cua.getState({ emit: false }); nodeRepl.write('ready');" +#: The approval Codex asks of its host before it acts on an app, which names it. +APPROVAL = re.compile(r'Allow (?:Computer Use|Codex) to use "?(?P[^"?]+)"?\?') +#: The key of the metadata Codex reads its sandbox from on each call. +SANDBOX_KEY = "codex/sandbox-state-meta" + + +@computer_driver(name="codex-computer-use", + supports=[{"os": {"type": "macos", "display": ["aqua"]}, "arch": ["arm64"]}]) +class CodexComputerUse(MacOSDriver): + class Settings(ComputerSettings): + #: The apps Codex may act on, by name or bundle identifier, case aside; None for every app. + allowed_apps: list[str] | None = None + js_sandbox: bool = True + + def __init__(self) -> None: + super().__init__() + self._pinned: list[ReadyCheck] | None = None + self._repls: dict[str | None, _Repl] = {} + + def forwarded_tools(self) -> list[ForwardedTool]: + return [ForwardedTool(EnvironmentTool(name=t["name"], description=t["description"], + inputSchema=t["inputSchema"]), + functools.partial(self._forward, t["name"]), t.get("annotations")) + for t in _manifest()["tools"] if t["name"] in FORWARDED] + + @server_instructions + def codex_instructions(self) -> str: + return _manifest()["instructions"] + + async def ready_checks(self) -> list[ReadyCheck]: + """The pinned executables, the tools ``cua_repl`` lists and that it answers, checked until they pass once, and + MacOSDriver's TCC permissions every time.""" + if self._pinned is None or not all(check.ok for check in self._pinned): + self._pinned = await self._pinned_checks() + return [*self._pinned, *await super().ready_checks()] + + async def _forward(self, tool: str, /, **arguments: Any) -> CallToolResult: + """A call to one of Codex's tools on the caller's own REPL, answered as Codex answers it.""" + repl = self._repls.get(caller := ToolContext.current().caller.session) + if repl is None: + repl = self._repls[caller] = _Repl(self._server) + try: + return await repl.call(tool, arguments, self._sandbox()) + except Exception as e: + return CallToolResult(isError=True, content=[TextContent(type="text", text=( + f"Codex's cua_repl failed before it answered, so {tool} may have run: {type(e).__name__}: {e}"))]) + + @contextlib.asynccontextmanager + async def _server(self) -> AsyncIterator[ClientSession]: + """``cua_repl`` connected and initialized, its approval requests answered; it ends with the block.""" + server = StdioServerParameters(command=str(NODE), args=[str(LAUNCHER)], env=_environment(), + cwd=str(_workspace())) + async with (stdio_client(server) as (read, write), + ClientSession(read, write, elicitation_callback=self._approve) as session): + await session.initialize() + yield session + + async def _approve(self, context: RequestContext, params: ElicitRequestParams) -> ElicitResult: + """Codex's request to act on an app, approved when ``allowed_apps`` names the app or names none; anything else + it asks is declined.""" + message = getattr(params, "message", "") + approved = False + if named := APPROVAL.fullmatch(message.strip()): + meta = params.meta.model_dump() if params.meta else {} + names = {named["app"].strip().lower(), str((meta.get("tool_params") or {}).get("app") or "").lower()} + allowed = self.settings.allowed_apps + approved = allowed is None or bool(names & {app.lower() for app in allowed}) + logger.info("%s %r", "approved" if approved else "declined", message) + return ElicitResult(action="accept", content={}) if approved else ElicitResult(action="decline") + + def _sandbox(self) -> dict: + """The metadata that sets the sandbox Codex runs the call's JavaScript in.""" + profile = _read_only() if self.settings.js_sandbox else {"type": "disabled"} + return {SANDBOX_KEY: {"sandboxCwd": _workspace().as_uri(), "permissionProfile": profile}} + + async def _pinned_checks(self) -> list[ReadyCheck]: + digests = await asyncio.to_thread(_digests) + differ = sorted(path for path, digest in digests.items() if digest != EXECUTABLES[path]) + checks = [ReadyCheck("Codex Computer Use", not differ, f"ChatGPT {VERSION}, as pinned" if not differ else + f"these differ from the pinned ChatGPT {VERSION}: {', '.join(differ)}")] + if differ: + return checks + try: + async with self._server() as session: + listed = (await session.list_tools()).tools + probed = await session.call_tool("js", {"code": PROBE}, meta=self._sandbox()) + except Exception as e: + return [*checks, ReadyCheck("Codex tools", False, f"cua_repl didn't answer: {type(e).__name__}: {e}")] + running = {t.name: t.inputSchema for t in listed} + pinned = {t["name"]: t["inputSchema"] for t in _manifest()["tools"]} + changed = sorted(name for name in running.keys() | pinned.keys() if running.get(name) != pinned.get(name)) + answer = " ".join(c.text for c in probed.content if isinstance(c, TextContent)) + return [*checks, + ReadyCheck("Codex tools", not changed, f"{len(running)} tools, as pinned" if not changed else + f"these tools differ from {MANIFEST.name}: {', '.join(changed)}"), + ReadyCheck("Codex answers", not probed.isError and "ready" in answer, + "it read the computer's apps" if not probed.isError else f"cua.getState() failed: {answer}")] + + +class _Repl: + """One ``cua_repl`` server, held open by a task of its own, since a stdio client is entered and left in one task, + and reached from any through a queue, one call at a time. A server that stops is started afresh on the next call, + without the last one's bindings.""" + + def __init__(self, open_server: Callable[[], contextlib.AbstractAsyncContextManager[ClientSession]]) -> None: + self._open = open_server + self._calls: asyncio.Queue = asyncio.Queue() + self._task: asyncio.Task | None = None + + async def call(self, tool: str, arguments: dict, meta: dict) -> CallToolResult: + if self._task is None or self._task.done(): + self._task = asyncio.create_task(self._serve()) + answer = asyncio.get_running_loop().create_future() + await self._calls.put((tool, arguments, meta, answer)) + return await answer + + async def _serve(self) -> None: + stopped: BaseException = ConnectionError("cua_repl stopped") + try: + async with self._open() as session: + while True: + tool, arguments, meta, answer = await self._calls.get() + try: + result = await session.call_tool(tool, arguments, meta=meta) + except McpError as e: + _settle(answer, error=e) + if e.error.code == CONNECTION_CLOSED: + raise + continue + except Exception as e: + _settle(answer, error=e) + raise + _settle(answer, result=result) + except Exception as e: + stopped = e + finally: + while not self._calls.empty(): + *_, answer = self._calls.get_nowait() + _settle(answer, error=stopped) + + +def _settle(answer: asyncio.Future, *, result: Any = None, error: BaseException | None = None) -> None: + if answer.done(): + return + if error is not None: + answer.set_exception(error) + else: + answer.set_result(result) + + +@functools.cache +def _manifest() -> dict: + return json.loads(MANIFEST.read_text()) + + +def _codex_home() -> Path: + return Path.home() / ".codex" + + +def _read_only() -> dict: + """Codex's managed sandbox profile: everything readable, only Codex's folder and the temporary ones writable, and + no network.""" + special = [{"path": {"type": "special", "value": {"kind": kind}}, "access": access} + for kind, access in (("root", "read"), ("tmpdir", "write"), ("slash_tmp", "write"))] + codex = {"path": {"type": "path", "path": str(_codex_home())}, "access": "write"} + return {"type": "managed", "network": "restricted", + "file_system": {"type": "restricted", "entries": [*special, codex]}} + + +def _workspace() -> Path: + """The folder ``cua_repl`` runs in, Codex's sandbox's working folder, inside the one it may write.""" + folder = _codex_home() / "agentenv" + folder.mkdir(parents=True, exist_ok=True) + return folder + + +def _environment() -> dict[str, str]: + """``cua_repl``'s environment, as the ChatGPT app builds it for the plugin, with the computer surface alone.""" + codex, modules = str(_codex_home()), str(NODE_MODULES) + return {**{name: os.environ[name] for name in KEPT if name in os.environ}, "PATH": PATH, + "CODEX_HOME": codex, + "CODEX_CLI_PATH": str(CODEX_CLI), + "CUA_REPL_ENABLED_SURFACES": "computer", + "CUA_REPL_NODE_REPL_PATH": str(NODE_REPL), + "NODE_REPL_INSTRUCTIONS_USE_CASE_COMPUTER_USE": "Control desktop apps on macOS through Computer Use.", + "NODE_REPL_NATIVE_PIPE_CONNECT_TIMEOUT_MS": "1000", + "NODE_REPL_NODE_MODULE_DIRS": modules, + "NODE_REPL_NODE_PATH": str(NODE), + "NODE_REPL_TRUSTED_CODE_PATHS": f"{codex}:{modules}", + "NODE_REPL_TRUSTED_SERVICES": json.dumps({"sky": "@oai/sky/service"}), + "SKY_CUA_SERVICE_PATH": str(HELPER)} + + +def _digests() -> dict[str, str | None]: + """Each pinned executable's sha256, None for one that isn't there.""" + roots = {"ChatGPT.app": APP, "Codex Computer Use.app": HELPER} + found = {} + for path in EXECUTABLES: + root, _, rest = path.partition("/") + file = roots[root] / rest + found[path] = hashlib.sha256(file.read_bytes()).hexdigest() if file.is_file() else None + return found diff --git a/src/agent_env/drivers/envs/codex-computer-use/env.toml b/src/agent_env/drivers/envs/codex-computer-use/env.toml new file mode 100644 index 000000000..fa5622d76 --- /dev/null +++ b/src/agent_env/drivers/envs/codex-computer-use/env.toml @@ -0,0 +1,15 @@ +type = "computer_driver" +module = "driver.py:CodexComputerUse" +port = 18765 + +[install.macos] +python = "requirements-macos.txt" +downloads = [ + { url = "https://persistent.oaistatic.com/codex-app-prod/ChatGPT-darwin-arm64-26.930.51102.zip", sha256 = "7001df293fd478b2a4e2d8b5f4ea3aae43a2e21d168960256b2f31ac4ee43f6f", to = "ChatGPT.zip", arch = "arm64" }, +] +run = [{ script = "install.sh", scope = "session" }] +grants = [ + { executable = "venv/bin/python", services = ["accessibility", "post_event", "screen_capture"] }, + { app = "/opt/agentenv/codex-computer-use/Codex Computer Use.app", services = ["accessibility", "post_event", "screen_capture"] }, +] +services = [{ name = "driver", command = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:CodexComputerUse" }] diff --git a/src/agent_env/drivers/envs/codex-computer-use/install.sh b/src/agent_env/drivers/envs/codex-computer-use/install.sh new file mode 100755 index 000000000..e4bd7e816 --- /dev/null +++ b/src/agent_env/drivers/envs/codex-computer-use/install.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# The pinned ChatGPT app, from the zip the install downloaded and checked, unpacked beside the driver and never run, so it +# never updates itself; and Codex Computer Use's helper app copied out of it to where the layer's grant names it, a path +# that doesn't depend on this folder's name. As the session user, whom the helper must belong to. +set -eu +ditto -x -k ChatGPT.zip . +rm -f ChatGPT.zip +helper="ChatGPT.app/Contents/Resources/cua_node/lib/node_modules/@oai/sky/Codex Computer Use.app" +mkdir -p /opt/agentenv/codex-computer-use +rm -rf "/opt/agentenv/codex-computer-use/Codex Computer Use.app" +ditto "$helper" "/opt/agentenv/codex-computer-use/Codex Computer Use.app" +xattr -dr com.apple.quarantine ChatGPT.app /opt/agentenv/codex-computer-use || true diff --git a/src/agent_env/drivers/envs/codex-computer-use/mcp.json b/src/agent_env/drivers/envs/codex-computer-use/mcp.json new file mode 100644 index 000000000..a919f71c3 --- /dev/null +++ b/src/agent_env/drivers/envs/codex-computer-use/mcp.json @@ -0,0 +1,105 @@ +{ + "instructions": "UI automation through cua_repl using the initialized cua API.", + "tools": [ + { + "name": "js", + "description": "Control native apps or browsers on the user’s computer by reading or operating UI. Prefer purpose-built skills, connectors, APIs, or CLIs when available.\n\nOn the first invocation of `cua_repl`, or after resetting it, execute exactly one of the API calls shown below, optionally assigning its result to a variable. Do not add other API calls, waits, or snapshots to that invocation.\nThe tool result will include documentation and, when creating or selecting a tab or selecting an app, its initial UI state. Selecting a browser does not open a tab. Read that result before continuing.\nUse only APIs described in the tool instructions or returned documentation.\n\nWhen you need an inventory of available apps, browsers, and tabs, get a snapshot of all enabled surfaces. Otherwise, use the relevant entry point below:\n\n```javascript\nawait cua.getState();\n```\n\nBrowser APIs are disabled.\n\nIf the user specifies an app to use, get the app by name, bundle ID, or path:\n\n```javascript\nlet app = await cua.getApp(\"Example App\");\n```\n\nTo add other content to the tool result, use `nodeRepl.write(value)` for text or other values and `await nodeRepl.emitImage(image)` for images. The APIs listed above already display their documentation or UI state; do not wrap their results in `write` or `emitImage`.\n\nIf your context begins with a summary of an existing computer use task, call `await cua.rewriteDocumentation()` before continuing the computer use task to ensure you have a complete view of the necessary documentation.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "code": { + "description": "JavaScript to execute using the initialized cua_repl runtime.", + "type": "string" + }, + "timeout_ms": { + "description": "Optional execution timeout in milliseconds. Defaults to 30000 (30 seconds) when omitted.", + "minimum": 1, + "type": "integer" + }, + "title": { + "description": "Short user-facing description of what the code does.", + "maxLength": 80, + "minLength": 1, + "type": "string" + } + }, + "required": [ + "code" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + } + }, + { + "name": "js_add_node_module_dir", + "description": "Add an absolute `node_modules` directory for package imports. The directory remains available after `js_reset`.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "path": { + "description": "Absolute path to a node_modules directory to add to Node package resolution.", + "minLength": 1, + "type": "string" + } + }, + "required": [ + "path" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + } + }, + { + "name": "js_reset", + "description": "Reset the persistent cua_repl JavaScript session. All JavaScript bindings are discarded. The next cua_repl.js call initializes a fresh runtime for the enabled surfaces. This does not close browser tabs or native apps, or erase their state.", + "inputSchema": { + "additionalProperties": false, + "properties": {}, + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + } + }, + { + "name": "turn_ended", + "description": "Notify trusted libraries that a Codex turn ended. Repeated notifications for the same session and turn are ignored.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "hook_event_name": { + "minLength": 1, + "type": "string" + }, + "session_id": { + "minLength": 1, + "type": "string" + }, + "turn_id": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "hook_event_name", + "session_id", + "turn_id" + ], + "type": "object" + }, + "annotations": { + "idempotentHint": true + } + } + ] +} diff --git a/src/agent_env/drivers/envs/codex-computer-use/requirements-macos.txt b/src/agent_env/drivers/envs/codex-computer-use/requirements-macos.txt new file mode 100644 index 000000000..839959600 --- /dev/null +++ b/src/agent_env/drivers/envs/codex-computer-use/requirements-macos.txt @@ -0,0 +1,4 @@ +# What a composed macOS computer installs into the driver's venv: the protocol with MacOSDriver, whose screen and input +# need its macos extra, and the MCP client the forwarded tools reach Codex's cua_repl server with. A release pins it to +# the version it ships. The ChatGPT app is the pinned download, whose executables' sha256 the driver checks. +agentenv-framework-protocol[computer,macos]>=0.1.459 diff --git a/tst/unit/bundle/shipped_bundles_test.py b/tst/unit/bundle/shipped_bundles_test.py index 5e069f1d3..357d5708f 100644 --- a/tst/unit/bundle/shipped_bundles_test.py +++ b/tst/unit/bundle/shipped_bundles_test.py @@ -63,6 +63,7 @@ def test_the_shipped_env_aliases_name_the_mail_env_the_validator_deploys_the_dri "cua-driver": "@local/agentenv-framework/drivers/cua-driver", "page": "@local/agentenv-framework/drivers/page", "agent-desktop": "@local/agentenv-framework/drivers/agent-desktop", "macos-harness": "@local/agentenv-framework/drivers/macos-harness", + "codex-computer-use": "@local/agentenv-framework/drivers/codex-computer-use", "chromium": "@local/agentenv-framework/browsers/chromium"} assert A2AAgent.VALIDATION_ENV_ID == "mail" diff --git a/tst/unit/drivers/codex_computer_use_driver_test.py b/tst/unit/drivers/codex_computer_use_driver_test.py new file mode 100644 index 000000000..77cf7ab92 --- /dev/null +++ b/tst/unit/drivers/codex_computer_use_driver_test.py @@ -0,0 +1,246 @@ +"""The codex-computer-use driver against a fake cua_repl: agents get its js and js_reset as pinned, with its +instructions, and not the tools it serves only the ChatGPT app; each call carries Codex's read-only sandbox unless the +setting turns it off; Codex's approval requests are answered from allowed_apps by name or bundle identifier, every app +approved when it names none, and nothing else; each agent session keeps a REPL of its own across calls, one that stops +starting afresh; cua_repl gets the app's environment and none of the driver's; and readiness checks the pinned +executables, the tools cua_repl lists and that it answers.""" +from __future__ import annotations + +import contextlib +import hashlib +import importlib.util +import json +import sys +import textwrap +from pathlib import Path + +import pytest +from mcp.shared.memory import create_connected_server_and_client_session + +import agent_env +from agentenv_protocol import SESSION_META_KEY +from agentenv_protocol.computer import Platform, _macos + +DRIVER = Path(agent_env.__file__).parent / "drivers" / "envs" / "codex-computer-use" / "driver.py" +FAKE = textwrap.dedent('''\ + import json, os + from pathlib import Path + import anyio + import mcp.types as types + from mcp.server.lowlevel import Server + from mcp.server.stdio import stdio_server + from mcp.shared.message import ServerMessageMetadata + state = Path(os.environ["HOME"]) / "fake" + (state / f"env-{os.getpid()}.json").write_text(json.dumps(dict(os.environ))) + manifest = json.loads((state / "tools.json").read_text()) + server = Server("cua_repl", instructions=manifest["instructions"]) + bindings = {} + + @server.list_tools() + async def tools(): + return [types.Tool(name=t["name"], description=t["description"], inputSchema=t["inputSchema"]) + for t in manifest["tools"]] + + async def ask(message, app): + context = server.request_context + meta = types.RequestParams.Meta(**({"tool_params": {"app": app}} if app else {})) + result = await context.session.send_request( + types.ServerRequest(types.ElicitRequest(params=types.ElicitRequestFormParams( + message=message, requestedSchema={"type": "object", "properties": {}}, _meta=meta))), + types.ElicitResult, metadata=ServerMessageMetadata(related_request_id=context.request_id)) + return result.action + + @server.call_tool(validate_input=False) + async def call(name, arguments): + context = server.request_context + with open(state / "calls.jsonl", "a") as out: + out.write(json.dumps({"pid": os.getpid(), "name": name, "arguments": arguments, + "meta": context.meta.model_dump() if context.meta else {}}) + "\\n") + code = arguments.get("code", "") + if name == "js_reset": + bindings.clear() + text = "reset" + elif code.startswith("app "): + _, display, bundle = code.split(" ") + text = await ask(f'Allow Computer Use to use "{display}"?', bundle) + elif code == "audio": + text = await ask("Allow Computer Use to record computer audio?", None) + elif code.startswith("set "): + bindings["x"] = code[4:] + text = "set" + elif code == "get": + text = f"{bindings.get('x')} in {os.getpid()}" + elif code == "die": + os._exit(1) + elif "cua.getState" in code: + text = "ready" if not (state / "unready").exists() else "no apps" + else: + text = "ran" + return [types.TextContent(type="text", text=text)] + + async def main(): + async with stdio_server() as (read, write): + await server.run(read, write, server.create_initialization_options()) + + anyio.run(main) +''') +MACOS = Platform(os="macos", version="26.5", arch="arm64", display="aqua") + + +class Granted: + def missing(self) -> list[str]: + return [] + + +@pytest.fixture +def module(monkeypatch, tmp_path): + monkeypatch.delenv("AGENTENV_COMPUTER_SETTINGS", raising=False) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("AGENTENV_CONTROL_TOKEN", "a-deploys-secret") + monkeypatch.setattr(_macos, "Quartz", Granted) + monkeypatch.setattr(sys, "platform", "darwin") + monkeypatch.setattr("agentenv_protocol.computer.driver.detect", lambda: MACOS) + spec = importlib.util.spec_from_file_location("codex_computer_use_driver", DRIVER) + loaded = importlib.util.module_from_spec(spec) + spec.loader.exec_module(loaded) + (tmp_path / "fake").mkdir() + (tmp_path / "fake" / "tools.json").write_text(loaded.MANIFEST.read_text()) + app, helper = tmp_path / "ChatGPT.app", tmp_path / "Codex Computer Use.app" + node = app / "Contents" / "Resources" / "cua_node" / "bin" / "node" + node.parent.mkdir(parents=True) + node.write_text(f"#!{sys.executable}\n{FAKE}") + node.chmod(0o755) + pinned = {} + for path in loaded.EXECUTABLES: + root, _, rest = path.partition("/") + file = (app if root == "ChatGPT.app" else helper) / rest + if not file.exists(): + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(f"the pinned {file.name}") + pinned[path] = hashlib.sha256(file.read_bytes()).hexdigest() + monkeypatch.setattr(loaded, "APP", app) + monkeypatch.setattr(loaded, "HELPER", helper) + monkeypatch.setattr(loaded, "NODE", node) + monkeypatch.setattr(loaded, "EXECUTABLES", pinned) + return loaded + + +def _driver(module, monkeypatch, **settings): + monkeypatch.setenv("AGENTENV_COMPUTER_SETTINGS", json.dumps(settings)) + return module.CodexComputerUse() + + +@contextlib.asynccontextmanager +async def mcp_session(driver): + async with create_connected_server_and_client_session(driver.mcp or driver.create_app()) as session: + yield session + + +def _calls(home: Path) -> list[dict]: + path = home / "fake" / "calls.jsonl" + return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else [] + + +def _text(result) -> str: + return " ".join(c.text for c in result.content if c.type == "text") + + +@pytest.mark.asyncio +async def test_agents_get_codexs_js_and_js_reset_as_pinned_with_its_instructions_and_not_the_apps_own_tools( + module, monkeypatch): + driver = _driver(module, monkeypatch) + async with mcp_session(driver) as session: + listed = {t.name: t for t in (await session.list_tools()).tools} + + pinned = {t["name"]: t for t in module._manifest()["tools"]} + assert {"js", "js_reset"} <= set(listed) and not {"js_add_node_module_dir", "turn_ended"} & set(listed) + assert listed["js"].inputSchema == pinned["js"]["inputSchema"] + assert listed["js"].description == pinned["js"]["description"] + assert driver.mcp.instructions.startswith(module._manifest()["instructions"]) + + +@pytest.mark.asyncio +async def test_each_call_runs_in_codexs_read_only_sandbox_unless_the_setting_turns_it_off( + module, monkeypatch, tmp_path): + async with mcp_session(_driver(module, monkeypatch)) as session: + await session.call_tool("js", {"code": "1 + 1"}) + async with mcp_session(_driver(module, monkeypatch, js_sandbox=False)) as session: + await session.call_tool("js", {"code": "1 + 1"}) + + sandboxed, open_ = (call["meta"]["codex/sandbox-state-meta"] for call in _calls(tmp_path)) + profile = sandboxed["permissionProfile"] + entries = {json.dumps(e["path"], sort_keys=True): e["access"] for e in profile["file_system"]["entries"]} + assert profile["type"] == "managed" and profile["network"] == "restricted" + assert entries == {'{"type": "special", "value": {"kind": "root"}}': "read", + '{"type": "special", "value": {"kind": "tmpdir"}}': "write", + '{"type": "special", "value": {"kind": "slash_tmp"}}': "write", + json.dumps({"path": str(tmp_path / ".codex"), "type": "path"}, sort_keys=True): "write"} + assert sandboxed["sandboxCwd"] == (tmp_path / ".codex" / "agentenv").as_uri() + assert open_["permissionProfile"] == {"type": "disabled"} + + +@pytest.mark.asyncio +async def test_approvals_follow_allowed_apps_by_name_or_bundle_identifier_and_nothing_else_is_approved( + module, monkeypatch): + async with mcp_session(_driver(module, monkeypatch, allowed_apps=["calculator", "com.apple.TextEdit"])) as session: + named = await session.call_tool("js", {"code": "app Calculator com.apple.calculator"}) + by_id = await session.call_tool("js", {"code": "app TextEdit com.apple.TextEdit"}) + other = await session.call_tool("js", {"code": "app Finder com.apple.finder"}) + audio = await session.call_tool("js", {"code": "audio"}) + async with mcp_session(_driver(module, monkeypatch)) as session: + any_app = await session.call_tool("js", {"code": "app Finder com.apple.finder"}) + no_audio = await session.call_tool("js", {"code": "audio"}) + + assert [_text(r) for r in (named, by_id, other, audio)] == ["accept", "accept", "decline", "decline"] + assert [_text(r) for r in (any_app, no_audio)] == ["accept", "decline"] + + +@pytest.mark.asyncio +async def test_each_agent_session_keeps_a_repl_of_its_own_and_one_that_stops_starts_afresh( + module, monkeypatch, tmp_path): + async with mcp_session(_driver(module, monkeypatch)) as session: + for caller, value in (("a", "1"), ("b", "2")): + await session.call_tool("js", {"code": f"set {value}"}, meta={SESSION_META_KEY: caller}) + a = _text(await session.call_tool("js", {"code": "get"}, meta={SESSION_META_KEY: "a"})) + b = _text(await session.call_tool("js", {"code": "get"}, meta={SESSION_META_KEY: "b"})) + died = await session.call_tool("js", {"code": "die"}, meta={SESSION_META_KEY: "a"}) + after = _text(await session.call_tool("js", {"code": "get"}, meta={SESSION_META_KEY: "a"})) + + (a_value, _, a_pid), (b_value, _, b_pid) = a.partition(" in "), b.partition(" in ") + assert (a_value, b_value) == ("1", "2") and a_pid != b_pid + assert died.isError and "may have run" in _text(died) + assert after.startswith("None in ") and after != f"None in {a_pid}" + + +@pytest.mark.asyncio +async def test_cua_repl_gets_the_apps_environment_and_none_of_the_drivers(module, monkeypatch, tmp_path): + async with mcp_session(_driver(module, monkeypatch)) as session: + await session.call_tool("js", {"code": "1 + 1"}) + + (env_file,) = (tmp_path / "fake").glob("env-*.json") + env = json.loads(env_file.read_text()) + modules = str(module.NODE_MODULES) + assert "AGENTENV_CONTROL_TOKEN" not in env and env["PATH"] == "/usr/bin:/bin:/usr/sbin:/sbin" + assert env["CUA_REPL_ENABLED_SURFACES"] == "computer" and env["SKY_CUA_SERVICE_PATH"] == str(module.HELPER) + assert env["NODE_REPL_TRUSTED_SERVICES"] == '{"sky": "@oai/sky/service"}' + assert env["NODE_REPL_TRUSTED_CODE_PATHS"] == f"{tmp_path / '.codex'}:{modules}" + assert env["CODEX_HOME"] == str(tmp_path / ".codex") and env["NODE_REPL_NODE_PATH"] == str(module.NODE) + + +@pytest.mark.asyncio +async def test_readiness_checks_the_pinned_executables_the_tools_cua_repl_lists_and_that_it_answers( + module, monkeypatch, tmp_path): + driver = _driver(module, monkeypatch) + ready = await driver.ready_checks() + three = {**module._manifest(), "tools": module._manifest()["tools"][:3]} + (tmp_path / "fake" / "tools.json").write_text(json.dumps(three)) + driver._pinned = None + tools = (await driver.ready_checks())[1] + (module.HELPER / "Contents" / "MacOS" / "SkyComputerUseService").write_text("another build") + driver._pinned = None + (changed, *_) = await driver.ready_checks() + + assert [(c.name, c.ok) for c in ready] == [("Codex Computer Use", True), ("Codex tools", True), + ("Codex answers", True), ("macOS permissions", True)] + assert not tools.ok and tools.detail == "these tools differ from mcp.json: turn_ended" + assert not changed.ok and "Codex Computer Use.app/Contents/MacOS/SkyComputerUseService" in changed.detail From 1c900dac0afae9c4542698b919ce185d99f6547c Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 8 Nov 2026 20:23:42 -0800 Subject: [PATCH 2/5] fix(drivers): close codex-computer-use's idle REPLs, and don't call js read-only Nothing tells the driver when an agent session ends, so each session's cua_repl stayed open for the driver's life: a REPL left unused for 30 minutes is now closed by a sweep, and past eight the least recently used is closed when a new one starts, never one in a call, so active sessions keep their bindings. js keeps the schema and description Codex serves, but its hints say it changes apps and reaches the world (readOnlyHint false, destructiveHint and openWorldHint true), where Codex marks it read-only. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/drivers/README.md | 6 +- .../drivers/envs/codex-computer-use/driver.py | 85 ++++++++++++++----- .../drivers/codex_computer_use_driver_test.py | 42 +++++++++ 3 files changed, 111 insertions(+), 22 deletions(-) diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index 2d789fa6a..83e4ece54 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -133,8 +133,10 @@ never run, so it never updates itself, and its helper app is copied to `/opt/age `[install.macos]` grants it Accessibility, Post Event and Screen Recording, as it grants the driver's own Python. The driver starts `cua_repl` as the app would, on the app's own Node with the environment the app builds and the computer surface alone, one server for each agent session, since the REPL keeps its bindings between calls; one that stops is -started afresh on the next call. Agents get the server's instructions and its `js` and `js_reset` tools, as `mcp.json` -pins them; `js_add_node_module_dir` and `turn_ended`, which it serves the ChatGPT app, are hidden. Codex asks its host +started afresh on the next call, and nothing says when a session ends, so one left unused for 30 minutes, or past the +eight kept, is closed, the least recently used first. Agents get the server's instructions and its `js` and `js_reset` +tools, as `mcp.json` pins them, `js` marked as acting on apps where Codex marks it read-only; `js_add_node_module_dir` +and `turn_ended`, which it serves the ChatGPT app, are hidden. Codex asks its host before each action on an app, and the driver answers: it approves the apps its `allowed_apps` setting names, by name or bundle identifier, case aside, or every app when it names none, and nothing else, recording the computer's audio included. Each call runs under Codex's own sandbox, read-only but for `~/.codex` and the temporary folders and with no diff --git a/src/agent_env/drivers/envs/codex-computer-use/driver.py b/src/agent_env/drivers/envs/codex-computer-use/driver.py index 5284a00c5..64ce898c0 100644 --- a/src/agent_env/drivers/envs/codex-computer-use/driver.py +++ b/src/agent_env/drivers/envs/codex-computer-use/driver.py @@ -3,14 +3,15 @@ accessibility trees and acts on them in the background through the plugin's helper app. The pinned app is unpacked beside the driver and never run, so it never updates itself; the driver starts ``cua_repl`` as the app would, on the app's own Node with the environment the app builds, one server for each agent session, since the REPL keeps its -bindings between calls. Agents get the server's instructions and its ``js`` and ``js_reset`` tools as pinned -(``mcp.json``); the two it serves only the ChatGPT app are hidden. Codex asks its host before each action on an app, -and the driver answers: it approves the apps the ``allowed_apps`` setting names, by name or bundle identifier, or every -app when it names none, and nothing else, recording the computer's audio included. Each call runs under Codex's own -sandbox, read-only but for Codex's folder and the temporary ones and with no network, unless the ``js_sandbox`` setting -is off; the apps it drives are outside it. No ChatGPT sign-in is needed. The computer/v1 operations are MacOSDriver's; -this process's Python is granted Accessibility, Post Event and Screen Recording, as the helper app is. It needs -``agentenv-framework-protocol[computer,macos]`` and the pinned ChatGPT app.""" +bindings between calls, closed once it's left unused or past the most kept, since nothing says when a session ends. +Agents get the server's instructions and its ``js`` and ``js_reset`` tools as pinned (``mcp.json``), ``js`` marked as +acting on apps where Codex marks it read-only; the two it serves only the ChatGPT app are hidden. Codex asks its host +before each action on an app, and the driver answers: it approves the apps the ``allowed_apps`` setting names, by name +or bundle identifier, or every app when it names none, and nothing else, recording the computer's audio included. Each +call runs under Codex's own sandbox, read-only but for Codex's folder and the temporary ones and with no network, unless +the ``js_sandbox`` setting is off; the apps it drives are outside it. No ChatGPT sign-in is needed. The computer/v1 +operations are MacOSDriver's; this process's Python is granted Accessibility, Post Event and Screen Recording, as the +helper app is. It needs ``agentenv-framework-protocol[computer,macos]`` and the pinned ChatGPT app.""" from __future__ import annotations import asyncio @@ -21,6 +22,7 @@ import logging import os import re +import time from pathlib import Path from typing import Any, AsyncIterator, Callable @@ -60,6 +62,13 @@ MANIFEST = HERE / "mcp.json" #: The tools agents get; the others are the ChatGPT app's own. FORWARDED = ("js", "js_reset") +#: The hints agents get where Codex's own undersell a tool: ``js`` acts on apps, and through them on anything. +ANNOTATIONS = {"js": {"readOnlyHint": False, "destructiveHint": True, "openWorldHint": True}} +#: How long a session's REPL is kept unused before it's closed, since nothing says when a session ends; how many are +#: kept at most, the least recently used closed first; and how often the idle ones are looked for. +IDLE_S, MAX_REPLS, SWEEP_S = 1800, 8, 60 +#: How long a closed REPL's server has to end before it's cancelled. +CLOSE_TIMEOUT_S = 10 #: What of the driver's environment ``cua_repl`` gets: the session user's own; none of a deploy's values. KEPT = ("HOME", "USER", "LOGNAME", "TMPDIR", "LANG") #: The PATH it runs, and launches apps, with. @@ -83,12 +92,15 @@ class Settings(ComputerSettings): def __init__(self) -> None: super().__init__() self._pinned: list[ReadyCheck] | None = None + #: Each session's REPL, the least recently used first. self._repls: dict[str | None, _Repl] = {} + self._sweeper: asyncio.Task | None = None def forwarded_tools(self) -> list[ForwardedTool]: return [ForwardedTool(EnvironmentTool(name=t["name"], description=t["description"], inputSchema=t["inputSchema"]), - functools.partial(self._forward, t["name"]), t.get("annotations")) + functools.partial(self._forward, t["name"]), + {**(t.get("annotations") or {}), **ANNOTATIONS.get(t["name"], {})}) for t in _manifest()["tools"] if t["name"] in FORWARDED] @server_instructions @@ -103,16 +115,34 @@ async def ready_checks(self) -> list[ReadyCheck]: return [*self._pinned, *await super().ready_checks()] async def _forward(self, tool: str, /, **arguments: Any) -> CallToolResult: - """A call to one of Codex's tools on the caller's own REPL, answered as Codex answers it.""" - repl = self._repls.get(caller := ToolContext.current().caller.session) - if repl is None: - repl = self._repls[caller] = _Repl(self._server) + """A call to one of Codex's tools on the caller's own REPL, answered as Codex answers it. Other sessions' REPLs + left idle, or past the most kept, are closed first.""" + repl = self._repls.pop(caller := ToolContext.current().caller.session, None) + await self._evict(keep=MAX_REPLS - 1) + repl = self._repls[caller] = repl or _Repl(self._server) + if self._sweeper is None or self._sweeper.done(): + self._sweeper = asyncio.create_task(self._sweep()) try: return await repl.call(tool, arguments, self._sandbox()) except Exception as e: return CallToolResult(isError=True, content=[TextContent(type="text", text=( f"Codex's cua_repl failed before it answered, so {tool} may have run: {type(e).__name__}: {e}"))]) + async def _evict(self, keep: int = MAX_REPLS) -> None: + """Close the REPLs idle for ``IDLE_S`` and, past ``keep``, the least recently used, none of them in a call.""" + now = time.monotonic() + idle = [caller for caller, repl in self._repls.items() if not repl.busy and now - repl.used > IDLE_S] + spare = [caller for caller, repl in self._repls.items() if not repl.busy and caller not in idle] + closing = [self._repls.pop(caller) for caller in [*idle, *spare[:max(0, len(self._repls) - len(idle) - keep)]]] + for repl in closing: + await repl.close() + + async def _sweep(self) -> None: + """Close idle REPLs every ``SWEEP_S`` while there are any.""" + while self._repls: + await asyncio.sleep(SWEEP_S) + await self._evict() + @contextlib.asynccontextmanager async def _server(self) -> AsyncIterator[ClientSession]: """``cua_repl`` connected and initialized, its approval requests answered; it ends with the block.""" @@ -168,26 +198,41 @@ async def _pinned_checks(self) -> list[ReadyCheck]: class _Repl: """One ``cua_repl`` server, held open by a task of its own, since a stdio client is entered and left in one task, and reached from any through a queue, one call at a time. A server that stops is started afresh on the next call, - without the last one's bindings.""" + without the last one's bindings; ``close`` ends it for good.""" def __init__(self, open_server: Callable[[], contextlib.AbstractAsyncContextManager[ClientSession]]) -> None: self._open = open_server self._calls: asyncio.Queue = asyncio.Queue() self._task: asyncio.Task | None = None + #: When a call last ended, on the monotonic clock, and how many are queued or running. + self.used, self.busy = time.monotonic(), 0 async def call(self, tool: str, arguments: dict, meta: dict) -> CallToolResult: if self._task is None or self._task.done(): self._task = asyncio.create_task(self._serve()) answer = asyncio.get_running_loop().create_future() - await self._calls.put((tool, arguments, meta, answer)) - return await answer + self.busy += 1 + try: + await self._calls.put((tool, arguments, meta, answer)) + return await answer + finally: + self.busy -= 1 + self.used = time.monotonic() + + async def close(self) -> None: + """The server ended, its process with it, waiting at most ``CLOSE_TIMEOUT_S`` before cancelling it.""" + if self._task is None or self._task.done(): + return + await self._calls.put(None) + with contextlib.suppress(TimeoutError): + await asyncio.wait_for(self._task, CLOSE_TIMEOUT_S) async def _serve(self) -> None: stopped: BaseException = ConnectionError("cua_repl stopped") try: async with self._open() as session: - while True: - tool, arguments, meta, answer = await self._calls.get() + while (call := await self._calls.get()) is not None: + tool, arguments, meta, answer = call try: result = await session.call_tool(tool, arguments, meta=meta) except McpError as e: @@ -203,8 +248,8 @@ async def _serve(self) -> None: stopped = e finally: while not self._calls.empty(): - *_, answer = self._calls.get_nowait() - _settle(answer, error=stopped) + if (call := self._calls.get_nowait()) is not None: + _settle(call[-1], error=stopped) def _settle(answer: asyncio.Future, *, result: Any = None, error: BaseException | None = None) -> None: diff --git a/tst/unit/drivers/codex_computer_use_driver_test.py b/tst/unit/drivers/codex_computer_use_driver_test.py index 77cf7ab92..9f64563cc 100644 --- a/tst/unit/drivers/codex_computer_use_driver_test.py +++ b/tst/unit/drivers/codex_computer_use_driver_test.py @@ -6,10 +6,12 @@ executables, the tools cua_repl lists and that it answers.""" from __future__ import annotations +import asyncio import contextlib import hashlib import importlib.util import json +import os import sys import textwrap from pathlib import Path @@ -145,6 +147,16 @@ def _text(result) -> str: return " ".join(c.text for c in result.content if c.type == "text") +async def _gone(pid: int) -> bool: + for _ in range(200): + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + await asyncio.sleep(0.01) + return False + + @pytest.mark.asyncio async def test_agents_get_codexs_js_and_js_reset_as_pinned_with_its_instructions_and_not_the_apps_own_tools( module, monkeypatch): @@ -155,6 +167,8 @@ async def test_agents_get_codexs_js_and_js_reset_as_pinned_with_its_instructions pinned = {t["name"]: t for t in module._manifest()["tools"]} assert {"js", "js_reset"} <= set(listed) and not {"js_add_node_module_dir", "turn_ended"} & set(listed) assert listed["js"].inputSchema == pinned["js"]["inputSchema"] + assert listed["js"].annotations.readOnlyHint is False and listed["js"].annotations.destructiveHint is True + assert listed["js_reset"].annotations.readOnlyHint is True assert listed["js"].description == pinned["js"]["description"] assert driver.mcp.instructions.startswith(module._manifest()["instructions"]) @@ -244,3 +258,31 @@ async def test_readiness_checks_the_pinned_executables_the_tools_cua_repl_lists_ ("Codex answers", True), ("macOS permissions", True)] assert not tools.ok and tools.detail == "these tools differ from mcp.json: turn_ended" assert not changed.ok and "Codex Computer Use.app/Contents/MacOS/SkyComputerUseService" in changed.detail + + +@pytest.mark.asyncio +async def test_past_the_most_repls_kept_the_least_recently_used_is_closed_and_the_others_keep_their_bindings( + module, monkeypatch): + monkeypatch.setattr(module, "MAX_REPLS", 2) + async with mcp_session(_driver(module, monkeypatch)) as session: + async def js(caller: str, code: str) -> str: + return _text(await session.call_tool("js", {"code": code}, meta={SESSION_META_KEY: caller})) + + for caller in "abc": + await js(caller, f"set {caller}") + b, c, a = await js("b", "get"), await js("c", "get"), await js("a", "get") + + assert a.startswith("None in ") and b.startswith("b in ") and c.startswith("c in ") + + +@pytest.mark.asyncio +async def test_a_repl_left_idle_is_closed_without_another_call_and_its_session_starts_afresh(module, monkeypatch): + monkeypatch.setattr(module, "SWEEP_S", 0.05) + monkeypatch.setattr(module, "IDLE_S", 0.2) + async with mcp_session(_driver(module, monkeypatch)) as session: + await session.call_tool("js", {"code": "set 1"}, meta={SESSION_META_KEY: "a"}) + before = _text(await session.call_tool("js", {"code": "get"}, meta={SESSION_META_KEY: "a"})) + closed = await _gone(int(before.rpartition(" in ")[2])) + after = _text(await session.call_tool("js", {"code": "get"}, meta={SESSION_META_KEY: "a"})) + + assert before.startswith("1 in ") and closed and after.startswith("None in ") From 3e14f5143975078c3e26f0f0cccd6e62b4c8042c Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 8 Nov 2026 20:31:04 -0800 Subject: [PATCH 3/5] fix(drivers): calls a session makes together share one codex-computer-use REPL Closing other sessions' REPLs awaited between finding the caller's REPL and putting it back, so a second call from the same session made a REPL of its own and the first call then replaced it. Eviction now takes REPLs out at once and closes them in the background, so nothing awaits before a call is queued. Co-Authored-By: Claude Opus 5.5 --- .../drivers/envs/codex-computer-use/driver.py | 31 ++++++++++++------- .../drivers/codex_computer_use_driver_test.py | 16 ++++++++++ 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/src/agent_env/drivers/envs/codex-computer-use/driver.py b/src/agent_env/drivers/envs/codex-computer-use/driver.py index 64ce898c0..e0ef1ecde 100644 --- a/src/agent_env/drivers/envs/codex-computer-use/driver.py +++ b/src/agent_env/drivers/envs/codex-computer-use/driver.py @@ -95,6 +95,8 @@ def __init__(self) -> None: #: Each session's REPL, the least recently used first. self._repls: dict[str | None, _Repl] = {} self._sweeper: asyncio.Task | None = None + #: The closes under way, held so they aren't collected before they end. + self._closing: set[asyncio.Task] = set() def forwarded_tools(self) -> list[ForwardedTool]: return [ForwardedTool(EnvironmentTool(name=t["name"], description=t["description"], @@ -116,10 +118,11 @@ async def ready_checks(self) -> list[ReadyCheck]: async def _forward(self, tool: str, /, **arguments: Any) -> CallToolResult: """A call to one of Codex's tools on the caller's own REPL, answered as Codex answers it. Other sessions' REPLs - left idle, or past the most kept, are closed first.""" - repl = self._repls.pop(caller := ToolContext.current().caller.session, None) - await self._evict(keep=MAX_REPLS - 1) - repl = self._repls[caller] = repl or _Repl(self._server) + left idle, or past the most kept, are closed. Nothing here awaits before the call is queued, so calls a session + makes together find one REPL.""" + repl = self._repls.pop(caller := ToolContext.current().caller.session, None) or _Repl(self._server) + self._evict(keep=MAX_REPLS - 1) + self._repls[caller] = repl if self._sweeper is None or self._sweeper.done(): self._sweeper = asyncio.create_task(self._sweep()) try: @@ -128,20 +131,24 @@ async def _forward(self, tool: str, /, **arguments: Any) -> CallToolResult: return CallToolResult(isError=True, content=[TextContent(type="text", text=( f"Codex's cua_repl failed before it answered, so {tool} may have run: {type(e).__name__}: {e}"))]) - async def _evict(self, keep: int = MAX_REPLS) -> None: - """Close the REPLs idle for ``IDLE_S`` and, past ``keep``, the least recently used, none of them in a call.""" + def _evict(self, keep: int = MAX_REPLS) -> None: + """Take out the REPLs idle for ``IDLE_S`` and, past ``keep``, the least recently used, none of them in a call, + and close them in the background.""" now = time.monotonic() - idle = [caller for caller, repl in self._repls.items() if not repl.busy and now - repl.used > IDLE_S] - spare = [caller for caller, repl in self._repls.items() if not repl.busy and caller not in idle] - closing = [self._repls.pop(caller) for caller in [*idle, *spare[:max(0, len(self._repls) - len(idle) - keep)]]] - for repl in closing: - await repl.close() + free = [caller for caller, repl in self._repls.items() if not repl.busy] + idle = [caller for caller in free if now - self._repls[caller].used > IDLE_S] + over = max(0, len(self._repls) - len(idle) - keep) + spare = [caller for caller in free if caller not in idle][:over] + for caller in [*idle, *spare]: + closing = asyncio.create_task(self._repls.pop(caller).close()) + self._closing.add(closing) + closing.add_done_callback(self._closing.discard) async def _sweep(self) -> None: """Close idle REPLs every ``SWEEP_S`` while there are any.""" while self._repls: await asyncio.sleep(SWEEP_S) - await self._evict() + self._evict() @contextlib.asynccontextmanager async def _server(self) -> AsyncIterator[ClientSession]: diff --git a/tst/unit/drivers/codex_computer_use_driver_test.py b/tst/unit/drivers/codex_computer_use_driver_test.py index 9f64563cc..0df9f401f 100644 --- a/tst/unit/drivers/codex_computer_use_driver_test.py +++ b/tst/unit/drivers/codex_computer_use_driver_test.py @@ -286,3 +286,19 @@ async def test_a_repl_left_idle_is_closed_without_another_call_and_its_session_s after = _text(await session.call_tool("js", {"code": "get"}, meta={SESSION_META_KEY: "a"})) assert before.startswith("1 in ") and closed and after.startswith("None in ") + + +@pytest.mark.asyncio +async def test_calls_a_session_makes_together_while_another_repl_closes_share_one_repl( + module, monkeypatch, tmp_path): + monkeypatch.setattr(module, "MAX_REPLS", 1) + async with mcp_session(_driver(module, monkeypatch)) as session: + async def js(caller: str, code: str) -> str: + return _text(await session.call_tool("js", {"code": code}, meta={SESSION_META_KEY: caller})) + + await js("a", "set a") + await asyncio.gather(js("b", "set b"), js("b", "get")) + after = await js("b", "get") + + pids = {call["pid"] for call in _calls(tmp_path) if call["arguments"]["code"] in ("set b", "get")} + assert pids == {int(after.rpartition(" in ")[2])} and after.startswith("b in ") From 37ed4de71195ee7fc98192813f3dfa798dec45c1 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 8 Nov 2026 20:39:03 -0800 Subject: [PATCH 4/5] refactor(drivers): codex-computer-use reads approvals from their metadata and checks its pins agree - Approvals take the app's bundle identifier and name from the request's metadata, and the name its message quotes only beside them, so a reworded message no longer declines every app; a request that names no app is a warning. - computer-ready compares cua_repl's instructions and every field mcp.json pins for its tools, not only their input schemas. - The app's paths are relative to APP, install.sh names the helper's folder once, the README no longer repeats the version, and a unit test checks that env.toml, install.sh and driver.py agree on the version and the helper's path. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/drivers/README.md | 41 +++--- .../drivers/envs/codex-computer-use/driver.py | 82 +++++++----- .../envs/codex-computer-use/install.sh | 11 +- .../drivers/codex_computer_use_driver_test.py | 122 ++++++++++++------ 4 files changed, 159 insertions(+), 97 deletions(-) diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index 83e4ece54..51f57f3ba 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -127,25 +127,28 @@ as `macos-harness`. `envs/codex-computer-use/` is OpenAI's Codex Computer Use as a driver, for the same macOS computers. It is the ChatGPT app's `unified-computer-use` plugin, whose MCP server, `cua_repl`, runs the JavaScript an agent sends against OpenAI's -`cua` API, which reads apps' accessibility trees and acts on them in the background through the plugin's helper app. -ChatGPT 26.930.51102, OpenAI's versioned download checked by sha256, is unpacked beside the driver by `install.sh` and -never run, so it never updates itself, and its helper app is copied to `/opt/agentenv/codex-computer-use/`, where -`[install.macos]` grants it Accessibility, Post Event and Screen Recording, as it grants the driver's own Python. The -driver starts `cua_repl` as the app would, on the app's own Node with the environment the app builds and the computer -surface alone, one server for each agent session, since the REPL keeps its bindings between calls; one that stops is -started afresh on the next call, and nothing says when a session ends, so one left unused for 30 minutes, or past the -eight kept, is closed, the least recently used first. Agents get the server's instructions and its `js` and `js_reset` -tools, as `mcp.json` pins them, `js` marked as acting on apps where Codex marks it read-only; `js_add_node_module_dir` -and `turn_ended`, which it serves the ChatGPT app, are hidden. Codex asks its host -before each action on an app, and the driver answers: it approves the apps its `allowed_apps` setting names, by name or -bundle identifier, case aside, or every app when it names none, and nothing else, recording the computer's audio -included. Each call runs under Codex's own sandbox, read-only but for `~/.codex` and the temporary folders and with no -network, unless its `js_sandbox` setting is off; the apps it drives are outside it, and it still lets the JavaScript -read files and run commands, so a bench that wants the GUI alone guards for that. No ChatGPT sign-in is needed. Its -`computer_*` tools are agent-desktop's, from `MacOSDriver`. `computer-ready` checks the executables' sha256, that -`cua_repl` lists the tools `mcp.json` pins and answers `cua.getState()`, and the TCC permissions. To move it to another -ChatGPT version, pin the zip's URL and sha256 in `env.toml` and the executables' in `driver.py`'s `EXECUTABLES`, and -take `mcp.json` again from what that version's `cua_repl` says at connect and lists. Name it as `codex-computer-use`. +`cua` API, which reads apps' accessibility trees and acts on them in the background through the plugin's helper app. The +ChatGPT version `driver.py` pins as `VERSION`, OpenAI's versioned download checked by sha256, is unpacked beside the +driver by `install.sh` and never run, so it never updates itself, and its helper app is copied to +`/opt/agentenv/codex-computer-use/`, where `[install.macos]` grants it Accessibility, Post Event and Screen Recording, +as it grants the driver's own Python. The driver starts `cua_repl` as the app would, on the app's own Node with the +environment the app builds and the computer surface alone, one server for each agent session, since the REPL keeps its +bindings between calls; one that stops is started afresh on the next call, and nothing says when a session ends, so one +left unused for 30 minutes, or past the eight kept, is closed, the least recently used first. Agents get the server's +instructions and its `js` and `js_reset` tools, as `mcp.json` pins them, `js` marked as acting on apps where Codex marks +it read-only; `js_add_node_module_dir` and `turn_ended`, which it serves the ChatGPT app, are hidden. Codex asks its +host before each action on an app, and the driver answers: it approves the apps its `allowed_apps` setting names, by +name or bundle identifier, case aside, as the request's metadata or its message gives them, or every app when it names +none, and nothing else, recording the computer's audio included, a request that names no app logged as a warning. Each +call runs under Codex's own sandbox, read-only but for `~/.codex` and the temporary folders and with no network, unless +its `js_sandbox` setting is off; the apps it drives are outside it, and it still lets the JavaScript read files and run +commands, so a bench that wants the GUI alone guards for that. No ChatGPT sign-in is needed. Its `computer_*` tools are +agent-desktop's, from `MacOSDriver`. `computer-ready` checks the executables' sha256, that `cua_repl` gives the +instructions and lists the tools `mcp.json` pins and answers `cua.getState()`, and the TCC permissions. To move it to +another ChatGPT version, set `VERSION`, pin the zip's URL and sha256 in `env.toml` and the executables' in `driver.py`'s +`EXECUTABLES` (a unit test checks that `env.toml`, `install.sh` and `driver.py` agree on the version and the helper's +path), and take `mcp.json` again from what that version's `cua_repl` says at connect and lists. Name it as +`codex-computer-use`. This bundle has no tasks of its own: other bundles pull its drivers by alias, and `agent-env run drivers` has nothing to run. diff --git a/src/agent_env/drivers/envs/codex-computer-use/driver.py b/src/agent_env/drivers/envs/codex-computer-use/driver.py index e0ef1ecde..455ce6a9a 100644 --- a/src/agent_env/drivers/envs/codex-computer-use/driver.py +++ b/src/agent_env/drivers/envs/codex-computer-use/driver.py @@ -29,7 +29,8 @@ from mcp import ClientSession, McpError, StdioServerParameters from mcp.client.stdio import stdio_client from mcp.shared.context import RequestContext -from mcp.types import CONNECTION_CLOSED, CallToolResult, ElicitRequestParams, ElicitResult, TextContent +from mcp.types import (CONNECTION_CLOSED, CallToolResult, ElicitRequestParams, ElicitResult, InitializeResult, + TextContent) from agentenv_protocol import EnvironmentTool, ForwardedTool, ToolContext, server_instructions from agentenv_protocol.computer import ComputerSettings, MacOSDriver, ReadyCheck, computer_driver @@ -37,15 +38,15 @@ logger = logging.getLogger(__name__) HERE = Path(__file__).resolve().parent -#: The pinned ChatGPT app, as install.sh unpacks it, and what of it starts Codex's REPL. +#: The pinned ChatGPT app, as install.sh unpacks it, and where in it what starts Codex's REPL is. VERSION = "26.930.51102" APP = HERE / "ChatGPT.app" -RESOURCES = APP / "Contents" / "Resources" -NODE = RESOURCES / "cua_node" / "bin" / "node" -NODE_REPL = RESOURCES / "cua_node" / "bin" / "node_repl" -NODE_MODULES = RESOURCES / "cua_node" / "lib" / "node_modules" +CUA_NODE = Path("Contents", "Resources", "cua_node") +NODE = CUA_NODE / "bin" / "node" +NODE_REPL = CUA_NODE / "bin" / "node_repl" +NODE_MODULES = CUA_NODE / "lib" / "node_modules" LAUNCHER = NODE_MODULES / "@oai" / "cua-repl" / "bin" / "cua-repl.mjs" -CODEX_CLI = RESOURCES / "codex-cli" / "CodexCLI.app" / "Contents" / "MacOS" / "codex" +CODEX_CLI = Path("Contents", "Resources", "codex-cli", "CodexCLI.app", "Contents", "MacOS", "codex") #: Codex Computer Use's helper app, where install.sh copies it and the layer's grant names it. HELPER = Path("/opt/agentenv/codex-computer-use/Codex Computer Use.app") #: The sha256 of each executable at the pinned version, by its path under the app or the helper. @@ -75,8 +76,9 @@ PATH = "/usr/bin:/bin:/usr/sbin:/sbin" #: The JavaScript the readiness check runs: an inventory, which needs no app's approval. PROBE = "await cua.getState({ emit: false }); nodeRepl.write('ready');" -#: The approval Codex asks of its host before it acts on an app, which names it. -APPROVAL = re.compile(r'Allow (?:Computer Use|Codex) to use "?(?P[^"?]+)"?\?') +#: The message of the approval Codex asks of its host before it acts on an app, which names it as the request's +#: metadata does. +APPROVAL =re.compile(r'Allow (?:Computer Use|Codex) to use "?(?P[^"?]+)"?\?') #: The key of the metadata Codex reads its sandbox from on each call. SANDBOX_KEY = "codex/sandbox-state-meta" @@ -151,26 +153,26 @@ async def _sweep(self) -> None: self._evict() @contextlib.asynccontextmanager - async def _server(self) -> AsyncIterator[ClientSession]: - """``cua_repl`` connected and initialized, its approval requests answered; it ends with the block.""" - server = StdioServerParameters(command=str(NODE), args=[str(LAUNCHER)], env=_environment(), + async def _server(self) -> AsyncIterator[tuple[ClientSession, InitializeResult]]: + """``cua_repl`` connected and initialized, with what it said at connect, its approval requests answered; it + ends with the block.""" + server = StdioServerParameters(command=str(APP / NODE), args=[str(APP / LAUNCHER)], env=_environment(), cwd=str(_workspace())) async with (stdio_client(server) as (read, write), ClientSession(read, write, elicitation_callback=self._approve) as session): - await session.initialize() - yield session + yield session, await session.initialize() async def _approve(self, context: RequestContext, params: ElicitRequestParams) -> ElicitResult: """Codex's request to act on an app, approved when ``allowed_apps`` names the app or names none; anything else - it asks is declined.""" + it asks, to record the computer's audio say, is declined.""" message = getattr(params, "message", "") - approved = False - if named := APPROVAL.fullmatch(message.strip()): - meta = params.meta.model_dump() if params.meta else {} - names = {named["app"].strip().lower(), str((meta.get("tool_params") or {}).get("app") or "").lower()} - allowed = self.settings.allowed_apps - approved = allowed is None or bool(names & {app.lower() for app in allowed}) - logger.info("%s %r", "approved" if approved else "declined", message) + names = _app_names(message, params.meta.model_dump() if params.meta else {}) + allowed = self.settings.allowed_apps + approved = bool(names) and (allowed is None or bool(names & {app.lower() for app in allowed})) + if names: + logger.info("%s %r", "approved" if approved else "declined", message) + else: + logger.warning("declined %r, which names no app", message) return ElicitResult(action="accept", content={}) if approved else ElicitResult(action="decline") def _sandbox(self) -> dict: @@ -186,18 +188,20 @@ async def _pinned_checks(self) -> list[ReadyCheck]: if differ: return checks try: - async with self._server() as session: + async with self._server() as (session, initialized): listed = (await session.list_tools()).tools probed = await session.call_tool("js", {"code": PROBE}, meta=self._sandbox()) except Exception as e: return [*checks, ReadyCheck("Codex tools", False, f"cua_repl didn't answer: {type(e).__name__}: {e}")] - running = {t.name: t.inputSchema for t in listed} - pinned = {t["name"]: t["inputSchema"] for t in _manifest()["tools"]} - changed = sorted(name for name in running.keys() | pinned.keys() if running.get(name) != pinned.get(name)) + running = {t.name: t.model_dump(mode="json", exclude_none=True) for t in listed} + pinned = {t["name"]: t for t in _manifest()["tools"]} + changed = ["instructions"] if initialized.instructions != _manifest()["instructions"] else [] + changed += sorted(name for name in running.keys() | pinned.keys() if name not in running or name not in pinned + or any(running[name].get(key) != value for key, value in pinned[name].items())) answer = " ".join(c.text for c in probed.content if isinstance(c, TextContent)) return [*checks, - ReadyCheck("Codex tools", not changed, f"{len(running)} tools, as pinned" if not changed else - f"these tools differ from {MANIFEST.name}: {', '.join(changed)}"), + ReadyCheck("Codex tools", not changed, f"its instructions and {len(running)} tools, as pinned" + if not changed else f"these differ from {MANIFEST.name}: {', '.join(changed)}"), ReadyCheck("Codex answers", not probed.isError and "ready" in answer, "it read the computer's apps" if not probed.isError else f"cua.getState() failed: {answer}")] @@ -207,7 +211,8 @@ class _Repl: and reached from any through a queue, one call at a time. A server that stops is started afresh on the next call, without the last one's bindings; ``close`` ends it for good.""" - def __init__(self, open_server: Callable[[], contextlib.AbstractAsyncContextManager[ClientSession]]) -> None: + def __init__(self, open_server: Callable[[], contextlib.AbstractAsyncContextManager[ + tuple[ClientSession, InitializeResult]]]) -> None: self._open = open_server self._calls: asyncio.Queue = asyncio.Queue() self._task: asyncio.Task | None = None @@ -237,7 +242,7 @@ async def close(self) -> None: async def _serve(self) -> None: stopped: BaseException = ConnectionError("cua_repl stopped") try: - async with self._open() as session: + async with self._open() as (session, _): while (call := await self._calls.get()) is not None: tool, arguments, meta, answer = call try: @@ -273,6 +278,15 @@ def _manifest() -> dict: return json.loads(MANIFEST.read_text()) +def _app_names(message: str, meta: dict) -> set[str]: + """The app an approval request is for, case aside: its bundle identifier and name as the request's metadata gives + them, and the name its message quotes; none for a request that isn't for an app.""" + shown = [field.get("value") for field in meta.get("tool_params_display") or [] if field.get("name") == "app"] + named = APPROVAL.fullmatch(message.strip()) + names = [(meta.get("tool_params") or {}).get("app"), *shown, named and named["app"]] + return {str(name).strip().lower() for name in names if name} + + def _codex_home() -> Path: return Path.home() / ".codex" @@ -296,16 +310,16 @@ def _workspace() -> Path: def _environment() -> dict[str, str]: """``cua_repl``'s environment, as the ChatGPT app builds it for the plugin, with the computer surface alone.""" - codex, modules = str(_codex_home()), str(NODE_MODULES) + codex, modules = str(_codex_home()), str(APP / NODE_MODULES) return {**{name: os.environ[name] for name in KEPT if name in os.environ}, "PATH": PATH, "CODEX_HOME": codex, - "CODEX_CLI_PATH": str(CODEX_CLI), + "CODEX_CLI_PATH": str(APP / CODEX_CLI), "CUA_REPL_ENABLED_SURFACES": "computer", - "CUA_REPL_NODE_REPL_PATH": str(NODE_REPL), + "CUA_REPL_NODE_REPL_PATH": str(APP / NODE_REPL), "NODE_REPL_INSTRUCTIONS_USE_CASE_COMPUTER_USE": "Control desktop apps on macOS through Computer Use.", "NODE_REPL_NATIVE_PIPE_CONNECT_TIMEOUT_MS": "1000", "NODE_REPL_NODE_MODULE_DIRS": modules, - "NODE_REPL_NODE_PATH": str(NODE), + "NODE_REPL_NODE_PATH": str(APP / NODE), "NODE_REPL_TRUSTED_CODE_PATHS": f"{codex}:{modules}", "NODE_REPL_TRUSTED_SERVICES": json.dumps({"sky": "@oai/sky/service"}), "SKY_CUA_SERVICE_PATH": str(HELPER)} diff --git a/src/agent_env/drivers/envs/codex-computer-use/install.sh b/src/agent_env/drivers/envs/codex-computer-use/install.sh index e4bd7e816..ecdd32d1b 100755 --- a/src/agent_env/drivers/envs/codex-computer-use/install.sh +++ b/src/agent_env/drivers/envs/codex-computer-use/install.sh @@ -5,8 +5,9 @@ set -eu ditto -x -k ChatGPT.zip . rm -f ChatGPT.zip -helper="ChatGPT.app/Contents/Resources/cua_node/lib/node_modules/@oai/sky/Codex Computer Use.app" -mkdir -p /opt/agentenv/codex-computer-use -rm -rf "/opt/agentenv/codex-computer-use/Codex Computer Use.app" -ditto "$helper" "/opt/agentenv/codex-computer-use/Codex Computer Use.app" -xattr -dr com.apple.quarantine ChatGPT.app /opt/agentenv/codex-computer-use || true +helper="Codex Computer Use.app" +dest=/opt/agentenv/codex-computer-use +mkdir -p "$dest" +rm -rf "$dest/$helper" +ditto "ChatGPT.app/Contents/Resources/cua_node/lib/node_modules/@oai/sky/$helper" "$dest/$helper" +xattr -dr com.apple.quarantine ChatGPT.app "$dest" || true diff --git a/tst/unit/drivers/codex_computer_use_driver_test.py b/tst/unit/drivers/codex_computer_use_driver_test.py index 0df9f401f..69398edbd 100644 --- a/tst/unit/drivers/codex_computer_use_driver_test.py +++ b/tst/unit/drivers/codex_computer_use_driver_test.py @@ -1,9 +1,10 @@ """The codex-computer-use driver against a fake cua_repl: agents get its js and js_reset as pinned, with its instructions, and not the tools it serves only the ChatGPT app; each call carries Codex's read-only sandbox unless the -setting turns it off; Codex's approval requests are answered from allowed_apps by name or bundle identifier, every app -approved when it names none, and nothing else; each agent session keeps a REPL of its own across calls, one that stops -starting afresh; cua_repl gets the app's environment and none of the driver's; and readiness checks the pinned -executables, the tools cua_repl lists and that it answers.""" +setting turns it off; Codex's approval requests are answered from allowed_apps by name or bundle identifier, from the +request's metadata or its message, every app approved when it names none, and nothing else; each agent session keeps a +REPL of its own across calls, one that stops starting afresh; cua_repl gets the app's environment and none of the +driver's; readiness checks the pinned executables, what cua_repl says and lists and that it answers; and the driver's +files agree on the pinned version.""" from __future__ import annotations import asyncio @@ -14,6 +15,7 @@ import os import sys import textwrap +import tomllib from pathlib import Path import pytest @@ -40,12 +42,14 @@ @server.list_tools() async def tools(): - return [types.Tool(name=t["name"], description=t["description"], inputSchema=t["inputSchema"]) - for t in manifest["tools"]] + return [types.Tool(**t) for t in manifest["tools"]] - async def ask(message, app): + async def ask(message, tool=None, app=None, shown=None): context = server.request_context - meta = types.RequestParams.Meta(**({"tool_params": {"app": app}} if app else {})) + meta = types.RequestParams.Meta(**({} if tool is None else { + "codex_approval_kind": "mcp_tool_call", "connector_id": "computer-use", "tool_name": tool, + "tool_params": {"app": app} if app else {}, + "tool_params_display": [{"display_name": "App", "name": "app", "value": shown}] if shown else []})) result = await context.session.send_request( types.ServerRequest(types.ElicitRequest(params=types.ElicitRequestFormParams( message=message, requestedSchema={"type": "object", "properties": {}}, _meta=meta))), @@ -63,10 +67,15 @@ async def call(name, arguments): bindings.clear() text = "reset" elif code.startswith("app "): - _, display, bundle = code.split(" ") - text = await ask(f'Allow Computer Use to use "{display}"?', bundle) + _, shown, bundle = code.split(" ") + text = await ask(f'Allow Computer Use to use "{shown}"?', "get_app_state", bundle, shown) + elif code.startswith("reworded "): + _, shown, bundle = code.split(" ") + text = await ask(f"Codex would like to control {shown}.", "get_app_state", bundle, shown) + elif code.startswith("bare "): + text = await ask(f'Allow Computer Use to use "{code[5:]}"?') elif code == "audio": - text = await ask("Allow Computer Use to record computer audio?", None) + text = await ask("Allow Computer Use to record computer audio?", "start_audio_recording") elif code.startswith("set "): bindings["x"] = code[4:] text = "set" @@ -95,7 +104,8 @@ def missing(self) -> list[str]: @pytest.fixture -def module(monkeypatch, tmp_path): +def pinned(monkeypatch, tmp_path): + """The driver's module, as it ships.""" monkeypatch.delenv("AGENTENV_COMPUTER_SETTINGS", raising=False) monkeypatch.setenv("HOME", str(tmp_path)) monkeypatch.setenv("AGENTENV_CONTROL_TOKEN", "a-deploys-secret") @@ -105,26 +115,31 @@ def module(monkeypatch, tmp_path): spec = importlib.util.spec_from_file_location("codex_computer_use_driver", DRIVER) loaded = importlib.util.module_from_spec(spec) spec.loader.exec_module(loaded) + return loaded + + +@pytest.fixture +def module(pinned, monkeypatch, tmp_path): + """The driver's module on a fake ChatGPT app whose Node is the fake cua_repl.""" (tmp_path / "fake").mkdir() - (tmp_path / "fake" / "tools.json").write_text(loaded.MANIFEST.read_text()) + (tmp_path / "fake" / "tools.json").write_text(pinned.MANIFEST.read_text()) app, helper = tmp_path / "ChatGPT.app", tmp_path / "Codex Computer Use.app" - node = app / "Contents" / "Resources" / "cua_node" / "bin" / "node" + node = app / pinned.NODE node.parent.mkdir(parents=True) node.write_text(f"#!{sys.executable}\n{FAKE}") node.chmod(0o755) - pinned = {} - for path in loaded.EXECUTABLES: + digests = {} + for path in pinned.EXECUTABLES: root, _, rest = path.partition("/") file = (app if root == "ChatGPT.app" else helper) / rest if not file.exists(): file.parent.mkdir(parents=True, exist_ok=True) file.write_text(f"the pinned {file.name}") - pinned[path] = hashlib.sha256(file.read_bytes()).hexdigest() - monkeypatch.setattr(loaded, "APP", app) - monkeypatch.setattr(loaded, "HELPER", helper) - monkeypatch.setattr(loaded, "NODE", node) - monkeypatch.setattr(loaded, "EXECUTABLES", pinned) - return loaded + digests[path] = hashlib.sha256(file.read_bytes()).hexdigest() + monkeypatch.setattr(pinned, "APP", app) + monkeypatch.setattr(pinned, "HELPER", helper) + monkeypatch.setattr(pinned, "EXECUTABLES", digests) + return pinned def _driver(module, monkeypatch, **settings): @@ -195,18 +210,23 @@ async def test_each_call_runs_in_codexs_read_only_sandbox_unless_the_setting_tur @pytest.mark.asyncio async def test_approvals_follow_allowed_apps_by_name_or_bundle_identifier_and_nothing_else_is_approved( - module, monkeypatch): + module, monkeypatch, caplog): async with mcp_session(_driver(module, monkeypatch, allowed_apps=["calculator", "com.apple.TextEdit"])) as session: - named = await session.call_tool("js", {"code": "app Calculator com.apple.calculator"}) - by_id = await session.call_tool("js", {"code": "app TextEdit com.apple.TextEdit"}) - other = await session.call_tool("js", {"code": "app Finder com.apple.finder"}) - audio = await session.call_tool("js", {"code": "audio"}) + async def js(code: str) -> str: + return _text(await session.call_tool("js", {"code": code})) + + named, by_id = await js("app Calculator com.apple.calculator"), await js("app TextEdit com.apple.TextEdit") + other, audio = await js("app Finder com.apple.finder"), await js("audio") + reworded, bare = await js("reworded Calculator com.apple.calc"), await js("bare Calculator") async with mcp_session(_driver(module, monkeypatch)) as session: - any_app = await session.call_tool("js", {"code": "app Finder com.apple.finder"}) - no_audio = await session.call_tool("js", {"code": "audio"}) + any_app = _text(await session.call_tool("js", {"code": "app Finder com.apple.finder"})) + no_audio = _text(await session.call_tool("js", {"code": "audio"})) - assert [_text(r) for r in (named, by_id, other, audio)] == ["accept", "accept", "decline", "decline"] - assert [_text(r) for r in (any_app, no_audio)] == ["accept", "decline"] + assert [named, by_id, other, audio] == ["accept", "accept", "decline", "decline"] + assert [reworded, bare] == ["accept", "accept"] + assert [any_app, no_audio] == ["accept", "decline"] + warned = [r.getMessage() for r in caplog.records if r.name == module.logger.name and r.levelname == "WARNING"] + assert len(warned) == 2 and all("record computer audio" in message for message in warned) @pytest.mark.asyncio @@ -233,33 +253,57 @@ async def test_cua_repl_gets_the_apps_environment_and_none_of_the_drivers(module (env_file,) = (tmp_path / "fake").glob("env-*.json") env = json.loads(env_file.read_text()) - modules = str(module.NODE_MODULES) + modules = str(module.APP / module.NODE_MODULES) assert "AGENTENV_CONTROL_TOKEN" not in env and env["PATH"] == "/usr/bin:/bin:/usr/sbin:/sbin" assert env["CUA_REPL_ENABLED_SURFACES"] == "computer" and env["SKY_CUA_SERVICE_PATH"] == str(module.HELPER) assert env["NODE_REPL_TRUSTED_SERVICES"] == '{"sky": "@oai/sky/service"}' assert env["NODE_REPL_TRUSTED_CODE_PATHS"] == f"{tmp_path / '.codex'}:{modules}" - assert env["CODEX_HOME"] == str(tmp_path / ".codex") and env["NODE_REPL_NODE_PATH"] == str(module.NODE) + assert env["CODEX_HOME"] == str(tmp_path / ".codex") and env["NODE_REPL_NODE_PATH"] == str(module.APP / module.NODE) @pytest.mark.asyncio -async def test_readiness_checks_the_pinned_executables_the_tools_cua_repl_lists_and_that_it_answers( +async def test_readiness_checks_the_pinned_executables_what_cua_repl_says_and_lists_and_that_it_answers( module, monkeypatch, tmp_path): driver = _driver(module, monkeypatch) + manifest = module._manifest() + + async def tools_check(served: dict): + (tmp_path / "fake" / "tools.json").write_text(json.dumps(served)) + driver._pinned = None + return (await driver.ready_checks())[1] + ready = await driver.ready_checks() - three = {**module._manifest(), "tools": module._manifest()["tools"][:3]} - (tmp_path / "fake" / "tools.json").write_text(json.dumps(three)) - driver._pinned = None - tools = (await driver.ready_checks())[1] + fewer = await tools_check({**manifest, "tools": manifest["tools"][:3]}) + described = await tools_check({"instructions": "other", "tools": [ + {**t, "description": "other"} if t["name"] == "js" else t for t in manifest["tools"]]}) + hinted = await tools_check({**manifest, "tools": [ + {**t, "annotations": {}} if t["name"] == "js_reset" else t for t in manifest["tools"]]}) (module.HELPER / "Contents" / "MacOS" / "SkyComputerUseService").write_text("another build") driver._pinned = None (changed, *_) = await driver.ready_checks() assert [(c.name, c.ok) for c in ready] == [("Codex Computer Use", True), ("Codex tools", True), ("Codex answers", True), ("macOS permissions", True)] - assert not tools.ok and tools.detail == "these tools differ from mcp.json: turn_ended" + assert (fewer.ok, fewer.detail) == (False, "these differ from mcp.json: turn_ended") + assert (described.ok, described.detail) == (False, "these differ from mcp.json: instructions, js") + assert (hinted.ok, hinted.detail) == (False, "these differ from mcp.json: js_reset") assert not changed.ok and "Codex Computer Use.app/Contents/MacOS/SkyComputerUseService" in changed.detail +def test_env_toml_install_sh_and_the_driver_agree_on_the_pinned_version_and_the_helpers_path(pinned): + """A move to another ChatGPT version, or the helper to another folder, has to change them together.""" + folder = DRIVER.parent + macos = tomllib.loads((folder / "env.toml").read_text())["install"]["macos"] + install = (folder / "install.sh").read_text() + + (download,) = macos["downloads"] + assert download["url"].endswith(f"/ChatGPT-darwin-arm64-{pinned.VERSION}.zip") + assert [grant["app"] for grant in macos["grants"] if "app" in grant] == [str(pinned.HELPER)] + assert f'helper="{pinned.HELPER.name}"\n' in install and f"dest={pinned.HELPER.parent}\n" in install + assert pinned.APP == folder / "ChatGPT.app" and all( + path.startswith(("ChatGPT.app/", f"{pinned.HELPER.name}/")) for path in pinned.EXECUTABLES) + + @pytest.mark.asyncio async def test_past_the_most_repls_kept_the_least_recently_used_is_closed_and_the_others_keep_their_bindings( module, monkeypatch): From cfca9f601dd5f3bea0ee0bc8c0e313d386e3beaf Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 8 Nov 2026 21:31:50 -0800 Subject: [PATCH 5/5] fix(drivers): a codex-computer-use call right after its REPL stops starts a new one A stopped server was still ending when the session's next call came, so the call joined its queue and failed with the error it ended on, as on Linux CI. Each server now has its own queue, let go as soon as it takes no more calls, so the next call starts a server afresh; close ends every server still ending. The test slows the old server's end so the next call always arrives meanwhile. Co-Authored-By: Claude Opus 5.5 --- .../drivers/envs/codex-computer-use/driver.py | 76 +++++++++++-------- .../drivers/codex_computer_use_driver_test.py | 14 +++- 2 files changed, 59 insertions(+), 31 deletions(-) diff --git a/src/agent_env/drivers/envs/codex-computer-use/driver.py b/src/agent_env/drivers/envs/codex-computer-use/driver.py index 455ce6a9a..ea6773c42 100644 --- a/src/agent_env/drivers/envs/codex-computer-use/driver.py +++ b/src/agent_env/drivers/envs/codex-computer-use/driver.py @@ -207,62 +207,78 @@ async def _pinned_checks(self) -> list[ReadyCheck]: class _Repl: - """One ``cua_repl`` server, held open by a task of its own, since a stdio client is entered and left in one task, - and reached from any through a queue, one call at a time. A server that stops is started afresh on the next call, - without the last one's bindings; ``close`` ends it for good.""" + """A session's ``cua_repl``, one server at a time, each held open by a task of its own, since a stdio client is + entered and left in one task, and reached from any through that server's queue, one call at a time. A server that + stops is let go at once, while it still ends, and the next call starts one afresh, without the last one's bindings; + ``close`` ends them for good.""" def __init__(self, open_server: Callable[[], contextlib.AbstractAsyncContextManager[ tuple[ClientSession, InitializeResult]]]) -> None: self._open = open_server - self._calls: asyncio.Queue = asyncio.Queue() - self._task: asyncio.Task | None = None + #: The queue of the server taking calls, None when none is; and the tasks of every server not yet ended. + self._calls: asyncio.Queue | None = None + self._serving: set[asyncio.Task] = set() #: When a call last ended, on the monotonic clock, and how many are queued or running. self.used, self.busy = time.monotonic(), 0 async def call(self, tool: str, arguments: dict, meta: dict) -> CallToolResult: - if self._task is None or self._task.done(): - self._task = asyncio.create_task(self._serve()) + if self._calls is None: + self._calls = asyncio.Queue() + serving = asyncio.create_task(self._serve(self._calls)) + self._serving.add(serving) + serving.add_done_callback(self._serving.discard) answer = asyncio.get_running_loop().create_future() self.busy += 1 try: - await self._calls.put((tool, arguments, meta, answer)) + self._calls.put_nowait((tool, arguments, meta, answer)) return await answer finally: self.busy -= 1 self.used = time.monotonic() async def close(self) -> None: - """The server ended, its process with it, waiting at most ``CLOSE_TIMEOUT_S`` before cancelling it.""" - if self._task is None or self._task.done(): - return - await self._calls.put(None) - with contextlib.suppress(TimeoutError): - await asyncio.wait_for(self._task, CLOSE_TIMEOUT_S) - - async def _serve(self) -> None: + """Every server ended, its process with it, waiting at most ``CLOSE_TIMEOUT_S`` before cancelling one.""" + if self._calls is not None: + self._calls.put_nowait(None) + self._calls = None + if serving := set(self._serving): + _, late = await asyncio.wait(serving, timeout=CLOSE_TIMEOUT_S) + for task in late: + task.cancel() + + async def _serve(self, calls: asyncio.Queue) -> None: stopped: BaseException = ConnectionError("cua_repl stopped") try: async with self._open() as (session, _): - while (call := await self._calls.get()) is not None: - tool, arguments, meta, answer = call - try: - result = await session.call_tool(tool, arguments, meta=meta) - except McpError as e: - _settle(answer, error=e) - if e.error.code == CONNECTION_CLOSED: + try: + while (call := await calls.get()) is not None: + tool, arguments, meta, answer = call + try: + result = await session.call_tool(tool, arguments, meta=meta) + except McpError as e: + _settle(answer, error=e) + if e.error.code == CONNECTION_CLOSED: + raise + continue + except Exception as e: + _settle(answer, error=e) raise - continue - except Exception as e: - _settle(answer, error=e) - raise - _settle(answer, result=result) + _settle(answer, result=result) + finally: + self._let_go(calls) except Exception as e: stopped = e finally: - while not self._calls.empty(): - if (call := self._calls.get_nowait()) is not None: + self._let_go(calls) + while not calls.empty(): + if (call := calls.get_nowait()) is not None: _settle(call[-1], error=stopped) + def _let_go(self, calls: asyncio.Queue) -> None: + """The next call to a new server, not to this one, which takes no more.""" + if self._calls is calls: + self._calls = None + def _settle(answer: asyncio.Future, *, result: Any = None, error: BaseException | None = None) -> None: if answer.done(): diff --git a/tst/unit/drivers/codex_computer_use_driver_test.py b/tst/unit/drivers/codex_computer_use_driver_test.py index 69398edbd..d7efc5bad 100644 --- a/tst/unit/drivers/codex_computer_use_driver_test.py +++ b/tst/unit/drivers/codex_computer_use_driver_test.py @@ -230,8 +230,20 @@ async def js(code: str) -> str: @pytest.mark.asyncio -async def test_each_agent_session_keeps_a_repl_of_its_own_and_one_that_stops_starts_afresh( +async def test_each_agent_session_keeps_a_repl_of_its_own_and_one_that_stops_starts_afresh_while_it_still_ends( module, monkeypatch, tmp_path): + """A server that stops takes a while to end; the session's next call starts a new one rather than wait on it.""" + serve = module.CodexComputerUse._server + + @contextlib.asynccontextmanager + async def slow_to_end(self): + async with serve(self) as opened: + try: + yield opened + finally: + await asyncio.sleep(0.5) + + monkeypatch.setattr(module.CodexComputerUse, "_server", slow_to_end) async with mcp_session(_driver(module, monkeypatch)) as session: for caller, value in (("a", "1"), ("b", "2")): await session.call_tool("js", {"code": f"set {value}"}, meta={SESSION_META_KEY: caller})