From 2731d7a9a44602cf709338d74285bbaf12154229 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 9 Nov 2026 10:27:33 -0800 Subject: [PATCH 1/2] feat(drivers): cua-driver on Omarchy: its install, Omarchy's shortcuts in the computer skill, and a Hyprland crash as an infrastructure failure cua-driver's supports names Omarchy's Wayland desktop on amd64, which installs [install."linux/omarchy"]: requirements-omarchy.txt, with cua-driver at Hyprland's own pin (0.44.0), and the two services; Omarchy's base carries the Hyprland plugin. The computer skill's shortcuts are keyed by platform, a distribution's replacing its OS type's, and Omarchy's are its Super chords: workspaces, tiling, the launcher, its menu and its universal copy and paste. On Hyprland cua-driver's notes say that only Super chords reach the desktop's shortcuts. When another Hyprland runs in place of the one the driver started with, as Hyprland's watchdog brings one back in safe mode after a crash, the plugin check fails for good (final) and every input fails as an infrastructure failure. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 5 +- .../src/agentenv_protocol/computer/skill.py | 19 +++++- .../tests/test_computer_driver.py | 12 ++++ src/agent_env/drivers/README.md | 42 +++++++------ .../drivers/envs/cua-driver/driver.py | 3 + .../drivers/envs/cua-driver/env.toml | 7 +++ .../drivers/envs/cua-driver/hyprland.py | 25 ++++++-- .../envs/cua-driver/requirements-omarchy.txt | 7 +++ tst/unit/drivers/cua_driver_test.py | 60 +++++++++++++++++-- 9 files changed, 147 insertions(+), 33 deletions(-) create mode 100644 src/agent_env/drivers/envs/cua-driver/requirements-omarchy.txt diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 9b2a47067..ce561f047 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -587,8 +587,9 @@ driver has them, through the `served_tools`, `served_extensions` and `served_ope features, among them `screen`: `desktop`, or `page` for a driver whose screen is a website browser's active tab, which a computer env pairs with exactly one website browser. The driver serves its skill through `skill/v1` (an `@agent_skill` handler on the base class): a SKILL.md written from the computer as it booted (its platform, session user and home, - the live screen size, what its input takes, its OS's shortcuts) and ending with the driver - class's `notes`, markdown for what its card can't say. A driver whose software ships a skill of + the live screen size, what its input takes, its platform's shortcuts, a distribution's where it + has its own, as Omarchy does) and ending with the driver class's `notes`, markdown for what its + card can't say. A driver whose software ships a skill of its own returns it from `bundled_skill()` (its description, body and other files), and the computer skill leads with it, the computer's part following. - **Framework extensions.** `computer-ready/v1` (`status`: real checks of the display and a diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/skill.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/skill.py index 63b28b417..2085b3ea4 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/skill.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/skill.py @@ -10,11 +10,13 @@ from typing import TYPE_CHECKING from .keys import PRINTABLE_KEYS +from .platform import entry_for, platform_key if TYPE_CHECKING: from .driver import ComputerDriverEnvironment -#: The chords each OS's desktops keep for the everyday actions, as ``computer_key`` takes them. +#: The chords each platform's desktops keep for the everyday actions, as ``computer_key`` takes them, by platform key: a +#: distribution's replace its OS type's. SHORTCUTS = { "linux": """\ - Copy, cut and paste: `['Control', 'c']`, `['Control', 'x']`, `['Control', 'v']`; in a terminal, with `'Shift'` too. @@ -22,6 +24,19 @@ - Switch windows `['Alt', 'Tab']`; close the focused one `['Alt', 'F4']`. - In a file manager, `['Control', 'l']` takes a typed path, `['F2']` renames and `['Control', 'Shift', 'n']` makes a folder. +""", + "linux/omarchy": """\ +- Workspaces: `['Super', '1']` to `['Super', '9']` switch to one; `['Super', 'Shift', '1']` to `['Super', 'Shift', '9']` + move the focused window there. +- Windows tile side by side. `['Super', 'ArrowLeft']` and the other arrows focus the window that way, and with + `'Shift'` too move it there; `['Super', 'w']` closes the focused window, `['Super', 'f']` makes it full screen and + `['Super', 't']` floats or tiles it. +- `['Super', 'Enter']` opens a terminal, `['Super', 'Shift', 'f']` the file manager and `['Super', 'Shift', 'b']` the + browser; `['Super', 'Alt', ' ']` the app launcher, where a typed name and Enter start an app; `['Super', ' ']` + Omarchy's menu; `['Super', 'k']` the list of every shortcut. +- In apps: copy, cut and paste `['Control', 'c']`, `['Control', 'x']`, `['Control', 'v']`, or in any window, a terminal + too, `['Super', 'c']`, `['Super', 'x']`, `['Super', 'v']`; undo `['Control', 'z']`, save `['Control', 's']`, select + all `['Control', 'a']`, find `['Control', 'f']`. """, } _SESSIONS = {"x11": "an X11 session", "wayland": "a Wayland session"} @@ -56,7 +71,7 @@ async def skill(driver: ComputerDriverEnvironment) -> tuple[str, str, dict[str, "## Input", _input(driver), ] - if shortcuts := SHORTCUTS.get(platform.os or ""): + if shortcuts := entry_for(SHORTCUTS, platform_key(platform.os, platform.distribution)): sections += ["## Shortcuts", shortcuts.rstrip()] if notes := driver.notes.strip(): sections += ["## Notes", notes] diff --git a/packages/agentenv-protocol/tests/test_computer_driver.py b/packages/agentenv-protocol/tests/test_computer_driver.py index 470283cd4..c447ff478 100644 --- a/packages/agentenv-protocol/tests/test_computer_driver.py +++ b/packages/agentenv-protocol/tests/test_computer_driver.py @@ -221,6 +221,18 @@ async def test_the_skill_is_written_from_the_computer_as_it_booted_under_the_nam assert refused.status_code == 400 and "lowercase letters, digits and single hyphens" in refused.text +@pytest.mark.asyncio +async def test_the_skills_shortcuts_are_its_distributions_own_where_it_has_them(monkeypatch): + monkeypatch.setattr("agentenv_protocol.computer.skill.getpass.getuser", lambda: "omarchy") + omarchy = Platform(os="linux", distribution="omarchy", version="4.5.0", arch="amd64", display="wayland") + async with http_client(FakeDriver(platform=omarchy)) as http: + skill_md = (await http.get("/agentenv/ext/skill/get", params={"name": "computer-desktop"})).json()["skill_md"] + + shortcuts = skill_md.partition("## Shortcuts")[2] + assert "- Omarchy 4.5.0 (amd64), a Wayland session" in skill_md + assert "`['Super', 'Alt', ' ']` the app launcher" in shortcuts and "`['Alt', 'Tab']`" not in skill_md + + @pytest.mark.asyncio async def test_a_driver_whose_software_ships_a_skill_leads_with_it_and_the_computers_part_follows(monkeypatch): class Bundling(FakeDriver): diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index df95be372..a1066acab 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -80,11 +80,12 @@ cua-driver's UI Automation tree, whose walk of it fails. On a Hyprland desktop (Wayland), as Omarchy's is, cua-driver runs its Wayland backend (`CUA_DRIVER_RS_ENABLE_WAYLAND=1`) and sends input through the seats of Hyprland's `cua-hyprland-plugin`. Hyprland's session turns this on -(`HYPRLAND_INSTANCE_SIGNATURE`), not any Wayland one, since the driver reaches Hyprland's IPC; `supports` names no -Wayland desktop yet, so gate 1 doesn't match it to one. It runs cua-driver 0.44.0, its own pin, since 0.38.0 refuses -background input there, the delivery cua-driver's own tools default to: `computer-ready` checks its wheel's executable -(`hyprland/amd64` in `EXECUTABLES`) and the tools `mcp-hyprland.json` pins, while `requirements.txt` keeps 0.38.0 for -the other desktops, so a Hyprland computer installs 0.44.0 itself. The daemon starts once the compositor's socket +(`HYPRLAND_INSTANCE_SIGNATURE`), not any Wayland one, since the driver reaches Hyprland's IPC; `supports` names +Omarchy's Wayland desktop on amd64, which installs `[install."linux/omarchy"]`: `requirements-omarchy.txt` and the two +services, with no Hyprland plugin, which Omarchy's base carries. It runs cua-driver 0.44.0, its own pin, since 0.38.0 +refuses background input there, the delivery cua-driver's own tools default to: `computer-ready` checks its wheel's +executable (`hyprland/amd64` in `EXECUTABLES`) and the tools `mcp-hyprland.json` pins, which `requirements-omarchy.txt` +installs, while `requirements.txt` keeps 0.38.0 for the other desktops. The daemon starts once the compositor's socket answers, on a socket in the session user's runtime folder. What the driver asks Hyprland (`hyprctl -j`) is in `hyprland.py`, beside `driver.py`. Text goes to the desktop in the foreground, where Hyprland's shortcuts, a menu holding the keyboard and non-ASCII text all work. A key goes to the window Hyprland has focused, in the foreground, @@ -92,23 +93,26 @@ where Backspace, the arrows and Home/End edit text as a person's keys do, which a US keyboard, the only layout the plugin takes there, so a shifted character goes as Shift and the key it's on. Super and keys held with it, Hyprland's own shortcuts, go to the desktop, as does a key with nothing focused and one the window path refuses because a layer such as Omarchy's menu holds the keyboard, punctuation there by its keysym name: -given the character, cua-driver falls back to libei, which can hang its input worker. Clicks, drags and scrolls stay -desktop-scope. Keys aimed at a window need the seat's keymap to be the US layout's, and an input method such as -Omarchy's fcitx5 takes on the keymap that desktop text uploads, after which cua-driver refuses them -(`unsupported_layout`): the driver then puts the layout back with a chord of the layout's own keys at the desktop, -Super+Ctrl+Alt+Shift+F12, which it binds to nothing in Hyprland first (`hyprctl eval`), so Hyprland keeps it from every -window, and sends the key again. It presses the chord only while its own bind alone has it: with the chord bound to -anything else, the key fails as cua-driver refused it. `computer-ready` also checks that the plugin is loaded, enabled, -built for this Hyprland and taking input (`transport.ready`), and every input, a `computer_*` operation or one of -cua-driver's own tools that isn't read-only, checks it again first and fails as `device_unavailable`, sending nothing, -when it isn't. The runtime runs ready checks only until they first pass, and a Hyprland restarted after a crash runs in -safe mode with no plugins, where input is accepted but never arrives. A plugin loaded again into a running Hyprland has -no input seat until the desktop restarts, and a driver that outlives its Hyprland can't reach the next one, which has a -new instance signature, so the driver's services have to restart with the graphical session. +given the character, cua-driver falls back to libei, which can hang its input worker. The computer skill tells agents so +(`notes`): Super chords reach the desktop's shortcuts, which the protocol's skill lists for Omarchy, and other chords, +such as Alt+Tab, go to the focused window. Clicks, drags and scrolls stay desktop-scope. Keys aimed at a window need the +seat's keymap to be the US layout's, and an input method such as Omarchy's fcitx5 takes on the keymap that desktop text +uploads, after which cua-driver refuses them (`unsupported_layout`): the driver then puts the layout back with a chord +of the layout's own keys at the desktop, Super+Ctrl+Alt+Shift+F12, which it binds to nothing in Hyprland first (`hyprctl +eval`), so Hyprland keeps it from every window, and sends the key again. It presses the chord only while its own bind +alone has it: with the chord bound to anything else, the key fails as cua-driver refused it. `computer-ready` also +checks that the plugin is loaded, enabled, built for this Hyprland and taking input (`transport.ready`), and every +input, a `computer_*` operation or one of cua-driver's own tools that isn't read-only, checks it again first and fails +as `device_unavailable`, sending nothing, when it isn't. The runtime runs ready checks only until they first pass. A +plugin loaded again into a running Hyprland has no input seat until the desktop restarts. After a crash, Hyprland's +watchdog brings it back within the same session, in safe mode with no plugins and under a new instance signature, so +nothing restarts the driver with it: once another Hyprland runs in place of the one the driver started with (`hyprctl +instances`), the plugin check fails for good (`final`) and every input fails as an infrastructure failure. The driver's +services otherwise restart with the graphical session, as after a logout. To move it to another cua-driver version: -1. Pin the version in `requirements.txt`. +1. Pin the version in `requirements.txt`, and Hyprland's own in `requirements-omarchy.txt`. 2. Regenerate `mcp.json` from what that version's MCP server, `cua-driver mcp`, says at connect and lists: its `instructions`, each tool's `name`, `description`, `inputSchema`, `annotations` and `outputSchema`, and each resource's `uri`, `name`, `description` and `mimeType`. That takes a running daemon, so do it on a Linux test diff --git a/src/agent_env/drivers/envs/cua-driver/driver.py b/src/agent_env/drivers/envs/cua-driver/driver.py index 1d6cd3baa..abd1f6273 100644 --- a/src/agent_env/drivers/envs/cua-driver/driver.py +++ b/src/agent_env/drivers/envs/cua-driver/driver.py @@ -127,9 +127,12 @@ @computer_driver(name="cua-driver", supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}, + {"os": {"type": "linux", "distribution": ["omarchy"], "display": ["wayland"]}, + "arch": ["amd64"]}, {"os": {"type": "macos", "display": ["aqua"]}, "arch": ["arm64"]}, {"os": {"type": "windows", "display": ["dwm"]}, "arch": ["amd64"]}]) class CuaDriver(ComputerDriverEnvironment): + notes = hyprland.NOTES if HYPRLAND else "" buttons = ("left", "right", "middle") max_click_count = 3 max_scroll_amount = 50 diff --git a/src/agent_env/drivers/envs/cua-driver/env.toml b/src/agent_env/drivers/envs/cua-driver/env.toml index 4d0a05609..ce2a6d9f0 100644 --- a/src/agent_env/drivers/envs/cua-driver/env.toml +++ b/src/agent_env/drivers/envs/cua-driver/env.toml @@ -9,6 +9,13 @@ services = [ { name = "driver", command = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:CuaDriver" }, ] +[install."linux/omarchy"] +python = "requirements-omarchy.txt" +services = [ + { name = "daemon", command = "venv/bin/python driver.py" }, + { name = "driver", command = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:CuaDriver" }, +] + [install.macos] python = "requirements-macos.txt" downloads = [ diff --git a/src/agent_env/drivers/envs/cua-driver/hyprland.py b/src/agent_env/drivers/envs/cua-driver/hyprland.py index 2c981e0ab..86710d19d 100644 --- a/src/agent_env/drivers/envs/cua-driver/hyprland.py +++ b/src/agent_env/drivers/envs/cua-driver/hyprland.py @@ -6,6 +6,7 @@ import asyncio import contextlib import json +import os import string import subprocess @@ -13,6 +14,10 @@ #: How long a request to Hyprland's IPC waits for its answer. TIMEOUT_S = 5 +#: Why a computer whose Hyprland restarted under its driver has failed for good: Hyprland's watchdog brings it back after +#: a crash, in safe mode with no plugins, within the same session, so nothing restarts the driver with it. +REPLACED = ("Hyprland restarted under this driver, as its watchdog does after a crash, in safe mode with no plugins, so " + "the computer can't take input again: an infrastructure failure") #: How keys and text go to the desktop: in the foreground, through the plugin's seat, so Hyprland's own shortcuts see #: them first and then whatever has the keyboard gets them, a window or a layer such as a menu. Without it a key with #: modifiers goes through libei, which Hyprland doesn't serve. @@ -34,6 +39,10 @@ "+": "=", "{": "[", "}": "]", "|": "\\", ":": ";", '"': "'", "<": ",", ">": ".", "?": "/", "~": "`", **{letter: letter.lower() for letter in string.ascii_uppercase}, } +#: What the computer skill tells an agent of how its keys go here: the desktop's shortcuts are Super chords, which the +#: driver sends to Hyprland, and every other key goes to the focused window, past Hyprland's own binds. +NOTES = ("Keys held with Super go to the desktop, where its shortcuts take them; every other key or chord goes to the " + "focused window, so a desktop shortcut without Super, such as Alt+Tab, doesn't fire.") #: What cua-driver says when a key aimed at a window is refused because a layer, such as a menu, holds the keyboard. GRABBED = "foreground_grab" #: What it says when one is refused because the seat's keymap isn't the US layout's: an input method such as Omarchy's @@ -90,13 +99,14 @@ def _chord_binds() -> list[dict]: async def plugin() -> ReadyCheck: """That Hyprland has cua-driver's plugin loaded, enabled, built for it and taking input; one restarted after a crash runs in safe mode, with no plugins, where input is sent but never arrives, and a plugin loaded again into a - running Hyprland answers but has no input seat until the desktop restarts. When Hyprland doesn't answer at all, as - once it restarts under a new instance signature after the driver started, it says so instead.""" + running Hyprland answers but has no input seat until the desktop restarts. When the Hyprland the driver started with + doesn't answer and another runs in its place, the check fails for good (``REPLACED``).""" status = await asyncio.to_thread(hyprctl, "cua:status") transport = (status or {}).get("transport") or {} if status is None and await asyncio.to_thread(hyprctl, "activeworkspace") is None: - problem = ("Hyprland doesn't answer on its IPC; if it restarted after this driver started, restart the driver " - "with it") + if await asyncio.to_thread(_replaced): + return ReadyCheck("cua-driver plugin", False, REPLACED, final=True) + problem = "Hyprland doesn't answer on its IPC" elif status is None: problem = "Hyprland hasn't loaded cua-driver's plugin" elif status.get("configured") is not True: @@ -110,6 +120,13 @@ async def plugin() -> ReadyCheck: return ReadyCheck("cua-driver plugin", False, problem) +def _replaced() -> bool: + """Whether another Hyprland runs in this session in place of the one whose instance signature the driver has.""" + instances = hyprctl("instances") or [] + ours = os.environ.get("HYPRLAND_INSTANCE_SIGNATURE") + return bool(instances) and all(instance.get("instance") != ours for instance in instances) + + async def require_plugin() -> None: """Input refused unless cua-driver's plugin can take it: the runtime runs ready checks only until they first pass, and input sent while Hyprland runs without the plugin is accepted but never arrives.""" diff --git a/src/agent_env/drivers/envs/cua-driver/requirements-omarchy.txt b/src/agent_env/drivers/envs/cua-driver/requirements-omarchy.txt new file mode 100644 index 000000000..0a462625e --- /dev/null +++ b/src/agent_env/drivers/envs/cua-driver/requirements-omarchy.txt @@ -0,0 +1,7 @@ +# What a composed Omarchy computer installs into the driver's venv: cua-driver at Hyprland's own pin, whose executable +# the driver checks (EXECUTABLES["hyprland/amd64"]), with Omarchy's base carrying the Hyprland plugin it drives input +# through; python-xlib, which driver.py imports on any Linux. A release pins agentenv-framework-protocol to the version +# it ships. +agentenv-framework-protocol[computer]>=0.1.444 +cua-driver==0.44.0 +python-xlib>=0.33,<0.34 diff --git a/tst/unit/drivers/cua_driver_test.py b/tst/unit/drivers/cua_driver_test.py index a5ecec3e0..e991f5d83 100644 --- a/tst/unit/drivers/cua_driver_test.py +++ b/tst/unit/drivers/cua_driver_test.py @@ -14,7 +14,9 @@ import types import stat import tempfile +import tomllib from pathlib import Path +from types import SimpleNamespace import httpx import pytest @@ -23,8 +25,10 @@ from PIL import Image as PILImage import agent_env +from agent_env.env.envs.computer.utils.install import for_os, installs from agentenv_protocol import SESSION_META_KEY from agentenv_protocol.computer import DeliveryFailed, DeviceUnavailable, Platform, Unsupported +from agentenv_protocol.computer.platform import supported DRIVER = Path(agent_env.__file__).parent / "drivers" / "envs" / "cua-driver" / "driver.py" @@ -135,9 +139,14 @@ def sync(self): def module(monkeypatch): """driver.py loaded as it is on an X11 desktop on Linux, whatever runs the tests, with X stood in for; a test of another platform sets that platform's flags itself.""" + monkeypatch.delenv("HYPRLAND_INSTANCE_SIGNATURE", raising=False) + return _loaded(monkeypatch) + + +def _loaded(monkeypatch): + """driver.py loaded on Linux, in a Hyprland session when ``HYPRLAND_INSTANCE_SIGNATURE`` is set.""" monkeypatch.delenv("ENVIRONMENT_NAME", raising=False) monkeypatch.delenv("AGENTENV_COMPUTER_SETTINGS", raising=False) - monkeypatch.delenv("HYPRLAND_INSTANCE_SIGNATURE", raising=False) xlib = types.ModuleType("Xlib") xlib.display = types.ModuleType("Xlib.display") xlib.display.Display = FakeKeymap @@ -682,12 +691,14 @@ async def test_on_hyprland_readiness_checks_its_own_pin_and_every_time_that_hypr digests = iter([module.EXECUTABLES["amd64"], module.EXECUTABLES["hyprland/amd64"]]) monkeypatch.setattr(module, "_executable_sha256", lambda: next(digests)) x11 = await driver.ready_checks() + monkeypatch.setenv("HYPRLAND_INSTANCE_SIGNATURE", "ours") hyprland.answers.update({ "cua:status": [PLUGIN_LOADED, None, {**PLUGIN_LOADED, "configured": False}, - {**PLUGIN_LOADED, "abi": {"match": False}}, PLUGIN_RELOADED, None], - "activeworkspace": [{"id": 1}, None]}) + {**PLUGIN_LOADED, "abi": {"match": False}}, PLUGIN_RELOADED, None, None], + "activeworkspace": [{"id": 1}, None, None], + "instances": [[{"instance": "ours"}], [{"instance": "after-a-crash"}]]}) - checks = [await driver.ready_checks() for _ in range(6)] + checks = [await driver.ready_checks() for _ in range(7)] assert (x11[0].name, x11[0].ok) == ("cua-driver", False) assert [(c.name, c.ok) for c in checks[0]] == [("cua-driver", True), ("cua-driver tools", True), @@ -698,8 +709,10 @@ async def test_on_hyprland_readiness_checks_its_own_pin_and_every_time_that_hypr (False, "cua-driver's plugin was built for another Hyprland"), (False, "cua-driver's plugin isn't taking input: input seat lifetime unavailable; restart the desktop before " "loading a replacement plugin"), - (False, "Hyprland doesn't answer on its IPC; if it restarted after this driver started, restart the driver " - "with it")] + (False, "Hyprland doesn't answer on its IPC"), + (False, module.hyprland.REPLACED)] + assert [c[-1].final for c in checks] == [False] * 6 + [True] + assert module.hyprland.REPLACED.endswith("an infrastructure failure") assert sum(1 for _, r in daemon.requests if r["method"] == "list") == 2 @@ -723,3 +736,38 @@ def execve(path, argv, env): (argv, env), = started assert argv[:2] == ["/opt/cua-driver", "serve"] and env["CUA_DRIVER_RS_ENABLE_WAYLAND"] == "1" assert env["DO_NOT_TRACK"] == "1" + + +@pytest.mark.asyncio +async def test_on_hyprland_input_once_another_hyprland_runs_in_place_of_the_drivers_fails_as_an_infrastructure_failure( + module, daemon, hyprland, monkeypatch): + monkeypatch.setenv("HYPRLAND_INSTANCE_SIGNATURE", "ours") + hyprland.answers.update({"cua:status": None, "activeworkspace": None, + "instances": [[{"instance": "after-a-crash"}]] * 2}) + + with pytest.raises(DeviceUnavailable, match="an infrastructure failure"): + await module.CuaDriver().press_keys(["Escape"]) + + assert daemon.calls() == [] + + +def test_in_a_hyprland_session_the_driver_pins_hyprlands_manifest_and_tells_agents_how_keys_go(module, monkeypatch): + monkeypatch.setenv("HYPRLAND_INSTANCE_SIGNATURE", "ours") + monkeypatch.setenv("XDG_RUNTIME_DIR", "/run/user/1000") + loaded = _loaded(monkeypatch) + + assert loaded.MANIFEST.name == "mcp-hyprland.json" and loaded.CuaDriver.notes == loaded.hyprland.NOTES + assert module.MANIFEST.name == "mcp.json" and module.CuaDriver.notes == "" + + +def test_an_omarchy_computer_installs_cua_driver_at_hyprlands_pin_and_the_driver_supports_omarchy(module): + found, problems = installs(tomllib.loads((DRIVER.parent / "env.toml").read_text())["install"]) + omarchy, ubuntu = (SimpleNamespace(type="linux", distribution=name) for name in ("omarchy", "ubuntu")) + + assert not problems + assert str(for_os(found, omarchy).python) == "requirements-omarchy.txt" + assert str(for_os(found, ubuntu).python) == "requirements.txt" + assert for_os(found, omarchy).services == for_os(found, ubuntu).services + assert "cua-driver==0.44.0" in (DRIVER.parent / "requirements-omarchy.txt").read_text().splitlines() + platforms = [platform for entry in module.CuaDriver.supports for platform in supported(entry)] + assert Platform(os="linux", distribution="omarchy", arch="amd64", display="wayland") in platforms From 73a44c6418ca16c8bbc637b22efde4ab7fef4e54 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 9 Nov 2026 10:34:45 -0800 Subject: [PATCH 2/2] refactor(drivers): one desktop key for cua-driver's pins, shared Linux app chords, and the Hyprland docs by topic cua-driver's manifest and its executable's pin are now chosen by one desktop key (macos, windows, hyprland, or none for X11) instead of two chains that had to agree. Omarchy's shortcuts reuse Linux's app chords rather than repeating them, the Linux text unchanged. The Omarchy install's test checks for an exact cua-driver pin, not a version, and the drivers README's Hyprland section is split into its install, its input, its keymap reset and its readiness. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/computer/skill.py | 14 ++--- src/agent_env/drivers/README.md | 52 ++++++++++--------- .../drivers/envs/cua-driver/driver.py | 14 +++-- tst/unit/drivers/cua_driver_test.py | 4 +- 4 files changed, 49 insertions(+), 35 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/skill.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/skill.py index 2085b3ea4..1c8f15ea9 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/skill.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/skill.py @@ -15,12 +15,15 @@ if TYPE_CHECKING: from .driver import ComputerDriverEnvironment +#: The chords Linux apps keep for editing, whatever their desktop. +_LINUX_APPS = """\ +- Copy, cut and paste: `['Control', 'c']`, `['Control', 'x']`, `['Control', 'v']`; in a terminal, with `'Shift'` too. +- Undo `['Control', 'z']`, save `['Control', 's']`, select all `['Control', 'a']`, find `['Control', 'f']`. +""" #: The chords each platform's desktops keep for the everyday actions, as ``computer_key`` takes them, by platform key: a #: distribution's replace its OS type's. SHORTCUTS = { - "linux": """\ -- Copy, cut and paste: `['Control', 'c']`, `['Control', 'x']`, `['Control', 'v']`; in a terminal, with `'Shift'` too. -- Undo `['Control', 'z']`, save `['Control', 's']`, select all `['Control', 'a']`, find `['Control', 'f']`. + "linux": _LINUX_APPS + """\ - Switch windows `['Alt', 'Tab']`; close the focused one `['Alt', 'F4']`. - In a file manager, `['Control', 'l']` takes a typed path, `['F2']` renames and `['Control', 'Shift', 'n']` makes a folder. @@ -34,9 +37,8 @@ - `['Super', 'Enter']` opens a terminal, `['Super', 'Shift', 'f']` the file manager and `['Super', 'Shift', 'b']` the browser; `['Super', 'Alt', ' ']` the app launcher, where a typed name and Enter start an app; `['Super', ' ']` Omarchy's menu; `['Super', 'k']` the list of every shortcut. -- In apps: copy, cut and paste `['Control', 'c']`, `['Control', 'x']`, `['Control', 'v']`, or in any window, a terminal - too, `['Super', 'c']`, `['Super', 'x']`, `['Super', 'v']`; undo `['Control', 'z']`, save `['Control', 's']`, select - all `['Control', 'a']`, find `['Control', 'f']`. +""" + _LINUX_APPS + """\ +- `['Super', 'c']`, `['Super', 'x']` and `['Super', 'v']` copy, cut and paste in any window, a terminal too. """, } _SESSIONS = {"x11": "an X11 session", "wayland": "a Wayland session"} diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index a1066acab..1fd670a22 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -85,30 +85,34 @@ Omarchy's Wayland desktop on amd64, which installs `[install."linux/omarchy"]`: services, with no Hyprland plugin, which Omarchy's base carries. It runs cua-driver 0.44.0, its own pin, since 0.38.0 refuses background input there, the delivery cua-driver's own tools default to: `computer-ready` checks its wheel's executable (`hyprland/amd64` in `EXECUTABLES`) and the tools `mcp-hyprland.json` pins, which `requirements-omarchy.txt` -installs, while `requirements.txt` keeps 0.38.0 for the other desktops. The daemon starts once the compositor's socket -answers, on a socket in the session user's runtime folder. What the driver asks Hyprland (`hyprctl -j`) is in -`hyprland.py`, beside `driver.py`. Text goes to the desktop in the foreground, where Hyprland's shortcuts, a menu -holding the keyboard and non-ASCII text all work. A key goes to the window Hyprland has focused, in the foreground, -where Backspace, the arrows and Home/End edit text as a person's keys do, which at the desktop they don't, pressed as on -a US keyboard, the only layout the plugin takes there, so a shifted character goes as Shift and the key it's on. Super -and keys held with it, Hyprland's own shortcuts, go to the desktop, as does a key with nothing focused and one the -window path refuses because a layer such as Omarchy's menu holds the keyboard, punctuation there by its keysym name: -given the character, cua-driver falls back to libei, which can hang its input worker. The computer skill tells agents so -(`notes`): Super chords reach the desktop's shortcuts, which the protocol's skill lists for Omarchy, and other chords, -such as Alt+Tab, go to the focused window. Clicks, drags and scrolls stay desktop-scope. Keys aimed at a window need the -seat's keymap to be the US layout's, and an input method such as Omarchy's fcitx5 takes on the keymap that desktop text -uploads, after which cua-driver refuses them (`unsupported_layout`): the driver then puts the layout back with a chord -of the layout's own keys at the desktop, Super+Ctrl+Alt+Shift+F12, which it binds to nothing in Hyprland first (`hyprctl -eval`), so Hyprland keeps it from every window, and sends the key again. It presses the chord only while its own bind -alone has it: with the chord bound to anything else, the key fails as cua-driver refused it. `computer-ready` also -checks that the plugin is loaded, enabled, built for this Hyprland and taking input (`transport.ready`), and every -input, a `computer_*` operation or one of cua-driver's own tools that isn't read-only, checks it again first and fails -as `device_unavailable`, sending nothing, when it isn't. The runtime runs ready checks only until they first pass. A -plugin loaded again into a running Hyprland has no input seat until the desktop restarts. After a crash, Hyprland's -watchdog brings it back within the same session, in safe mode with no plugins and under a new instance signature, so -nothing restarts the driver with it: once another Hyprland runs in place of the one the driver started with (`hyprctl -instances`), the plugin check fails for good (`final`) and every input fails as an infrastructure failure. The driver's -services otherwise restart with the graphical session, as after a logout. +installs, while `requirements.txt` keeps 0.38.0 for the other desktops. + +The daemon starts once the compositor's socket answers, on a socket in the session user's runtime folder. What the +driver asks Hyprland (`hyprctl -j`) is in `hyprland.py`, beside `driver.py`. Text goes to the desktop in the foreground, +where Hyprland's shortcuts, a menu holding the keyboard and non-ASCII text all work. A key goes to the window Hyprland +has focused, in the foreground, where Backspace, the arrows and Home/End edit text as a person's keys do, which at the +desktop they don't, pressed as on a US keyboard, the only layout the plugin takes there, so a shifted character goes as +Shift and the key it's on. Super and keys held with it, Hyprland's own shortcuts, go to the desktop, as does a key with +nothing focused and one the window path refuses because a layer such as Omarchy's menu holds the keyboard, punctuation +there by its keysym name: given the character, cua-driver falls back to libei, which can hang its input worker. The +computer skill tells agents so (`notes`): Super chords reach the desktop's shortcuts, which the protocol's skill lists +for Omarchy, and other chords, such as Alt+Tab, go to the focused window. Clicks, drags and scrolls stay desktop-scope. + +Keys aimed at a window need the seat's keymap to be the US layout's, and an input method such as Omarchy's fcitx5 takes +on the keymap that desktop text uploads, after which cua-driver refuses them (`unsupported_layout`): the driver then +puts the layout back with a chord of the layout's own keys at the desktop, Super+Ctrl+Alt+Shift+F12, which it binds to +nothing in Hyprland first (`hyprctl eval`), so Hyprland keeps it from every window, and sends the key again. It presses +the chord only while its own bind alone has it: with the chord bound to anything else, the key fails as cua-driver +refused it. + +`computer-ready` also checks that the plugin is loaded, enabled, built for this Hyprland and taking input +(`transport.ready`), and every input, a `computer_*` operation or one of cua-driver's own tools that isn't read-only, +checks it again first and fails as `device_unavailable`, sending nothing, when it isn't. The runtime runs ready checks +only until they first pass. A plugin loaded again into a running Hyprland has no input seat until the desktop restarts. +After a crash, Hyprland's watchdog brings it back within the same session, in safe mode with no plugins and under a new +instance signature, so nothing restarts the driver with it: once another Hyprland runs in place of the one the driver +started with (`hyprctl instances`), the plugin check fails for good (`final`) and every input fails as an infrastructure +failure. The driver's services otherwise restart with the graphical session, as after a logout. To move it to another cua-driver version: diff --git a/src/agent_env/drivers/envs/cua-driver/driver.py b/src/agent_env/drivers/envs/cua-driver/driver.py index abd1f6273..9345fb0a2 100644 --- a/src/agent_env/drivers/envs/cua-driver/driver.py +++ b/src/agent_env/drivers/envs/cua-driver/driver.py @@ -67,6 +67,14 @@ #: its Hyprland plugin; what follows reaches Hyprland's IPC, so it's Hyprland's session that turns it on, not any #: Wayland one. HYPRLAND = sys.platform == "linux" and bool(os.environ.get("HYPRLAND_INSTANCE_SIGNATURE")) + + +def _desktop() -> str | None: + """The desktop with a cua-driver build of its own, whose name keys its manifest (``mcp-.json``) and its + executables' sha256s (``/``); None on an X11 desktop, whose are the default's.""" + return "macos" if MACOS else "windows" if WINDOWS else "hyprland" if HYPRLAND else None + + #: Where cua-driver's daemon listens: on Hyprland in the session user's runtime folder, which the session's units share. if WINDOWS: SOCKET = r"\\.\pipe\agentenv-cua-driver" @@ -78,8 +86,7 @@ SOCKET = "/run/agentenv/cua-driver.sock" #: What cua-driver's own MCP server (``cua-driver mcp``) says at connect and lists at the pinned version on each OS: its #: instructions, its tools, which the running daemon must list the same, and its skill resources. -MANIFEST = Path(__file__).with_name("mcp-macos.json" if MACOS else "mcp-windows.json" if WINDOWS else - "mcp-hyprland.json" if HYPRLAND else "mcp.json") +MANIFEST = Path(__file__).with_name(f"mcp-{_desktop()}.json" if _desktop() else "mcp.json") #: Where the release's zip is unpacked on Windows, beside this file. WINDOWS_EXECUTABLE = Path(__file__).with_name("cua-driver") / "cua-driver.exe" #: CuaDriver.app's executable, which macOS runs the daemon as. @@ -322,8 +329,7 @@ async def _pinned_checks(self) -> list[ReadyCheck]: return [ReadyCheck("cua-driver", False, "its daemon isn't answering yet")] arch = self.platform.arch or "" digest = await asyncio.to_thread(_executable_sha256) - expected = EXECUTABLES.get(f"macos/{arch}" if MACOS else f"windows/{arch}" if WINDOWS else - f"hyprland/{arch}" if HYPRLAND else arch) + expected = EXECUTABLES.get(f"{desktop}/{arch}" if (desktop := _desktop()) else arch) running = {t["name"]: t["input_schema"] for t in listed} pinned = {t["name"]: t["inputSchema"] for t in _manifest()["tools"]} differ = sorted(name for name in running.keys() | pinned.keys() if running.get(name) != pinned.get(name)) diff --git a/tst/unit/drivers/cua_driver_test.py b/tst/unit/drivers/cua_driver_test.py index e991f5d83..c1d3e59a7 100644 --- a/tst/unit/drivers/cua_driver_test.py +++ b/tst/unit/drivers/cua_driver_test.py @@ -9,6 +9,7 @@ import importlib.util import io import json +import re import socket import sys import types @@ -768,6 +769,7 @@ def test_an_omarchy_computer_installs_cua_driver_at_hyprlands_pin_and_the_driver assert str(for_os(found, omarchy).python) == "requirements-omarchy.txt" assert str(for_os(found, ubuntu).python) == "requirements.txt" assert for_os(found, omarchy).services == for_os(found, ubuntu).services - assert "cua-driver==0.44.0" in (DRIVER.parent / "requirements-omarchy.txt").read_text().splitlines() + pins = (DRIVER.parent / "requirements-omarchy.txt").read_text().splitlines() + assert any(re.fullmatch(r"cua-driver==\d+(\.\d+)+", line) for line in pins) platforms = [platform for entry in module.CuaDriver.supports for platform in supported(entry)] assert Platform(os="linux", distribution="omarchy", arch="amd64", display="wayland") in platforms