Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions packages/agentenv-protocol/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -587,8 +587,9 @@ driver has them, through the `served_tools`, `served_extensions` and `served_ope
features, among them `screen`: `desktop`, or `page` for a driver whose screen is a website
browser's active tab, which a computer env pairs with exactly one website browser. The driver serves its skill through `skill/v1` (an `@agent_skill` handler on the base
class): a SKILL.md written from the computer as it booted (its platform, session user and home,
the live screen size, what its input takes, its OS's shortcuts) and ending with the driver
class's `notes`, markdown for what its card can't say. A driver whose software ships a skill of
the live screen size, what its input takes, its platform's shortcuts, a distribution's where it
has its own, as Omarchy does) and ending with the driver class's `notes`, markdown for what its
card can't say. A driver whose software ships a skill of
its own returns it from `bundled_skill()` (its description, body and other files), and the
computer skill leads with it, the computer's part following.
- **Framework extensions.** `computer-ready/v1` (`status`: real checks of the display and a
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,35 @@
from typing import TYPE_CHECKING

from .keys import PRINTABLE_KEYS
from .platform import entry_for, platform_key

if TYPE_CHECKING:
from .driver import ComputerDriverEnvironment

#: The chords each OS's desktops keep for the everyday actions, as ``computer_key`` takes them.
SHORTCUTS = {
"linux": """\
#: The chords Linux apps keep for editing, whatever their desktop.
_LINUX_APPS = """\
- Copy, cut and paste: `['Control', 'c']`, `['Control', 'x']`, `['Control', 'v']`; in a terminal, with `'Shift'` too.
- Undo `['Control', 'z']`, save `['Control', 's']`, select all `['Control', 'a']`, find `['Control', 'f']`.
"""
#: The chords each platform's desktops keep for the everyday actions, as ``computer_key`` takes them, by platform key: a
#: distribution's replace its OS type's.
SHORTCUTS = {
"linux": _LINUX_APPS + """\
- Switch windows `['Alt', 'Tab']`; close the focused one `['Alt', 'F4']`.
- In a file manager, `['Control', 'l']` takes a typed path, `['F2']` renames and `['Control', 'Shift', 'n']` makes a
folder.
""",
"linux/omarchy": """\
- Workspaces: `['Super', '1']` to `['Super', '9']` switch to one; `['Super', 'Shift', '1']` to `['Super', 'Shift', '9']`
move the focused window there.
- Windows tile side by side. `['Super', 'ArrowLeft']` and the other arrows focus the window that way, and with
`'Shift'` too move it there; `['Super', 'w']` closes the focused window, `['Super', 'f']` makes it full screen and
`['Super', 't']` floats or tiles it.
- `['Super', 'Enter']` opens a terminal, `['Super', 'Shift', 'f']` the file manager and `['Super', 'Shift', 'b']` the
browser; `['Super', 'Alt', ' ']` the app launcher, where a typed name and Enter start an app; `['Super', ' ']`
Omarchy's menu; `['Super', 'k']` the list of every shortcut.
""" + _LINUX_APPS + """\
- `['Super', 'c']`, `['Super', 'x']` and `['Super', 'v']` copy, cut and paste in any window, a terminal too.
""",
}
_SESSIONS = {"x11": "an X11 session", "wayland": "a Wayland session"}
Expand Down Expand Up @@ -56,7 +73,7 @@ async def skill(driver: ComputerDriverEnvironment) -> tuple[str, str, dict[str,
"## Input",
_input(driver),
]
if shortcuts := SHORTCUTS.get(platform.os or ""):
if shortcuts := entry_for(SHORTCUTS, platform_key(platform.os, platform.distribution)):
sections += ["## Shortcuts", shortcuts.rstrip()]
if notes := driver.notes.strip():
sections += ["## Notes", notes]
Expand Down
12 changes: 12 additions & 0 deletions packages/agentenv-protocol/tests/test_computer_driver.py
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,18 @@ async def test_the_skill_is_written_from_the_computer_as_it_booted_under_the_nam
assert refused.status_code == 400 and "lowercase letters, digits and single hyphens" in refused.text


@pytest.mark.asyncio
async def test_the_skills_shortcuts_are_its_distributions_own_where_it_has_them(monkeypatch):
monkeypatch.setattr("agentenv_protocol.computer.skill.getpass.getuser", lambda: "omarchy")
omarchy = Platform(os="linux", distribution="omarchy", version="4.5.0", arch="amd64", display="wayland")
async with http_client(FakeDriver(platform=omarchy)) as http:
skill_md = (await http.get("/agentenv/ext/skill/get", params={"name": "computer-desktop"})).json()["skill_md"]

shortcuts = skill_md.partition("## Shortcuts")[2]
assert "- Omarchy 4.5.0 (amd64), a Wayland session" in skill_md
assert "`['Super', 'Alt', ' ']` the app launcher" in shortcuts and "`['Alt', 'Tab']`" not in skill_md


@pytest.mark.asyncio
async def test_a_driver_whose_software_ships_a_skill_leads_with_it_and_the_computers_part_follows(monkeypatch):
class Bundling(FakeDriver):
Expand Down
60 changes: 34 additions & 26 deletions src/agent_env/drivers/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,35 +80,43 @@ cua-driver's UI Automation tree, whose walk of it fails.

On a Hyprland desktop (Wayland), as Omarchy's is, cua-driver runs its Wayland backend (`CUA_DRIVER_RS_ENABLE_WAYLAND=1`)
and sends input through the seats of Hyprland's `cua-hyprland-plugin`. Hyprland's session turns this on
(`HYPRLAND_INSTANCE_SIGNATURE`), not any Wayland one, since the driver reaches Hyprland's IPC; `supports` names no
Wayland desktop yet, so gate 1 doesn't match it to one. It runs cua-driver 0.44.0, its own pin, since 0.38.0 refuses
background input there, the delivery cua-driver's own tools default to: `computer-ready` checks its wheel's executable
(`hyprland/amd64` in `EXECUTABLES`) and the tools `mcp-hyprland.json` pins, while `requirements.txt` keeps 0.38.0 for
the other desktops, so a Hyprland computer installs 0.44.0 itself. The daemon starts once the compositor's socket
answers, on a socket in the session user's runtime folder. What the driver asks Hyprland (`hyprctl -j`) is in
`hyprland.py`, beside `driver.py`. Text goes to the desktop in the foreground, where Hyprland's shortcuts, a menu
holding the keyboard and non-ASCII text all work. A key goes to the window Hyprland has focused, in the foreground,
where Backspace, the arrows and Home/End edit text as a person's keys do, which at the desktop they don't, pressed as on
a US keyboard, the only layout the plugin takes there, so a shifted character goes as Shift and the key it's on. Super
and keys held with it, Hyprland's own shortcuts, go to the desktop, as does a key with nothing focused and one the
window path refuses because a layer such as Omarchy's menu holds the keyboard, punctuation there by its keysym name:
given the character, cua-driver falls back to libei, which can hang its input worker. Clicks, drags and scrolls stay
desktop-scope. Keys aimed at a window need the seat's keymap to be the US layout's, and an input method such as
Omarchy's fcitx5 takes on the keymap that desktop text uploads, after which cua-driver refuses them
(`unsupported_layout`): the driver then puts the layout back with a chord of the layout's own keys at the desktop,
Super+Ctrl+Alt+Shift+F12, which it binds to nothing in Hyprland first (`hyprctl eval`), so Hyprland keeps it from every
window, and sends the key again. It presses the chord only while its own bind alone has it: with the chord bound to
anything else, the key fails as cua-driver refused it. `computer-ready` also checks that the plugin is loaded, enabled,
built for this Hyprland and taking input (`transport.ready`), and every input, a `computer_*` operation or one of
cua-driver's own tools that isn't read-only, checks it again first and fails as `device_unavailable`, sending nothing,
when it isn't. The runtime runs ready checks only until they first pass, and a Hyprland restarted after a crash runs in
safe mode with no plugins, where input is accepted but never arrives. A plugin loaded again into a running Hyprland has
no input seat until the desktop restarts, and a driver that outlives its Hyprland can't reach the next one, which has a
new instance signature, so the driver's services have to restart with the graphical session.
(`HYPRLAND_INSTANCE_SIGNATURE`), not any Wayland one, since the driver reaches Hyprland's IPC; `supports` names
Omarchy's Wayland desktop on amd64, which installs `[install."linux/omarchy"]`: `requirements-omarchy.txt` and the two
services, with no Hyprland plugin, which Omarchy's base carries. It runs cua-driver 0.44.0, its own pin, since 0.38.0
refuses background input there, the delivery cua-driver's own tools default to: `computer-ready` checks its wheel's
executable (`hyprland/amd64` in `EXECUTABLES`) and the tools `mcp-hyprland.json` pins, which `requirements-omarchy.txt`
installs, while `requirements.txt` keeps 0.38.0 for the other desktops.

The daemon starts once the compositor's socket answers, on a socket in the session user's runtime folder. What the
driver asks Hyprland (`hyprctl -j`) is in `hyprland.py`, beside `driver.py`. Text goes to the desktop in the foreground,
where Hyprland's shortcuts, a menu holding the keyboard and non-ASCII text all work. A key goes to the window Hyprland
has focused, in the foreground, where Backspace, the arrows and Home/End edit text as a person's keys do, which at the
desktop they don't, pressed as on a US keyboard, the only layout the plugin takes there, so a shifted character goes as
Shift and the key it's on. Super and keys held with it, Hyprland's own shortcuts, go to the desktop, as does a key with
nothing focused and one the window path refuses because a layer such as Omarchy's menu holds the keyboard, punctuation
there by its keysym name: given the character, cua-driver falls back to libei, which can hang its input worker. The
computer skill tells agents so (`notes`): Super chords reach the desktop's shortcuts, which the protocol's skill lists
for Omarchy, and other chords, such as Alt+Tab, go to the focused window. Clicks, drags and scrolls stay desktop-scope.

Keys aimed at a window need the seat's keymap to be the US layout's, and an input method such as Omarchy's fcitx5 takes
on the keymap that desktop text uploads, after which cua-driver refuses them (`unsupported_layout`): the driver then
puts the layout back with a chord of the layout's own keys at the desktop, Super+Ctrl+Alt+Shift+F12, which it binds to
nothing in Hyprland first (`hyprctl eval`), so Hyprland keeps it from every window, and sends the key again. It presses
the chord only while its own bind alone has it: with the chord bound to anything else, the key fails as cua-driver
refused it.

`computer-ready` also checks that the plugin is loaded, enabled, built for this Hyprland and taking input
(`transport.ready`), and every input, a `computer_*` operation or one of cua-driver's own tools that isn't read-only,
checks it again first and fails as `device_unavailable`, sending nothing, when it isn't. The runtime runs ready checks
only until they first pass. A plugin loaded again into a running Hyprland has no input seat until the desktop restarts.
After a crash, Hyprland's watchdog brings it back within the same session, in safe mode with no plugins and under a new
instance signature, so nothing restarts the driver with it: once another Hyprland runs in place of the one the driver
started with (`hyprctl instances`), the plugin check fails for good (`final`) and every input fails as an infrastructure
failure. The driver's services otherwise restart with the graphical session, as after a logout.

To move it to another cua-driver version:

1. Pin the version in `requirements.txt`.
1. Pin the version in `requirements.txt`, and Hyprland's own in `requirements-omarchy.txt`.
2. Regenerate `mcp.json` from what that version's MCP server, `cua-driver mcp`, says at connect and lists: its
`instructions`, each tool's `name`, `description`, `inputSchema`, `annotations` and `outputSchema`, and each
resource's `uri`, `name`, `description` and `mimeType`. That takes a running daemon, so do it on a Linux test
Expand Down
17 changes: 13 additions & 4 deletions src/agent_env/drivers/envs/cua-driver/driver.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,14 @@
#: its Hyprland plugin; what follows reaches Hyprland's IPC, so it's Hyprland's session that turns it on, not any
#: Wayland one.
HYPRLAND = sys.platform == "linux" and bool(os.environ.get("HYPRLAND_INSTANCE_SIGNATURE"))


def _desktop() -> str | None:
"""The desktop with a cua-driver build of its own, whose name keys its manifest (``mcp-<desktop>.json``) and its
executables' sha256s (``<desktop>/<arch>``); None on an X11 desktop, whose are the default's."""
return "macos" if MACOS else "windows" if WINDOWS else "hyprland" if HYPRLAND else None


#: Where cua-driver's daemon listens: on Hyprland in the session user's runtime folder, which the session's units share.
if WINDOWS:
SOCKET = r"\\.\pipe\agentenv-cua-driver"
Expand All @@ -78,8 +86,7 @@
SOCKET = "/run/agentenv/cua-driver.sock"
#: What cua-driver's own MCP server (``cua-driver mcp``) says at connect and lists at the pinned version on each OS: its
#: instructions, its tools, which the running daemon must list the same, and its skill resources.
MANIFEST = Path(__file__).with_name("mcp-macos.json" if MACOS else "mcp-windows.json" if WINDOWS else
"mcp-hyprland.json" if HYPRLAND else "mcp.json")
MANIFEST = Path(__file__).with_name(f"mcp-{_desktop()}.json" if _desktop() else "mcp.json")
#: Where the release's zip is unpacked on Windows, beside this file.
WINDOWS_EXECUTABLE = Path(__file__).with_name("cua-driver") / "cua-driver.exe"
#: CuaDriver.app's executable, which macOS runs the daemon as.
Expand Down Expand Up @@ -127,9 +134,12 @@

@computer_driver(name="cua-driver",
supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]},
{"os": {"type": "linux", "distribution": ["omarchy"], "display": ["wayland"]},
"arch": ["amd64"]},
{"os": {"type": "macos", "display": ["aqua"]}, "arch": ["arm64"]},
{"os": {"type": "windows", "display": ["dwm"]}, "arch": ["amd64"]}])
class CuaDriver(ComputerDriverEnvironment):
notes = hyprland.NOTES if HYPRLAND else ""
buttons = ("left", "right", "middle")
max_click_count = 3
max_scroll_amount = 50
Expand Down Expand Up @@ -319,8 +329,7 @@ async def _pinned_checks(self) -> list[ReadyCheck]:
return [ReadyCheck("cua-driver", False, "its daemon isn't answering yet")]
arch = self.platform.arch or ""
digest = await asyncio.to_thread(_executable_sha256)
expected = EXECUTABLES.get(f"macos/{arch}" if MACOS else f"windows/{arch}" if WINDOWS else
f"hyprland/{arch}" if HYPRLAND else arch)
expected = EXECUTABLES.get(f"{desktop}/{arch}" if (desktop := _desktop()) else arch)
running = {t["name"]: t["input_schema"] for t in listed}
pinned = {t["name"]: t["inputSchema"] for t in _manifest()["tools"]}
differ = sorted(name for name in running.keys() | pinned.keys() if running.get(name) != pinned.get(name))
Expand Down
7 changes: 7 additions & 0 deletions src/agent_env/drivers/envs/cua-driver/env.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,13 @@ services = [
{ name = "driver", command = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:CuaDriver" },
]

[install."linux/omarchy"]
python = "requirements-omarchy.txt"
services = [
{ name = "daemon", command = "venv/bin/python driver.py" },
{ name = "driver", command = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:CuaDriver" },
]

[install.macos]
python = "requirements-macos.txt"
downloads = [
Expand Down
25 changes: 21 additions & 4 deletions src/agent_env/drivers/envs/cua-driver/hyprland.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,18 @@
import asyncio
import contextlib
import json
import os
import string
import subprocess

from agentenv_protocol.computer import DeviceUnavailable, ReadyCheck

#: How long a request to Hyprland's IPC waits for its answer.
TIMEOUT_S = 5
#: Why a computer whose Hyprland restarted under its driver has failed for good: Hyprland's watchdog brings it back after
#: a crash, in safe mode with no plugins, within the same session, so nothing restarts the driver with it.
REPLACED = ("Hyprland restarted under this driver, as its watchdog does after a crash, in safe mode with no plugins, so "
"the computer can't take input again: an infrastructure failure")
#: How keys and text go to the desktop: in the foreground, through the plugin's seat, so Hyprland's own shortcuts see
#: them first and then whatever has the keyboard gets them, a window or a layer such as a menu. Without it a key with
#: modifiers goes through libei, which Hyprland doesn't serve.
Expand All @@ -34,6 +39,10 @@
"+": "=", "{": "[", "}": "]", "|": "\\", ":": ";", '"': "'", "<": ",", ">": ".", "?": "/", "~": "`",
**{letter: letter.lower() for letter in string.ascii_uppercase},
}
#: What the computer skill tells an agent of how its keys go here: the desktop's shortcuts are Super chords, which the
#: driver sends to Hyprland, and every other key goes to the focused window, past Hyprland's own binds.
NOTES = ("Keys held with Super go to the desktop, where its shortcuts take them; every other key or chord goes to the "
"focused window, so a desktop shortcut without Super, such as Alt+Tab, doesn't fire.")
Comment on lines +44 to +45

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Notes omit keyboard delivery exceptions

NOTES says every chord without Super goes to the focused window. But _press_on_hyprland sends it to the desktop when no window is focused or a menu holds the keyboard. Mention those exceptions so agents know ordinary keys can still reach menus and the empty desktop.

Suggested change
NOTES = ("Keys held with Super go to the desktop, where its shortcuts take them; every other key or chord goes to the "
"focused window, so a desktop shortcut without Super, such as Alt+Tab, doesn't fire.")
NOTES = ("Keys held with Super go to the desktop, where its shortcuts take them. Other keys and chords go to the "
"focused window, bypassing desktop shortcuts such as Alt+Tab. When no window is focused or a menu holds the "
"keyboard, they go to the desktop instead.")
Prompt To Fix With AI
This is a comment left during a code review.
Path: src/agent_env/drivers/envs/cua-driver/hyprland.py
Line: 44-45

Comment:
**Notes omit keyboard delivery exceptions**

`NOTES` says every chord without Super goes to the focused window. But `_press_on_hyprland` sends it to the desktop when no window is focused or a menu holds the keyboard. Mention those exceptions so agents know ordinary keys can still reach menus and the empty desktop.

```suggestion
NOTES = ("Keys held with Super go to the desktop, where its shortcuts take them. Other keys and chords go to the "
         "focused window, bypassing desktop shortcuts such as Alt+Tab. When no window is focused or a menu holds the "
         "keyboard, they go to the desktop instead.")
```

---

For each issue above, determine whether it is valid and should be fixed. If so, fix it directly.

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

Fix in Cursor Fix in Claude Code Fix in Codex

#: What cua-driver says when a key aimed at a window is refused because a layer, such as a menu, holds the keyboard.
GRABBED = "foreground_grab"
#: What it says when one is refused because the seat's keymap isn't the US layout's: an input method such as Omarchy's
Expand Down Expand Up @@ -90,13 +99,14 @@ def _chord_binds() -> list[dict]:
async def plugin() -> ReadyCheck:
"""That Hyprland has cua-driver's plugin loaded, enabled, built for it and taking input; one restarted after a
crash runs in safe mode, with no plugins, where input is sent but never arrives, and a plugin loaded again into a
running Hyprland answers but has no input seat until the desktop restarts. When Hyprland doesn't answer at all, as
once it restarts under a new instance signature after the driver started, it says so instead."""
running Hyprland answers but has no input seat until the desktop restarts. When the Hyprland the driver started with
doesn't answer and another runs in its place, the check fails for good (``REPLACED``)."""
status = await asyncio.to_thread(hyprctl, "cua:status")
transport = (status or {}).get("transport") or {}
if status is None and await asyncio.to_thread(hyprctl, "activeworkspace") is None:
problem = ("Hyprland doesn't answer on its IPC; if it restarted after this driver started, restart the driver "
"with it")
if await asyncio.to_thread(_replaced):
return ReadyCheck("cua-driver plugin", False, REPLACED, final=True)
problem = "Hyprland doesn't answer on its IPC"
elif status is None:
problem = "Hyprland hasn't loaded cua-driver's plugin"
elif status.get("configured") is not True:
Expand All @@ -110,6 +120,13 @@ async def plugin() -> ReadyCheck:
return ReadyCheck("cua-driver plugin", False, problem)


def _replaced() -> bool:
"""Whether another Hyprland runs in this session in place of the one whose instance signature the driver has."""
instances = hyprctl("instances") or []
ours = os.environ.get("HYPRLAND_INSTANCE_SIGNATURE")
return bool(instances) and all(instance.get("instance") != ours for instance in instances)


async def require_plugin() -> None:
"""Input refused unless cua-driver's plugin can take it: the runtime runs ready checks only until they first pass,
and input sent while Hyprland runs without the plugin is accepted but never arrives."""
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# What a composed Omarchy computer installs into the driver's venv: cua-driver at Hyprland's own pin, whose executable
# the driver checks (EXECUTABLES["hyprland/amd64"]), with Omarchy's base carrying the Hyprland plugin it drives input
# through; python-xlib, which driver.py imports on any Linux. A release pins agentenv-framework-protocol to the version
# it ships.
agentenv-framework-protocol[computer]>=0.1.444
cua-driver==0.44.0
python-xlib>=0.33,<0.34
Loading
Loading