Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions packages/agentenv-protocol/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -692,15 +692,21 @@ driver booted with a control token is probed with it: `--token`, by default
| Level | Checks | Needs on the computer |
|---|---|---|
| `basic` | P0 ready and a screenshot; P1 the card and tool schemas; P2 native frames, regions and JPEG; P5 refusals and their codes; X the declared extensions (`computer-display`, `computer-spaces`, `computer-keyboard`, `computer-exec`, `data/get`, per-role tool access, the pointer) | nothing |
| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add`, and a recording's segments matching their files with no gap, a repeated `request_id` joining it and `max_concurrent` held; on a page, X the page tools: a prompt answered, a file uploaded, a reload, navigation, back and forward, and tabs | Linux, `python3` with Tk (on a page, `python3` and `computer-page/v1`), a 1280x800 screen or larger, `computer-exec` and `data/add` |
| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add`, and a recording's segments matching their files with no gap, a repeated `request_id` joining it and `max_concurrent` held; on a page, X the page tools: a prompt answered, a file uploaded, a reload, navigation, back and forward, and tabs | Linux, `python3` with Tk (on Wayland, with PyGObject and GTK 3 instead; on a page, `python3` and `computer-page/v1`), a 1280x800 screen or larger, `computer-exec` and `data/add` |

At `full` the probe writes the Fixture Desk, a Tk window (`conformance/fixture.py`), into
`~/.agentenv-probe` with `data/add` and starts it with `computer-exec`. It aims at the boxes the
At `full` the probe writes the Fixture Desk, a Tk window (`conformance/fixture.py`), and its layout
(`conformance/desk.py`) into `~/.agentenv-probe` with `data/add` and starts it with `computer-exec`. It aims at the boxes the
window's widgets report, reads back what reached them, and then removes the fixture. On X11 the
window is maximized and kept above other windows, raising itself again when one covers it, so a
window an app keeps on top doesn't take the clicks aimed at it; elsewhere it is full screen. When the
fixture can't be staged, the `full` checks are unverified rather than failed.

On a driver whose platform's display is Wayland, the desk is drawn with GTK 3 as a native Wayland client
(`conformance/wayland_fixture.py`, staged beside `desk.py` in the Tk window's place), since Tk runs
only under XWayland. It goes full screen, takes its origin from Hyprland's IPC, keeps the clipboard
through wl-clipboard. There the probe selects a text box's text with GTK's own Ctrl+A rather than
Tk's Ctrl+/.

On a driver whose card says `screen: "page"`, the desk is an HTML page with the same widgets at the
same places (`conformance/page_fixture.py`). The probe starts its server on loopback, which writes
what the page reports to the same files, and opens the desk with `computer_tabs` in a tab of its
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,8 +116,15 @@ def __init__(self, driver: Driver, session: ClientSession, ready_timeout_s: floa
self.layout: dict = {}
self.fixture_pid: str | None = None
self.page = self.features.get("screen") == "page"
self.fixture = "page_fixture.py" if self.page else "fixture.py"
self.windows = (self.features.get("platform") or {}).get("os") == "windows"
platform = self.features.get("platform") or {}
#: Whether the desk is the GTK one: on Wayland, where Tk runs only under XWayland, which a Wayland driver may
#: not reach.
self.gtk = not self.page and platform.get("display") == "wayland"
self.fixture = "page_fixture.py" if self.page else "wayland_fixture.py" if self.gtk else "fixture.py"
self.windows = platform.get("os") == "windows"
#: The chord that selects a text box's text: Tk's Ctrl+/, which the Tk desk binds everywhere, or on the GTK
#: desk GTK's own Ctrl+A.
self.select_all = ["Control", "a"] if self.gtk else ["Control", "/"]
self.desk = ""
self.tab: str | None = None
self.opened: list[str] = []
Expand Down Expand Up @@ -562,12 +569,13 @@ def _grants(self) -> None:
# full: the fixture

async def _stage(self) -> str | None:
"""The Fixture Desk started, or what stopped it: the Tk desk, or on a page, the HTML desk's server, with the
desk opened in a tab of the probe's own."""
source = resources.files(__package__).joinpath(self.fixture).read_bytes()
added = await self.driver.add_file(f"{STAGE}/{self.fixture}", source)
if "error" in added:
return f"data/add refused it: {added['error'].get('message')}"
"""The Fixture Desk started, or what stopped it: the Tk desk or on Wayland the GTK desk, either beside the layout
they share (desk.py), or on a page, the HTML desk's server, with the desk opened in a tab of the probe's own."""
for file in [self.fixture] if self.page else [self.fixture, "desk.py"]:
source = resources.files(__package__).joinpath(file).read_bytes()
added = await self.driver.add_file(f"{STAGE}/{file}", source)
if "error" in added:
return f"data/add refused it: {added['error'].get('message')}"
if self.windows:
await self.driver.run(*WIN_REMOVE, f"{STAGE}/state")
started = await self.driver.run(*WIN_START, f"{STAGE}/{self.fixture}", f"{STAGE}/state", f"{STAGE}/fixture.log")
Expand Down Expand Up @@ -781,7 +789,7 @@ async def focus(box: str) -> None:

async def clear() -> None:
await focus("name")
await self.act("computer_key", {"keys": ["Control", "/"]})
await self.act("computer_key", {"keys": self.select_all})
await self.act("computer_key", {"keys": ["Backspace"]})

async def unicode() -> str | None:
Expand All @@ -796,7 +804,7 @@ async def events() -> str | None:

async def chord() -> str | None:
await focus("name")
await self.act("computer_key", {"keys": ["Control", "/"]})
await self.act("computer_key", {"keys": self.select_all})
await self.act("computer_key", {"keys": ["Backspace"]})
return await self.settled(lambda s: s["name"] == "", lambda s: s["name"])

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
"""The Fixture Desk's layout and what it records, shared by the Tk desk (fixture.py) and the GTK desk
(wayland_fixture.py), so the two can't drift apart. The probe stages it beside whichever desk it starts. It imports no
toolkit, since a Wayland computer may have no Tk."""
from __future__ import annotations

import json
import os
from pathlib import Path

#: The colours a grid cell cycles through, one more on each click.
COLOURS = ("#ffffff", "#ff0000", "#00aa00", "#0000ff")
CELL = 50
GRID = 8
#: Single-pixel markers on a black field, at odd and even pixels: (x, y) within the field, then the colour.
MARKERS = ((11, 11, "#ff0000"), (12, 11, "#00ff00"), (11, 12, "#0000ff"), (12, 12, "#ffff00"), (37, 20, "#ff00ff"),
(20, 37, "#00ffff"))
CLIPBOARD = "agentenv-probe clipboard"
MIN_SIZE = (1280, 800)
#: Where each widget's top left goes, the name box's label at "label" and the list's frame at "list".
AT = {"label": (40, 40), "name": (140, 40), "save": (560, 34), "subscribe": (40, 100), "notes": (760, 40),
"list": (40, 160), "grid": (400, 160), "pad": (840, 160), "wheel": (840, 500), "field": (1080, 500)}
#: The widgets sized in pixels: the list's frame, the drawing pad, the wheel and the marker field.
SIZE = {"list": {"width": 300, "height": 400}, "pad": {"width": 400, "height": 300},
"wheel": {"width": 200, "height": 120}, "field": {"width": 48, "height": 48}}
#: The list's rows, Item 1 to Item ITEMS.
ITEMS = 100
#: How many of the latest keys and drags state.json keeps, and about how many points of a drag's path.
KEYS_KEPT, DRAGS_KEPT, PATH_POINTS = 100, 20, 50
#: How often the desk reads inbox.txt.
INBOX_EVERY_MS = 200


def new_state(width: int, height: int, windowing: str) -> dict:
"""What state.json holds before anything reaches the desk."""
return {"name": "", "notes": "", "keys": [], "saved_count": 0, "subscribed": False, "selected": None,
"scroll_top": 0.0, "cells": {}, "right_clicks": 0, "double_clicks": 0, "drags": [],
"wheel": {"up": 0, "down": 0, "left": 0, "right": 0}, "clipboard": None, "loaded": None,
"screen": [width, height], "windowing": windowing}


def write_state(state_dir: Path, state: dict) -> None:
"""state.json replaced whole, so the probe never reads half of one."""
part = state_dir / ".state.json.part"
part.write_text(json.dumps(state))
os.replace(part, state_dir / "state.json")


def write_layout(state_dir: Path, boxes: dict, row_height: int, field: tuple[int, int]) -> None:
"""layout.json: each widget's box on the screen, a list row's height, the markers' pixels from the field's top left,
and the cell size and clipboard text the probe checks against."""
layout = {**boxes, "list_row_height": row_height,
"markers": [[field[0] + x, field[1] + y, colour] for x, y, colour in MARKERS], "cell": CELL,
"clipboard": CLIPBOARD}
(state_dir / "layout.json").write_text(json.dumps(layout))


def record_key(state: dict, key: str) -> None:
state["keys"] = (state["keys"] + [key])[-KEYS_KEPT:]


def record_drag(state: dict, stroke: list) -> None:
"""A finished drag: its ends, its number of points and its path thinned to about PATH_POINTS points."""
path = stroke[::max(1, len(stroke) // PATH_POINTS)] + stroke[-1:]
drag = {"start": stroke[0], "end": stroke[-1], "points": len(stroke), "path": path}
state["drags"] = (state["drags"] + [drag])[-DRAGS_KEPT:]
Original file line number Diff line number Diff line change
Expand Up @@ -3,26 +3,19 @@
whatever lands in <state-dir>/inbox.txt, to <state-dir>/state.json, so the probe can aim at it and read back what
arrived. It records the same effects under X11, macOS's Aqua and Windows, whose Tk names keys, binds select-all and
reports the wheel differently. On X11 it stays above an app's windows (``keep_on_top``); elsewhere it's full screen.
Run as ``python3 fixture.py <state-dir>``; it needs a 1280x800 screen or larger."""
Its layout and what it records are desk.py's, which the GTK desk shares. Run as ``python3 fixture.py <state-dir>``
beside desk.py; it needs a 1280x800 screen or larger."""
from __future__ import annotations

import ctypes
import json
import os
import sys
import time
import tkinter as tk
from pathlib import Path

#: The colours a grid cell cycles through, one more on each click.
COLOURS = ("#ffffff", "#ff0000", "#00aa00", "#0000ff")
CELL = 50
GRID = 8
#: Single-pixel markers on a black field, at odd and even pixels: (x, y) within the field, then the colour.
MARKERS = ((11, 11, "#ff0000"), (12, 11, "#00ff00"), (11, 12, "#0000ff"), (12, 12, "#ffff00"), (37, 20, "#ff00ff"),
(20, 37, "#00ffff"))
CLIPBOARD = "agentenv-probe clipboard"
MIN_SIZE = (1280, 800)
from desk import (AT, CELL, CLIPBOARD, COLOURS, GRID, INBOX_EVERY_MS, ITEMS, MARKERS, MIN_SIZE, SIZE, new_state,
record_drag, record_key, write_layout, write_state)

#: X11's keysyms for the keys Aqua's Tk names by their character.
KEYSYMS = {",": "comma", " ": "space"}
#: A wheel notch's <MouseWheel> delta where Tk scales the wheel: Windows', and Tk 9's everywhere.
Expand All @@ -46,15 +39,10 @@ def main(state_dir: Path) -> None:
else:
root.attributes("-fullscreen", True)
root.configure(bg="#f0f0f0")
state: dict = {"name": "", "notes": "", "keys": [], "saved_count": 0, "subscribed": False, "selected": None,
"scroll_top": 0.0, "cells": {}, "right_clicks": 0, "double_clicks": 0, "drags": [],
"wheel": {"up": 0, "down": 0, "left": 0, "right": 0}, "clipboard": None, "loaded": None,
"screen": [width, height], "windowing": root.tk.call("tk", "windowingsystem")}
state = new_state(width, height, root.tk.call("tk", "windowingsystem"))

def save() -> None:
part = state_dir / ".state.json.part"
part.write_text(json.dumps(state))
os.replace(part, state_dir / "state.json")
write_state(state_dir, state)

if width < MIN_SIZE[0] or height < MIN_SIZE[1]:
state["error"] = f"the screen is {width}x{height}; the fixture needs {MIN_SIZE[0]}x{MIN_SIZE[1]}"
Expand All @@ -63,13 +51,13 @@ def save() -> None:
return

font = ("DejaVu Sans", 16)
tk.Label(root, text="Name:", font=font, bg="#f0f0f0").place(x=40, y=40)
_place(tk.Label(root, text="Name:", font=font, bg="#f0f0f0"), "label")
name = tk.Entry(root, font=font, width=30)
name.place(x=140, y=40)
_place(name, "name")

def key(event) -> None:
state["name"] = name.get()
state["keys"] = (state["keys"] + [KEYSYMS.get(event.keysym, event.keysym)])[-100:]
record_key(state, KEYSYMS.get(event.keysym, event.keysym))
state["clipboard"] = _clipboard(root)
save()

Expand All @@ -88,7 +76,7 @@ def on_save() -> None:
save()

button = tk.Button(root, text="Save", font=font, command=on_save, width=8)
button.place(x=560, y=34)
_place(button, "save")

subscribed = tk.BooleanVar(value=False)

Expand All @@ -98,10 +86,10 @@ def on_subscribe() -> None:

subscribe = tk.Checkbutton(root, text="Subscribe", font=font, variable=subscribed, command=on_subscribe,
bg="#f0f0f0")
subscribe.place(x=40, y=100)
_place(subscribe, "subscribe")

notes = tk.Text(root, font=("DejaVu Sans", 14), width=28, height=3)
notes.place(x=760, y=40)
_place(notes, "notes")

def note(event) -> None:
state["notes"] = notes.get("1.0", "end-1c")
Expand All @@ -110,7 +98,7 @@ def note(event) -> None:
notes.bind("<KeyRelease>", note)

frame = tk.Frame(root)
frame.place(x=40, y=160, width=300, height=400)
_place(frame, "list", **SIZE["list"])
listbox = tk.Listbox(frame, font=("DejaVu Sans", 14), exportselection=False)
listbox.pack(side="left", fill="both", expand=True)
bar = tk.Scrollbar(frame, command=listbox.yview)
Expand All @@ -122,7 +110,7 @@ def scrolled(first, last) -> None:
save()

listbox.configure(yscrollcommand=scrolled)
for i in range(1, 101):
for i in range(1, ITEMS + 1):
listbox.insert("end", f"Item {i}")

def select(event) -> None:
Expand All @@ -133,7 +121,7 @@ def select(event) -> None:
listbox.bind("<<ListboxSelect>>", select)

grid = tk.Canvas(root, width=GRID * CELL, height=GRID * CELL, bg="#ffffff", highlightthickness=0)
grid.place(x=400, y=160)
_place(grid, "grid")
for row in range(GRID):
for col in range(GRID):
grid.create_rectangle(col * CELL, row * CELL, (col + 1) * CELL, (row + 1) * CELL, fill=COLOURS[0],
Expand All @@ -155,8 +143,8 @@ def double(event) -> None:
grid.bind("<Button-3>", lambda e: cell(e, True))
grid.bind("<Double-Button-1>", double)

pad = tk.Canvas(root, width=400, height=300, bg="#fffbe6", highlightthickness=0)
pad.place(x=840, y=160)
pad = tk.Canvas(root, **SIZE["pad"], bg="#fffbe6", highlightthickness=0)
_place(pad, "pad")
stroke: list = []

def press(event) -> None:
Expand All @@ -170,17 +158,16 @@ def motion(event) -> None:
def release(event) -> None:
if stroke:
stroke.append((event.x, event.y))
state["drags"] = (state["drags"] + [{"start": stroke[0], "end": stroke[-1], "points": len(stroke),
"path": stroke[::max(1, len(stroke) // 50)] + stroke[-1:]}])[-20:]
record_drag(state, stroke)
save()
stroke.clear()

pad.bind("<ButtonPress-1>", press)
pad.bind("<B1-Motion>", motion)
pad.bind("<ButtonRelease-1>", release)

wheel = tk.Canvas(root, width=200, height=120, bg="#e6f0ff", highlightthickness=0)
wheel.place(x=840, y=500)
wheel = tk.Canvas(root, **SIZE["wheel"], bg="#e6f0ff", highlightthickness=0)
_place(wheel, "wheel")

def wheeled(horizontal: bool, steps: int) -> None:
way = ("left" if horizontal else "up") if steps > 0 else ("right" if horizontal else "down")
Expand All @@ -202,8 +189,8 @@ def scrolled(delta: int) -> None:
if tk.TkVersion >= 8.7:
wheel.bind("<TouchpadScroll>", lambda e: scrolled(e.delta))

field = tk.Canvas(root, width=48, height=48, bg="#000000", highlightthickness=0, bd=0)
field.place(x=1080, y=500)
field = tk.Canvas(root, **SIZE["field"], bg="#000000", highlightthickness=0, bd=0)
_place(field, "field")
for x, y, colour in MARKERS:
field.create_rectangle(x, y, x + 1, y + 1, fill=colour, outline="", width=0)

Expand All @@ -212,28 +199,29 @@ def poll_inbox() -> None:
if message.exists() and (text := message.read_text()) != state["loaded"]:
state["loaded"] = text
save()
root.after(200, poll_inbox)
root.after(INBOX_EVERY_MS, poll_inbox)

def write_layout() -> None:
def layout() -> None:
root.update_idletasks()
boxes = {key: [w.winfo_rootx(), w.winfo_rooty(), w.winfo_width(), w.winfo_height()]
for key, w in (("name", name), ("save", button), ("subscribe", subscribe), ("notes", notes),
("list", listbox), ("grid", grid), ("pad", pad), ("wheel", wheel), ("field", field))}
boxes["list_row_height"] = listbox.bbox(0)[3] if listbox.bbox(0) else 0
boxes["markers"] = [[field.winfo_rootx() + x, field.winfo_rooty() + y, colour] for x, y, colour in MARKERS]
boxes["cell"] = CELL
boxes["clipboard"] = CLIPBOARD
save()
(state_dir / "layout.json").write_text(json.dumps(boxes))
write_layout(state_dir, boxes, listbox.bbox(0)[3] if listbox.bbox(0) else 0,
(field.winfo_rootx(), field.winfo_rooty()))

root.clipboard_clear()
root.clipboard_append(CLIPBOARD)
state["clipboard"] = CLIPBOARD
root.after(300, write_layout)
root.after(300, layout)
root.after(500, poll_inbox)
root.mainloop()


def _place(widget: tk.Widget, at: str, **size: int) -> None:
widget.place(x=AT[at][0], y=AT[at][1], **size)


def keep_on_top(root: tk.Tk) -> None:
"""Keep the fixture over the windows an app opens, which would take the probe's clicks: maximized and above other
windows rather than full screen, since xfwm4 keeps a window an app puts above over a full-screen one, and raised
Expand Down
Loading
Loading