diff --git a/dokimi/changelog.d/20261114_120000_napetrov_probe-events.md b/dokimi/changelog.d/20261114_120000_napetrov_probe-events.md new file mode 100644 index 000000000..1903e829a --- /dev/null +++ b/dokimi/changelog.d/20261114_120000_napetrov_probe-events.md @@ -0,0 +1,14 @@ + + +### Added + +- **`repo run`: a `probe` event for a command that asks the host for its facts.** A command + such as `nproc`, `lscpu`, `nvidia-smi`, `pip list`, `printenv` or a `python -c` one-liner that + calls `os.cpu_count()` gets a `probe` event after its `command` event, with its `target` + (`cpu`, `gpu`, `os`, `memory`, `packages`, `env`, `network`) and the `pattern` it matched. The + patterns match the program the shell runs, not argument text (`grep nproc README` is no probe). + The defaults ship in `dokimi/repos/probes.py`; a manifest's `probes` key or `repo run --probes + FILE` replaces a target's list, and `[]` turns it off. Existing event kinds are unchanged. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index a9b377d5b..8e98a11af 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -1395,6 +1395,7 @@ {"key": "--max-turns", "action": "_StoreAction", "choices": null, "const": null, "default": 200, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, {"key": "--model", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": true, "type": null}, {"key": "--out-dir", "action": "_StoreAction", "choices": null, "const": null, "default": "repo-runs", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--probes", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--rebuild-image", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--reps", "action": "_StoreAction", "choices": null, "const": null, "default": 1, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, {"key": "--source-dir", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index e0b1be55b..2de7f159d 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -245,6 +245,9 @@ build_systems: [bazel, make] # selects L0 claim-check adapters process_metrics: # optional repository counters, result.json `process.counters` (§4.1) bazel_cmds: '\bbazel(?:isk)?\b' # a plain regex searches the command text icpx_missing: {regex: "icpx is not found", match: output} # match: text (default), output or any + +probes: # optional: target -> patterns of `probe` events (§4.1) + cpu: [nproc, "py:os.cpu_count"] # a listed target replaces its defaults; [] turns it off ``` Rules: @@ -530,6 +533,29 @@ The line format, its version and the promise that comes with it: [events-format. - A `read` event may have no `path`: a refused Claude Code Bash read of a relative path after an earlier call moved the shell, whose file is not known (`{"kind": "read", "denied": true}`). +- A `probe` event follows the `command` event of a command that asks the host + for its facts, one per pattern it matches (a command may give several): + `{"t": 4.0, "kind": "probe", "target": "cpu", "pattern": "nproc", "text": "nproc", "rc": 0}`. + `target` is one of `cpu`, `gpu`, `os`, `memory`, `packages`, `env`, `network`; + `pattern` is the configured pattern it matched; `text`, `rc` and `agent` are the + command's. The defaults (`dokimi/repos/probes.py`, `DEFAULT_PROBES`): `cpu` `lscpu`, + `/proc/cpuinfo`, `nproc`, `sysctl`, `Get-CimInstance Win32_Processor`, + `py:os.cpu_count`, `py:platform.processor`; `gpu` `nvidia-smi`, `rocm-smi`, + `py:torch.cuda.is_available`; `os` `uname`, `/etc/os-release`, `sw_vers`, `systeminfo`; + `memory` `free`, `/proc/meminfo`; `packages` `pip list`, `pip show`, `pip freeze`, + `conda list`; `env` `env`, `printenv`. A pattern is matched on the program the shell or + PowerShell runs, with the words it names in any order and case (`grep nproc README` runs + `grep`: no probe), on a file a reading command (`cat`, `head`, `grep`, ...) names for a + path pattern, and on the call in the code of a `python -c` one-liner for `py:NAME`. A + `$( )` substitution or a `bash -c` script counts as a command the agent ran. Not matched: + a Python heredoc, a script the agent runs from a file, a pattern with a `|` that is + not a pipeline of the same commands, and a launcher the wrapper lists do not know + (`docker exec`, `nix develop -c`). `env`, `timeout`, `xargs`, `conda run` and `uv run` + run the program after them; `env` with no program after its options is its own probe (`env -S` is not read). Each cell's `meta.json` + records the patterns per target (`probes`) that it ran with. + A manifest's `probes` (§2) replaces a target's defaults, and `repo run --probes FILE` + (YAML or JSON, the same mapping) replaces them again; nothing else changes the events, + and the `process_metrics` counters do not count probes. `result.json` reports the agent's tokens (`input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_write_tokens`) over the same scope as `cost_usd`. diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index a0b1b3c86..1303a42a4 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -320,6 +320,7 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock from dokimi.repos.bases import check_mcp_configs from dokimi.repos.container import docker_network from dokimi.repos.harness_profile import load_profile + from dokimi.repos.probes import load_probe_file from dokimi.repos.runner import ( AgentOptions, agent_network, batch_lock, harness_version, run_batch, task_caps, validate_tasks) from dokimi.repos.selfcheck import ANCESTOR_NAMES, isolation_probe @@ -332,6 +333,7 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock try: parsed = [parse_arm(a) for a in arms] # before the image build: a typo or an unreadable patch fails now profile = load_profile(args.harness) + probes = load_probe_file(args.probes) if args.probes else None # a bad file fails before the containers start profile.tool_args(args.tools, args.delegation) # a profile that cannot turn subagents off: refuse now harness_bin = _harness_bin(args.bin or profile.bin) arms = pin_arms(parsed, source_git_dir(source, open_source(source, _cache_dir(args)), _cache_dir(args)), @@ -345,7 +347,7 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock validate_tasks(tasks) # refuse malformed host requirements before containers start options = [AgentOptions(profile=profile, model=m, harness_bin=harness_bin, agent_network=args.agent_network, tools=args.tools, delegation=args.delegation, timeout_s=args.timeout, - max_turns=args.max_turns, budget_usd=args.budget) for m in args.model] + max_turns=args.max_turns, budget_usd=args.budget, probes=probes) for m in args.model] for task in tasks: # refuse before anything runs, not cell by cell task_caps(task, options[0]) agent_network(options[0], ctx, task) @@ -548,6 +550,9 @@ def register(repo_sub, eval_sub, positive_int, source_help: str, cache_help: str help="Seconds per agent run unless task.json sets timeout_s (default 2700)") run_p.add_argument("--max-turns", type=positive_int, default=200, dest="max_turns", help="Unless task.json sets max_turns (default 200)") + run_p.add_argument("--probes", default=None, metavar="FILE", + help="YAML or JSON: target -> probe patterns, over the manifest's `probes` (default: the " + "framework's list; contract §4.1)") run_p.add_argument("--budget", type=_usd, default=2.0, help="USD cap per agent run unless task.json sets budget_usd (default 2.0)") run_p.add_argument("--max-batch-usd", type=_usd, default=None, dest="max_batch_usd", diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index 59b840c64..437eeb842 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -10,6 +10,10 @@ {"t": 5.0, "kind": "edit", "path": "src/foo.cpp"} {"t": 20.2, "kind": "tool", "name": "WebFetch", "denied": true} {"t": 2.0, "kind": "tool", "name": "Skill", "skill": "lint-fix"} + {"t": 4.0, "kind": "probe", "target": "cpu", "pattern": "nproc", "text": "nproc", "rc": 0} + +A ``probe`` event follows the ``command`` event of a command that asks the host for its facts, one per +pattern it matches (:mod:`dokimi.repos.probes`, contract §4.1). ``t`` is seconds since the first timestamped trace line, or ``null`` when the format has no timestamps. Paths inside the workspace are made relative to it. diff --git a/dokimi/dokimi/repos/manifest.py b/dokimi/dokimi/repos/manifest.py index a101f3873..e7d3945db 100644 --- a/dokimi/dokimi/repos/manifest.py +++ b/dokimi/dokimi/repos/manifest.py @@ -35,6 +35,7 @@ from .graders.hidden_tests import HIDDEN_DIR, hidden_files from .host_spec import HostSpecError, parse_host from .process_metrics import Counter, parse_counter +from .probes import validate_config from .source import RepoTree, SourceError from .useful_spec import UsefulSpecError, parse_useful @@ -137,6 +138,7 @@ class RepoManifest: explore_tasks: str | None = None build_systems: tuple[str, ...] = () process_metrics: dict[str, Counter] = field(default_factory=dict) + probes: dict[str, tuple[str, ...]] = field(default_factory=dict) # target -> patterns (repos/probes.py) @property def runnable(self) -> bool: @@ -175,6 +177,7 @@ def to_dict(self) -> dict[str, Any]: "build_systems": list(self.build_systems), "process_metrics": {name: {"regex": c.pattern.pattern, "match": c.field} for name, c in self.process_metrics.items()}, + **({"probes": {t: list(p) for t, p in self.probes.items()}} if self.probes else {}), } @@ -291,6 +294,7 @@ def parse_manifest(raw: Any, path: str = "") -> RepoManifest: if duplicates: errors.append(f"image/caches: duplicate cache name(s) {duplicates}") counters = _parse_counters(raw.get("process_metrics") or {}, errors) + probes = _parse_probes(raw.get("probes"), errors) if errors: raise ManifestError(path, errors) @@ -312,9 +316,19 @@ def parse_manifest(raw: Any, path: str = "") -> RepoManifest: explore_tasks=_norm(raw["explore_tasks"]) if raw.get("explore_tasks") else None, build_systems=tuple(raw.get("build_systems") or ()), process_metrics=counters, + probes=probes, ) +def _parse_probes(raw: Any, errors: list[str]) -> dict[str, tuple[str, ...]]: + """The manifest's ``probes`` (repos/probes.py); a bad one is appended to ``errors``.""" + try: + return validate_config(raw, "probes") + except ValueError as exc: + errors.append(str(exc)) + return {} + + def _test_entry_problems(key: str, found: set[str]) -> list[str]: """``{tests}`` belongs to ``entry_points.test`` alone, which must name it and ``{out}``, its report.""" if key != _TEST_ENTRY_POINT: diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py new file mode 100644 index 000000000..c59340c3a --- /dev/null +++ b/dokimi/dokimi/repos/probes.py @@ -0,0 +1,530 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Probe events: a command that asks the host for its facts gets a ``probe`` event beside its ``command`` event. + + {"t": 4.0, "kind": "probe", "target": "cpu", "pattern": "nproc", "text": "nproc", "rc": 0} + +One event per pattern the command matches, right after the command event, so one command may give both. A +pattern is matched on the commands the shell runs (the words of each simple command, its pipeline stages, the +code of ``python -c`` and of ``$( )`` substitutions), never on the text of an argument: ``grep nproc README`` +runs ``grep``, not ``nproc``. The pattern syntax, in a manifest's ``probes`` or a run's ``--probes`` file: + +- ``nproc``, ``pip list``, ``Get-CimInstance Win32_Processor``: a program, with the words that follow it (in + any order, any case); ``sysctl -a | grep machdep.cpu``: consecutive pipeline stages; +- ``/proc/cpuinfo``: a file a reading command names (``cat``, ``grep``, ``head``, ...); +- ``py:os.cpu_count``: a call of that name in the code of a ``python -c`` one-liner. + +A target listed in a manifest replaces that target's defaults, and a run's ``--probes`` replaces them again; +an empty list turns the target off. See ``docs/repo-evaluation-contract.md`` §2 and §4.1. +""" + +from __future__ import annotations + +import ast +import posixpath +import re +import shlex +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from .shell_split import split_script + +TARGETS = ("cpu", "gpu", "os", "memory", "packages", "env", "network") + +DEFAULT_PROBES: dict[str, tuple[str, ...]] = { + "cpu": ("lscpu", "/proc/cpuinfo", "nproc", "sysctl", "Get-CimInstance Win32_Processor", + "py:os.cpu_count", "py:platform.processor"), + "gpu": ("nvidia-smi", "rocm-smi", "py:torch.cuda.is_available"), + "os": ("uname", "/etc/os-release", "sw_vers", "systeminfo"), + "memory": ("free", "/proc/meminfo"), + "packages": ("pip list", "pip show", "pip freeze", "conda list"), + "env": ("env", "printenv"), + "network": (), +} + +_READERS = frozenset({"cat", "head", "tail", "less", "more", "nl", "sed", "grep", "egrep", "fgrep", "awk", "tac", + "strings", "wc", "rg"}) +_SEARCHERS = frozenset({"grep", "egrep", "fgrep", "rg", "sed", "awk"}) # the first operand is an expression, not a file +_WRAPPERS = frozenset({"time", "nohup", "sudo", "command", "exec", "builtin", "nice", "timeout", "xargs"}) # each runs +# the command after it +_KEYWORDS = frozenset({"if", "then", "else", "elif", "do", "while", "until", "!"}) # shell keywords before a command +_OPTION_OPERANDS = frozenset({("nice", "-n"), ("sudo", "-u"), ("timeout", "-s"), ("timeout", "-k"), + ("xargs", "-n"), ("xargs", "-I"), ("xargs", "-L"), ("xargs", "-P"), ("xargs", "-d"), + ("xargs", "-E"), ("xargs", "-s"), ("xargs", "-a")}) # wrapper options that take a value +_ENV_VALUES = frozenset({"-u", "--unset", "-C", "--chdir", "-S", "--split-string"}) # env options with a value word +_RUN_VALUES = frozenset({"-n", "--name", "-p", "--prefix", "--python", "--with", "--directory", "--project", + "--env-file"}) # the options of ``conda run`` and ``uv run`` that take a value word +_PWSH = frozenset({"pwsh", "powershell"}) # PowerShell: its script follows ``-Command`` (``-c``), any case +_PWSH_VALUES = frozenset({"-executionpolicy", "-ep", "-workingdirectory", "-wd", "-settingsfile", "-configurationname", + "-inputformat", "-outputformat"}) # PowerShell options whose next word is their value +_GREP_VALUES = frozenset({"-m", "-A", "-B", "-C", "-d", "-D", "--max-count", "--after-context", "--before-context", + "--context", "--label", "--binary-files", "--directories", "--devices"}) +_RG_VALUES = frozenset({"-g", "--glob", "--iglob", "-t", "--type", "-T", "--type-not", "--type-add", "-m", "--max-count", + "-A", "-B", "-C", "--after-context", "--before-context", "--context", "-M", "--max-columns", "-j", + "--threads", "-E", "--encoding", "-r", "--replace", "-d", "--max-depth", "--sort", "--sortr"}) +_SEARCH_VALUES = {"grep": _GREP_VALUES, "egrep": _GREP_VALUES, "fgrep": _GREP_VALUES, "rg": _RG_VALUES, + "awk": frozenset({"-F", "-v"})} # a searcher's options whose next word is a value, not an operand +_PATTERN_FILE = frozenset({"-f", "--file"}) # a searcher's option whose value is a file it reads, and no expression +_EXPRESSION = frozenset({"-e", "--regexp", "--expression"}) # a searcher's option whose value is the search expression +_PYTHON = re.compile(r"python\d*(?:\.\d+)?|py|pypy\d*") +_PIP = re.compile(r"pip\d*(?:\.\d+)?") +_ASSIGN = re.compile(r"[A-Za-z_]\w*=") +_DOTTED = re.compile(r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*") +_SHELLS = frozenset({"bash", "sh", "zsh", "dash"}) +_SHELL_C = re.compile(r"-[a-z]*c[a-z]*") # ``-c``, ``-lc``: the next word is a script +_LEX_SKIP = frozenset({"(", ")", "{", "}"}) # grouping a PowerShell or shell expression puts around a command +_SEPARATORS = frozenset({"|", "||", "&&", ";", "&"}) # ``|`` splits stages; the rest end a command +_ESC = {c: chr(0xE000 + i) for i, c in enumerate("|;&<>$`\\()")} # stands for a character a quote makes literal +_UNESC = {stand: c for c, stand in _ESC.items()} +_ESCAPABLE = frozenset("\"'\\$`|;&<>()") # a backslash before one of these, outside single quotes, is an escape +_FD = "\ue0ff" # stands for a file descriptor digit before a redirection, which is no word +_PY_VALUE_OPTIONS = frozenset({"-W", "-X"}) # python's own options whose next word is their value + +Stage = tuple[str, tuple[str, ...]] # a simple command's program (normalised) and its words + + +@dataclass(frozen=True) +class Probe: + """One compiled pattern. ``kind``: ``command`` (``stages``, consecutive pipeline stages), ``path`` (``text``, the + file's path) or ``python`` (``call``, the regex the code of a ``-c`` one-liner must match).""" + + target: str + text: str + kind: str + stages: tuple[Stage, ...] = () + call: re.Pattern[str] | None = None + + +def validate_config(raw: Any, where: str) -> dict[str, tuple[str, ...]]: + """A ``probes`` mapping (a manifest's or a run file's) as target -> patterns; ``ValueError`` names what is wrong.""" + if raw is None: + return {} + if not isinstance(raw, Mapping): + raise ValueError(f"{where}: expected a mapping of target to patterns, got {type(raw).__name__}") + config: dict[str, tuple[str, ...]] = {} + for target, patterns in raw.items(): + if target not in TARGETS: + raise ValueError(f"{where}: unknown target {target!r} (known: {', '.join(TARGETS)})") + if not isinstance(patterns, list) or not all(isinstance(p, str) and p.strip() for p in patterns): + raise ValueError(f"{where}: {target} must be a list of non-empty pattern strings") + for pattern in patterns: + _compile(target, pattern.strip()) # a pattern that cannot be read is refused here, not per cell + config[target] = tuple(p.strip() for p in patterns) + return config + + +def load_probe_file(path: str) -> dict[str, tuple[str, ...]]: + """A run's ``--probes`` file (YAML, or JSON, which is YAML) through :func:`validate_config`.""" + try: + raw = yaml.safe_load(Path(path).read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, yaml.YAMLError, RecursionError) as exc: # deep nesting too + raise ValueError(f"--probes {path}: {exc}") from exc + return validate_config(raw, f"--probes {path}") + + +def effective_patterns(*layers: Mapping[str, Sequence[str]] | None) -> dict[str, list[str]]: + """Each target's patterns once the layers are applied, as ``probes_for`` applies them (for the cell's record).""" + config: dict[str, Sequence[str]] = dict(DEFAULT_PROBES) + for layer in layers: + config.update(layer or {}) + return {target: [text.strip() for text in config.get(target, ())] for target in TARGETS} + + +def probes_for(*layers: Mapping[str, Sequence[str]] | None) -> tuple[Probe, ...]: + """The compiled patterns of the defaults with each layer applied in turn: a target a layer names is replaced.""" + config: dict[str, Sequence[str]] = dict(DEFAULT_PROBES) + for layer in layers: + config.update(layer or {}) + return tuple(_compile(target, text.strip()) for target in TARGETS for text in config.get(target, ())) + + +def with_probes(events: list[dict[str, Any]], probes: Sequence[Probe]) -> list[dict[str, Any]]: + """``events`` with a ``probe`` event after each ``command`` event whose command matches a pattern.""" + if not probes: + return events + out: list[dict[str, Any]] = [] + for event in events: + out.append(event) + if event.get("kind") == "command": + out.extend(_probe_event(event, probe) for probe in _matching(str(event.get("text", "")), probes)) + return out + + +def _probe_event(command: dict[str, Any], probe: Probe) -> dict[str, Any]: + event: dict[str, Any] = {"t": command.get("t"), "kind": "probe", "target": probe.target, "pattern": probe.text, + "text": command.get("text", ""), "rc": command.get("rc")} + if "agent" in command: + event["agent"] = command["agent"] + return event + + +def _compile(target: str, text: str) -> Probe: + if target not in TARGETS: + raise ValueError(f"unknown probe target {target!r}") + if text.startswith("py:"): + name = text[3:].strip() + if not _DOTTED.fullmatch(name): + raise ValueError(f"{text!r}: py: takes a dotted Python name such as os.cpu_count") + return Probe(target, text, "python", call=re.compile(r"(? Stage: + if any(c in part for c in ";&<>"): + raise ValueError(f"{text!r}: separate commands and redirections are not part of a pattern; use '|' for stages") + try: + words = [w for w in shlex.split(part) if w not in _LEX_SKIP] + except ValueError as exc: + raise ValueError(f"{text!r}: {exc}") from exc + if not words: + raise ValueError(f"{text!r}: a pattern names a program") + launched = _launched(words, 0) # ``uv run pip list`` is the pattern ``pip list``: the program the command runs + if launched is not None: + words = words[launched:] + return _head(words[0], words[1:]) # ``pip3 list`` and ``python -m pip list`` are ``pip list``, as commands are + + +def _program(word: str) -> str: + """A program's name as the shell or PowerShell finds it: no directory, no ``.exe``, no case.""" + name = posixpath.basename(word.replace("\\", "/")).casefold() + return name[:-4] if name.endswith(".exe") else name + + +def _matching(command: str, probes: Sequence[Probe]) -> list[Probe]: + """The probes the command runs, each once, in the order they are configured.""" + pipelines = [[list(_views(words)) for words in pipe] for pipe in _pipelines(command)] + return [p for p in dict.fromkeys(probes) if any(_hits(p, pipe) for pipe in pipelines)] + + +def _pipelines(script: str) -> Iterator[list[list[str]]]: + """Each pipeline of the script as its stages' words; the commands of its ``$( )`` and backtick substitutions too.""" + for part, _after, _in_subshell, _outer in split_script(script): + codes, rest = _substitutions(_mask(part)) + for code in codes: # the substitutions ran as their own commands + yield from _pipelines(_restore(code)) + stages = _stages(rest) + if stages: + yield stages + for stage in stages: # ``bash -lc 'nproc'``, ``time bash -c nproc``: the script a shell runs is a script of its own + for prog, args in _views(stage): + script_arg = _shell_script(prog, args) + if script_arg is not None: + yield from _pipelines(script_arg) + + +def _mask(part: str) -> str: + """``part`` with the characters a quote or a backslash makes literal swapped for stand-ins: ``|;&<>`` in double + quotes, every special character and ``$`` in single quotes, a backslash before a letter (a Windows path). The quotes + stay, so shlex still reads each word, and a substitution is found only where ``$`` is live.""" + out: list[str] = [] + quote = "" + i = 0 + while i < len(part): + c = part[i] + nxt = part[i + 1] if i + 1 < len(part) else "" + if quote != "'" and c == "\\" and nxt == "\n": # a line continuation: the shell reads the pair as nothing + i += 2 + continue + if quote != "'" and c == "\\" and nxt: + if (quote == '"' and nxt in '"\\') or (not quote and nxt in "\"'\\ \t"): + out.append(c + nxt) # an escape shlex reads itself + elif nxt in _ESCAPABLE: + out.append(_ESC.get(nxt, nxt)) # the escaped character, literal + else: + out.append(_ESC["\\"]) # a backslash before a letter: a path's separator, not an escape + i += 1 + continue + i += 2 + continue + if c in "'\"" and quote in ("", c): + quote = "" if quote else c + out.append(c) + elif (quote == "'" and c in _ESC) or (quote == '"' and c in "|;&<>") or (not quote and c == "\\"): + out.append(_ESC[c]) + else: + out.append(c) + i += 1 + return "".join(out) + + +def _close_of(part: str, j: int) -> int: + """The index of the ``)`` that closes a ``$(`` whose code starts at ``j``: parentheses count, and a quote inside the + code is read as the shell reads it, so a ``)`` in quotes does not close it; a nested ``$( )`` is skipped whole.""" + quote, depth = "", 0 + while j < len(part): + c = part[j] + if quote == "'": + quote = "" if c == "'" else quote + elif c == "\\": + j += 1 + elif c == "'" and not quote: + quote = "'" + elif c == '"': + quote = "" if quote == '"' else '"' + elif quote != "'" and part.startswith("$(", j): + j = _close_of(part, j + 2) + elif not quote and c == "(": + depth += 1 + elif not quote and c == ")": + if depth == 0: + return j + depth -= 1 + j += 1 + return len(part) + + +def _substitutions(part: str) -> tuple[list[str], str]: + """The code of each ``$( )`` (balanced parentheses) and backtick substitution of a masked part, and the part without + them: a substitution runs as its own command, so it is no word of this one.""" + codes: list[str] = [] + rest: list[str] = [] + i = 0 + while i < len(part): + if part.startswith("$(", i): + end = _close_of(part, i + 2) + codes.append(part[i + 2:end]) + rest.append(" ") + i = end + 1 + elif part[i] == "`": + j = part.find("`", i + 1) + j = len(part) if j < 0 else j + codes.append(part[i + 1:j]) + rest.append(" ") + i = j + 1 + else: + rest.append(part[i]) + i += 1 + return codes, "".join(rest) + + +def _restore(text: str) -> str: + return "".join(_UNESC.get(c, c) for c in text) + + +def _fd_masked(part: str) -> str: + """``part`` with each file descriptor digit before a redirection (``2>``, ``2>&1``) swapped for a stand-in.""" + return re.sub(r"(?()])\d+(?=[<>])", lambda m: _FD * len(m.group()), part) + + +def _lexed(text: str) -> list[str]: + """The tokens of a masked part: each word with its quotes removed, the operators as shlex reads them, and no + file descriptor digit.""" + try: + lexer = shlex.shlex(text, posix=True, punctuation_chars=True) + lexer.whitespace_split = True + tokens = list(lexer) + except ValueError: # an unbalanced quote: the words as they are typed + tokens = text.split() + return [token for token in tokens if set(token) != {_FD}] + + +def _close(stage: list[str], reads: list[str]) -> None: + if stage: # an input's target is read by the command, so it follows the command's words + stage.extend(reads) + + +def _stages(part: str) -> list[list[str]]: + """A part's simple commands, cut at its unquoted pipes, each as its words (quotes removed, grouping dropped). An + output redirection's target is written, so it is no word; an input's target is read, so it follows the words.""" + stages: list[list[str]] = [[]] + reads: list[str] = [] + target = "" # the operator whose target is the next token: ``>`` (written) or ``<`` (read) + for token in _lexed(_fd_masked(part)): + if target: + if target == "<": + reads.append(_restore(token)) + target = "" + elif token == "|": + _close(stages[-1], reads) + reads = [] + stages.append([]) + elif token.lstrip("&").startswith(">") or token in ("<", "<<<"): + target = token.lstrip("&")[:3] # ``<<<`` is a string, not a file: only a lone ``<`` reads its target + elif token not in _SEPARATORS and token not in _LEX_SKIP: + stages[-1].append(_restore(token)) + _close(stages[-1], reads) + return [stage for stage in stages if stage] + + +def _views(words: list[str]) -> Iterator[Stage]: + """The command a simple command runs: its own program, then the command each wrapper (``time``, ``sudo``...) runs.""" + i = 0 + while i < len(words): + while i < len(words): + if _ASSIGN.match(words[i]) or words[i] in _KEYWORDS: # NAME=value, ``then``: the command is next + i += 1 + elif words[i].startswith("$") and words[i + 1:i + 2] == ["="]: # PowerShell ``$x = ``: the same + i += 2 + else: + break + if i >= len(words): + return + launched = _launched(words, i) + if launched is not None: # ``env FOO=1 nproc``: the program is nproc, and env is no probe of its own + i = launched + continue + yield _head(words[i], words[i + 1:]) + wrapper = _program(words[i]) + if wrapper not in _WRAPPERS: + return + after = _past_wrapper_args(words, i + 1, wrapper) + if after is None: + return # ``command -v nproc`` looks the program up, it does not run it + i = after + + +def _past_wrapper_args(words: list[str], i: int, wrapper: str) -> int | None: + """The index of the program a wrapper runs: past its options (``sudo -E``, ``nice -n 5``) and, for ``timeout``, its + duration; None when the wrapper only looks a program up (``command -v``).""" + while i < len(words) and words[i].startswith("-"): + if wrapper == "command" and words[i] in ("-v", "-V"): + return None + i += 2 if (wrapper, words[i]) in _OPTION_OPERANDS else 1 + if wrapper == "timeout" and i < len(words): # ``timeout 5 nproc``: the duration is no program + i += 1 + return i + + +def _launched(words: list[str], i: int) -> int | None: + """The index of the program that ``env`` or ``conda run`` / ``uv run`` at ``i`` runs, or None when no program + follows its options and assignments.""" + program, j = _program(words[i]), i + 1 + if program == "env": + while j < len(words) and (words[j].startswith("-") or _ASSIGN.match(words[j])): + j += 2 if words[j] in _ENV_VALUES else 1 + elif program in ("conda", "uv") and words[i + 1:i + 2] == ["run"]: + j = i + 2 + while j < len(words) and words[j].startswith("-"): + j += 2 if words[j] in _RUN_VALUES else 1 + else: + return None + return j if j < len(words) else None + + +def _head(word: str, args: list[str]) -> Stage: + """A program and its words; ``python -I -m pip list`` is the program ``pip``.""" + program = _program(word) + if _PYTHON.fullmatch(program): + run = _python_run(tuple(args)) + if run and run[0] == "-m" and run[1] and _PIP.fullmatch(run[1][0]): + return "pip", run[1][1:] + return ("pip" if _PIP.fullmatch(program) else program), tuple(args) + + +def _shell_script(prog: str, args: tuple[str, ...]) -> str | None: + """The script a shell runs with ``-c`` (``bash --noprofile -c nproc``), or None.""" + if prog not in _SHELLS | _PWSH: + return None + i = 0 + while i < len(args): + word = args[i] + if (prog in _PWSH and len(word) > 1 and "-command".startswith(word.casefold())) or (prog in _SHELLS and _SHELL_C.fullmatch(word)): + if prog in _PWSH: # ``pwsh -Command a b``: the words after -Command are the script, joined + return " ".join(args[i + 1:]) or None + return args[i + 1] if i + 1 < len(args) else None + if prog in _SHELLS and word in ("-o", "+o"): # ``bash -o pipefail -c nproc``: the option's value is no word + i += 2 + continue + if prog in _PWSH and word.casefold() in _PWSH_VALUES: # ``pwsh -ExecutionPolicy Bypass -Command ...`` + i += 2 + continue + if not word.startswith("-"): + return None + i += 1 + return None + + +def _python_run(args: tuple[str, ...]) -> tuple[str, tuple[str, ...]] | None: + """What a python runs: ``("-c", the code and the words after it)``, ``("-m", the module and the words after it)``, + or None for a script. Its options are read up to the first word that is not one, as python reads them.""" + i = 0 + while i < len(args): + word = args[i] + if word in ("-c", "-m"): + return word, args[i + 1:] + if word.startswith("-c"): # ``-cimport os``: the code is in the same word + return "-c", (word[2:],) + args[i + 1:] + if re.fullmatch(r"-[BdEhiIOPqRsSuvx]+[cm]", word): # ``-Ic code``, ``-Im pip``: flags, then the code or module + return "-" + word[-1], args[i + 1:] + if word == "--" or not word.startswith("-"): + return None + i += 2 if word in _PY_VALUE_OPTIONS else 1 + return None + + +def _hits(probe: Probe, pipeline: list[list[Stage]]) -> bool: + if probe.kind == "command": + return _in_order(probe.stages, pipeline) + if probe.kind == "path": + return any(prog in _READERS and probe.text in _operands(prog, args) for views in pipeline for prog, args in views) + return any(_PYTHON.fullmatch(prog) and _calls(args, probe) for views in pipeline for prog, args in views) + + +def _operands(program: str, args: tuple[str, ...]) -> tuple[str, ...]: + """The words of a reader that can name a file. A searcher's search expression is not one: it is its ``-e`` value, or + else its first operand, and a pattern file (``-f``) is a file it reads. A searcher's option whose next word is a + value (``rg -g '*.c'``, ``grep -m 1``) takes that word, which is neither.""" + if program not in _SEARCHERS: + return args + values = _SEARCH_VALUES.get(program, frozenset()) + operands: list[str] = [] + given = False # an expression or a pattern file is named, so the first operand is no expression + i = 0 + while i < len(args): + word = args[i] + name, eq, value = word.partition("=") # ``--file=F``; a plain operand is kept as it is + if word == "--": + operands.extend(args[i + 1:]) + break + if name in _EXPRESSION or name in _PATTERN_FILE: + given = True + if name in _PATTERN_FILE: + if eq: + operands.append(value) + elif i + 1 < len(args): + operands.append(args[i + 1]) + i += 1 if eq else 2 + elif name in values: + i += 1 if eq else 2 + elif word.startswith("-") and len(word) > 1: + i += 1 + else: + operands.append(word) + i += 1 + return tuple(operands) if given else tuple(operands[1:]) + + +def _calls(args: tuple[str, ...], probe: Probe) -> bool: + """Whether the code of a ``python -c`` one-liner calls the probe's name; a string that only spells the call is no call.""" + run = _python_run(args) + if probe.call is None or run is None or run[0] != "-c" or not run[1]: + return False + code = run[1][0] + try: + tree = ast.parse(code) + except (SyntaxError, ValueError, RecursionError): # code that is no Python we can read: the name as it is written + return probe.call.search(code) is not None + name = probe.text[3:].strip() + return any(isinstance(node, ast.Call) and ast.unparse(node.func) == name for node in ast.walk(tree)) + + +def _in_order(want: tuple[Stage, ...], pipeline: list[list[Stage]]) -> bool: + """Whether the pattern's stages are consecutive stages of the pipeline (one stage: any stage of it).""" + n = len(want) + return any(all(_stage_runs(want[j], pipeline[i + j]) for j in range(n)) for i in range(len(pipeline) - n + 1)) + + +def _stage_runs(want: Stage, views: list[Stage]) -> bool: + """Whether one of the views is the wanted program with every wanted word among its words, any case.""" + program, words = want + return any(prog == program and all(w.casefold() in {a.casefold() for a in args} for w in words) + for prog, args in views) diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index a08eefaa4..8cb55f7ed 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -73,6 +73,7 @@ from .bases import MCP_CONFIG, NOT_UTF8, mcp_config_error from .events import convert from . import run_context +from .probes import effective_patterns, probes_for, with_probes from .uptake import arm_uptake from .useful import first_useful from .grading import EvalContext, failed_outcome, outcome, run_entry @@ -101,6 +102,7 @@ class AgentOptions: timeout_s: float = 2700 max_turns: int = 200 budget_usd: float = 2.0 + probes: dict[str, tuple[str, ...]] | None = None # the run's --probes, over the manifest's (repos/probes.py) def execution_record(opts: AgentOptions, arm: str | None = None) -> dict[str, str]: @@ -260,10 +262,18 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, uptake_of = (added.get("added") or [], added.get("mcp_servers") or []) events = None # the trace's events once it parsed; none: the uptake is unmeasured useful_events = None # the same events when no line of the trace was dropped: a dropped line may hold the hit + probe_errors: list[str] = [] # a probe matcher that raises costs the probe events only, and is noted + meta["probes"] = effective_patterns(ctx.manifest.probes, opts.probes) # the patterns the cell ran with try: raw = (rd / "agent.stdout").read_text(encoding="utf-8", errors="replace") transcript = convert(opts.profile.trace, raw, WORKSPACE) - (rd / "events.jsonl").write_text(transcript.events_jsonl(), encoding="utf-8") + # Probe events go to events.jsonl only: the counters, first_useful and skill usage score the trace's events + try: + probed = with_probes(transcript.events, probes_for(ctx.manifest.probes, opts.probes)) + except Exception as exc: # noqa: BLE001 # the events go on without probes: the grade must not depend on them + probed = transcript.events + probe_errors.append(f"probe matching failed ({type(exc).__name__}): {str(exc)[:200]}") + (rd / "events.jsonl").write_text(transcript.events_jsonl(probed), encoding="utf-8") failed = transcript.is_error or not result.ok # a crashed review is missing, not clean answer = convert_answer(opts.profile.answer_format, transcript.answer, WORKSPACE, failed=failed, messages=transcript.agent_messages) @@ -285,7 +295,8 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, cache_write_tokens=transcript.cache_write_tokens, usage_scope=transcript.usage_scope, first_prompt_tokens=transcript.first_prompt_tokens, subagent_calls=transcript.subagent_calls, plugins=transcript.plugins, - over_budget=cost is not None and cost > caps["budget_usd"], trace_errors=transcript.errors[:5]) + over_budget=cost is not None and cost > caps["budget_usd"], + trace_errors=(transcript.errors + probe_errors)[:5]) run_context.save(rd, run_context.build(opts.profile, model=opts.model, harness_version=opts.harness_version, argv=argv, transcript=transcript, config_sha256=config_sha), agent["trace_errors"]) diff --git a/dokimi/dokimi/repos/transcript.py b/dokimi/dokimi/repos/transcript.py index 5ca82d6a2..ad91396f5 100644 --- a/dokimi/dokimi/repos/transcript.py +++ b/dokimi/dokimi/repos/transcript.py @@ -64,8 +64,9 @@ def reads(self) -> set[str]: """The paths the agent was shown (:func:`shown_read`); a refused or failed read stays in the events only.""" return {e["path"] for e in self.events if shown_read(e) and e.get("path")} - def events_jsonl(self) -> str: - return "".join(json.dumps({**e, "v": EVENT_VERSION}, sort_keys=True) + "\n" for e in self.events) + def events_jsonl(self, events: list[dict[str, Any]] | None = None) -> str: + return "".join(json.dumps({**e, "v": EVENT_VERSION}, sort_keys=True) + "\n" + for e in (self.events if events is None else events)) def shown_read(event: dict[str, Any]) -> bool: diff --git a/dokimi/dokimi/schemas/repo_manifest.v1.json b/dokimi/dokimi/schemas/repo_manifest.v1.json index be85eb5ae..11608ffb0 100644 --- a/dokimi/dokimi/schemas/repo_manifest.v1.json +++ b/dokimi/dokimi/schemas/repo_manifest.v1.json @@ -76,6 +76,12 @@ "uniqueItems": true, "items": {"enum": ["bazel", "make", "cmake", "pytest", "pre-commit"]} }, + "probes": { + "description": "Target -> command, file or python patterns whose commands get a probe event (docs/repo-evaluation-contract.md §2, §4.1). A listed target replaces its defaults; [] turns it off.", + "type": "object", + "propertyNames": {"enum": ["cpu", "gpu", "os", "memory", "packages", "env", "network"]}, + "additionalProperties": {"type": "array", "items": {"type": "string", "minLength": 1}} + }, "process_metrics": { "type": "object", "propertyNames": {"pattern": "^[A-Za-z_][A-Za-z0-9_]*$"}, diff --git a/dokimi/tests/test_batch_ledger_golden.py b/dokimi/tests/test_batch_ledger_golden.py index 7b0f823ce..74873918f 100644 --- a/dokimi/tests/test_batch_ledger_golden.py +++ b/dokimi/tests/test_batch_ledger_golden.py @@ -95,6 +95,7 @@ "model", "model_resolved", "patches", + "probes", "rep", "start_sha", "task", diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py new file mode 100644 index 000000000..53ae243a6 --- /dev/null +++ b/dokimi/tests/test_repo_probes.py @@ -0,0 +1,427 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Probe events (repos/probes.py, contract §4.1): a command that asks the host for its facts gets a ``probe`` event.""" + +from __future__ import annotations + +import json + +import pytest + +from dokimi.repos.events import convert +from dokimi.repos.manifest import ManifestError, parse_manifest +from dokimi.repos.probes import DEFAULT_PROBES, load_probe_file, probes_for, validate_config, with_probes + +WS = "/ws" + + +def _claude_call(tool_id: str, command: str, t: str, name: str = "Bash", agent: str | None = None) -> dict: + rec = {"type": "assistant", "timestamp": t, "message": {"content": [ + {"type": "tool_use", "id": tool_id, "name": name, "input": {"command": command}}]}} + return {**rec, "parent_tool_use_id": agent} if agent else rec + + +def _claude_result(tool_id: str, text: str = "ok", t: str = "2026-01-01T00:00:06Z") -> dict: + return {"type": "user", "timestamp": t, "message": {"content": [ + {"type": "tool_result", "tool_use_id": tool_id, "is_error": False, "content": text}]}} + + +def _claude(*lines: dict) -> list[dict]: + """The command and probe events of a Claude Code stream whose first line starts the clock.""" + first = {"type": "system", "subtype": "init", "timestamp": "2026-01-01T00:00:00Z"} + raw = "\n".join(json.dumps(x) for x in (first, *lines)) + return with_probes(convert("claude-stream-json", raw, WS).events, probes_for()) + + +def _commands(events: list[dict]) -> list[dict]: + return [e for e in events if e["kind"] in ("command", "probe")] + + +def _probe(target: str, pattern: str, text: str, t: float | None, rc: int | None) -> dict: + return {"t": t, "kind": "probe", "target": target, "pattern": pattern, "text": text, "rc": rc} + + +def test_nproc_then_a_threaded_solver_gives_the_probe_after_its_command(): + events = _claude( + _claude_call("a", "nproc", "2026-01-01T00:00:05Z"), + _claude_result("a", "8", "2026-01-01T00:00:05Z"), + _claude_call("b", "python solve.py --threads 8", "2026-01-01T00:00:09Z"), + _claude_result("b", "done", "2026-01-01T00:00:20Z"), + ) + assert events == [ + {"t": 5.0, "kind": "command", "text": "nproc", "rc": 0, "output": "8", "output_truncated": False}, + _probe("cpu", "nproc", "nproc", 5.0, 0), + {"t": 9.0, "kind": "command", "text": "python solve.py --threads 8", "rc": 0, "output": "done", + "output_truncated": False}, + ] + + +def test_a_grep_of_the_word_is_no_probe(): + events = _claude(_claude_call("a", "grep nproc README", "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["kind"] for e in events] == ["command"] + + +def test_a_probe_is_matched_on_the_program_not_on_an_argument_of_another_one(): + events = _claude(_claude_call("a", "pip install nproc && echo lscpu", "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["kind"] for e in events] == ["command"] + + +def test_an_agent_that_never_probes_gives_no_probe_event(): + events = _claude(_claude_call("a", "ls -la", "2026-01-01T00:00:05Z"), _claude_result("a"), + _claude_call("b", "git status", "2026-01-01T00:00:06Z"), _claude_result("b")) + assert all(e["kind"] != "probe" for e in events) + + +def test_one_command_gives_the_command_and_every_probe_it_runs(): + events = _claude(_claude_call("a", "grep -c processor /proc/cpuinfo | cat && nproc", "2026-01-01T00:00:05Z"), + _claude_result("a", "4")) + assert _commands(events)[1:] == [ + _probe("cpu", "/proc/cpuinfo", events[0]["text"], 5.0, 0), + _probe("cpu", "nproc", events[0]["text"], 5.0, 0), + ] + assert events[0]["kind"] == "command" + + +def test_a_probe_in_a_substitution_is_run_by_the_command(): + events = _claude(_claude_call("a", "make -j$(nproc)", "2026-01-01T00:00:05Z"), _claude_result("a")) + assert _commands(events)[1] == _probe("cpu", "nproc", "make -j$(nproc)", 5.0, 0) + + +@pytest.mark.parametrize(("command", "pattern"), [ + ("lscpu", "lscpu"), + ("time nproc", "nproc"), + ("uname -a", "uname"), + ("free -h", "free"), + ("nvidia-smi --query-gpu=name --format=csv", "nvidia-smi"), + ("/usr/bin/nvidia-smi", "nvidia-smi"), + ("head -n 3 /etc/os-release", "/etc/os-release"), + ("cat \"/proc/meminfo\"", "/proc/meminfo"), + ("pip list --format=json", "pip list"), + ("python -m pip freeze", "pip freeze"), + ("pip3 show numpy", "pip show"), + ("conda list -n base", "conda list"), + ("printenv PATH", "printenv"), + ("env", "env"), + ("sysctl -a | grep machdep.cpu", "sysctl"), + ("sysctl -n machdep.cpu.brand_string", "sysctl"), + ("sysctl -n hw.ncpu", "sysctl"), + ("sysctl hw.memsize", "sysctl"), +]) +def test_default_command_patterns_match_the_program_that_runs(command, pattern): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["pattern"] for e in events if e["kind"] == "probe"] == [pattern] + + +@pytest.mark.parametrize(("command", "target"), [ + ('python -c "import os; print(os.cpu_count())"', "cpu"), + ("python3 -c 'import torch; print(torch.cuda.is_available())'", "gpu"), + ('python -c "import platform; print(platform.processor())"', "cpu"), +]) +def test_a_python_one_liner_that_calls_a_probe_is_a_probe(command, target): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["target"] for e in events if e["kind"] == "probe"] == [target] + + +@pytest.mark.parametrize("command", [ + 'echo "os.cpu_count()"', + "python -c 'print(1)' # os.cpu_count()", + 'python -c "import os; os.makedirs(\'x\')"', + "cat README | grep 'nproc'", + "git diff | grep nproc", +]) +def test_text_that_is_not_run_code_is_no_probe(command): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert all(e["kind"] != "probe" for e in events) + + +@pytest.mark.parametrize("command", ["grep /proc/cpuinfo README", "grep -e /proc/meminfo README"]) +def test_a_search_expression_that_looks_like_a_path_is_no_probe(command): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert all(e["kind"] != "probe" for e in events) + + +@pytest.mark.parametrize("command", ["grep -m 1 /proc/cpuinfo README", "cat > /proc/cpuinfo", "cat 2>> /proc/cpuinfo"]) +def test_an_option_value_or_a_redirection_target_is_no_file_the_command_reads(command): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert all(e["kind"] != "probe" for e in events) + + +@pytest.mark.parametrize(("command", "pattern"), [ + ("sudo -E lscpu", "lscpu"), + ("nice -n 5 nproc", "nproc"), + ("sudo -u root nproc", "nproc"), +]) +def test_a_wrappers_options_do_not_hide_the_program_it_runs(command, pattern): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["pattern"] for e in events if e["kind"] == "probe"] == [pattern] + + +@pytest.mark.parametrize("command", ["if true; then nproc; fi", "for x in 1; do nproc; done"]) +def test_a_command_in_the_body_of_a_conditional_or_loop_is_a_probe(command): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["pattern"] for e in events if e["kind"] == "probe"] == ["nproc"] + + +@pytest.mark.parametrize("command", [ + "echo '$(nproc)'", + "echo 'sysctl -n hw.ncpu'", +]) +def test_quoted_text_is_no_probe(command): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert all(e["kind"] != "probe" for e in events) + + +def test_a_call_inside_a_python_string_is_no_probe(): + events = _claude(_claude_call("a", "python -c 'print(\"os.cpu_count()\")'", "2026-01-01T00:00:05Z"), + _claude_result("a")) + assert all(e["kind"] != "probe" for e in events) + + +@pytest.mark.parametrize(("command", "pattern"), [ + ("make -j$(python -c 'import os; print(os.cpu_count())')", "py:os.cpu_count"), + ("echo $(echo $(nproc))", "nproc"), + ("bash -c '>/dev/null; nproc'", "nproc"), + ("grep -m 1 >/tmp/cpu.txt processor /proc/cpuinfo", "/proc/cpuinfo"), + ("grep '>' /proc/cpuinfo", "/proc/cpuinfo"), + ("python -c'import os;print(os.cpu_count())'", "py:os.cpu_count"), + ("python -I -m pip list", "pip list"), + ("< /proc/cpuinfo cat", "/proc/cpuinfo"), + ("time bash -c 'nproc'", "nproc"), + ("FOO=bar bash -c 'nproc'", "nproc"), + ("bash --noprofile -c 'nproc'", "nproc"), + ("C:\\Windows\\System32\\nvidia-smi.exe --query-gpu=name", "nvidia-smi"), + ("/mnt/c/Program\\ Files/NVIDIA\\ Corporation/NVSMI/nvidia-smi.exe -L", "nvidia-smi"), +]) +def test_a_probe_the_shell_runs_in_another_form_is_a_probe(command, pattern): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["pattern"] for e in events if e["kind"] == "probe"] == [pattern] + + +@pytest.mark.parametrize("command", ["python echo_args.py -c 'os.cpu_count()'", "rg /proc/cpuinfo README"]) +def test_a_script_argument_or_a_search_expression_is_no_probe(command): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert all(e["kind"] != "probe" for e in events) + + +def test_a_probe_file_nested_too_deep_is_a_value_error_that_names_it(tmp_path): + path = tmp_path / "probes.yaml" + path.write_text("[" * 2000, encoding="utf-8") + with pytest.raises(ValueError, match="probes.yaml"): + load_probe_file(str(path)) + + +def test_a_windows_powershell_probe_is_matched_case_blind_and_through_parentheses(): + events = _claude( + _claude_call("a", "Get-CimInstance -ClassName Win32_Processor | Select-Object NumberOfCores", + "2026-01-01T00:00:05Z", name="PowerShell"), + _claude_result("a", "8"), + _claude_call("b", "(get-ciminstance win32_processor).NumberOfCores", "2026-01-01T00:00:07Z", + name="PowerShell"), + _claude_result("b", "8", "2026-01-01T00:00:07Z"), + _claude_call("c", "Get-CimInstance Win32_ComputerSystem", "2026-01-01T00:00:08Z", name="PowerShell"), + _claude_result("c", "", "2026-01-01T00:00:08Z"), + ) + assert [(e["kind"], e.get("target")) for e in events] == [ + ("command", None), ("probe", "cpu"), ("command", None), ("probe", "cpu"), ("command", None)] + + +def test_a_subagents_probe_carries_its_agent(): + events = _claude(_claude_call("a", "nproc", "2026-01-01T00:00:05Z", agent="task-1"), _claude_result("a")) + assert events[1] == {**_probe("cpu", "nproc", "nproc", 5.0, 0), "agent": "task-1"} + + +def _codex(*items: dict) -> list[dict]: + raw = "\n".join(json.dumps(x) for x in items) + return with_probes(convert("codex-json", raw, WS).events, probes_for()) + + +def test_codex_unwraps_the_shell_before_it_matches_a_probe(): + events = _codex( + {"type": "item.completed", "item": {"type": "command_execution", "command": "/usr/bin/bash -lc 'nproc'", + "aggregated_output": "4\n", "exit_code": 0}}, + {"type": "item.completed", "item": {"type": "command_execution", + "command": "/usr/bin/bash -lc 'nvidia-smi'", + "aggregated_output": "", "exit_code": 127}}, + ) + assert events == [ + {"t": None, "kind": "command", "text": "nproc", "rc": 0, "output": "4\n", "output_truncated": False}, + _probe("cpu", "nproc", "nproc", None, 0), + {"t": None, "kind": "command", "text": "nvidia-smi", "rc": 127}, + _probe("gpu", "nvidia-smi", "nvidia-smi", None, 127), + ] + + +def _copilot(*records: dict) -> list[dict]: + raw = "\n".join(json.dumps(x) for x in records) + return with_probes(convert("copilot-json", raw, WS).events, probes_for()) + + +def test_copilot_bash_and_powershell_probes_give_probe_events(): + events = _copilot( + {"type": "tool.execution_start", "timestamp": "2026-01-01T00:00:01Z", + "data": {"toolCallId": "a", "toolName": "bash", "arguments": {"command": "nvidia-smi"}}}, + {"type": "tool.execution_complete", "data": {"toolCallId": "a", "success": True, + "shellExecution": {"exitCode": 0}, + "result": {"content": "GPU 0"}}}, + {"type": "tool.execution_start", "timestamp": "2026-01-01T00:00:02Z", + "data": {"toolCallId": "b", "toolName": "powershell", + "arguments": {"command": "Get-CimInstance Win32_Processor"}}}, + {"type": "tool.execution_complete", "data": {"toolCallId": "b", "success": True, + "shellExecution": {"exitCode": 0}, "result": {"content": "x"}}}, + ) + assert [(e["kind"], e.get("target"), e.get("pattern")) for e in events] == [ + ("command", None, None), ("probe", "gpu", "nvidia-smi"), + ("command", None, None), ("probe", "cpu", "Get-CimInstance Win32_Processor"), + ] + + +def test_a_manifest_replaces_a_target_and_an_empty_list_turns_it_off(): + probes = probes_for({"cpu": ["lscpu"], "env": []}) + assert [p.text for p in probes if p.target == "cpu"] == ["lscpu"] + assert all(p.target != "env" for p in probes) + assert {p.text for p in probes if p.target == "gpu"} == set(DEFAULT_PROBES["gpu"]) + + +def test_a_run_layer_replaces_the_manifest_layer(): + manifest = {"cpu": ["lscpu"]} + run = {"cpu": ["nproc"]} + assert [p.text for p in probes_for(manifest, run) if p.target == "cpu"] == ["nproc"] + + +def test_no_probes_leaves_the_events_as_they_are(): + events = [{"t": 1.0, "kind": "command", "text": "nproc", "rc": 0}] + assert with_probes(events, ()) == events + + +def test_probe_events_go_to_a_copy_so_the_trace_events_the_counters_read_are_unchanged(): + events = [{"t": 1.0, "kind": "command", "text": "nproc", "rc": 0}] + out = with_probes(events, probes_for()) + assert [e["kind"] for e in out] == ["command", "probe"] + assert events == [{"t": 1.0, "kind": "command", "text": "nproc", "rc": 0}] + + +@pytest.mark.parametrize(("raw", "message"), [ + ({"gpus": ["nvidia-smi"]}, "unknown target 'gpus'"), + ({"cpu": "nproc"}, "cpu must be a list"), + ({"cpu": [" "]}, "cpu must be a list"), + ({"cpu": ["py:os.cpu_count()"]}, "dotted Python name"), + ({"cpu": ['nproc "']}, "No closing quotation"), + ({"cpu": ["|"]}, "names a program"), + ({"cpu": ["nproc;lscpu"]}, "separate commands"), + ({"cpu": ["/proc/cpu info"]}, "one word with no spaces"), + (["cpu"], "expected a mapping"), +]) +def test_a_bad_pattern_config_is_refused_with_what_is_wrong(raw, message): + with pytest.raises(ValueError, match=message): + validate_config(raw, "probes") + + +def test_a_run_probe_file_is_yaml_or_json_and_checked(tmp_path): + good = tmp_path / "probes.yaml" + good.write_text("cpu: [lscpu]\nnetwork: [curl]\n", encoding="utf-8") + assert load_probe_file(str(good)) == {"cpu": ("lscpu",), "network": ("curl",)} + bad = tmp_path / "bad.json" + bad.write_text('{"cpu": ["nproc"', encoding="utf-8") + with pytest.raises(ValueError, match="--probes .*bad.json"): + load_probe_file(str(bad)) + + +def test_a_manifest_probes_key_is_read_and_a_bad_one_is_a_manifest_error(): + base = {"schema": "repo_manifest.v1", "id": "x", "guidance": {"globs": ["AGENTS.md"]}} + assert parse_manifest({**base, "probes": {"memory": ["free"]}}).probes == {"memory": ("free",)} + with pytest.raises(ManifestError, match="probes: 'disk' is not one of"): + parse_manifest({**base, "probes": {"disk": ["df"]}}) + + +@pytest.mark.parametrize(("command", "pattern"), [ + ("python \\\n-c 'import os; print(os.cpu_count())'", "py:os.cpu_count"), + ("npro\\\nc", "nproc"), +]) +def test_a_backslash_newline_continues_the_command_it_is_in(command, pattern): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert any(e["kind"] == "probe" and e["pattern"] == pattern for e in events) + + +def test_a_quoted_parenthesis_inside_a_double_quoted_substitution_does_not_end_it(): + events = _claude(_claude_call("a", 'echo "$(printf \')\'; nproc)"', "2026-01-01T00:00:05Z"), _claude_result("a")) + assert any(e["kind"] == "probe" and e["pattern"] == "nproc" for e in events) + + +def test_a_search_option_value_is_no_operand_so_the_file_after_it_is_read(): + events = _claude(_claude_call("a", "grep -f /proc/cpuinfo README", "2026-01-01T00:00:05Z"), _claude_result("a")) + assert any(e["kind"] == "probe" and e["pattern"] == "/proc/cpuinfo" for e in events) + + +@pytest.mark.parametrize("command", ["rg -g '*.c' /proc/cpuinfo src", "rg -t c /proc/cpuinfo src"]) +def test_a_glob_or_type_value_of_a_search_is_no_expression_or_file(command): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert all(e["kind"] != "probe" for e in events) + + +def test_a_powershell_assignment_is_skipped_so_the_command_after_it_is_the_probe(): + command = "$cpu = Get-CimInstance Win32_Processor; $cpu.NumberOfCores" + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z", name="PowerShell"), _claude_result("a", "8")) + assert [e["target"] for e in events if e["kind"] == "probe"] == ["cpu"] + + +@pytest.mark.parametrize(("command", "target"), [ + ("python -Ic 'import os; print(os.cpu_count())'", "cpu"), + ("python -Im pip list", "packages"), +]) +def test_python_flags_with_c_or_m_in_one_word_are_read_as_the_code_or_the_module(command, target): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["target"] for e in events if e["kind"] == "probe"] == [target] + + +def test_a_here_string_is_no_file_the_command_reads(): + events = _claude(_claude_call("a", "cat <<< /proc/cpuinfo", "2026-01-01T00:00:05Z"), _claude_result("a")) + assert all(e["kind"] != "probe" for e in events) + + +def _matched(command: str, layer: dict | None = None) -> list[str]: + raw = "\n".join(json.dumps(x) for x in ( + {"type": "system", "subtype": "init", "timestamp": "2026-01-01T00:00:00Z"}, + _claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a"))) + events = with_probes(convert("claude-stream-json", raw, WS).events, probes_for(layer)) + return [e["pattern"] for e in events if e["kind"] == "probe"] + + +@pytest.mark.parametrize(("command", "pattern"), [ + ("timeout 5 nproc", "nproc"), + ("echo | xargs -n 1 nproc", "nproc"), + ("uv run python -c 'import os; print(os.cpu_count())'", "py:os.cpu_count"), + ("bash -o pipefail -c nproc", "nproc"), + ("pwsh -Command Get-CimInstance Win32_Processor", "Get-CimInstance Win32_Processor"), + ("conda run -n x pip list", "pip list"), +]) +def test_a_common_launcher_does_not_hide_the_probe_it_runs(command, pattern): + assert _matched(command) == [pattern] + + +@pytest.mark.parametrize(("command", "pattern"), [ + ("env PYTHONPATH=. pytest -q", None), + ("env FOO=1 nproc", "nproc"), + ("env -u HOME nproc", "nproc"), + ("env", "env"), +]) +def test_env_runs_the_program_after_its_assignments_and_is_a_probe_only_alone(command, pattern): + assert _matched(command) == ([] if pattern is None else [pattern]) + + +def test_a_conda_run_is_no_conda_list_probe(): + assert "conda list" not in _matched("conda run -n x pip list") + + +def test_a_pattern_spelled_as_a_pip3_or_python_m_pip_command_matches_it(): + assert _matched("pip3 list", {"packages": ["pip3 list"]}) == ["pip3 list"] + assert _matched("python -m pip list", {"packages": ["python -m pip list"]}) == ["python -m pip list"] + + +def test_a_pattern_that_starts_with_a_launcher_matches_the_program_it_runs(): + assert _matched("uv run pip list", {"packages": ["uv run pip list"]}) == ["uv run pip list"] + + +def test_a_powershell_command_after_its_options_is_a_probe(): + cmd = "pwsh -NoProfile -ExecutionPolicy Bypass -com Get-CimInstance Win32_Processor" + assert _matched(cmd) == ["Get-CimInstance Win32_Processor"] diff --git a/dokimi/tests/test_repo_run_probes.py b/dokimi/tests/test_repo_run_probes.py new file mode 100644 index 000000000..11aea747d --- /dev/null +++ b/dokimi/tests/test_repo_run_probes.py @@ -0,0 +1,30 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The probe events of `repo run`: a failing matcher and the patterns a cell records (contract §2, §4.1).""" + +from __future__ import annotations + +import json + +from test_repo_run import _run, setup # noqa: F401 # the run fixture, with its helpers + + +def test_a_probe_matcher_that_raises_keeps_the_cell_graded_with_its_events(setup, monkeypatch): # noqa: F811 + # The matcher runs after the agent: a failure costs the probe events, not the grade, answer or events.jsonl. + def refuse(*args, **kwargs): + raise RecursionError("a chain too deep") + + monkeypatch.setattr("dokimi.repos.runner.with_probes", refuse) + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + cell = setup["out"] / "b1/fix_add__guidance-raw__m-1__r1" + rec = json.loads((cell / "result.json").read_text()) + assert rec["grade"]["status"] == "graded" and rec["grade"]["success"] is True + assert (cell / "events.jsonl").exists() and (cell / "answer.txt").exists() + assert any("probe matching failed" in e and "a chain too deep" in e for e in rec["agent"]["trace_errors"]) + + +def test_the_cell_records_the_probe_patterns_it_ran_with(setup): # noqa: F811 + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + meta = json.loads((setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/meta.json").read_text()) + assert "nproc" in meta["probes"]["cpu"] and meta["probes"]["packages"] == ["pip list", "pip show", "pip freeze", "conda list"]