From d3360cd2ef53e4d83f96d751190df6ce2795eaa7 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 14 Nov 2026 21:46:58 -0800 Subject: [PATCH 01/15] feat(repo): probe events for commands that ask the host for its facts A command that runs nproc, lscpu, nvidia-smi, pip list, printenv, a python -c one-liner calling os.cpu_count(), or a PowerShell Get-CimInstance Win32_Processor gets a probe event after its command event, one per pattern it matches. Patterns match the program the shell runs, never argument text, so grep nproc README is no probe. The defaults ship in repos/probes.py. A manifest's probes key and repo run --probes FILE replace a target's list; [] turns it off. Existing event kinds are unchanged. Signed-off-by: Nikolay Petrov --- .../20261114_120000_napetrov_probe-events.md | 14 + dokimi/docs/reference/cli-interface.json | 1 + dokimi/docs/repo-evaluation-contract.md | 23 ++ dokimi/dokimi/commands/repo_run.py | 7 +- dokimi/dokimi/repos/events.py | 4 + dokimi/dokimi/repos/manifest.py | 14 + dokimi/dokimi/repos/probes.py | 249 +++++++++++++++++ dokimi/dokimi/repos/runner.py | 6 +- dokimi/dokimi/repos/transcript.py | 4 +- dokimi/dokimi/schemas/repo_manifest.v1.json | 6 + dokimi/tests/test_repo_probes.py | 255 ++++++++++++++++++ 11 files changed, 579 insertions(+), 4 deletions(-) create mode 100644 dokimi/changelog.d/20261114_120000_napetrov_probe-events.md create mode 100644 dokimi/dokimi/repos/probes.py create mode 100644 dokimi/tests/test_repo_probes.py diff --git a/dokimi/changelog.d/20261114_120000_napetrov_probe-events.md b/dokimi/changelog.d/20261114_120000_napetrov_probe-events.md new file mode 100644 index 000000000..1903e829a --- /dev/null +++ b/dokimi/changelog.d/20261114_120000_napetrov_probe-events.md @@ -0,0 +1,14 @@ + + +### Added + +- **`repo run`: a `probe` event for a command that asks the host for its facts.** A command + such as `nproc`, `lscpu`, `nvidia-smi`, `pip list`, `printenv` or a `python -c` one-liner that + calls `os.cpu_count()` gets a `probe` event after its `command` event, with its `target` + (`cpu`, `gpu`, `os`, `memory`, `packages`, `env`, `network`) and the `pattern` it matched. The + patterns match the program the shell runs, not argument text (`grep nproc README` is no probe). + The defaults ship in `dokimi/repos/probes.py`; a manifest's `probes` key or `repo run --probes + FILE` replaces a target's list, and `[]` turns it off. Existing event kinds are unchanged. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index a87a843cc..d6257c963 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -1394,6 +1394,7 @@ {"key": "--max-turns", "action": "_StoreAction", "choices": null, "const": null, "default": 200, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, {"key": "--model", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": true, "type": null}, {"key": "--out-dir", "action": "_StoreAction", "choices": null, "const": null, "default": "repo-runs", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--probes", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--rebuild-image", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--reps", "action": "_StoreAction", "choices": null, "const": null, "default": 1, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, {"key": "--source-dir", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 5c1c0de52..ec570a8b8 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -244,6 +244,9 @@ build_systems: [bazel, make] # selects L0 claim-check adapters process_metrics: # optional repository counters, result.json `process.counters` (§4.1) bazel_cmds: '\bbazel(?:isk)?\b' # a plain regex searches the command text icpx_missing: {regex: "icpx is not found", match: output} # match: text (default), output or any + +probes: # optional: target -> patterns of `probe` events (§4.1) + cpu: [nproc, "py:os.cpu_count"] # a listed target replaces its defaults; [] turns it off ``` Rules: @@ -527,6 +530,26 @@ reads the upstream base (`base_sha`) and the workspace start commit - A `read` event may have no `path`: a refused Claude Code Bash read of a relative path after an earlier call moved the shell, whose file is not known (`{"kind": "read", "denied": true}`). +- A `probe` event follows the `command` event of a command that asks the host + for its facts, one per pattern it matches (a command may give several): + `{"t": 4.0, "kind": "probe", "target": "cpu", "pattern": "nproc", "text": "nproc", "rc": 0}`. + `target` is one of `cpu`, `gpu`, `os`, `memory`, `packages`, `env`, `network`; + `pattern` is the configured pattern it matched; `text`, `rc` and `agent` are the + command's. The defaults (`dokimi/repos/probes.py`, `DEFAULT_PROBES`): `cpu` `lscpu`, + `/proc/cpuinfo`, `nproc`, `sysctl -a | grep machdep.cpu`, `Get-CimInstance Win32_Processor`, + `py:os.cpu_count`, `py:platform.processor`; `gpu` `nvidia-smi`, `rocm-smi`, + `py:torch.cuda.is_available`; `os` `uname`, `/etc/os-release`, `sw_vers`, `systeminfo`; + `memory` `free`, `/proc/meminfo`; `packages` `pip list`, `pip show`, `pip freeze`, + `conda list`; `env` `env`, `printenv`. A pattern is matched on the program the shell or + PowerShell runs, with the words it names in any order and case (`grep nproc README` runs + `grep`: no probe), on a file a reading command (`cat`, `head`, `grep`, ...) names for a + path pattern, and on the call in the code of a `python -c` one-liner for `py:NAME`. A + `$( )` substitution or a `bash -c` script counts as a command the agent ran. Not matched: + a Python heredoc, a script the agent runs from a file, and a pattern with a `|` that is + not a pipeline of the same commands. + A manifest's `probes` (§2) replaces a target's defaults, and `repo run --probes FILE` + (YAML or JSON, the same mapping) replaces them again; nothing else changes the events, + and the `process_metrics` counters do not count probes. `result.json` reports the agent's tokens (`input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_write_tokens`) over the same scope as `cost_usd`. diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index 3f9368856..a6621b9cb 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -319,6 +319,7 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock from dokimi.repos.bases import check_mcp_configs from dokimi.repos.container import docker_network from dokimi.repos.harness_profile import load_profile + from dokimi.repos.probes import load_probe_file from dokimi.repos.runner import ( AgentOptions, agent_network, batch_lock, harness_version, run_batch, task_caps, validate_tasks) from dokimi.repos.selfcheck import ANCESTOR_NAMES, isolation_probe @@ -331,6 +332,7 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock try: parsed = [parse_arm(a) for a in arms] # before the image build: a typo or an unreadable patch fails now profile = load_profile(args.harness) + probes = load_probe_file(args.probes) if args.probes else None # a bad file fails before the containers start profile.tool_args(args.tools, args.delegation) # a profile that cannot turn subagents off: refuse now harness_bin = _harness_bin(args.bin or profile.bin) arms = pin_arms(parsed, source_git_dir(source, open_source(source, _cache_dir(args)), _cache_dir(args)), @@ -344,7 +346,7 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock validate_tasks(tasks) # refuse malformed host requirements before containers start options = [AgentOptions(profile=profile, model=m, harness_bin=harness_bin, agent_network=args.agent_network, tools=args.tools, delegation=args.delegation, timeout_s=args.timeout, - max_turns=args.max_turns, budget_usd=args.budget) for m in args.model] + max_turns=args.max_turns, budget_usd=args.budget, probes=probes) for m in args.model] for task in tasks: # refuse before anything runs, not cell by cell task_caps(task, options[0]) agent_network(options[0], ctx, task) @@ -547,6 +549,9 @@ def register(repo_sub, eval_sub, positive_int, source_help: str, cache_help: str help="Seconds per agent run unless task.json sets timeout_s (default 2700)") run_p.add_argument("--max-turns", type=positive_int, default=200, dest="max_turns", help="Unless task.json sets max_turns (default 200)") + run_p.add_argument("--probes", default=None, metavar="FILE", + help="YAML or JSON: target -> probe patterns, over the manifest's `probes` (default: the " + "framework's list; contract §4.1)") run_p.add_argument("--budget", type=_usd, default=2.0, help="USD cap per agent run unless task.json sets budget_usd (default 2.0)") run_p.add_argument("--max-batch-usd", type=_usd, default=None, dest="max_batch_usd", diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index 956171d08..05ec07ae0 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -10,6 +10,10 @@ {"t": 5.0, "kind": "edit", "path": "src/foo.cpp"} {"t": 20.2, "kind": "tool", "name": "WebFetch", "denied": true} {"t": 2.0, "kind": "tool", "name": "Skill", "skill": "lint-fix"} + {"t": 4.0, "kind": "probe", "target": "cpu", "pattern": "nproc", "text": "nproc", "rc": 0} + +A ``probe`` event follows the ``command`` event of a command that asks the host for its facts, one per +pattern it matches (:mod:`dokimi.repos.probes`, contract §4.1). ``t`` is seconds since the first timestamped trace line, or ``null`` when the format has no timestamps. Paths inside the workspace are made relative to it. diff --git a/dokimi/dokimi/repos/manifest.py b/dokimi/dokimi/repos/manifest.py index 9dc481491..884bdd504 100644 --- a/dokimi/dokimi/repos/manifest.py +++ b/dokimi/dokimi/repos/manifest.py @@ -35,6 +35,7 @@ from .graders.hidden_tests import HIDDEN_DIR, hidden_files from .host_spec import HostSpecError, parse_host from .process_metrics import Counter, parse_counter +from .probes import validate_config from .source import RepoTree, SourceError from .useful_spec import UsefulSpecError, parse_useful @@ -137,6 +138,7 @@ class RepoManifest: explore_tasks: str | None = None build_systems: tuple[str, ...] = () process_metrics: dict[str, Counter] = field(default_factory=dict) + probes: dict[str, tuple[str, ...]] = field(default_factory=dict) # target -> patterns (repos/probes.py) @property def runnable(self) -> bool: @@ -175,6 +177,7 @@ def to_dict(self) -> dict[str, Any]: "build_systems": list(self.build_systems), "process_metrics": {name: {"regex": c.pattern.pattern, "match": c.field} for name, c in self.process_metrics.items()}, + **({"probes": {t: list(p) for t, p in self.probes.items()}} if self.probes else {}), } @@ -291,6 +294,7 @@ def parse_manifest(raw: Any, path: str = "") -> RepoManifest: if duplicates: errors.append(f"image/caches: duplicate cache name(s) {duplicates}") counters = _parse_counters(raw.get("process_metrics") or {}, errors) + probes = _parse_probes(raw.get("probes"), errors) if errors: raise ManifestError(path, errors) @@ -312,9 +316,19 @@ def parse_manifest(raw: Any, path: str = "") -> RepoManifest: explore_tasks=_norm(raw["explore_tasks"]) if raw.get("explore_tasks") else None, build_systems=tuple(raw.get("build_systems") or ()), process_metrics=counters, + probes=probes, ) +def _parse_probes(raw: Any, errors: list[str]) -> dict[str, tuple[str, ...]]: + """The manifest's ``probes`` (repos/probes.py); a bad one is appended to ``errors``.""" + try: + return validate_config(raw, "probes") + except ValueError as exc: + errors.append(str(exc)) + return {} + + def _test_entry_problems(key: str, found: set[str]) -> list[str]: """``{tests}`` belongs to ``entry_points.test`` alone, which must name it and ``{out}``, its report.""" if key != _TEST_ENTRY_POINT: diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py new file mode 100644 index 000000000..d872761e4 --- /dev/null +++ b/dokimi/dokimi/repos/probes.py @@ -0,0 +1,249 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Probe events: a command that asks the host for its facts gets a ``probe`` event beside its ``command`` event. + + {"t": 4.0, "kind": "probe", "target": "cpu", "pattern": "nproc", "text": "nproc", "rc": 0} + +One event per pattern the command matches, right after the command event, so one command may give both. A +pattern is matched on the commands the shell runs (the words of each simple command, its pipeline stages, the +code of ``python -c`` and of ``$( )`` substitutions), never on the text of an argument: ``grep nproc README`` +runs ``grep``, not ``nproc``. The pattern syntax, in a manifest's ``probes`` or a run's ``--probes`` file: + +- ``nproc``, ``pip list``, ``Get-CimInstance Win32_Processor``: a program, with the words that follow it (in + any order, any case); ``sysctl -a | grep machdep.cpu``: consecutive pipeline stages; +- ``/proc/cpuinfo``: a file a reading command names (``cat``, ``grep``, ``head``, ...); +- ``py:os.cpu_count``: a call of that name in the code of a ``python -c`` one-liner. + +A target listed in a manifest replaces that target's defaults, and a run's ``--probes`` replaces them again; +an empty list turns the target off. See ``docs/repo-evaluation-contract.md`` §2 and §4.1. +""" + +from __future__ import annotations + +import posixpath +import re +import shlex +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from .shell_split import split_script + +TARGETS = ("cpu", "gpu", "os", "memory", "packages", "env", "network") + +DEFAULT_PROBES: dict[str, tuple[str, ...]] = { + "cpu": ("lscpu", "/proc/cpuinfo", "nproc", "sysctl -a | grep machdep.cpu", "Get-CimInstance Win32_Processor", + "py:os.cpu_count", "py:platform.processor"), + "gpu": ("nvidia-smi", "rocm-smi", "py:torch.cuda.is_available"), + "os": ("uname", "/etc/os-release", "sw_vers", "systeminfo"), + "memory": ("free", "/proc/meminfo"), + "packages": ("pip list", "pip show", "pip freeze", "conda list"), + "env": ("env", "printenv"), + "network": (), +} + +_READERS = frozenset({"cat", "head", "tail", "less", "more", "nl", "sed", "grep", "egrep", "fgrep", "awk", "tac", + "strings", "wc", "rg"}) +_WRAPPERS = frozenset({"time", "nohup", "sudo", "command", "exec", "builtin", "nice"}) # each runs the command after it +_PYTHON = re.compile(r"python\d*(?:\.\d+)?|py|pypy\d*") +_PIP = re.compile(r"pip\d*(?:\.\d+)?") +_ASSIGN = re.compile(r"[A-Za-z_]\w*=") +_DOTTED = re.compile(r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*") +_SUBSTITUTION = re.compile(r"\$\(([^()]*)\)|`([^`]*)`") +_SHELLS = frozenset({"bash", "sh", "zsh", "dash"}) +_SHELL_C = re.compile(r"-[a-z]*c[a-z]*") # ``-c``, ``-lc``: the next word is a script +_LEX_SKIP = frozenset({"(", ")", "{", "}"}) # grouping a PowerShell or shell expression puts around a command +_SEPARATORS = frozenset({"|", "||", "&&", ";", "&"}) # ``|`` splits stages; the rest end a command + +Stage = tuple[str, tuple[str, ...]] # a simple command's program (normalised) and its words + + +@dataclass(frozen=True) +class Probe: + """One compiled pattern. ``kind``: ``command`` (``stages``, consecutive pipeline stages), ``path`` (``text``, the + file's path) or ``python`` (``call``, the regex the code of a ``-c`` one-liner must match).""" + + target: str + text: str + kind: str + stages: tuple[Stage, ...] = () + call: re.Pattern[str] | None = None + + +def validate_config(raw: Any, where: str) -> dict[str, tuple[str, ...]]: + """A ``probes`` mapping (a manifest's or a run file's) as target -> patterns; ``ValueError`` names what is wrong.""" + if raw is None: + return {} + if not isinstance(raw, Mapping): + raise ValueError(f"{where}: expected a mapping of target to patterns, got {type(raw).__name__}") + config: dict[str, tuple[str, ...]] = {} + for target, patterns in raw.items(): + if target not in TARGETS: + raise ValueError(f"{where}: unknown target {target!r} (known: {', '.join(TARGETS)})") + if not isinstance(patterns, list) or not all(isinstance(p, str) and p.strip() for p in patterns): + raise ValueError(f"{where}: {target} must be a list of non-empty pattern strings") + for pattern in patterns: + _compile(target, pattern.strip()) # a pattern that cannot be read is refused here, not per cell + config[target] = tuple(p.strip() for p in patterns) + return config + + +def load_probe_file(path: str) -> dict[str, tuple[str, ...]]: + """A run's ``--probes`` file (YAML, or JSON, which is YAML) through :func:`validate_config`.""" + try: + raw = yaml.safe_load(Path(path).read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc: + raise ValueError(f"--probes {path}: {exc}") from exc + return validate_config(raw, f"--probes {path}") + + +def probes_for(*layers: Mapping[str, Sequence[str]] | None) -> tuple[Probe, ...]: + """The compiled patterns of the defaults with each layer applied in turn: a target a layer names is replaced.""" + config: dict[str, Sequence[str]] = dict(DEFAULT_PROBES) + for layer in layers: + config.update(layer or {}) + return tuple(_compile(target, text.strip()) for target in TARGETS for text in config.get(target, ())) + + +def with_probes(events: list[dict[str, Any]], probes: Sequence[Probe]) -> list[dict[str, Any]]: + """``events`` with a ``probe`` event after each ``command`` event whose command matches a pattern.""" + if not probes: + return events + out: list[dict[str, Any]] = [] + for event in events: + out.append(event) + if event.get("kind") == "command": + out.extend(_probe_event(event, probe) for probe in _matching(str(event.get("text", "")), probes)) + return out + + +def _probe_event(command: dict[str, Any], probe: Probe) -> dict[str, Any]: + event: dict[str, Any] = {"t": command.get("t"), "kind": "probe", "target": probe.target, "pattern": probe.text, + "text": command.get("text", ""), "rc": command.get("rc")} + if "agent" in command: + event["agent"] = command["agent"] + return event + + +def _compile(target: str, text: str) -> Probe: + if target not in TARGETS: + raise ValueError(f"unknown probe target {target!r}") + if text.startswith("py:"): + name = text[3:].strip() + if not _DOTTED.fullmatch(name): + raise ValueError(f"{text!r}: py: takes a dotted Python name such as os.cpu_count") + return Probe(target, text, "python", call=re.compile(r"(? Stage: + if any(c in part for c in ";&<>"): + raise ValueError(f"{text!r}: separate commands and redirections are not part of a pattern; use '|' for stages") + try: + words = [w for w in shlex.split(part) if w not in _LEX_SKIP] + except ValueError as exc: + raise ValueError(f"{text!r}: {exc}") from exc + if not words: + raise ValueError(f"{text!r}: a pattern names a program") + return _program(words[0]), tuple(words[1:]) + + +def _program(word: str) -> str: + """A program's name as the shell or PowerShell finds it: no directory, no ``.exe``, no case.""" + name = posixpath.basename(word.replace("\\", "/")).casefold() + return name[:-4] if name.endswith(".exe") else name + + +def _matching(command: str, probes: Sequence[Probe]) -> list[Probe]: + """The probes the command runs, each once, in the order they are configured.""" + pipelines = [[list(_views(words)) for words in pipe] for pipe in _pipelines(command)] + return [p for p in dict.fromkeys(probes) if any(_hits(p, pipe) for pipe in pipelines)] + + +def _pipelines(script: str) -> Iterator[list[list[str]]]: + """Each pipeline of the script as its stages' words; the commands of its ``$( )`` and backtick substitutions too.""" + for part, _after, _in_subshell, _outer in split_script(script): + for inner in _SUBSTITUTION.finditer(part): + yield from _pipelines(inner.group(1) if inner.group(1) is not None else inner.group(2)) + stages = _stages(_SUBSTITUTION.sub(" ", part)) # the substitutions ran as their own commands, above + if stages: + yield stages + for stage in stages: # ``bash -lc 'nproc'``: the script a shell runs is a script of its own + if len(stage) > 2 and _program(stage[0]) in _SHELLS and _SHELL_C.fullmatch(stage[1]): + yield from _pipelines(stage[2]) + + +def _stages(part: str) -> list[list[str]]: + """A part's simple commands, cut at its unquoted pipes, each as its words (quotes removed, grouping dropped).""" + try: + lexer = shlex.shlex(part, posix=True, punctuation_chars=True) + lexer.whitespace_split = True + tokens = list(lexer) + except ValueError: # an unbalanced quote: the words as they are typed + tokens = part.split() + stages: list[list[str]] = [[]] + for token in tokens: + if token == "|": + stages.append([]) + elif token not in _SEPARATORS and token not in _LEX_SKIP: + stages[-1].append(token) + return [stage for stage in stages if stage] + + +def _views(words: list[str]) -> Iterator[Stage]: + """The command a simple command runs: its own program, then the command each wrapper (``time``, ``sudo``...) runs.""" + i = 0 + while i < len(words): + while i < len(words) and _ASSIGN.match(words[i]): # NAME=value words set variables for the command after them + i += 1 + if i >= len(words): + return + yield _head(words[i], words[i + 1:]) + if _program(words[i]) not in _WRAPPERS: + return + i += 1 + + +def _head(word: str, args: list[str]) -> Stage: + """A program and its words; ``python -m pip list`` is the program ``pip``.""" + program = _program(word) + if _PYTHON.fullmatch(program) and args[:1] == ["-m"] and len(args) > 1 and _PIP.fullmatch(args[1]): + return "pip", tuple(args[2:]) + return ("pip" if _PIP.fullmatch(program) else program), tuple(args) + + +def _hits(probe: Probe, pipeline: list[list[Stage]]) -> bool: + if probe.kind == "command": + return _in_order(probe.stages, pipeline) + if probe.kind == "path": + return any(prog in _READERS and probe.text in args for views in pipeline for prog, args in views) + return any(_PYTHON.fullmatch(prog) and _calls(args, probe) for views in pipeline for prog, args in views) + + +def _calls(args: tuple[str, ...], probe: Probe) -> bool: + """Whether the code of a ``python -c`` one-liner calls the probe's name.""" + if "-c" not in args or probe.call is None: + return False + code = args[args.index("-c") + 1:args.index("-c") + 2] + return bool(code) and probe.call.search(code[0]) is not None + + +def _in_order(want: tuple[Stage, ...], pipeline: list[list[Stage]]) -> bool: + """Whether the pattern's stages are consecutive stages of the pipeline (one stage: any stage of it).""" + n = len(want) + return any(all(_stage_runs(want[j], pipeline[i + j]) for j in range(n)) for i in range(len(pipeline) - n + 1)) + + +def _stage_runs(want: Stage, views: list[Stage]) -> bool: + """Whether one of the views is the wanted program with every wanted word among its words, any case.""" + program, words = want + return any(prog == program and all(w.casefold() in {a.casefold() for a in args} for w in words) + for prog, args in views) diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index a5809f29a..95f11623a 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -70,6 +70,7 @@ ) from .bases import MCP_CONFIG, NOT_UTF8, mcp_config_error from .events import convert +from .probes import probes_for, with_probes from .uptake import arm_uptake from .useful import first_useful from .grading import EvalContext, failed_outcome, outcome, run_entry @@ -98,6 +99,7 @@ class AgentOptions: timeout_s: float = 2700 max_turns: int = 200 budget_usd: float = 2.0 + probes: dict[str, tuple[str, ...]] | None = None # the run's --probes, over the manifest's (repos/probes.py) def execution_record(opts: AgentOptions, arm: str | None = None) -> dict[str, str]: @@ -259,7 +261,9 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, try: raw = (rd / "agent.stdout").read_text(encoding="utf-8", errors="replace") transcript = convert(opts.profile.trace, raw, WORKSPACE) - (rd / "events.jsonl").write_text(transcript.events_jsonl(), encoding="utf-8") + # Probe events go to events.jsonl only: the counters, first_useful and skill usage score the trace's events + probed = with_probes(transcript.events, probes_for(ctx.manifest.probes, opts.probes)) + (rd / "events.jsonl").write_text(transcript.events_jsonl(probed), encoding="utf-8") failed = transcript.is_error or not result.ok # a crashed review is missing, not clean answer = convert_answer(opts.profile.answer_format, transcript.answer, WORKSPACE, failed=failed, messages=transcript.agent_messages) diff --git a/dokimi/dokimi/repos/transcript.py b/dokimi/dokimi/repos/transcript.py index 7d9408625..bd56e981e 100644 --- a/dokimi/dokimi/repos/transcript.py +++ b/dokimi/dokimi/repos/transcript.py @@ -59,8 +59,8 @@ def reads(self) -> set[str]: """The paths the agent was shown (:func:`shown_read`); a refused or failed read stays in the events only.""" return {e["path"] for e in self.events if shown_read(e) and e.get("path")} - def events_jsonl(self) -> str: - return "".join(json.dumps(e, sort_keys=True) + "\n" for e in self.events) + def events_jsonl(self, events: list[dict[str, Any]] | None = None) -> str: + return "".join(json.dumps(e, sort_keys=True) + "\n" for e in (self.events if events is None else events)) def shown_read(event: dict[str, Any]) -> bool: diff --git a/dokimi/dokimi/schemas/repo_manifest.v1.json b/dokimi/dokimi/schemas/repo_manifest.v1.json index be85eb5ae..11608ffb0 100644 --- a/dokimi/dokimi/schemas/repo_manifest.v1.json +++ b/dokimi/dokimi/schemas/repo_manifest.v1.json @@ -76,6 +76,12 @@ "uniqueItems": true, "items": {"enum": ["bazel", "make", "cmake", "pytest", "pre-commit"]} }, + "probes": { + "description": "Target -> command, file or python patterns whose commands get a probe event (docs/repo-evaluation-contract.md §2, §4.1). A listed target replaces its defaults; [] turns it off.", + "type": "object", + "propertyNames": {"enum": ["cpu", "gpu", "os", "memory", "packages", "env", "network"]}, + "additionalProperties": {"type": "array", "items": {"type": "string", "minLength": 1}} + }, "process_metrics": { "type": "object", "propertyNames": {"pattern": "^[A-Za-z_][A-Za-z0-9_]*$"}, diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py new file mode 100644 index 000000000..3c9b96a7c --- /dev/null +++ b/dokimi/tests/test_repo_probes.py @@ -0,0 +1,255 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Probe events (repos/probes.py, contract §4.1): a command that asks the host for its facts gets a ``probe`` event.""" + +from __future__ import annotations + +import json + +import pytest + +from dokimi.repos.events import convert +from dokimi.repos.manifest import ManifestError, parse_manifest +from dokimi.repos.probes import DEFAULT_PROBES, load_probe_file, probes_for, validate_config, with_probes + +WS = "/ws" + + +def _claude_call(tool_id: str, command: str, t: str, name: str = "Bash", agent: str | None = None) -> dict: + rec = {"type": "assistant", "timestamp": t, "message": {"content": [ + {"type": "tool_use", "id": tool_id, "name": name, "input": {"command": command}}]}} + return {**rec, "parent_tool_use_id": agent} if agent else rec + + +def _claude_result(tool_id: str, text: str = "ok", t: str = "2026-01-01T00:00:06Z") -> dict: + return {"type": "user", "timestamp": t, "message": {"content": [ + {"type": "tool_result", "tool_use_id": tool_id, "is_error": False, "content": text}]}} + + +def _claude(*lines: dict) -> list[dict]: + """The command and probe events of a Claude Code stream whose first line starts the clock.""" + first = {"type": "system", "subtype": "init", "timestamp": "2026-01-01T00:00:00Z"} + raw = "\n".join(json.dumps(x) for x in (first, *lines)) + return with_probes(convert("claude-stream-json", raw, WS).events, probes_for()) + + +def _commands(events: list[dict]) -> list[dict]: + return [e for e in events if e["kind"] in ("command", "probe")] + + +def _probe(target: str, pattern: str, text: str, t: float | None, rc: int | None) -> dict: + return {"t": t, "kind": "probe", "target": target, "pattern": pattern, "text": text, "rc": rc} + + +def test_nproc_then_a_threaded_solver_gives_the_probe_after_its_command(): + events = _claude( + _claude_call("a", "nproc", "2026-01-01T00:00:05Z"), + _claude_result("a", "8", "2026-01-01T00:00:05Z"), + _claude_call("b", "python solve.py --threads 8", "2026-01-01T00:00:09Z"), + _claude_result("b", "done", "2026-01-01T00:00:20Z"), + ) + assert events == [ + {"t": 5.0, "kind": "command", "text": "nproc", "rc": 0, "output": "8", "output_truncated": False}, + _probe("cpu", "nproc", "nproc", 5.0, 0), + {"t": 9.0, "kind": "command", "text": "python solve.py --threads 8", "rc": 0, "output": "done", + "output_truncated": False}, + ] + + +def test_a_grep_of_the_word_is_no_probe(): + events = _claude(_claude_call("a", "grep nproc README", "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["kind"] for e in events] == ["command"] + + +def test_a_probe_is_matched_on_the_program_not_on_an_argument_of_another_one(): + events = _claude(_claude_call("a", "pip install nproc && echo lscpu", "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["kind"] for e in events] == ["command"] + + +def test_an_agent_that_never_probes_gives_no_probe_event(): + events = _claude(_claude_call("a", "ls -la", "2026-01-01T00:00:05Z"), _claude_result("a"), + _claude_call("b", "git status", "2026-01-01T00:00:06Z"), _claude_result("b")) + assert all(e["kind"] != "probe" for e in events) + + +def test_one_command_gives_the_command_and_every_probe_it_runs(): + events = _claude(_claude_call("a", "grep -c processor /proc/cpuinfo | cat && nproc", "2026-01-01T00:00:05Z"), + _claude_result("a", "4")) + assert _commands(events)[1:] == [ + _probe("cpu", "/proc/cpuinfo", events[0]["text"], 5.0, 0), + _probe("cpu", "nproc", events[0]["text"], 5.0, 0), + ] + assert events[0]["kind"] == "command" + + +def test_a_probe_in_a_substitution_is_run_by_the_command(): + events = _claude(_claude_call("a", "make -j$(nproc)", "2026-01-01T00:00:05Z"), _claude_result("a")) + assert _commands(events)[1] == _probe("cpu", "nproc", "make -j$(nproc)", 5.0, 0) + + +@pytest.mark.parametrize(("command", "pattern"), [ + ("lscpu", "lscpu"), + ("time nproc", "nproc"), + ("uname -a", "uname"), + ("free -h", "free"), + ("nvidia-smi --query-gpu=name --format=csv", "nvidia-smi"), + ("/usr/bin/nvidia-smi", "nvidia-smi"), + ("head -n 3 /etc/os-release", "/etc/os-release"), + ("cat \"/proc/meminfo\"", "/proc/meminfo"), + ("pip list --format=json", "pip list"), + ("python -m pip freeze", "pip freeze"), + ("pip3 show numpy", "pip show"), + ("conda list -n base", "conda list"), + ("printenv PATH", "printenv"), + ("env", "env"), + ("sysctl -a | grep machdep.cpu", "sysctl -a | grep machdep.cpu"), +]) +def test_default_command_patterns_match_the_program_that_runs(command, pattern): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["pattern"] for e in events if e["kind"] == "probe"] == [pattern] + + +@pytest.mark.parametrize(("command", "target"), [ + ('python -c "import os; print(os.cpu_count())"', "cpu"), + ("python3 -c 'import torch; print(torch.cuda.is_available())'", "gpu"), + ('python -c "import platform; print(platform.processor())"', "cpu"), +]) +def test_a_python_one_liner_that_calls_a_probe_is_a_probe(command, target): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["target"] for e in events if e["kind"] == "probe"] == [target] + + +@pytest.mark.parametrize("command", [ + 'echo "os.cpu_count()"', + "python -c 'print(1)' # os.cpu_count()", + 'python -c "import os; os.makedirs(\'x\')"', + "cat README | grep 'nproc'", + "git diff | grep nproc", +]) +def test_text_that_is_not_run_code_is_no_probe(command): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert all(e["kind"] != "probe" for e in events) + + +def test_a_windows_powershell_probe_is_matched_case_blind_and_through_parentheses(): + events = _claude( + _claude_call("a", "Get-CimInstance -ClassName Win32_Processor | Select-Object NumberOfCores", + "2026-01-01T00:00:05Z", name="PowerShell"), + _claude_result("a", "8"), + _claude_call("b", "(get-ciminstance win32_processor).NumberOfCores", "2026-01-01T00:00:07Z", + name="PowerShell"), + _claude_result("b", "8", "2026-01-01T00:00:07Z"), + _claude_call("c", "Get-CimInstance Win32_ComputerSystem", "2026-01-01T00:00:08Z", name="PowerShell"), + _claude_result("c", "", "2026-01-01T00:00:08Z"), + ) + assert [(e["kind"], e.get("target")) for e in events] == [ + ("command", None), ("probe", "cpu"), ("command", None), ("probe", "cpu"), ("command", None)] + + +def test_a_subagents_probe_carries_its_agent(): + events = _claude(_claude_call("a", "nproc", "2026-01-01T00:00:05Z", agent="task-1"), _claude_result("a")) + assert events[1] == {**_probe("cpu", "nproc", "nproc", 5.0, 0), "agent": "task-1"} + + +def _codex(*items: dict) -> list[dict]: + raw = "\n".join(json.dumps(x) for x in items) + return with_probes(convert("codex-json", raw, WS).events, probes_for()) + + +def test_codex_unwraps_the_shell_before_it_matches_a_probe(): + events = _codex( + {"type": "item.completed", "item": {"type": "command_execution", "command": "/usr/bin/bash -lc 'nproc'", + "aggregated_output": "4\n", "exit_code": 0}}, + {"type": "item.completed", "item": {"type": "command_execution", + "command": "/usr/bin/bash -lc 'nvidia-smi'", + "aggregated_output": "", "exit_code": 127}}, + ) + assert events == [ + {"t": None, "kind": "command", "text": "nproc", "rc": 0, "output": "4\n", "output_truncated": False}, + _probe("cpu", "nproc", "nproc", None, 0), + {"t": None, "kind": "command", "text": "nvidia-smi", "rc": 127}, + _probe("gpu", "nvidia-smi", "nvidia-smi", None, 127), + ] + + +def _copilot(*records: dict) -> list[dict]: + raw = "\n".join(json.dumps(x) for x in records) + return with_probes(convert("copilot-json", raw, WS).events, probes_for()) + + +def test_copilot_bash_and_powershell_probes_give_probe_events(): + events = _copilot( + {"type": "tool.execution_start", "timestamp": "2026-01-01T00:00:01Z", + "data": {"toolCallId": "a", "toolName": "bash", "arguments": {"command": "nvidia-smi"}}}, + {"type": "tool.execution_complete", "data": {"toolCallId": "a", "success": True, + "shellExecution": {"exitCode": 0}, + "result": {"content": "GPU 0"}}}, + {"type": "tool.execution_start", "timestamp": "2026-01-01T00:00:02Z", + "data": {"toolCallId": "b", "toolName": "powershell", + "arguments": {"command": "Get-CimInstance Win32_Processor"}}}, + {"type": "tool.execution_complete", "data": {"toolCallId": "b", "success": True, + "shellExecution": {"exitCode": 0}, "result": {"content": "x"}}}, + ) + assert [(e["kind"], e.get("target"), e.get("pattern")) for e in events] == [ + ("command", None, None), ("probe", "gpu", "nvidia-smi"), + ("command", None, None), ("probe", "cpu", "Get-CimInstance Win32_Processor"), + ] + + +def test_a_manifest_replaces_a_target_and_an_empty_list_turns_it_off(): + probes = probes_for({"cpu": ["lscpu"], "env": []}) + assert [p.text for p in probes if p.target == "cpu"] == ["lscpu"] + assert all(p.target != "env" for p in probes) + assert {p.text for p in probes if p.target == "gpu"} == set(DEFAULT_PROBES["gpu"]) + + +def test_a_run_layer_replaces_the_manifest_layer(): + manifest = {"cpu": ["lscpu"]} + run = {"cpu": ["nproc"]} + assert [p.text for p in probes_for(manifest, run) if p.target == "cpu"] == ["nproc"] + + +def test_no_probes_leaves_the_events_as_they_are(): + events = [{"t": 1.0, "kind": "command", "text": "nproc", "rc": 0}] + assert with_probes(events, ()) == events + + +def test_probe_events_go_to_a_copy_so_the_trace_events_the_counters_read_are_unchanged(): + events = [{"t": 1.0, "kind": "command", "text": "nproc", "rc": 0}] + out = with_probes(events, probes_for()) + assert [e["kind"] for e in out] == ["command", "probe"] + assert events == [{"t": 1.0, "kind": "command", "text": "nproc", "rc": 0}] + + +@pytest.mark.parametrize(("raw", "message"), [ + ({"gpus": ["nvidia-smi"]}, "unknown target 'gpus'"), + ({"cpu": "nproc"}, "cpu must be a list"), + ({"cpu": [" "]}, "cpu must be a list"), + ({"cpu": ["py:os.cpu_count()"]}, "dotted Python name"), + ({"cpu": ['nproc "']}, "No closing quotation"), + ({"cpu": ["|"]}, "names a program"), + ({"cpu": ["nproc;lscpu"]}, "separate commands"), + ({"cpu": ["/proc/cpu info"]}, "one word with no spaces"), + (["cpu"], "expected a mapping"), +]) +def test_a_bad_pattern_config_is_refused_with_what_is_wrong(raw, message): + with pytest.raises(ValueError, match=message): + validate_config(raw, "probes") + + +def test_a_run_probe_file_is_yaml_or_json_and_checked(tmp_path): + good = tmp_path / "probes.yaml" + good.write_text("cpu: [lscpu]\nnetwork: [curl]\n", encoding="utf-8") + assert load_probe_file(str(good)) == {"cpu": ("lscpu",), "network": ("curl",)} + bad = tmp_path / "bad.json" + bad.write_text('{"cpu": ["nproc"', encoding="utf-8") + with pytest.raises(ValueError, match="--probes .*bad.json"): + load_probe_file(str(bad)) + + +def test_a_manifest_probes_key_is_read_and_a_bad_one_is_a_manifest_error(): + base = {"schema": "repo_manifest.v1", "id": "x", "guidance": {"globs": ["AGENTS.md"]}} + assert parse_manifest({**base, "probes": {"memory": ["free"]}}).probes == {"memory": ("free",)} + with pytest.raises(ManifestError, match="probes: 'disk' is not one of"): + parse_manifest({**base, "probes": {"disk": ["df"]}}) From d7ee57a6384d699a1bcf43686011c188e2ae3b70 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 15 Nov 2026 11:27:28 +0000 Subject: [PATCH 02/15] Pin quoted text that is still read as a probe with xfail tests Signed-off-by: Anton Volkov --- dokimi/tests/test_repo_probes.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index 3c9b96a7c..c5fd08054 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -132,6 +132,17 @@ def test_text_that_is_not_run_code_is_no_probe(command): assert all(e["kind"] != "probe" for e in events) +@pytest.mark.xfail(strict=True, reason="quoted text is still searched for substitutions and pipes") +@pytest.mark.parametrize("command", [ + "echo '$(nproc)'", + "sysctl -a '|' grep machdep.cpu", + "python -c 'print(\"os.cpu_count()\")'", +]) +def test_quoted_text_is_no_probe(command): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert all(e["kind"] != "probe" for e in events) + + def test_a_windows_powershell_probe_is_matched_case_blind_and_through_parentheses(): events = _claude( _claude_call("a", "Get-CimInstance -ClassName Win32_Processor | Select-Object NumberOfCores", From 40021607603c84f229c8ab7c60efc6203051ec70 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 15 Nov 2026 11:27:44 +0000 Subject: [PATCH 03/15] Skip wrapper options before taking the program a wrapper runs Signed-off-by: Anton Volkov --- dokimi/dokimi/repos/probes.py | 8 +++++++- dokimi/tests/test_repo_probes.py | 10 ++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py index d872761e4..afd2c6da3 100644 --- a/dokimi/dokimi/repos/probes.py +++ b/dokimi/dokimi/repos/probes.py @@ -49,6 +49,7 @@ _READERS = frozenset({"cat", "head", "tail", "less", "more", "nl", "sed", "grep", "egrep", "fgrep", "awk", "tac", "strings", "wc", "rg"}) _WRAPPERS = frozenset({"time", "nohup", "sudo", "command", "exec", "builtin", "nice"}) # each runs the command after it +_OPTION_OPERANDS = frozenset({("nice", "-n"), ("sudo", "-u")}) # wrapper options that take a value word _PYTHON = re.compile(r"python\d*(?:\.\d+)?|py|pypy\d*") _PIP = re.compile(r"pip\d*(?:\.\d+)?") _ASSIGN = re.compile(r"[A-Za-z_]\w*=") @@ -207,9 +208,14 @@ def _views(words: list[str]) -> Iterator[Stage]: if i >= len(words): return yield _head(words[i], words[i + 1:]) - if _program(words[i]) not in _WRAPPERS: + wrapper = _program(words[i]) + if wrapper not in _WRAPPERS: return i += 1 + while i < len(words) and words[i].startswith("-"): # ``sudo -E``, ``nice -n 5``: options, then the program + if wrapper == "command" and words[i] in ("-v", "-V"): + return # ``command -v nproc`` looks the program up, it does not run it + i += 2 if (wrapper, words[i]) in _OPTION_OPERANDS else 1 def _head(word: str, args: list[str]) -> Stage: diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index c5fd08054..9b0d6693f 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -132,6 +132,16 @@ def test_text_that_is_not_run_code_is_no_probe(command): assert all(e["kind"] != "probe" for e in events) +@pytest.mark.parametrize(("command", "pattern"), [ + ("sudo -E lscpu", "lscpu"), + ("nice -n 5 nproc", "nproc"), + ("sudo -u root nproc", "nproc"), +]) +def test_a_wrappers_options_do_not_hide_the_program_it_runs(command, pattern): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["pattern"] for e in events if e["kind"] == "probe"] == [pattern] + + @pytest.mark.xfail(strict=True, reason="quoted text is still searched for substitutions and pipes") @pytest.mark.parametrize("command", [ "echo '$(nproc)'", From 8a0885fd6ecc36a75e756fca7b36b08491eb7c9a Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 15 Nov 2026 11:28:24 +0000 Subject: [PATCH 04/15] Leave a searcher's search expression out of the path probe match Signed-off-by: Anton Volkov --- dokimi/dokimi/repos/probes.py | 13 ++++++++++++- dokimi/tests/test_repo_probes.py | 6 ++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py index afd2c6da3..f6923618f 100644 --- a/dokimi/dokimi/repos/probes.py +++ b/dokimi/dokimi/repos/probes.py @@ -48,6 +48,7 @@ _READERS = frozenset({"cat", "head", "tail", "less", "more", "nl", "sed", "grep", "egrep", "fgrep", "awk", "tac", "strings", "wc", "rg"}) +_SEARCHERS = frozenset({"grep", "egrep", "fgrep", "rg", "sed", "awk"}) # the first operand is an expression, not a file _WRAPPERS = frozenset({"time", "nohup", "sudo", "command", "exec", "builtin", "nice"}) # each runs the command after it _OPTION_OPERANDS = frozenset({("nice", "-n"), ("sudo", "-u")}) # wrapper options that take a value word _PYTHON = re.compile(r"python\d*(?:\.\d+)?|py|pypy\d*") @@ -230,10 +231,20 @@ def _hits(probe: Probe, pipeline: list[list[Stage]]) -> bool: if probe.kind == "command": return _in_order(probe.stages, pipeline) if probe.kind == "path": - return any(prog in _READERS and probe.text in args for views in pipeline for prog, args in views) + return any(prog in _READERS and probe.text in _operands(prog, args) for views in pipeline for prog, args in views) return any(_PYTHON.fullmatch(prog) and _calls(args, probe) for views in pipeline for prog, args in views) +def _operands(program: str, args: tuple[str, ...]) -> tuple[str, ...]: + """The words of a reader that can name a file: a searcher's search expression (its ``-e`` values, else its first operand) is not one.""" + if program not in _SEARCHERS: + return args + if "-e" in args: + return tuple(a for i, a in enumerate(args) if a != "-e" and args[i - 1:i] != ("-e",)) + first = next((i for i, a in enumerate(args) if not a.startswith("-")), None) + return args if first is None else args[:first] + args[first + 1:] + + def _calls(args: tuple[str, ...], probe: Probe) -> bool: """Whether the code of a ``python -c`` one-liner calls the probe's name.""" if "-c" not in args or probe.call is None: diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index 9b0d6693f..d3fdecf09 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -132,6 +132,12 @@ def test_text_that_is_not_run_code_is_no_probe(command): assert all(e["kind"] != "probe" for e in events) +@pytest.mark.parametrize("command", ["grep /proc/cpuinfo README", "grep -e /proc/meminfo README"]) +def test_a_search_expression_that_looks_like_a_path_is_no_probe(command): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert all(e["kind"] != "probe" for e in events) + + @pytest.mark.parametrize(("command", "pattern"), [ ("sudo -E lscpu", "lscpu"), ("nice -n 5 nproc", "nproc"), From 57508d3b859421751e385e0173ba322977e2e189 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 15 Nov 2026 12:12:26 +0000 Subject: [PATCH 05/15] Match commands that follow a shell keyword such as then or do Signed-off-by: Anton Volkov --- dokimi/dokimi/repos/probes.py | 3 ++- dokimi/tests/test_repo_probes.py | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py index f6923618f..4af09d0f2 100644 --- a/dokimi/dokimi/repos/probes.py +++ b/dokimi/dokimi/repos/probes.py @@ -50,6 +50,7 @@ "strings", "wc", "rg"}) _SEARCHERS = frozenset({"grep", "egrep", "fgrep", "rg", "sed", "awk"}) # the first operand is an expression, not a file _WRAPPERS = frozenset({"time", "nohup", "sudo", "command", "exec", "builtin", "nice"}) # each runs the command after it +_KEYWORDS = frozenset({"if", "then", "else", "elif", "do", "while", "until", "!"}) # shell keywords before a command _OPTION_OPERANDS = frozenset({("nice", "-n"), ("sudo", "-u")}) # wrapper options that take a value word _PYTHON = re.compile(r"python\d*(?:\.\d+)?|py|pypy\d*") _PIP = re.compile(r"pip\d*(?:\.\d+)?") @@ -204,7 +205,7 @@ def _views(words: list[str]) -> Iterator[Stage]: """The command a simple command runs: its own program, then the command each wrapper (``time``, ``sudo``...) runs.""" i = 0 while i < len(words): - while i < len(words) and _ASSIGN.match(words[i]): # NAME=value words set variables for the command after them + while i < len(words) and (_ASSIGN.match(words[i]) or words[i] in _KEYWORDS): # NAME=value, ``then``: the command is next i += 1 if i >= len(words): return diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index d3fdecf09..aaa10e744 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -148,6 +148,12 @@ def test_a_wrappers_options_do_not_hide_the_program_it_runs(command, pattern): assert [e["pattern"] for e in events if e["kind"] == "probe"] == [pattern] +@pytest.mark.parametrize("command", ["if true; then nproc; fi", "for x in 1; do nproc; done"]) +def test_a_command_in_the_body_of_a_conditional_or_loop_is_a_probe(command): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["pattern"] for e in events if e["kind"] == "probe"] == ["nproc"] + + @pytest.mark.xfail(strict=True, reason="quoted text is still searched for substitutions and pipes") @pytest.mark.parametrize("command", [ "echo '$(nproc)'", From 2fe9ab06e0034776380cfb721194afa4c8b96fc8 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 15 Nov 2026 12:12:51 +0000 Subject: [PATCH 06/15] Leave option values and redirection targets out of a reader's files Signed-off-by: Anton Volkov --- dokimi/dokimi/repos/probes.py | 12 ++++++++++-- dokimi/tests/test_repo_probes.py | 6 ++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py index 4af09d0f2..b1e3e1039 100644 --- a/dokimi/dokimi/repos/probes.py +++ b/dokimi/dokimi/repos/probes.py @@ -52,6 +52,7 @@ _WRAPPERS = frozenset({"time", "nohup", "sudo", "command", "exec", "builtin", "nice"}) # each runs the command after it _KEYWORDS = frozenset({"if", "then", "else", "elif", "do", "while", "until", "!"}) # shell keywords before a command _OPTION_OPERANDS = frozenset({("nice", "-n"), ("sudo", "-u")}) # wrapper options that take a value word +_VALUE_OPTIONS = frozenset({"-m", "-A", "-B", "-C"}) # a searcher's options whose next word is a value, not an operand _PYTHON = re.compile(r"python\d*(?:\.\d+)?|py|pypy\d*") _PIP = re.compile(r"pip\d*(?:\.\d+)?") _ASSIGN = re.compile(r"[A-Za-z_]\w*=") @@ -193,9 +194,16 @@ def _stages(part: str) -> list[list[str]]: except ValueError: # an unbalanced quote: the words as they are typed tokens = part.split() stages: list[list[str]] = [[]] + skip = False for token in tokens: - if token == "|": + if skip: # the target of an output redirection is written, not read + skip = False + elif token == "|": stages.append([]) + elif token.lstrip("&").startswith(">"): + skip = True + if stages[-1][-1:] and stages[-1][-1].isdigit(): # ``2>``: the file descriptor + stages[-1].pop() elif token not in _SEPARATORS and token not in _LEX_SKIP: stages[-1].append(token) return [stage for stage in stages if stage] @@ -242,7 +250,7 @@ def _operands(program: str, args: tuple[str, ...]) -> tuple[str, ...]: return args if "-e" in args: return tuple(a for i, a in enumerate(args) if a != "-e" and args[i - 1:i] != ("-e",)) - first = next((i for i, a in enumerate(args) if not a.startswith("-")), None) + first = next((i for i, a in enumerate(args) if not a.startswith("-") and (args[i - 1] if i else "") not in _VALUE_OPTIONS), None) return args if first is None else args[:first] + args[first + 1:] diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index aaa10e744..3e864c08e 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -138,6 +138,12 @@ def test_a_search_expression_that_looks_like_a_path_is_no_probe(command): assert all(e["kind"] != "probe" for e in events) +@pytest.mark.parametrize("command", ["grep -m 1 /proc/cpuinfo README", "cat > /proc/cpuinfo", "cat 2>> /proc/cpuinfo"]) +def test_an_option_value_or_a_redirection_target_is_no_file_the_command_reads(command): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert all(e["kind"] != "probe" for e in events) + + @pytest.mark.parametrize(("command", "pattern"), [ ("sudo -E lscpu", "lscpu"), ("nice -n 5 nproc", "nproc"), From 658e2ac085b35346a4a246237bbf55493e6c3004 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 21:53:42 -0800 Subject: [PATCH 07/15] Read the probe forms that review found the matcher missing or inventing A backslash before a space outside quotes is an escape again, so a WSL path `/mnt/c/Program\ Files/...` keeps one word (self-review of this round). Substitutions are found by balanced parentheses; quoted text is no command, so '$(nproc)' and '|' inside quotes are words; a redirection's target is written, an input's is read; a python one-liner's options end at its script and -cCODE is a code; a shell launcher's -c script is read after time, env assignments and options; a Windows path keeps its backslashes; a call inside a string literal is no call; a probe file nested too deep is a ValueError. The rg search pin fails when rg leaves the searchers. Already fixed earlier in this PR: sudo -E lscpu (400216076) and a grep search expression that looks like a path (8a0885fd6). Thread: PRRT_kwDOVBg7g863m7XT Thread: PRRT_kwDOVBg7g863o9_8 Thread: PRRT_kwDOVBg7g863m7XW Thread: PRRT_kwDOVBg7g863m7XX Thread: PRRT_kwDOVBg7g863m7XY Thread: PRRT_kwDOVBg7g863m7XZ Thread: PRRT_kwDOVBg7g863o9_- Thread: PRRT_kwDOVBg7g863pFRV Thread: PRRT_kwDOVBg7g863pb2R Thread: PRRT_kwDOVBg7g863pb2V Thread: PRRT_kwDOVBg7g863qoo9 Thread: PRRT_kwDOVBg7g863qoo_ Thread: PRRT_kwDOVBg7g863qopB Thread: PRRT_kwDOVBg7g863qopD Thread: PRRT_kwDOVBg7g863qopE Thread: PRRT_kwDOVBg7g863qopJ Review: PRR_kwDOVBg7g88AAAABYEoIyQ Review: PRR_kwDOVBg7g88AAAABYFAfqQ Review: PRR_kwDOVBg7g88AAAABYFeYaQ Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/probes.py | 186 ++++++++++++++++++++++++++----- dokimi/tests/test_repo_probes.py | 41 ++++++- 2 files changed, 197 insertions(+), 30 deletions(-) diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py index b1e3e1039..33f03ba08 100644 --- a/dokimi/dokimi/repos/probes.py +++ b/dokimi/dokimi/repos/probes.py @@ -21,6 +21,7 @@ from __future__ import annotations +import ast import posixpath import re import shlex @@ -57,11 +58,15 @@ _PIP = re.compile(r"pip\d*(?:\.\d+)?") _ASSIGN = re.compile(r"[A-Za-z_]\w*=") _DOTTED = re.compile(r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*") -_SUBSTITUTION = re.compile(r"\$\(([^()]*)\)|`([^`]*)`") _SHELLS = frozenset({"bash", "sh", "zsh", "dash"}) _SHELL_C = re.compile(r"-[a-z]*c[a-z]*") # ``-c``, ``-lc``: the next word is a script _LEX_SKIP = frozenset({"(", ")", "{", "}"}) # grouping a PowerShell or shell expression puts around a command _SEPARATORS = frozenset({"|", "||", "&&", ";", "&"}) # ``|`` splits stages; the rest end a command +_ESC = {c: chr(0xE000 + i) for i, c in enumerate("|;&<>$`\\()")} # stands for a character a quote makes literal +_UNESC = {stand: c for c, stand in _ESC.items()} +_ESCAPABLE = frozenset("\"'\\$`|;&<>()") # a backslash before one of these, outside single quotes, is an escape +_FD = "\ue0ff" # stands for a file descriptor digit before a redirection, which is no word +_PY_VALUE_OPTIONS = frozenset({"-W", "-X"}) # python's own options whose next word is their value Stage = tuple[str, tuple[str, ...]] # a simple command's program (normalised) and its words @@ -100,7 +105,7 @@ def load_probe_file(path: str) -> dict[str, tuple[str, ...]]: """A run's ``--probes`` file (YAML, or JSON, which is YAML) through :func:`validate_config`.""" try: raw = yaml.safe_load(Path(path).read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc: + except (OSError, UnicodeDecodeError, yaml.YAMLError, RecursionError) as exc: # deep nesting too raise ValueError(f"--probes {path}: {exc}") from exc return validate_config(raw, f"--probes {path}") @@ -175,37 +180,126 @@ def _matching(command: str, probes: Sequence[Probe]) -> list[Probe]: def _pipelines(script: str) -> Iterator[list[list[str]]]: """Each pipeline of the script as its stages' words; the commands of its ``$( )`` and backtick substitutions too.""" for part, _after, _in_subshell, _outer in split_script(script): - for inner in _SUBSTITUTION.finditer(part): - yield from _pipelines(inner.group(1) if inner.group(1) is not None else inner.group(2)) - stages = _stages(_SUBSTITUTION.sub(" ", part)) # the substitutions ran as their own commands, above + codes, rest = _substitutions(_mask(part)) + for code in codes: # the substitutions ran as their own commands + yield from _pipelines(_restore(code)) + stages = _stages(rest) if stages: yield stages - for stage in stages: # ``bash -lc 'nproc'``: the script a shell runs is a script of its own - if len(stage) > 2 and _program(stage[0]) in _SHELLS and _SHELL_C.fullmatch(stage[1]): - yield from _pipelines(stage[2]) + for stage in stages: # ``bash -lc 'nproc'``, ``time bash -c nproc``: the script a shell runs is a script of its own + for prog, args in _views(stage): + script_arg = _shell_script(prog, args) + if script_arg is not None: + yield from _pipelines(script_arg) + + +def _mask(part: str) -> str: + """``part`` with the characters a quote or a backslash makes literal swapped for stand-ins: ``|;&<>`` in double + quotes, every special character and ``$`` in single quotes, a backslash before a letter (a Windows path). The quotes + stay, so shlex still reads each word, and a substitution is found only where ``$`` is live.""" + out: list[str] = [] + quote = "" + i = 0 + while i < len(part): + c = part[i] + nxt = part[i + 1] if i + 1 < len(part) else "" + if quote != "'" and c == "\\" and nxt: + if (quote == '"' and nxt in '"\\') or (not quote and nxt in "\"'\\ \t"): + out.append(c + nxt) # an escape shlex reads itself + elif nxt in _ESCAPABLE: + out.append(_ESC.get(nxt, nxt)) # the escaped character, literal + else: + out.append(_ESC["\\"]) # a backslash before a letter: a path's separator, not an escape + i += 1 + continue + i += 2 + continue + if c in "'\"" and quote in ("", c): + quote = "" if quote else c + out.append(c) + elif (quote == "'" and c in _ESC) or (quote == '"' and c in "|;&<>") or (not quote and c == "\\"): + out.append(_ESC[c]) + else: + out.append(c) + i += 1 + return "".join(out) -def _stages(part: str) -> list[list[str]]: - """A part's simple commands, cut at its unquoted pipes, each as its words (quotes removed, grouping dropped).""" +def _substitutions(part: str) -> tuple[list[str], str]: + """The code of each ``$( )`` (balanced parentheses) and backtick substitution of a masked part, and the part without + them: a substitution runs as its own command, so it is no word of this one.""" + codes: list[str] = [] + rest: list[str] = [] + i = 0 + while i < len(part): + if part.startswith("$(", i): + depth, j = 0, i + 1 + while j < len(part): + depth += (part[j] == "(") - (part[j] == ")") + if depth == 0: + break + j += 1 + codes.append(part[i + 2:j]) + rest.append(" ") + i = j + 1 + elif part[i] == "`": + j = part.find("`", i + 1) + j = len(part) if j < 0 else j + codes.append(part[i + 1:j]) + rest.append(" ") + i = j + 1 + else: + rest.append(part[i]) + i += 1 + return codes, "".join(rest) + + +def _restore(text: str) -> str: + return "".join(_UNESC.get(c, c) for c in text) + + +def _fd_masked(part: str) -> str: + """``part`` with each file descriptor digit before a redirection (``2>``, ``2>&1``) swapped for a stand-in.""" + return re.sub(r"(?()])\d+(?=[<>])", lambda m: _FD * len(m.group()), part) + + +def _lexed(text: str) -> list[str]: + """The tokens of a masked part: each word with its quotes removed, the operators as shlex reads them, and no + file descriptor digit.""" try: - lexer = shlex.shlex(part, posix=True, punctuation_chars=True) + lexer = shlex.shlex(text, posix=True, punctuation_chars=True) lexer.whitespace_split = True tokens = list(lexer) except ValueError: # an unbalanced quote: the words as they are typed - tokens = part.split() + tokens = text.split() + return [token for token in tokens if set(token) != {_FD}] + + +def _close(stage: list[str], reads: list[str]) -> None: + if stage: # an input's target is read by the command, so it follows the command's words + stage.extend(reads) + + +def _stages(part: str) -> list[list[str]]: + """A part's simple commands, cut at its unquoted pipes, each as its words (quotes removed, grouping dropped). An + output redirection's target is written, so it is no word; an input's target is read, so it follows the words.""" stages: list[list[str]] = [[]] - skip = False - for token in tokens: - if skip: # the target of an output redirection is written, not read - skip = False + reads: list[str] = [] + target = "" # the operator whose target is the next token: ``>`` (written) or ``<`` (read) + for token in _lexed(_fd_masked(part)): + if target: + if target == "<": + reads.append(_restore(token)) + target = "" elif token == "|": + _close(stages[-1], reads) + reads = [] stages.append([]) - elif token.lstrip("&").startswith(">"): - skip = True - if stages[-1][-1:] and stages[-1][-1].isdigit(): # ``2>``: the file descriptor - stages[-1].pop() + elif token.lstrip("&").startswith(">") or token == "<": + target = token.lstrip("&")[:1] elif token not in _SEPARATORS and token not in _LEX_SKIP: - stages[-1].append(token) + stages[-1].append(_restore(token)) + _close(stages[-1], reads) return [stage for stage in stages if stage] @@ -229,13 +323,43 @@ def _views(words: list[str]) -> Iterator[Stage]: def _head(word: str, args: list[str]) -> Stage: - """A program and its words; ``python -m pip list`` is the program ``pip``.""" + """A program and its words; ``python -I -m pip list`` is the program ``pip``.""" program = _program(word) - if _PYTHON.fullmatch(program) and args[:1] == ["-m"] and len(args) > 1 and _PIP.fullmatch(args[1]): - return "pip", tuple(args[2:]) + if _PYTHON.fullmatch(program): + run = _python_run(tuple(args)) + if run and run[0] == "-m" and run[1] and _PIP.fullmatch(run[1][0]): + return "pip", run[1][1:] return ("pip" if _PIP.fullmatch(program) else program), tuple(args) +def _shell_script(prog: str, args: tuple[str, ...]) -> str | None: + """The script a shell runs with ``-c`` (``bash --noprofile -c nproc``), or None.""" + if prog not in _SHELLS: + return None + for i, word in enumerate(args): + if _SHELL_C.fullmatch(word): + return args[i + 1] if i + 1 < len(args) else None + if not word.startswith("-"): + return None + return None + + +def _python_run(args: tuple[str, ...]) -> tuple[str, tuple[str, ...]] | None: + """What a python runs: ``("-c", the code and the words after it)``, ``("-m", the module and the words after it)``, + or None for a script. Its options are read up to the first word that is not one, as python reads them.""" + i = 0 + while i < len(args): + word = args[i] + if word in ("-c", "-m"): + return word, args[i + 1:] + if word.startswith("-c"): # ``-cimport os``: the code is in the same word + return "-c", (word[2:],) + args[i + 1:] + if word == "--" or not word.startswith("-"): + return None + i += 2 if word in _PY_VALUE_OPTIONS else 1 + return None + + def _hits(probe: Probe, pipeline: list[list[Stage]]) -> bool: if probe.kind == "command": return _in_order(probe.stages, pipeline) @@ -255,11 +379,17 @@ def _operands(program: str, args: tuple[str, ...]) -> tuple[str, ...]: def _calls(args: tuple[str, ...], probe: Probe) -> bool: - """Whether the code of a ``python -c`` one-liner calls the probe's name.""" - if "-c" not in args or probe.call is None: + """Whether the code of a ``python -c`` one-liner calls the probe's name; a string that only spells the call is no call.""" + run = _python_run(args) + if probe.call is None or run is None or run[0] != "-c" or not run[1]: return False - code = args[args.index("-c") + 1:args.index("-c") + 2] - return bool(code) and probe.call.search(code[0]) is not None + code = run[1][0] + try: + tree = ast.parse(code) + except (SyntaxError, ValueError, RecursionError): # code that is no Python we can read: the name as it is written + return probe.call.search(code) is not None + name = probe.text[3:].strip() + return any(isinstance(node, ast.Call) and ast.unparse(node.func) == name for node in ast.walk(tree)) def _in_order(want: tuple[Stage, ...], pipeline: list[list[Stage]]) -> bool: diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index 3e864c08e..c91c7f1be 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -160,17 +160,54 @@ def test_a_command_in_the_body_of_a_conditional_or_loop_is_a_probe(command): assert [e["pattern"] for e in events if e["kind"] == "probe"] == ["nproc"] -@pytest.mark.xfail(strict=True, reason="quoted text is still searched for substitutions and pipes") @pytest.mark.parametrize("command", [ "echo '$(nproc)'", "sysctl -a '|' grep machdep.cpu", - "python -c 'print(\"os.cpu_count()\")'", ]) def test_quoted_text_is_no_probe(command): events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) assert all(e["kind"] != "probe" for e in events) +def test_a_call_inside_a_python_string_is_no_probe(): + events = _claude(_claude_call("a", "python -c 'print(\"os.cpu_count()\")'", "2026-01-01T00:00:05Z"), + _claude_result("a")) + assert all(e["kind"] != "probe" for e in events) + + +@pytest.mark.parametrize(("command", "pattern"), [ + ("make -j$(python -c 'import os; print(os.cpu_count())')", "py:os.cpu_count"), + ("echo $(echo $(nproc))", "nproc"), + ("bash -c '>/dev/null; nproc'", "nproc"), + ("grep -m 1 >/tmp/cpu.txt processor /proc/cpuinfo", "/proc/cpuinfo"), + ("grep '>' /proc/cpuinfo", "/proc/cpuinfo"), + ("python -c'import os;print(os.cpu_count())'", "py:os.cpu_count"), + ("python -I -m pip list", "pip list"), + ("< /proc/cpuinfo cat", "/proc/cpuinfo"), + ("time bash -c 'nproc'", "nproc"), + ("FOO=bar bash -c 'nproc'", "nproc"), + ("bash --noprofile -c 'nproc'", "nproc"), + ("C:\\Windows\\System32\\nvidia-smi.exe --query-gpu=name", "nvidia-smi"), + ("/mnt/c/Program\\ Files/NVIDIA\\ Corporation/NVSMI/nvidia-smi.exe -L", "nvidia-smi"), +]) +def test_a_probe_the_shell_runs_in_another_form_is_a_probe(command, pattern): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["pattern"] for e in events if e["kind"] == "probe"] == [pattern] + + +@pytest.mark.parametrize("command", ["python echo_args.py -c 'os.cpu_count()'", "rg /proc/cpuinfo README"]) +def test_a_script_argument_or_a_search_expression_is_no_probe(command): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert all(e["kind"] != "probe" for e in events) + + +def test_a_probe_file_nested_too_deep_is_a_value_error_that_names_it(tmp_path): + path = tmp_path / "probes.yaml" + path.write_text("[" * 2000, encoding="utf-8") + with pytest.raises(ValueError, match="probes.yaml"): + load_probe_file(str(path)) + + def test_a_windows_powershell_probe_is_matched_case_blind_and_through_parentheses(): events = _claude( _claude_call("a", "Get-CimInstance -ClassName Win32_Processor | Select-Object NumberOfCores", From a445b7fb6364c905c1c590b1f33c83b41cd8a032 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 23:13:12 -0800 Subject: [PATCH 08/15] Address review: preserve POSIX backslash-newline continuations Thread: PRRT_kwDOVBg7g863zrh3 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/probes.py | 3 +++ dokimi/tests/test_repo_probes.py | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py index 33f03ba08..3b884e615 100644 --- a/dokimi/dokimi/repos/probes.py +++ b/dokimi/dokimi/repos/probes.py @@ -203,6 +203,9 @@ def _mask(part: str) -> str: while i < len(part): c = part[i] nxt = part[i + 1] if i + 1 < len(part) else "" + if quote != "'" and c == "\\" and nxt == "\n": # a line continuation: the shell reads the pair as nothing + i += 2 + continue if quote != "'" and c == "\\" and nxt: if (quote == '"' and nxt in '"\\') or (not quote and nxt in "\"'\\ \t"): out.append(c + nxt) # an escape shlex reads itself diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index c91c7f1be..775c3c5c7 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -329,3 +329,12 @@ def test_a_manifest_probes_key_is_read_and_a_bad_one_is_a_manifest_error(): assert parse_manifest({**base, "probes": {"memory": ["free"]}}).probes == {"memory": ("free",)} with pytest.raises(ManifestError, match="probes: 'disk' is not one of"): parse_manifest({**base, "probes": {"disk": ["df"]}}) + + +@pytest.mark.parametrize(("command", "pattern"), [ + ("python \\\n-c 'import os; print(os.cpu_count())'", "py:os.cpu_count"), + ("npro\\\nc", "nproc"), +]) +def test_a_backslash_newline_continues_the_command_it_is_in(command, pattern): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert any(e["kind"] == "probe" and e["pattern"] == pattern for e in events) From 815dbc94923029ecea9171e8edbe1d38004e3869 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 23:13:35 -0800 Subject: [PATCH 09/15] Address review: a quoted parenthesis does not end a substitution Thread: PRRT_kwDOVBg7g863zriH Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/probes.py | 37 +++++++++++++++++++++++++------- dokimi/tests/test_repo_probes.py | 5 +++++ 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py index 3b884e615..7db8ca1c3 100644 --- a/dokimi/dokimi/repos/probes.py +++ b/dokimi/dokimi/repos/probes.py @@ -228,6 +228,32 @@ def _mask(part: str) -> str: return "".join(out) +def _close_of(part: str, j: int) -> int: + """The index of the ``)`` that closes a ``$(`` whose code starts at ``j``: parentheses count, and a quote inside the + code is read as the shell reads it, so a ``)`` in quotes does not close it; a nested ``$( )`` is skipped whole.""" + quote, depth = "", 0 + while j < len(part): + c = part[j] + if quote == "'": + quote = "" if c == "'" else quote + elif c == "\\": + j += 1 + elif c == "'" and not quote: + quote = "'" + elif c == '"': + quote = "" if quote == '"' else '"' + elif quote != "'" and part.startswith("$(", j): + j = _close_of(part, j + 2) + elif not quote and c == "(": + depth += 1 + elif not quote and c == ")": + if depth == 0: + return j + depth -= 1 + j += 1 + return len(part) + + def _substitutions(part: str) -> tuple[list[str], str]: """The code of each ``$( )`` (balanced parentheses) and backtick substitution of a masked part, and the part without them: a substitution runs as its own command, so it is no word of this one.""" @@ -236,15 +262,10 @@ def _substitutions(part: str) -> tuple[list[str], str]: i = 0 while i < len(part): if part.startswith("$(", i): - depth, j = 0, i + 1 - while j < len(part): - depth += (part[j] == "(") - (part[j] == ")") - if depth == 0: - break - j += 1 - codes.append(part[i + 2:j]) + end = _close_of(part, i + 2) + codes.append(part[i + 2:end]) rest.append(" ") - i = j + 1 + i = end + 1 elif part[i] == "`": j = part.find("`", i + 1) j = len(part) if j < 0 else j diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index 775c3c5c7..6e69a1c11 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -338,3 +338,8 @@ def test_a_manifest_probes_key_is_read_and_a_bad_one_is_a_manifest_error(): def test_a_backslash_newline_continues_the_command_it_is_in(command, pattern): events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) assert any(e["kind"] == "probe" and e["pattern"] == pattern for e in events) + + +def test_a_quoted_parenthesis_inside_a_double_quoted_substitution_does_not_end_it(): + events = _claude(_claude_call("a", 'echo "$(printf \')\'; nproc)"', "2026-01-01T00:00:05Z"), _claude_result("a")) + assert any(e["kind"] == "probe" and e["pattern"] == "nproc" for e in events) From 6c007cf5d1513fdf77767d10071e0ad5550da639 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 23:14:09 -0800 Subject: [PATCH 10/15] Address review: a search option's value is no operand Thread: PRRT_kwDOVBg7g863zriW Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/probes.py | 44 +++++++++++++++++++++++++++----- dokimi/tests/test_repo_probes.py | 11 ++++++++ 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py index 7db8ca1c3..6a2eebc62 100644 --- a/dokimi/dokimi/repos/probes.py +++ b/dokimi/dokimi/repos/probes.py @@ -53,7 +53,15 @@ _WRAPPERS = frozenset({"time", "nohup", "sudo", "command", "exec", "builtin", "nice"}) # each runs the command after it _KEYWORDS = frozenset({"if", "then", "else", "elif", "do", "while", "until", "!"}) # shell keywords before a command _OPTION_OPERANDS = frozenset({("nice", "-n"), ("sudo", "-u")}) # wrapper options that take a value word -_VALUE_OPTIONS = frozenset({"-m", "-A", "-B", "-C"}) # a searcher's options whose next word is a value, not an operand +_GREP_VALUES = frozenset({"-m", "-A", "-B", "-C", "-d", "-D", "--max-count", "--after-context", "--before-context", + "--context", "--label", "--binary-files", "--directories", "--devices"}) +_RG_VALUES = frozenset({"-g", "--glob", "--iglob", "-t", "--type", "-T", "--type-not", "--type-add", "-m", "--max-count", + "-A", "-B", "-C", "--after-context", "--before-context", "--context", "-M", "--max-columns", "-j", + "--threads", "-E", "--encoding", "-r", "--replace", "-d", "--max-depth", "--sort", "--sortr"}) +_SEARCH_VALUES = {"grep": _GREP_VALUES, "egrep": _GREP_VALUES, "fgrep": _GREP_VALUES, "rg": _RG_VALUES, + "awk": frozenset({"-F", "-v"})} # a searcher's options whose next word is a value, not an operand +_PATTERN_FILE = frozenset({"-f", "--file"}) # a searcher's option whose value is a file it reads, and no expression +_EXPRESSION = frozenset({"-e", "--regexp", "--expression"}) # a searcher's option whose value is the search expression _PYTHON = re.compile(r"python\d*(?:\.\d+)?|py|pypy\d*") _PIP = re.compile(r"pip\d*(?:\.\d+)?") _ASSIGN = re.compile(r"[A-Za-z_]\w*=") @@ -393,13 +401,37 @@ def _hits(probe: Probe, pipeline: list[list[Stage]]) -> bool: def _operands(program: str, args: tuple[str, ...]) -> tuple[str, ...]: - """The words of a reader that can name a file: a searcher's search expression (its ``-e`` values, else its first operand) is not one.""" + """The words of a reader that can name a file. A searcher's search expression is not one: it is its ``-e`` value, or + else its first operand, and a pattern file (``-f``) is a file it reads. A searcher's option whose next word is a + value (``rg -g '*.c'``, ``grep -m 1``) takes that word, which is neither.""" if program not in _SEARCHERS: return args - if "-e" in args: - return tuple(a for i, a in enumerate(args) if a != "-e" and args[i - 1:i] != ("-e",)) - first = next((i for i, a in enumerate(args) if not a.startswith("-") and (args[i - 1] if i else "") not in _VALUE_OPTIONS), None) - return args if first is None else args[:first] + args[first + 1:] + values = _SEARCH_VALUES.get(program, frozenset()) + operands: list[str] = [] + given = False # an expression or a pattern file is named, so the first operand is no expression + i = 0 + while i < len(args): + word = args[i] + name, eq, value = word.partition("=") # ``--file=F``; a plain operand is kept as it is + if word == "--": + operands.extend(args[i + 1:]) + break + if name in _EXPRESSION or name in _PATTERN_FILE: + given = True + if name in _PATTERN_FILE: + if eq: + operands.append(value) + elif i + 1 < len(args): + operands.append(args[i + 1]) + i += 1 if eq else 2 + elif name in values: + i += 1 if eq else 2 + elif word.startswith("-") and len(word) > 1: + i += 1 + else: + operands.append(word) + i += 1 + return tuple(operands) if given else tuple(operands[1:]) def _calls(args: tuple[str, ...], probe: Probe) -> bool: diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index 6e69a1c11..2d1627e93 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -343,3 +343,14 @@ def test_a_backslash_newline_continues_the_command_it_is_in(command, pattern): def test_a_quoted_parenthesis_inside_a_double_quoted_substitution_does_not_end_it(): events = _claude(_claude_call("a", 'echo "$(printf \')\'; nproc)"', "2026-01-01T00:00:05Z"), _claude_result("a")) assert any(e["kind"] == "probe" and e["pattern"] == "nproc" for e in events) + + +def test_a_search_option_value_is_no_operand_so_the_file_after_it_is_read(): + events = _claude(_claude_call("a", "grep -f /proc/cpuinfo README", "2026-01-01T00:00:05Z"), _claude_result("a")) + assert any(e["kind"] == "probe" and e["pattern"] == "/proc/cpuinfo" for e in events) + + +@pytest.mark.parametrize("command", ["rg -g '*.c' /proc/cpuinfo src", "rg -t c /proc/cpuinfo src"]) +def test_a_glob_or_type_value_of_a_search_is_no_expression_or_file(command): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert all(e["kind"] != "probe" for e in events) From 7746c597c5210b579fe792bf0190e627f5f4ae25 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 08:18:14 -0800 Subject: [PATCH 11/15] Address review: PowerShell assignment, python -Ic, here-string A PowerShell $x = assignment is skipped; python -Ic code and -Im module read the code or module after the flags; <<< is a string, so the word after it is no file the command reads. Thread: PRRT_kwDOVBg7g8630gp- Thread: PRRT_kwDOVBg7g8630gqG Thread: PRRT_kwDOVBg7g8630gqL Review: PRR_kwDOVBg7g88AAAABYKTqPw Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/probes.py | 15 +++++++++++---- dokimi/tests/test_repo_probes.py | 20 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py index 6a2eebc62..245ba7ace 100644 --- a/dokimi/dokimi/repos/probes.py +++ b/dokimi/dokimi/repos/probes.py @@ -327,8 +327,8 @@ def _stages(part: str) -> list[list[str]]: _close(stages[-1], reads) reads = [] stages.append([]) - elif token.lstrip("&").startswith(">") or token == "<": - target = token.lstrip("&")[:1] + elif token.lstrip("&").startswith(">") or token in ("<", "<<<"): + target = token.lstrip("&")[:3] # ``<<<`` is a string, not a file: only a lone ``<`` reads its target elif token not in _SEPARATORS and token not in _LEX_SKIP: stages[-1].append(_restore(token)) _close(stages[-1], reads) @@ -339,8 +339,13 @@ def _views(words: list[str]) -> Iterator[Stage]: """The command a simple command runs: its own program, then the command each wrapper (``time``, ``sudo``...) runs.""" i = 0 while i < len(words): - while i < len(words) and (_ASSIGN.match(words[i]) or words[i] in _KEYWORDS): # NAME=value, ``then``: the command is next - i += 1 + while i < len(words): + if _ASSIGN.match(words[i]) or words[i] in _KEYWORDS: # NAME=value, ``then``: the command is next + i += 1 + elif words[i].startswith("$") and words[i + 1:i + 2] == ["="]: # PowerShell ``$x = ``: the same + i += 2 + else: + break if i >= len(words): return yield _head(words[i], words[i + 1:]) @@ -386,6 +391,8 @@ def _python_run(args: tuple[str, ...]) -> tuple[str, tuple[str, ...]] | None: return word, args[i + 1:] if word.startswith("-c"): # ``-cimport os``: the code is in the same word return "-c", (word[2:],) + args[i + 1:] + if re.fullmatch(r"-[BdEhiIOPqRsSuvx]+[cm]", word): # ``-Ic code``, ``-Im pip``: flags, then the code or module + return "-" + word[-1], args[i + 1:] if word == "--" or not word.startswith("-"): return None i += 2 if word in _PY_VALUE_OPTIONS else 1 diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index 2d1627e93..a63385f19 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -354,3 +354,23 @@ def test_a_search_option_value_is_no_operand_so_the_file_after_it_is_read(): def test_a_glob_or_type_value_of_a_search_is_no_expression_or_file(command): events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) assert all(e["kind"] != "probe" for e in events) + + +def test_a_powershell_assignment_is_skipped_so_the_command_after_it_is_the_probe(): + command = "$cpu = Get-CimInstance Win32_Processor; $cpu.NumberOfCores" + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z", name="PowerShell"), _claude_result("a", "8")) + assert [e["target"] for e in events if e["kind"] == "probe"] == ["cpu"] + + +@pytest.mark.parametrize(("command", "target"), [ + ("python -Ic 'import os; print(os.cpu_count())'", "cpu"), + ("python -Im pip list", "packages"), +]) +def test_python_flags_with_c_or_m_in_one_word_are_read_as_the_code_or_the_module(command, target): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["target"] for e in events if e["kind"] == "probe"] == [target] + + +def test_a_here_string_is_no_file_the_command_reads(): + events = _claude(_claude_call("a", "cat <<< /proc/cpuinfo", "2026-01-01T00:00:05Z"), _claude_result("a")) + assert all(e["kind"] != "probe" for e in events) From a3672c145a670c5c128b3c9f91ee58bca98ba429 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 13:47:51 -0800 Subject: [PATCH 12/15] Address review: a probe matcher that raises keeps the cell A matcher that raises (a deep chain hits RecursionError in ast.unparse) now costs the cell its probe events only: the grade, answer.txt and events.jsonl are written as before, and trace_errors notes the failure. Each cell's meta.json records the effective probe patterns per target. Thread: PRRT_kwDOVBg7g864ZUdO Thread: PRRT_kwDOVBg7g864ZUdg Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/runner.py | 13 ++++++++++--- dokimi/tests/test_repo_run.py | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index 68ba9d645..e47f1ce7d 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -72,7 +72,7 @@ from .bases import MCP_CONFIG, NOT_UTF8, mcp_config_error from .events import convert from . import run_context -from .probes import probes_for, with_probes +from .probes import effective_patterns, probes_for, with_probes from .uptake import arm_uptake from .useful import first_useful from .grading import EvalContext, failed_outcome, outcome, run_entry @@ -261,11 +261,17 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, uptake_of = (added.get("added") or [], added.get("mcp_servers") or []) events = None # the trace's events once it parsed; none: the uptake is unmeasured useful_events = None # the same events when no line of the trace was dropped: a dropped line may hold the hit + probe_errors: list[str] = [] # a probe matcher that raises costs the probe events only, and is noted + meta["probes"] = effective_patterns(ctx.manifest.probes, opts.probes) # the patterns the cell ran with try: raw = (rd / "agent.stdout").read_text(encoding="utf-8", errors="replace") transcript = convert(opts.profile.trace, raw, WORKSPACE) # Probe events go to events.jsonl only: the counters, first_useful and skill usage score the trace's events - probed = with_probes(transcript.events, probes_for(ctx.manifest.probes, opts.probes)) + try: + probed = with_probes(transcript.events, probes_for(ctx.manifest.probes, opts.probes)) + except Exception as exc: # noqa: BLE001 # the events go on without probes: the grade must not depend on them + probed = transcript.events + probe_errors.append(f"probe matching failed ({type(exc).__name__}): {str(exc)[:200]}") (rd / "events.jsonl").write_text(transcript.events_jsonl(probed), encoding="utf-8") failed = transcript.is_error or not result.ok # a crashed review is missing, not clean answer = convert_answer(opts.profile.answer_format, transcript.answer, WORKSPACE, failed=failed, @@ -288,7 +294,8 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, cache_write_tokens=transcript.cache_write_tokens, usage_scope=transcript.usage_scope, first_prompt_tokens=transcript.first_prompt_tokens, subagent_calls=transcript.subagent_calls, plugins=transcript.plugins, - over_budget=cost is not None and cost > caps["budget_usd"], trace_errors=transcript.errors[:5]) + over_budget=cost is not None and cost > caps["budget_usd"], + trace_errors=(transcript.errors + probe_errors)[:5]) run_context.save(rd, run_context.build(opts.profile, model=opts.model, harness_version=opts.harness_version, argv=argv, transcript=transcript, config_sha256=config_sha), agent["trace_errors"]) diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index 78c0f3f58..6044548a2 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -998,3 +998,23 @@ def bad_recall(run_dir): # the grader of one cell writes `bad` as its score met args = build_parser().parse_args(["repo", "compare", str(batch), "--baseline", "guidance:none"]) args.func(args) assert "review_add" in capsys.readouterr().out + + +def test_a_probe_matcher_that_raises_keeps_the_cell_graded_with_its_events(setup, monkeypatch): + # The matcher runs after the agent: a failure costs the probe events, not the grade, answer or events.jsonl. + def refuse(*args, **kwargs): + raise RecursionError("a chain too deep") + + monkeypatch.setattr("dokimi.repos.runner.with_probes", refuse) + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + cell = setup["out"] / "b1/fix_add__guidance-raw__m-1__r1" + rec = json.loads((cell / "result.json").read_text()) + assert rec["grade"]["status"] == "graded" and rec["grade"]["success"] is True + assert (cell / "events.jsonl").exists() and (cell / "answer.txt").exists() + assert any("probe matching failed" in e and "a chain too deep" in e for e in rec["agent"]["trace_errors"]) + + +def test_the_cell_records_the_probe_patterns_it_ran_with(setup): + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + meta = json.loads((setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/meta.json").read_text()) + assert "nproc" in meta["probes"]["cpu"] and meta["probes"]["packages"] == ["pip list", "pip show", "pip freeze", "conda list"] From 77061ba60a9d568b575c41b0c951741474c51b01 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 13:47:51 -0800 Subject: [PATCH 13/15] Address review: launchers, env and pip3 patterns match the probe they run env runs the program after its options and assignments, and is a probe only when no program follows. timeout, xargs, conda run, uv run, bash -o and pwsh -Command are read through, and a pattern spelled pip3 or python -m pip is matched as pip. The contract lists what is still not matched. Thread: PRRT_kwDOVBg7g864ZUdS Thread: PRRT_kwDOVBg7g864ZUdY Thread: PRRT_kwDOVBg7g864ZUdo Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 7 ++- dokimi/dokimi/repos/probes.py | 57 ++++++++++++++++++++++--- dokimi/tests/test_repo_probes.py | 39 +++++++++++++++++ 3 files changed, 95 insertions(+), 8 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index f57840298..3550519c6 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -546,8 +546,11 @@ reads the upstream base (`base_sha`) and the workspace start commit `grep`: no probe), on a file a reading command (`cat`, `head`, `grep`, ...) names for a path pattern, and on the call in the code of a `python -c` one-liner for `py:NAME`. A `$( )` substitution or a `bash -c` script counts as a command the agent ran. Not matched: - a Python heredoc, a script the agent runs from a file, and a pattern with a `|` that is - not a pipeline of the same commands. + a Python heredoc, a script the agent runs from a file, a pattern with a `|` that is + not a pipeline of the same commands, and a launcher the wrapper lists do not know + (`docker exec`, `nix develop -c`). `env`, `timeout`, `xargs`, `conda run` and `uv run` + run the program after them; `env` alone is its own probe. Each cell's `meta.json` + records the patterns per target (`probes`) that it ran with. A manifest's `probes` (§2) replaces a target's defaults, and `repo run --probes FILE` (YAML or JSON, the same mapping) replaces them again; nothing else changes the events, and the `process_metrics` counters do not count probes. diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py index 245ba7ace..9ff5f251f 100644 --- a/dokimi/dokimi/repos/probes.py +++ b/dokimi/dokimi/repos/probes.py @@ -50,9 +50,16 @@ _READERS = frozenset({"cat", "head", "tail", "less", "more", "nl", "sed", "grep", "egrep", "fgrep", "awk", "tac", "strings", "wc", "rg"}) _SEARCHERS = frozenset({"grep", "egrep", "fgrep", "rg", "sed", "awk"}) # the first operand is an expression, not a file -_WRAPPERS = frozenset({"time", "nohup", "sudo", "command", "exec", "builtin", "nice"}) # each runs the command after it +_WRAPPERS = frozenset({"time", "nohup", "sudo", "command", "exec", "builtin", "nice", "timeout", "xargs"}) # each runs +# the command after it _KEYWORDS = frozenset({"if", "then", "else", "elif", "do", "while", "until", "!"}) # shell keywords before a command -_OPTION_OPERANDS = frozenset({("nice", "-n"), ("sudo", "-u")}) # wrapper options that take a value word +_OPTION_OPERANDS = frozenset({("nice", "-n"), ("sudo", "-u"), ("timeout", "-s"), ("timeout", "-k"), + ("xargs", "-n"), ("xargs", "-I"), ("xargs", "-L"), ("xargs", "-P"), ("xargs", "-d"), + ("xargs", "-E"), ("xargs", "-s"), ("xargs", "-a")}) # wrapper options that take a value +_ENV_VALUES = frozenset({"-u", "--unset", "-C", "--chdir", "-S", "--split-string"}) # env options with a value word +_RUN_VALUES = frozenset({"-n", "--name", "-p", "--prefix", "--python", "--with", "--directory", "--project", + "--env-file"}) # the options of ``conda run`` and ``uv run`` that take a value word +_PWSH = frozenset({"pwsh", "powershell"}) # PowerShell: its script follows ``-Command`` (``-c``), any case _GREP_VALUES = frozenset({"-m", "-A", "-B", "-C", "-d", "-D", "--max-count", "--after-context", "--before-context", "--context", "--label", "--binary-files", "--directories", "--devices"}) _RG_VALUES = frozenset({"-g", "--glob", "--iglob", "-t", "--type", "-T", "--type-not", "--type-add", "-m", "--max-count", @@ -118,6 +125,14 @@ def load_probe_file(path: str) -> dict[str, tuple[str, ...]]: return validate_config(raw, f"--probes {path}") +def effective_patterns(*layers: Mapping[str, Sequence[str]] | None) -> dict[str, list[str]]: + """Each target's patterns once the layers are applied, as ``probes_for`` applies them (for the cell's record).""" + config: dict[str, Sequence[str]] = dict(DEFAULT_PROBES) + for layer in layers: + config.update(layer or {}) + return {target: [text.strip() for text in config.get(target, ())] for target in TARGETS} + + def probes_for(*layers: Mapping[str, Sequence[str]] | None) -> tuple[Probe, ...]: """The compiled patterns of the defaults with each layer applied in turn: a target a layer names is replaced.""" config: dict[str, Sequence[str]] = dict(DEFAULT_PROBES) @@ -170,7 +185,7 @@ def _stage(part: str, text: str) -> Stage: raise ValueError(f"{text!r}: {exc}") from exc if not words: raise ValueError(f"{text!r}: a pattern names a program") - return _program(words[0]), tuple(words[1:]) + return _head(words[0], words[1:]) # ``pip3 list`` and ``python -m pip list`` are ``pip list``, as commands are def _program(word: str) -> str: @@ -348,6 +363,10 @@ def _views(words: list[str]) -> Iterator[Stage]: break if i >= len(words): return + launched = _launched(words, i) + if launched is not None: # ``env FOO=1 nproc``: the program is nproc, and env is no probe of its own + i = launched + continue yield _head(words[i], words[i + 1:]) wrapper = _program(words[i]) if wrapper not in _WRAPPERS: @@ -357,6 +376,24 @@ def _views(words: list[str]) -> Iterator[Stage]: if wrapper == "command" and words[i] in ("-v", "-V"): return # ``command -v nproc`` looks the program up, it does not run it i += 2 if (wrapper, words[i]) in _OPTION_OPERANDS else 1 + if wrapper == "timeout" and i < len(words): # ``timeout 5 nproc``: the duration is no program + i += 1 + + +def _launched(words: list[str], i: int) -> int | None: + """The index of the program that ``env`` or ``conda run`` / ``uv run`` at ``i`` runs, or None when no program + follows its options and assignments.""" + program, j = _program(words[i]), i + 1 + if program == "env": + while j < len(words) and (words[j].startswith("-") or _ASSIGN.match(words[j])): + j += 2 if words[j] in _ENV_VALUES else 1 + elif program in ("conda", "uv") and words[i + 1:i + 2] == ["run"]: + j = i + 2 + while j < len(words) and words[j].startswith("-"): + j += 2 if words[j] in _RUN_VALUES else 1 + else: + return None + return j if j < len(words) else None def _head(word: str, args: list[str]) -> Stage: @@ -371,13 +408,21 @@ def _head(word: str, args: list[str]) -> Stage: def _shell_script(prog: str, args: tuple[str, ...]) -> str | None: """The script a shell runs with ``-c`` (``bash --noprofile -c nproc``), or None.""" - if prog not in _SHELLS: + if prog not in _SHELLS | _PWSH: return None - for i, word in enumerate(args): - if _SHELL_C.fullmatch(word): + i = 0 + while i < len(args): + word = args[i] + if (prog in _PWSH and word.casefold() in ("-c", "-command")) or (prog in _SHELLS and _SHELL_C.fullmatch(word)): + if prog in _PWSH: # ``pwsh -Command a b``: the words after -Command are the script, joined + return " ".join(args[i + 1:]) or None return args[i + 1] if i + 1 < len(args) else None + if prog in _SHELLS and word in ("-o", "+o"): # ``bash -o pipefail -c nproc``: the option's value is no word + i += 2 + continue if not word.startswith("-"): return None + i += 1 return None diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index a63385f19..030dfb270 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -374,3 +374,42 @@ def test_python_flags_with_c_or_m_in_one_word_are_read_as_the_code_or_the_module def test_a_here_string_is_no_file_the_command_reads(): events = _claude(_claude_call("a", "cat <<< /proc/cpuinfo", "2026-01-01T00:00:05Z"), _claude_result("a")) assert all(e["kind"] != "probe" for e in events) + + +def _matched(command: str, layer: dict | None = None) -> list[str]: + raw = "\n".join(json.dumps(x) for x in ( + {"type": "system", "subtype": "init", "timestamp": "2026-01-01T00:00:00Z"}, + _claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a"))) + events = with_probes(convert("claude-stream-json", raw, WS).events, probes_for(layer)) + return [e["pattern"] for e in events if e["kind"] == "probe"] + + +@pytest.mark.parametrize(("command", "pattern"), [ + ("timeout 5 nproc", "nproc"), + ("echo | xargs -n 1 nproc", "nproc"), + ("uv run python -c 'import os; print(os.cpu_count())'", "py:os.cpu_count"), + ("bash -o pipefail -c nproc", "nproc"), + ("pwsh -Command Get-CimInstance Win32_Processor", "Get-CimInstance Win32_Processor"), + ("conda run -n x pip list", "pip list"), +]) +def test_a_common_launcher_does_not_hide_the_probe_it_runs(command, pattern): + assert _matched(command) == [pattern] + + +@pytest.mark.parametrize(("command", "pattern"), [ + ("env PYTHONPATH=. pytest -q", None), + ("env FOO=1 nproc", "nproc"), + ("env -u HOME nproc", "nproc"), + ("env", "env"), +]) +def test_env_runs_the_program_after_its_assignments_and_is_a_probe_only_alone(command, pattern): + assert _matched(command) == ([] if pattern is None else [pattern]) + + +def test_a_conda_run_is_no_conda_list_probe(): + assert "conda list" not in _matched("conda run -n x pip list") + + +def test_a_pattern_spelled_as_a_pip3_or_python_m_pip_command_matches_it(): + assert _matched("pip3 list", {"packages": ["pip3 list"]}) == ["pip3 list"] + assert _matched("python -m pip list", {"packages": ["python -m pip list"]}) == ["python -m pip list"] From 39c94edb818baa0b5815e88adfa005e26727051c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 14:24:21 -0800 Subject: [PATCH 14/15] Self-review: a launcher pattern matches the program it runs A pattern that starts with env, conda run or uv run is read through the same launcher rule as a command, so "uv run pip list" matches the command "uv run pip list". PowerShell options with a value (-ExecutionPolicy) are skipped, and -com is -Command. The two run tests move to tests/test_repo_run_probes.py to keep test_repo_run.py under the file cap; _views takes its wrapper arguments from a helper (complexity cap). The golden meta keys gain "probes". Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 2 +- dokimi/dokimi/repos/probes.py | 31 ++++++++++++++++++------ dokimi/tests/test_batch_ledger_golden.py | 1 + dokimi/tests/test_repo_probes.py | 9 +++++++ dokimi/tests/test_repo_run.py | 20 --------------- dokimi/tests/test_repo_run_probes.py | 30 +++++++++++++++++++++++ 6 files changed, 65 insertions(+), 28 deletions(-) create mode 100644 dokimi/tests/test_repo_run_probes.py diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index b03eee8c0..4f872b572 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -551,7 +551,7 @@ The line format, its version and the promise that comes with it: [events-format. a Python heredoc, a script the agent runs from a file, a pattern with a `|` that is not a pipeline of the same commands, and a launcher the wrapper lists do not know (`docker exec`, `nix develop -c`). `env`, `timeout`, `xargs`, `conda run` and `uv run` - run the program after them; `env` alone is its own probe. Each cell's `meta.json` + run the program after them; `env` with no program after its options is its own probe (`env -S` is not read). Each cell's `meta.json` records the patterns per target (`probes`) that it ran with. A manifest's `probes` (§2) replaces a target's defaults, and `repo run --probes FILE` (YAML or JSON, the same mapping) replaces them again; nothing else changes the events, diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py index 9ff5f251f..cc8cc6591 100644 --- a/dokimi/dokimi/repos/probes.py +++ b/dokimi/dokimi/repos/probes.py @@ -60,6 +60,8 @@ _RUN_VALUES = frozenset({"-n", "--name", "-p", "--prefix", "--python", "--with", "--directory", "--project", "--env-file"}) # the options of ``conda run`` and ``uv run`` that take a value word _PWSH = frozenset({"pwsh", "powershell"}) # PowerShell: its script follows ``-Command`` (``-c``), any case +_PWSH_VALUES = frozenset({"-executionpolicy", "-ep", "-workingdirectory", "-wd", "-settingsfile", "-configurationname", + "-inputformat", "-outputformat"}) # PowerShell options whose next word is their value _GREP_VALUES = frozenset({"-m", "-A", "-B", "-C", "-d", "-D", "--max-count", "--after-context", "--before-context", "--context", "--label", "--binary-files", "--directories", "--devices"}) _RG_VALUES = frozenset({"-g", "--glob", "--iglob", "-t", "--type", "-T", "--type-not", "--type-add", "-m", "--max-count", @@ -185,6 +187,9 @@ def _stage(part: str, text: str) -> Stage: raise ValueError(f"{text!r}: {exc}") from exc if not words: raise ValueError(f"{text!r}: a pattern names a program") + launched = _launched(words, 0) # ``uv run pip list`` is the pattern ``pip list``: the program the command runs + if launched is not None: + words = words[launched:] return _head(words[0], words[1:]) # ``pip3 list`` and ``python -m pip list`` are ``pip list``, as commands are @@ -371,13 +376,22 @@ def _views(words: list[str]) -> Iterator[Stage]: wrapper = _program(words[i]) if wrapper not in _WRAPPERS: return + after = _past_wrapper_args(words, i + 1, wrapper) + if after is None: + return # ``command -v nproc`` looks the program up, it does not run it + i = after + + +def _past_wrapper_args(words: list[str], i: int, wrapper: str) -> int | None: + """The index of the program a wrapper runs: past its options (``sudo -E``, ``nice -n 5``) and, for ``timeout``, its + duration; None when the wrapper only looks a program up (``command -v``).""" + while i < len(words) and words[i].startswith("-"): + if wrapper == "command" and words[i] in ("-v", "-V"): + return None + i += 2 if (wrapper, words[i]) in _OPTION_OPERANDS else 1 + if wrapper == "timeout" and i < len(words): # ``timeout 5 nproc``: the duration is no program i += 1 - while i < len(words) and words[i].startswith("-"): # ``sudo -E``, ``nice -n 5``: options, then the program - if wrapper == "command" and words[i] in ("-v", "-V"): - return # ``command -v nproc`` looks the program up, it does not run it - i += 2 if (wrapper, words[i]) in _OPTION_OPERANDS else 1 - if wrapper == "timeout" and i < len(words): # ``timeout 5 nproc``: the duration is no program - i += 1 + return i def _launched(words: list[str], i: int) -> int | None: @@ -413,13 +427,16 @@ def _shell_script(prog: str, args: tuple[str, ...]) -> str | None: i = 0 while i < len(args): word = args[i] - if (prog in _PWSH and word.casefold() in ("-c", "-command")) or (prog in _SHELLS and _SHELL_C.fullmatch(word)): + if (prog in _PWSH and len(word) > 1 and "-command".startswith(word.casefold())) or (prog in _SHELLS and _SHELL_C.fullmatch(word)): if prog in _PWSH: # ``pwsh -Command a b``: the words after -Command are the script, joined return " ".join(args[i + 1:]) or None return args[i + 1] if i + 1 < len(args) else None if prog in _SHELLS and word in ("-o", "+o"): # ``bash -o pipefail -c nproc``: the option's value is no word i += 2 continue + if prog in _PWSH and word.casefold() in _PWSH_VALUES: # ``pwsh -ExecutionPolicy Bypass -Command ...`` + i += 2 + continue if not word.startswith("-"): return None i += 1 diff --git a/dokimi/tests/test_batch_ledger_golden.py b/dokimi/tests/test_batch_ledger_golden.py index 7b0f823ce..74873918f 100644 --- a/dokimi/tests/test_batch_ledger_golden.py +++ b/dokimi/tests/test_batch_ledger_golden.py @@ -95,6 +95,7 @@ "model", "model_resolved", "patches", + "probes", "rep", "start_sha", "task", diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index 030dfb270..99f4d8935 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -413,3 +413,12 @@ def test_a_conda_run_is_no_conda_list_probe(): def test_a_pattern_spelled_as_a_pip3_or_python_m_pip_command_matches_it(): assert _matched("pip3 list", {"packages": ["pip3 list"]}) == ["pip3 list"] assert _matched("python -m pip list", {"packages": ["python -m pip list"]}) == ["python -m pip list"] + + +def test_a_pattern_that_starts_with_a_launcher_matches_the_program_it_runs(): + assert _matched("uv run pip list", {"packages": ["uv run pip list"]}) == ["uv run pip list"] + + +def test_a_powershell_command_after_its_options_is_a_probe(): + cmd = "pwsh -NoProfile -ExecutionPolicy Bypass -com Get-CimInstance Win32_Processor" + assert _matched(cmd) == ["Get-CimInstance Win32_Processor"] diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index b32b7f6c6..f816d6b01 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -998,23 +998,3 @@ def bad_recall(run_dir): # the grader of one cell writes `bad` as its score met args = build_parser().parse_args(["repo", "compare", str(batch), "--baseline", "guidance:none"]) args.func(args) assert "review_add" in capsys.readouterr().out - - -def test_a_probe_matcher_that_raises_keeps_the_cell_graded_with_its_events(setup, monkeypatch): - # The matcher runs after the agent: a failure costs the probe events, not the grade, answer or events.jsonl. - def refuse(*args, **kwargs): - raise RecursionError("a chain too deep") - - monkeypatch.setattr("dokimi.repos.runner.with_probes", refuse) - _run(setup, "--arm", "guidance:raw", "--task", "fix_add") - cell = setup["out"] / "b1/fix_add__guidance-raw__m-1__r1" - rec = json.loads((cell / "result.json").read_text()) - assert rec["grade"]["status"] == "graded" and rec["grade"]["success"] is True - assert (cell / "events.jsonl").exists() and (cell / "answer.txt").exists() - assert any("probe matching failed" in e and "a chain too deep" in e for e in rec["agent"]["trace_errors"]) - - -def test_the_cell_records_the_probe_patterns_it_ran_with(setup): - _run(setup, "--arm", "guidance:raw", "--task", "fix_add") - meta = json.loads((setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/meta.json").read_text()) - assert "nproc" in meta["probes"]["cpu"] and meta["probes"]["packages"] == ["pip list", "pip show", "pip freeze", "conda list"] diff --git a/dokimi/tests/test_repo_run_probes.py b/dokimi/tests/test_repo_run_probes.py new file mode 100644 index 000000000..11aea747d --- /dev/null +++ b/dokimi/tests/test_repo_run_probes.py @@ -0,0 +1,30 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The probe events of `repo run`: a failing matcher and the patterns a cell records (contract §2, §4.1).""" + +from __future__ import annotations + +import json + +from test_repo_run import _run, setup # noqa: F401 # the run fixture, with its helpers + + +def test_a_probe_matcher_that_raises_keeps_the_cell_graded_with_its_events(setup, monkeypatch): # noqa: F811 + # The matcher runs after the agent: a failure costs the probe events, not the grade, answer or events.jsonl. + def refuse(*args, **kwargs): + raise RecursionError("a chain too deep") + + monkeypatch.setattr("dokimi.repos.runner.with_probes", refuse) + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + cell = setup["out"] / "b1/fix_add__guidance-raw__m-1__r1" + rec = json.loads((cell / "result.json").read_text()) + assert rec["grade"]["status"] == "graded" and rec["grade"]["success"] is True + assert (cell / "events.jsonl").exists() and (cell / "answer.txt").exists() + assert any("probe matching failed" in e and "a chain too deep" in e for e in rec["agent"]["trace_errors"]) + + +def test_the_cell_records_the_probe_patterns_it_ran_with(setup): # noqa: F811 + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + meta = json.loads((setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/meta.json").read_text()) + assert "nproc" in meta["probes"]["cpu"] and meta["probes"]["packages"] == ["pip list", "pip show", "pip freeze", "conda list"] From af6d2a811f30b406b7f960abad4da0cd6dc15a14 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 15:14:56 -0800 Subject: [PATCH 15/15] Address review: default cpu probe matches bare sysctl The default `sysctl -a | grep machdep.cpu` matched only that grep word, so `sysctl -n machdep.cpu.brand_string`, `sysctl -n hw.ncpu` and `sysctl hw.memsize` gave no probe. The default is now `sysctl`, which reads kernel facts in almost every use. Thread: PRRT_kwDOVBg7g864ZUdu Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 2 +- dokimi/dokimi/repos/probes.py | 2 +- dokimi/tests/test_repo_probes.py | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 4f872b572..2de7f159d 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -539,7 +539,7 @@ The line format, its version and the promise that comes with it: [events-format. `target` is one of `cpu`, `gpu`, `os`, `memory`, `packages`, `env`, `network`; `pattern` is the configured pattern it matched; `text`, `rc` and `agent` are the command's. The defaults (`dokimi/repos/probes.py`, `DEFAULT_PROBES`): `cpu` `lscpu`, - `/proc/cpuinfo`, `nproc`, `sysctl -a | grep machdep.cpu`, `Get-CimInstance Win32_Processor`, + `/proc/cpuinfo`, `nproc`, `sysctl`, `Get-CimInstance Win32_Processor`, `py:os.cpu_count`, `py:platform.processor`; `gpu` `nvidia-smi`, `rocm-smi`, `py:torch.cuda.is_available`; `os` `uname`, `/etc/os-release`, `sw_vers`, `systeminfo`; `memory` `free`, `/proc/meminfo`; `packages` `pip list`, `pip show`, `pip freeze`, diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py index cc8cc6591..c59340c3a 100644 --- a/dokimi/dokimi/repos/probes.py +++ b/dokimi/dokimi/repos/probes.py @@ -37,7 +37,7 @@ TARGETS = ("cpu", "gpu", "os", "memory", "packages", "env", "network") DEFAULT_PROBES: dict[str, tuple[str, ...]] = { - "cpu": ("lscpu", "/proc/cpuinfo", "nproc", "sysctl -a | grep machdep.cpu", "Get-CimInstance Win32_Processor", + "cpu": ("lscpu", "/proc/cpuinfo", "nproc", "sysctl", "Get-CimInstance Win32_Processor", "py:os.cpu_count", "py:platform.processor"), "gpu": ("nvidia-smi", "rocm-smi", "py:torch.cuda.is_available"), "os": ("uname", "/etc/os-release", "sw_vers", "systeminfo"), diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index 99f4d8935..53ae243a6 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -103,7 +103,10 @@ def test_a_probe_in_a_substitution_is_run_by_the_command(): ("conda list -n base", "conda list"), ("printenv PATH", "printenv"), ("env", "env"), - ("sysctl -a | grep machdep.cpu", "sysctl -a | grep machdep.cpu"), + ("sysctl -a | grep machdep.cpu", "sysctl"), + ("sysctl -n machdep.cpu.brand_string", "sysctl"), + ("sysctl -n hw.ncpu", "sysctl"), + ("sysctl hw.memsize", "sysctl"), ]) def test_default_command_patterns_match_the_program_that_runs(command, pattern): events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) @@ -162,7 +165,7 @@ def test_a_command_in_the_body_of_a_conditional_or_loop_is_a_probe(command): @pytest.mark.parametrize("command", [ "echo '$(nproc)'", - "sysctl -a '|' grep machdep.cpu", + "echo 'sysctl -n hw.ncpu'", ]) def test_quoted_text_is_no_probe(command): events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a"))